[ TU ] 5 Xl [ R " UR
SUR - SS9U l [ R " UR UR
SS9U l [ UR U l g )N Fbias) super__init__confignnLinear hidden_sizeintermediate_size gate_up_proj down_projr
hidden_act
activation_fnselfr& __class__s \/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/glm/modeling_glm.pyr% GlmMLP.__init__/ sn
IIf&8&8!f>V>V:V]b c6#;#;V=O=OV[\ #F$5$5 6
hidden_statesreturnc U R U5 nUR SSS9u p2X R U5 - nU R U5 $ )Nr! dim)r+ chunkr. r, )r0 r5 up_statesgates r2 forwardGlmMLP.forward7 sH %%m4 #//!/4 2 24 88 ~~i((r4 )r. r& r, r+ )
__name__
__module__ __qualname____firstlineno__r% torch FloatTensorr> __static_attributes__
__classcell__r1 s @r2 r r . s, 7)U%6%6 )5;L;L ) )r4 r r5 n_repr6 c U R u p#pEUS:X a U $ U SS2SS2SSS2SS24 R X#XU5 n U R X#U- XE5 $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r N)shapeexpandreshape)r5 rI batchnum_key_value_headsslenhead_dims r2 repeat_kvrR @ s_
2?1D1D.E z !!Qa"23::5W\dlmM e(CT TTr4 modulequerykeyvalueattention_maskscalingdropoutkwargsc @ [ X R 5 n[ X0R 5 n [ R " XR SS5 5 U- n
Ub" US S 2S S 2S S 2S UR
S 24 n X- n
[ R R U
S[ R S9R UR 5 n
[ R R XU R S9n
[ R " X5 n U R SS5 R 5 n X4$ )Nr! r r8 )r: dtype)ptrainingr )rR num_key_value_groupsrD matmul transposerK r'
functionalsoftmaxfloat32tor] rY r_
contiguous)
rS rT rU rV rW rX rY rZ
key_states value_states attn_weights causal_mask attn_outputs
r2 eager_attention_forwardrm L s 3 ; ; [ TU ] 5 Xl X l [ USUR
UR - 5 U l UR UR - U l U R S- U l
UR U l SU l [ R " UR
UR U R - UR S9U l [ R " UR
UR U R - UR S9U l [ R " UR
UR U R - UR S9U l [ R " UR U R - UR
SS9U l g )NrQ g Tr" F)r$ r% r& r getattrr) num_attention_headsrQ rO r` rX attention_dropout is_causalr' r( attention_biasq_projk_projv_projo_projr0 r& r r1 s r2 r% GlmAttention.__init__ s@
"
F4F4F&JdJd4de
$*$>$>&B\B\$\!}}d* !'!9!9 ii : :T]] JQWQfQf
ii : :T]] JQWQfQf
ii : :T]] JQWQfQf
ii : :T]] JFL^L^ejk r4 r5 position_embeddingsrW past_key_valuecache_positionrZ r6 c 4 UR S S n/ UQSPU R P7nU R U5 R U5 R SS5 n U R U5 R U5 R SS5 n
U R
U5 R U5 R SS5 n Uu p[ XX5 u pUb$ XUS.nUR XU R U5 u p[ nU R R S:w a [ U R R nU" U U U
U U4U R ( d SO U R U R S.UD6u nnUR" " / UQSP76 R% 5 nU R' U5 nUU4$ )Nr8 r r! )r| r{ r eager )rY rX )rK rQ r viewrb r r r updater rm r& _attn_implementationr r_ r rX rM rg r )r0 r5 r rW r r rZ input_shape hidden_shape query_statesrh ri r{ r| cache_kwargsattention_interfacerl rj s r2 r> GlmAttention.forward s $))#2. 88b8$--8 {{=166|DNNqRST [[ /44\BLLQPQR
{{=166|DNNqRST  RU#[ %#&nUL'5'<' rF rG rH s @r2 r r s Gly lXc] l l4 +/59
)) ||)) #5<< #= >)) ! . ))
! )) !!1!12
)) +,))
u||U\\) *)) ))r4 r RMSNormc 8 ^ \ rS rSrSU 4S jjrS rS rSrU =r$ )
GlmRMSNorm c > [ TU ] 5 [ R " [ R
" U5 5 U l X l g)z)
GlmRMSNorm is equivalent to T5LayerNorm
N)r$ r% r' ParameterrD onesweightvariance_epsilon)r0 r) epsr1 s r2 r% GlmRMSNorm.__init__ s/ ll5::k#:; # r4 c UR nUR [ R 5 nUR S5 R SSS9nU[ R " X0R - 5 - nU R UR U5 - $ )Nr! r8 T)keepdim) r] rf rD re powmeanrsqrtr r )r0 r5 input_dtypevariances r2 r> GlmRMSNorm.forward sw #)) %(( 7
$$Q',,R,>% H?T?T4T(UU
{{] - -k :::r4 c ^ [ U R R 5 SU R 3$ )Nz, eps=)r r rK r r0 s r2
extra_reprGlmRMSNorm.extra_repr s* ))*+6$2G2G1HIIr4 )r r )gư>) r@ rA rB rC r% r> r rF rG rH s @r2 r r s $;J Jr4 r c l ^ \ rS rSrSS\4U 4S jjjr\R " 5 \S 5 5 r Sr
U =r $ )GlmRotaryEmbedding r& c > [ TU ] 5 [ US5 ( aZ [ UR [
5 ( a; UR R
SUR R
S5 5 U l OSU l UR U l UR U l
Xl [ U R U l
U R U R U5 u o0l U R SUSS9 U R U l g )N rope_scaling rope_typetypedefaultinv_freqF)
persistent)r$ r% hasattr
isinstancer dictgetr max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr& r rope_init_fnattention_scalingregister_bufferr original_inv_freq)r0 r& devicer r1 s r2 r% GlmRotaryEmbedding.__init__ s
6> * *z&:M:Mt/T/T # 0 0 4 4[&BUBUBYBYZ`Ba bDN &DN"("@"@ $*$B$B! / ?+/+<+ GlmRotaryEmbedding.forward sR !MM$4- 8 > > @ G G HZHZ[\H]_acd e h hijiqiq r ,QaZ 8 > > @ '1!((--'E'E!((--[`J`ahhmmfk
^^ U
C&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d 4 44C'')d 4 44C D vvAGGv$cff177f&;;;
D
Cs
$BF
F.)r r& r r r r r r ) r@ rA rB rC r r% rD no_gradr r> rF rG rH s @r2 r r s6 /y / /" ]]_ < [ TU ] 5 UR U l [ XS9U l [ U5 U l [ UR UR S9U l [ UR UR S9U l
g )N)r& r r ) r$ r% r) r self_attnr mlpr rms_norm_epsinput_layernormpost_attention_layernormr s r2 r% GlmDecoderLayer.__init__ si
!--%VI&> )&*<*<&BUBU V (263E3E6K^K^(_%r4 r5 rW r} r use_cacher r rZ r6 c Un U R U5 nU R " SUUUUUUUS.UD6u pX- nUn U R U5 nU R U5 nX- nU$ )N)r5 rW r} r r r r )r r r r ) r0 r5 rW r} r r r r rZ residual_s r2 r> GlmDecoderLayer.forward s ! ,,];
>>
')%) ) 3
!0
! 55mD
/
0
r4 )r) r r r r )NNNFNN)r@ rA rB rC r r r% rD r r r r boolr r r r> rF rG rH s @r2 r r s `y `S ` 2637*.$)59KO || ! . u / /0
! D>
!!1!12 &eELL%,,,F&G H +,
u|| r4 r c R \ rS rSr% \\S' SrSrS/rS/r Sr
Sr Sr Sr
Sr\\S.rSrg )
GlmPreTrainedModeli? r& modelTr past_key_values)r5
attentionsr N)r@ rA rB rC r __annotations__base_model_prefix supports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr r _can_record_outputsrF r r4 r2 r r ? sQ &*#*+#4"5 N !"& ("r4 r c ^ \ rS rSrS\4U 4S jjr\\ SS\\ R S\\ R S\\ R S\\ S\\ R S \\ R S
\\ S \\ S \4S
jj5 5 rSrU =r$ )GlmModeliR r& c > [ TU ] U5 UR U l UR U l [
R " UR UR U R 5 U l [
R " [ UR 5 Vs/ sH n[ X5 PM sn5 U l
[ UR UR S9U l [# US9U l SU l U R) 5 g s snf )Nr )r& F)r$ r% pad_token_id padding_idx
vocab_sizer' Embeddingr) embed_tokens
ModuleListrangenum_hidden_layersr layersr r normr
rotary_embgradient_checkpointing post_initr s r2 r% GlmModel.__init__T s
!.. ++ LL):):F input_idsrW r} r
inputs_embedsr r rZ r6 c
8 US L US L- ( a [ S5 eUc U R U5 nU( a
Uc
[ 5 nUcD Ub UR 5 OSn [ R
" XUR S - UR S9nUc UR S5 n[ U R UUUUUS9n
Un U R X5 n U R S U R R H n
U
" U 4U
UUUU S.UD6n M U R U 5 n [ U US9$ )Nz:You must specify exactly one of input_ids or inputs_embedsr r )r )r& input_embedsrW r r r} )rW r} r r r )last_hidden_stater )
ValueErrorr r get_seq_lengthrD arangerK r rv r r& r r r r r )r0 r rW r} r r r r rZ past_seen_tokensrk r5 r
decoder_layers r2 r> GlmModel.forwardd sK
-t"; < YZ Z *.*;*;I*FM 0 *nO !CRC^ = = ?de +0<< ]5H5H5K"KTaThTh,N )33A6L(;;&)) +%
&
"oom J ![[)H4;;+H+H IM ) * ) . -$7
M J -0
& + +
r4 )r r r r r
r r )NNNNNNN)r@ rA rB rC r r% r r r rD r r r rE r r r r r> rF rG rH s @r2 r
r
R s y 151537+/5959$(8
E , ,-8
! .8
u / /0 8
"% 8
1 12
8
!!1!128
D>8
+,8
!8
8
r4 r
c ~ ^ \ rS rSrS/rSS0rSS/S/40rU 4S jrS rS r \
\ SS
\ \
R S \ \
R S \ \
R S
\ \ S\ \
R" S\ \
R S\ \ S\ \
R S\\\
R 4 S\\ S\4S jj5 5 rSrU =r$ )GlmForCausalLMi zlm_head.weightlm_head colwise_repr5 logitsc > [ TU ] U5 [ U5 U l UR U l [
R " UR UR SS9U l U R 5 g )NFr" )
r$ r% r
r r r' r( r) r' r r/ s r2 r% GlmForCausalLM.__init__ sU
f%
++yy!3!3V5F5FUS
r4 c Xl g r r )r0 decoders r2 set_decoderGlmForCausalLM.set_decoder s
r4 c U R $ r r- r s r2 get_decoderGlmForCausalLM.get_decoder s zzr4 r rW r} r r labelsr r logits_to_keeprZ r6 c
~ U R " SUUUUUUUS.U
D6n U R n [ U [ 5 ( a
[ U * S5 OU n
U R U SS2U
SS24 5 nSnUb) U R " SXU R R S.U
D6n[ UUU R U R U R S9$ )ac
Example:
```python
>>> from transformers import AutoTokenizer, GlmForCausalLM
>>> model = GlmForCausalLM.from_pretrained("meta-glm/Glm-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-glm/Glm-2-7b-hf")
>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")
>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r rW r} r r r r N)r) r4 r )lossr) r r5 r r )
r r r r slicer'
loss_functionr& r r r r5 r )r0 r rW r} r r r4 r r r5 rZ outputsr5
slice_indicesr) r7 s r2 r> GlmForCausalLM.forward s @ ,0:: ,
)% +' ) ,
,
11
8B>SV8W8W~ot4]k
mA}a,? @A %%pVt{{OeOepiopD% # 3 3!// ))
r4 )r' r r ) NNNNNNNNr )r@ rA rB rC _tied_weights_keys_tp_plan_pp_planr% r/ r2 r r r rD r r r rE r r r r r r r> rF rG rH s @r2 r& r& s: *+=)H_ -z:;H 151537+/59-1$(59348
E , ,-8
! .8
u / /0 8
"% 8
1 12
8
))*8
D>8
!!1!128
c5<< /08
+,8
8
8
r4 r& c \ rS rSrSrg) GlmForSequenceClassificationi r Nr@ rA rB rC rF r r4 r2 rA rA r4 rA c \ rS rSrSrg)GlmForTokenClassificationi r NrB r r4 r2 rE rE rC r4 rE )r r
r& rA rE )r )Nr )<typingr r r rD torch.nnr' activationsr cache_utilsr r
generationr
integrationsr
masking_utilsr modeling_layersr
r r modeling_outputsr r modeling_rope_utilsr r modeling_utilsr r processing_utilsr utilsr r r
utils.genericr configuration_glmr Moduler r r rR r rm rt r r r r r r r
r& rA rE __all__r r4 r2 rW s , - , ! . ) 7 /
P K F & I I / ()RYY )$ UU\\ U# U%,, U& % II% <<%
% << %
U\\* %
% % '(%46' TA)299 A)H Y'J J (J( <