
    <h7                        S SK JrJr  S SKrS SKJr  S SKJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJr  S	S
KJrJrJr  S	SKJr  S	SKJrJrJrJrJrJr  \R:                  " \5      r " S S\5      r  " S S\5      r!S r" " S S\5      r# " S S\5      r$ " S S\5      r% " S S\5      r& " S S\5      r' " S S\5      r(/ SQr)g)    )CallableOptionalN)TransformersKwargs   )Cache)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )LlamaPreTrainedModelLlamaRMSNormeager_attention_forward)
OlmoConfig)OlmoAttentionOlmoDecoderLayerOlmoForCausalLM	OlmoModelOlmoRotaryEmbeddingapply_rotary_pos_embc                      ^  \ rS rSrSrSrSSSSSSSS.rS	/S
/4SS/S/4S/S/4S.r                   SU 4S jjrSr	U =r
$ )Olmo2Config   a  
This is the configuration class to store the configuration of a [`Olmo2Model`]. It is used to instantiate an OLMo2
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the [allenai/Olmo2-7B-1124-hf](https://huggingface.co/allenai/Olmo2-7B-1124-hf).

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.


Args:
    vocab_size (`int`, *optional*, defaults to 50304):
        Vocabulary size of the Olmo2 model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`Olmo2Model`]
    hidden_size (`int`, *optional*, defaults to 4096):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 11008):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 32):
        Number of hidden layers in the Transformer decoder.
    num_attention_heads (`int`, *optional*, defaults to 32):
        Number of attention heads for each attention layer in the Transformer decoder.
    num_key_value_heads (`int`, *optional*):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `num_attention_heads`.
    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    max_position_embeddings (`int`, *optional*, defaults to 2048):
        The maximum sequence length that this model might ever be used with.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    pad_token_id (`int`, *optional*, defaults to 1):
        Padding token id.
    bos_token_id (`int`, *optional*):
        Beginning of stream token id.
    eos_token_id (`int`, *optional*, defaults to 50279):
        End of stream token id.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether to tie weight embeddings
    rope_theta (`float`, *optional*, defaults to 10000.0):
        The base period of the RoPE embeddings.
    rope_scaling (`Dict`, *optional*):
        Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
        strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
        `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
        `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
        these scaling strategies behave:
        https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
        experimental feature, subject to breaking API changes in future versions.
    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
        Whether to use a bias in the query, key, value and output projection layers during self-attention.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    rms_norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.

```python
>>> from transformers import Olmo2Model, Olmo2Config

>>> # Initializing a Olmo2 7B style configuration
>>> configuration = Olmo2Config()

>>> # Initializing a model from the Olmo2 7B style configuration
>>> model = Olmo2Model(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```
olmo2colwise_reprowwise_repcolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormc                    > [         TU ]  " S0 SU_SU_SU_SU_SU_SU_SU_SU_S	U	_S
U
_SU_SU_SU_SU_SU_SU_SU_SU_UD6  UU l        U ?g )N
vocab_sizehidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsnum_key_value_heads
hidden_actmax_position_embeddingsinitializer_range	use_cachepad_token_idbos_token_ideos_token_idtie_word_embeddings
rope_thetarope_scalingattention_biasattention_dropout )super__init__rms_norm_epsclip_qkv)selfr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r;   kwargs	__class__s                        _/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/olmo2/modular_olmo2.pyr:   Olmo2Config.__init__y   s    . 	 	
!	
#	
 0	
 0		

 !4	
 !4	
 "	
 %<	
 0	
  	
 &	
 &	
 &	
 !4	
 "	
  &!	
" *#	
$ 0'	
, )M    )r;   )i  i   i +      rC   Nsilui   g{Gz?T   Nig  Fg     @NF        gh㈵>)__name__
__module____qualname____firstlineno____doc__
model_typebase_model_tp_planbase_model_pp_planr:   __static_attributes____classcell__r?   s   @r@   r   r      s    KZ J%2%2%2%2"+ )"+ &(9:#%568IJ!"_$56   $!). .rB   r   c                       \ rS rSrS rSrg)Olmo2RMSNorm   c                    UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  U-  R                  U5      $ )Nr   T)keepdim)	dtypetotorchfloat32powmeanrsqrtvariance_epsilonweight)r=   r    input_dtypevariances       r@   forwardOlmo2RMSNorm.forward   sw    #))%((7 $$Q',,R,>%H?T?T4T(UUm+//<<rB   r8   N)rG   rH   rI   rJ   rc   rO   r8   rB   r@   rS   rS      s    =rB   rS   c                     U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrV   r   )dim)shaperZ   cat)xx1x2s      r@   rotate_halfrl      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rB   c                   L  ^  \ rS rSrSS\S\\   4U 4S jjjr  SS\R                  S\
\R                  \R                  4   S\\R                     S\\   S	\\R                     S
\\   S\
\R                  \\R                     \\
\R                        4   4S jjrSrU =r$ )Olmo2Attention   config	layer_idxc                    > [         TU ]  XS9  [        UR                  U R                  -  UR
                  5      U l        [        UR                  U R                  -  UR
                  5      U l        g )Nrq   )	r9   r:   rS   r*   head_dimr;   q_normr+   k_normr=   rp   rq   r?   s      r@   r:   Olmo2Attention.__init__   sY    5"6#=#=#MvObObc"6#=#=#MvObObcrB   r    position_embeddingsr!   past_key_valuecache_positionr>   returnc                 |   UR                   S S n/ UQSPU R                  P7nU R                  U R                  U5      5      n	U R	                  U R                  U5      5      n
U R                  U5      nU	R                  U5      R                  SS5      n	U
R                  U5      R                  SS5      n
UR                  U5      R                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	U
UU4U R                   (       d  SOU R"                  U R$                  S.UD6u  nnUR&                  " / UQSP76 R)                  5       nU R+                  U5      nUU4$ )NrV   rE   r   )sincosr{   eagerrF   )dropoutscaling)rg   rt   ru   q_projrv   k_projv_projview	transposer   updaterq   r   rp   _attn_implementationr   trainingr7   r   reshape
contiguouso_proj)r=   r    ry   r!   rz   r{   r>   input_shapehidden_shapequery_states
key_statesvalue_statesr   r~   cache_kwargsattention_interfaceattn_outputattn_weightss                     r@   rc   Olmo2Attention.forward   s    $))#2.88b8$--8{{4;;}#=>[[]!;<
{{=1#((6@@AF__\2<<QB
#((6@@AF&#7RU#[ %#&nUL'5'<'<ZW[WeWegs't$J(?;;++w6"9$++:Z:Z"[$7	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((rB   )rv   ru   )N)NN)rG   rH   rI   rJ   r   r   intr:   rZ   Tensortupler   
LongTensorr	   r   rc   rO   rP   rQ   s   @r@   rn   rn      s    d{ dx} d d +/59-)||-) #5<<#=>-) !.	-)
 !-) !!1!12-) +,-) 
u||Xell3XeELL>Q5RR	S-) -)rB   rn   c                   t  ^  \ rS rSrS\S\4U 4S jjr      SS\R                  S\	\R                     S\	\R                     S\	\   S	\	\   S
\	\R                     S\	\\R                  \R                  4      S\\   S\\R                   \	\\R                   \R                   4      4   4S jjrSrU =r$ )Olmo2DecoderLayer   rp   rq   c                    > [         TU ]  XS9  [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        XS9U l        U ?	g )Nrs   eps)rp   rq   )
r9   r:   rS   r'   r;   post_attention_layernormpost_feedforward_layernormrn   	self_attninput_layernormrw   s      r@   r:   Olmo2DecoderLayer.__init__   s[    5(4V5G5GVM`M`(a%*6v7I7IvObOb*c''vK rB   r    r!   position_idsrz   r/   r{   ry   r>   r|   c                     Un	U R                   " SUUUUUUUS.UD6u  pU R                  U5      nX-   nUn	U R                  U5      nU R                  U5      nX-   nU$ )N)r    r!   r   rz   r/   r{   ry   r8   )r   r   mlpr   )r=   r    r!   r   rz   r/   r{   ry   r>   residual_s              r@   rc   Olmo2DecoderLayer.forward   s     !>> 	
')%)) 3	
 	
 55mD 0 !/77F 0rB   )r   r   r   )NNNFNN)rG   rH   rI   rJ   r   r   r:   rZ   r   r   r   r   boolr   r	   r   FloatTensorrc   rO   rP   rQ   s   @r@   r   r      s    !{ !s ! 2637*.$)59KO|| !. u//0	
 ! D> !!1!12 &eELL%,,,F&GH +, 
u  (51B1BEDUDU1U+V"WW	X rB   r   c                       \ rS rSrSrg)Olmo2RotaryEmbeddingi!  r8   NrG   rH   rI   rJ   rO   r8   rB   r@   r   r   !      rB   r   c                       \ rS rSrSrg)Olmo2PreTrainedModeli%  r8   Nr   r8   rB   r@   r   r   %  r   rB   r   c                   0   ^  \ rS rSrS\4U 4S jjrSrU =r$ )
Olmo2Modeli+  rp   c           	        > [         TU ]  U5        [        UR                  UR                  S9U l        [        R                  " [        UR                  5       Vs/ sH  n[        X5      PM     sn5      U l        g s  snf )Nr   )r9   r:   rS   r'   r;   r$   nn
ModuleListranger)   r   r#   rw   s      r@   r:   Olmo2Model.__init__,  sb      !3!39L9LM	mmCHIaIaCbcCbiv1Cbc
cs   A<)r#   r$   )rG   rH   rI   rJ   r   r:   rO   rP   rQ   s   @r@   r   r   +  s    
{ 
 
rB   r   c                       \ rS rSrSrg)Olmo2ForCausalLMi5  r8   Nr   r8   rB   r@   r   r   5  r   rB   r   )r   r   r   r   )*typingr   r   rZ   torch.nnr   transformers.utils.genericr   cache_utilsr   modeling_utilsr   processing_utilsr	   utilsr
   llama.modeling_llamar   r   r   olmo.configuration_olmor   olmo.modeling_olmor   r   r   r   r   r   
get_loggerrG   loggerr   rS   rl   rn   r   r   r   r   r   __all__r8   rB   r@   <module>r      s    %   9   5 &  ^ ^ 0  
		H	%L* Lb=< =(3)] 3)r&( &R	. 		/ 	
 
	 	rB   