§
    ‚ŠtjZ"  ã                   ó  — d dl mZ d dlZd dlmZ d dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZ d
dlmZ d
dlmZmZmZmZmZmZmZ  ej         e!¦  «        Z" ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z# G d„ de¦  «        Z$ G d„ de¦  «        Z%d„ Z& G d„ de¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ d e¦  «        Z+ G d!„ d"e¦  «        Z,g d#¢Z-dS )$é    )ÚCallableN)Ústrict)ÚTransformersKwargsé   )ÚCache)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚloggingé   )ÚLlamaPreTrainedModelÚLlamaRMSNormÚeager_attention_forward)Ú
OlmoConfig)ÚOlmoAttentionÚOlmoDecoderLayerÚOlmoForCausalLMÚOlmoForSequenceClassificationÚ	OlmoModelÚOlmoRotaryEmbeddingÚapply_rotary_pos_embzallenai/Olmo2-7B-1124-hf)Ú
checkpointc                   ót   — e Zd ZU dZdZddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZdZee	d<    e
¦   «         ZdS )ÚOlmo2Configa�  
    Example:

    ```python
    >>> from transformers import Olmo2Model, Olmo2Config

    >>> # Initializing a Olmo2 7B style configuration
    >>> configuration = Olmo2Config()

    >>> # Initializing a model from the Olmo2 7B style configuration
    >>> model = Olmo2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úolmo2Úcolwise_gather_outputÚrowwise_split_inputÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormgñhãˆµøä>Úrms_norm_epsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planÚbase_model_pp_planr'   ÚfloatÚ__annotations__ÚAttributeErrorÚclip_qkv© ó    úe/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/olmo2/modular_olmo2.pyr   r   0   s�   € € € € € € ðð ð" €Jà%<Ø%<Ø%<Ø%:Ø"+Ø )Ø"+ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð €L�%ÐÐÑØˆ~ÑÔ€H€H€Hr4   r   c                   ó   — e Zd Zd„ ZdS )ÚOlmo2RMSNormc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |z                       |¦  «        S )Nr   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtoÚtorchÚfloat32ÚpowÚmeanÚrsqrtÚvariance_epsilonÚweight)Úselfr"   Úinput_dtypeÚvariances       r5   ÚforwardzOlmo2RMSNorm.forward[   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØ”˜mÑ+×/Ò/°Ñ<Ô<Ð<r4   N)r(   r)   r*   rG   r3   r4   r5   r7   r7   Z   s#   € € € € € ð=ð =ð =ð =ð =r4   r7   c                   ó   — e Zd ZdS )ÚOlmo2RotaryEmbeddingN©r(   r)   r*   r3   r4   r5   rI   rI   c   ó   € € € € € Ø€Dr4   rI   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr9   r   )Údim)Úshaper=   Úcat)ÚxÚx1Úx2s      r5   Úrotate_halfrS   g   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r4   c                   óÎ   ‡ — e Zd Zddededz  fˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         fd„Zˆ xZS )ÚOlmo2AttentionNÚconfigÚ	layer_idxc                 óê   •— t          ¦   «                              ||¬¦  «         t          |j        | j        z  |j        ¦  «        | _        t          |j        | j        z  |j        ¦  «        | _        d S )N©rW   )	ÚsuperÚ__init__r7   Únum_attention_headsÚhead_dimr'   Úq_normÚnum_key_value_headsÚk_norm©rD   rV   rW   Ú	__class__s      €r5   r[   zOlmo2Attention.__init__r   s`   ø€ Ý‰Œ×Ò˜¨9ÐÑ5Ô5Ð5Ý" 6Ô#=ÀÄÑ#MÈvÔObÑcÔcˆŒÝ" 6Ô#=ÀÄÑ#MÈvÔObÑcÔcˆŒˆˆr4   r"   Úposition_embeddingsr#   Úpast_key_valuesÚkwargsÚreturnc                 óz  — |j         d d…         }g |¢d‘| j        ‘R }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }	|                      |¦  «        }
|                     |¦  «                             dd¦  «        }|	                     |¦  «                             dd¦  «        }	|
                     |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�| 
                    |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr9   é   r   g        )ÚdropoutÚscaling)rN   r]   r^   Úq_projr`   Úk_projÚv_projÚviewÚ	transposer   ÚupdaterW   r   Úget_interfacerV   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrj   ÚreshapeÚ
contiguousÚo_proj)rD   r"   rc   r#   rd   re   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r5   rG   zOlmo2Attention.forwardw   sâ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ 4§;¢;¨}Ñ#=Ô#=Ñ>Ô>ˆØ—[’[ §¢¨]Ñ!;Ô!;Ñ<Ô<ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r4   )N)r(   r)   r*   r   Úintr[   r=   ÚTensorÚtupler   r	   r   rG   Ú__classcell__©rb   s   @r5   rU   rU   q   sé   ø€ € € € € ðdð d˜{ð d°s¸T±zð dð dð dð dð dð dð )-ð*)ð *)à”|ð*)ð # 5¤<°´Ð#=Ô>ð*)ð œ tÑ+ð	*)ð
  ™ð*)ð Ð+Ô,ð*)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r4   rU   c                   óÒ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚOlmo2DecoderLayerrV   rW   c                 óþ   •— t          ¦   «                              ||¬¦  «         t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          ||¬¦  «        | _        | `	d S )NrY   ©Úeps)rV   rW   )
rZ   r[   r7   Úhidden_sizer'   Úpost_attention_layernormÚpost_feedforward_layernormrU   Ú	self_attnÚinput_layernormra   s      €r5   r[   zOlmo2DecoderLayer.__init__¨   sv   ø€ Ý‰Œ×Ò˜¨9ÐÑ5Ô5Ð5Ý(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%Ý*6°vÔ7IÈvÔObÐ*cÑ*cÔ*cˆÔ'Ý'¨vÀÐKÑKÔKˆŒØÐ Ð Ð r4   NFr"   r#   Úposition_idsrd   Ú	use_cacherc   re   rf   c           
      óÎ   — |} | j         d||||||dœ|¤Ž\  }}	|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r"   r#   r‘   rd   r’   rc   r3   )r�   r�   ÚmlprŽ   )
rD   r"   r#   r‘   rd   r’   rc   re   ÚresidualÚ_s
             r5   rG   zOlmo2DecoderLayer.forward¯   s¡   € ð !ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø+ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆð !ˆØŸš Ñ/Ô/ˆØ×7Ò7¸ÑFÔFˆØ  =Ñ0ˆØÐr4   )NNNFN)r(   r)   r*   r   r‚   r[   r=   rƒ   Ú
LongTensorr   Úboolr„   r	   r   rG   r…   r†   s   @r5   rˆ   rˆ   §   s÷   ø€ € € € € ð!˜{ð !°sð !ð !ð !ð !ð !ð !ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r4   rˆ   c                   ó   — e Zd ZdS )ÚOlmo2PreTrainedModelNrJ   r3   r4   r5   rš   rš   Î   rK   r4   rš   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú
Olmo2ModelrV   c                 óú   •‡— t          ¦   «                              ‰¦  «         t          ‰j        ‰j        ¬¦  «        | _        t          j        ˆfd„t          ‰j	        ¦  «        D ¦   «         ¦  «        | _
        d S )NrŠ   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r3   )rˆ   )Ú.0rW   rV   s     €r5   ú
<listcomp>z'Olmo2Model.__init__.<locals>.<listcomp>Ù   s$   ø€ ÐcÐcÐc°iÕ˜v yÑ1Ô1ÐcÐcÐcr4   )rZ   r[   r7   rŒ   r'   r&   ÚnnÚ
ModuleListÚrangeÚnum_hidden_layersr%   )rD   rV   rb   s    `€r5   r[   zOlmo2Model.__init__Õ   so   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ô!3¸Ô9LÐMÑMÔMˆŒ	Ý”mØcÐcÐcÐcÅ5ÈÔIaÑCbÔCbÐcÑcÔcñ
ô 
ˆŒˆˆr4   )r(   r)   r*   r   r[   r…   r†   s   @r5   rœ   rœ   Ô   sD   ø€ € € € € ð
˜{ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r4   rœ   c                   ó   — e Zd ZdS )ÚOlmo2ForCausalLMNrJ   r3   r4   r5   r¦   r¦   Þ   rK   r4   r¦   c                   ó   — e Zd ZdS )ÚOlmo2ForSequenceClassificationNrJ   r3   r4   r5   r¨   r¨   â   rK   r4   r¨   )r   r¦   r¨   rœ   rš   ).Úcollections.abcr   r=   Útorch.nnr¡   Úhuggingface_hub.dataclassesr   Útransformers.utils.genericr   Úcache_utilsr   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   r   Úllama.modeling_llamar   r   r   Úolmo.configuration_olmor   Úolmo.modeling_olmor   r   r   r   r   r   r   Ú
get_loggerr(   Úloggerr   r7   rI   rS   rU   rˆ   rš   rœ   r¦   r¨   Ú__all__r3   r4   r5   ú<module>r·      s  ðð( %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .à 9Ð 9Ð 9Ð 9Ð 9Ð 9à  Ð  Ð  Ð  Ð  Ð  Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ð ^Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð5Ð6Ñ6Ô6Øð# ð # ð # ð # ð # �*ñ # ô # ñ „ñ 7Ô6ð# ðP=ð =ð =ð =ð =�<ñ =ô =ð =ð	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ð(ð (ð (ð0)ð 0)ð 0)ð 0)ð 0)�]ñ 0)ô 0)ð 0)ðl$ð $ð $ð $ð $Ð(ñ $ô $ð $ðN	ð 	ð 	ð 	ð 	Ð/ñ 	ô 	ð 	ð
ð 
ð 
ð 
ð 
�ñ 
ô 
ð 
ð	ð 	ð 	ð 	ð 	�ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð%Bñ 	ô 	ð 	ðð ð €€€r4   