§
    ‚ŠtjÛ  ã                   óô  — d dl mZ d dlZd dlmZ d dlmc mZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZmZmZ ddlmZ  ej         e!¦  «        Z" G d„ dej#        ¦  «        Z$ G d„ de¦  «        Z% G d„ de¦  «        Z& ed¦  «        d!d„¦   «         Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ de¦  «        Z+ G d„ de¦  «        Z,g d ¢Z-dS )"é    )ÚCallableNé   )ÚCache)Úuse_kernel_func_from_hub)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)Úlogging)Úmaybe_autocasté   )	ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaMLPÚ
LlamaModelÚLlamaRotaryEmbeddingÚeager_attention_forwardÚrotate_halfé   )Ú
OlmoConfigc                   óP   ‡ — e Zd ZdZdeddfˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚOlmoLayerNormz/LayerNorm but with no learnable weight or bias.Úhidden_sizeÚreturnNc                 óX   •— t          ¦   «                              ¦   «          |f| _        d S ©N)ÚsuperÚ__init__Únormalized_shape)Úselfr   Ú	__class__s     €úc/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/olmo/modular_olmo.pyr   zOlmoLayerNorm.__init__4   s)   ø€ Ý‰Œ×ÒÑÔÐØ!, ˆÔÐÐó    Úhidden_statesc                 ó®   — |j         }t          j        |                     t          j        ¬¦  «        | j        d d d¬¦  «                             |¦  «        S )N)Údtypegñhãˆµøä>)Úeps)r&   ÚFÚ
layer_normÚtoÚtorchÚfloat32r   )r    r$   Ú
orig_dtypes      r"   ÚforwardzOlmoLayerNorm.forward8   sS   € Ø"Ô(ˆ
ÝŒ|˜M×,Ò,µ5´=Ð,ÑAÔAÀ4ÔCXÐZ^Ð`dÐjnÐoÑoÔo×rÒrØñ
ô 
ð 	
r#   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r+   ÚTensorr.   Ú__classcell__©r!   s   @r"   r   r   1   sw   ø€ € € € € Ø9Ð9ð/ Cð /¨Dð /ð /ð /ð /ð /ð /ð
 U¤\ð 
°e´lð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r#   r   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚOlmoMLPc                 ó.  •— t          ¦   «                              |¦  «         t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        d S )NF)Úbias)	r   r   ÚnnÚLinearr   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_proj©r    Úconfigr!   s     €r"   r   zOlmoMLP.__init__@   s|   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ýœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒˆˆr#   )r/   r0   r1   r   r5   r6   s   @r"   r8   r8   ?   sA   ø€ € € € € ðYð Yð Yð Yð Yð Yð Yð Yð Yr#   r8   c                   óN   — e Zd Z ej        ¦   «         ed„ ¦   «         ¦   «         ZdS )ÚOlmoRotaryEmbeddingc                 óê  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   ||	fS )
Nr   éÿÿÿÿr   ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)Úinv_freqÚfloatÚexpandÚshaper*   ÚdeviceÚ
isinstanceÚtypeÚstrr
   Ú	transposer+   ÚcatÚcosÚattention_scalingÚsin)
r    ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrI   ÚfreqsÚembrV   rX   s
             r"   r.   zOlmoRotaryEmbedding.forwardJ   s•  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð
 �Cˆxˆs   ÃBE&Å&E*Å-E*N)r/   r0   r1   r+   Úno_gradr   r.   © r#   r"   rD   rD   I   s@   € € € € € Ø€U„]�_„_Øð
ð 
ñ Ôñ „_ð
ð 
ð 
r#   rD   Úrotary_pos_embc                 ó&  — | j         |j         }}|                     |¦  «        }|                     |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }|                     |¦  «        |                     |¦  «        fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r&   Ú	unsqueezer   r*   )	ÚqÚkrV   rX   Úunsqueeze_dimÚq_typeÚk_typeÚq_embedÚk_embeds	            r"   Úapply_rotary_pos_embrk   Y   sˆ   € ð& ”W˜aœgˆF€FØ
�-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�:Š:�fÑÔ˜wŸzšz¨&Ñ1Ô1Ð1Ð1r#   c                   óœ   — e Zd Z	 ddej        deej        ej        f         dej        dz  dedz  deej        ej        dz  f         f
d„ZdS )	ÚOlmoAttentionNr$   Úposition_embeddingsÚattention_maskÚpast_key_valuesr   c                 óN  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |¦  «        }	|                      |¦  «        }
| j        j        �„|                     | j        j         | j        j        ¬¦  «         |	                     | j        j         | j        j        ¬¦  «         |
                     | j        j         | j        j        ¬¦  «         |                     |¦  «         	                    dd¦  «        }|	                     |¦  «         	                    dd¦  «        }	|
                     |¦  «         	                    dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j        ¦  «        \  }	}
t          j        | j        j        t           ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrF   )ÚminÚmaxr   r   g        )ÚdropoutÚscaling)rO   Úhead_dimÚq_projÚk_projÚv_projrB   Úclip_qkvÚclamp_ÚviewrT   rk   ÚupdateÚ	layer_idxr   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutru   ÚreshapeÚ
contiguousÚo_proj)r    r$   rn   ro   rp   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrV   rX   Úattention_interfaceÚattn_outputÚattn_weightss                   r"   r.   zOlmoAttention.forwardu   sG  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆàŒ;ÔÐ+Ø×Ò T¤[Ô%9Ð$9¸t¼{Ô?SÐÑTÔTÐTØ×Ò 4¤;Ô#7Ð"7¸T¼[Ô=QÐÑRÔRÐRØ×Ò T¤[Ô%9Ð$9¸t¼{Ô?SÐÑTÔTÐTà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r#   r   )r/   r0   r1   r+   r4   Útupler   r.   r`   r#   r"   rm   rm   t   s�   € € € € € ð )-ð/)ð /)à”|ð/)ð # 5¤<°´Ð#=Ô>ð/)ð œ tÑ+ð	/)ð
  ™ð/)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð/)ð /)ð /)ð /)ð /)ð /)r#   rm   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚOlmoDecoderLayerrB   r~   c                 óÜ   •— t          ¦   «                              ||¦  «         t          |j        ¦  «        | _        t          |j        ¦  «        | _        t          ||¬¦  «        | _        d S )N)rB   r~   )r   r   r   r   Úinput_layernormÚpost_attention_layernormrm   Ú	self_attn)r    rB   r~   r!   s      €r"   r   zOlmoDecoderLayer.__init__¨   s[   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ý,¨VÔ-?Ñ@Ô@ˆÔÝ(5°fÔ6HÑ(IÔ(IˆÔ%Ý&¨fÀ	ÐJÑJÔJˆŒˆˆr#   )r/   r0   r1   r   r3   r   r5   r6   s   @r"   r‘   r‘   §   sW   ø€ € € € € ðK˜zð K°cð Kð Kð Kð Kð Kð Kð Kð Kð Kð Kr#   r‘   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	OlmoModelrB   c                 óì   •‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ¦  «        | _	        d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r`   )r‘   )Ú.0r~   rB   s     €r"   ú
<listcomp>z&OlmoModel.__init__.<locals>.<listcomp>³   s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbr#   )
r   r   r;   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersr   r   ÚnormrA   s    `€r"   r   zOlmoModel.__init__°   si   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ " &Ô"4Ñ5Ô5ˆŒ	ˆ	ˆ	r#   )r/   r0   r1   r   r   r5   r6   s   @r"   r—   r—   ¯   sD   ø€ € € € € ð6˜zð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6ð 6r#   r—   c                   ó   — e Zd ZdS )ÚOlmoForCausalLMN©r/   r0   r1   r`   r#   r"   r¢   r¢   ¸   ó   € € € € € Ø€Dr#   r¢   c                   ó   — e Zd ZdS )ÚOlmoForSequenceClassificationNr£   r`   r#   r"   r¦   r¦   ¼   r¤   r#   r¦   )r¢   r¦   r—   ÚOlmoPreTrainedModel)r   ).Úcollections.abcr   r+   Útorch.nnr;   Útorch.nn.functionalÚ
functionalr(   Úcache_utilsr   Úintegrationsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úutilsr	   Úutils.genericr
   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úconfiguration_olmor   Ú
get_loggerr/   ÚloggerÚModuler   r8   rD   rk   rm   r‘   r—   r¢   r¦   Ú__all__r`   r#   r"   ú<module>r¸      sú  ðð( %Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à  Ð  Ð  Ð  Ð  Ð  Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø Ð Ð Ð Ð Ð Ø +Ð +Ð +Ð +Ð +Ð +ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð +Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ð
ð 
ð 
ð 
ð 
�B”Iñ 
ô 
ð 
ðYð Yð Yð Yð Yˆhñ Yô Yð Yðð ð ð ð Ð.ñ ô ð ð  ÐÐ*Ñ+Ô+ð2ð 2ð 2ñ ,Ô+ð2ð40)ð 0)ð 0)ð 0)ð 0)�Nñ 0)ô 0)ð 0)ðfKð Kð Kð Kð KÐ(ñ Kô Kð Kð6ð 6ð 6ð 6ð 6�
ñ 6ô 6ð 6ð	ð 	ð 	ð 	ð 	Ð&ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$Bñ 	ô 	ð 	ðð ð €€€r#   