§
    ‚Štj³!  ã                   ó  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZmZ ddlmZ d	d
lmZ d	dlmZmZmZmZmZmZmZmZmZmZ d	dlmZ ddlmZ  ej         e!¦  «        Z"dZ#dZ$ G d„ de¦  «        Z%d„ Z& G d„ de¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ de¦  «        Z+ G d„ de¦  «        Z, G d „ d!e¦  «        Z- G d"„ d#e¦  «        Z. G d$„ d%e¦  «        Z/g d&¢Z0dS )'é    N)ÚCallable)Únné   )Úinitialization)ÚCache)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)Úloggingé   )ÚGemmaForCausalLM)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Ú
MistralMLPé   )ÚDiffLlamaConfigzkajuma/DiffLlama-0.3B-handcutr   c                   ó   — e Zd ZdS )ÚDiffLlamaMLPN©Ú__name__Ú
__module__Ú__qualname__© ó    úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/diffllama/modular_diffllama.pyr   r   2   ó   € € € € € Ø€Dr!   r   c                 ó<   — ddt          j        d| z  ¦  «        z  z
  S )Ngš™™™™™é?g333333ã?g333333Ó¿)ÚmathÚexp)Ú	layer_idxs    r"   Úlambda_init_fnr(   6   s!   € Ø�•t”x  yÑ 0Ñ1Ô1Ñ1Ñ1Ð1r!   c                   ó   — e Zd ZdS )ÚDiffLlamaRotaryEmbeddingNr   r    r!   r"   r*   r*   :   r#   r!   r*   c                   ó¾   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej        de	ej        ej        f         dej        dz  d	e
dz  d
e	ej        ej        f         f
d„Zˆ xZS )ÚDiffLlamaAttentionuÿ  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

    Computes ``(softmax(Q1 K1áµ€) - Î» Â· softmax(Q2 K2áµ€)) Â· V`` as two standard attention calls
    sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
    V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
    forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
    requires ``head_dim_v == head_dim_q``.
    NÚconfigr'   c                 ó  •— t          ¦   «                              ||¦  «         |j        dk    rt          d¦  «        ‚|j        �|j        dz  dk    rt          d|j        › d�¦  «        ‚t          |¦  «        | _        t          j        t          j
        d|j        | j        f¬¦  «        ¦  «        | _        t          j        t          j
        d|j        | j        f¬¦  «        ¦  «        | _        t          j        t          j
        d|j        | j        f¬¦  «        ¦  «        | _        t          j        t          j
        d|j        | j        f¬¦  «        ¦  «        | _        t          j        d| j        z  |j        d¬	¦  «        | _        d S )
Nç        z€DiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.r   r   z–DiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got ú.)ÚsizeF)ÚepsÚelementwise_affine)ÚsuperÚ__init__Úattention_dropoutÚ
ValueErrorÚnum_key_value_headsr(   Úlambda_initr   Ú	ParameterÚtorchÚnormalÚlambda_std_devÚhead_dimÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2ÚRMSNormÚrms_norm_epsÚ	groupnorm©Úselfr-   r'   Ú	__class__s      €r"   r5   zDiffLlamaAttention.__init__H   s|  ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ð
 Ô# cÒ)Ð)ÝðDñô ð ð Ô%Ð-°Ô1KÈaÑ1OÐSTÒ1TÐ1TÝðtØV\ÔVpðtð tð tñô ð õ *¨)Ñ4Ô4ˆÔÝœ¥e¤l°1°fÔ6KÐSWÔS`ÐRbÐ&cÑ&cÔ&cÑdÔdˆŒÝœ¥e¤l°1°fÔ6KÐSWÔS`ÐRbÐ&cÑ&cÔ&cÑdÔdˆŒÝœ¥e¤l°1°fÔ6KÐSWÔS`ÐRbÐ&cÑ&cÔ&cÑdÔdˆŒÝœ¥e¤l°1°fÔ6KÐSWÔS`ÐRbÐ&cÑ&cÔ&cÑdÔdˆŒÝœ A¨¬Ñ$5¸6Ô;NÐchÐiÑiÔiˆŒˆˆr!   Úhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚreturnc                 óˆ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
d„ t          j        |
dd¬¦  «        D ¦   «         \  }}t          j        | j        j        t           ¦  «        } || ||	||fd| j        dœ|¤Ž\  }} || ||	||fd| j        dœ|¤Ž\  }}t          j        ||gd¬¦  «        }t          j        |dd¬¦  «        \  }}t          j        t          j        | j        | j        z  dt          j        ¬¦  «        ¦  «                             |j        ¦  «        }t          j        t          j        | j        | j        z  dt          j        ¬¦  «        ¦  «                             |j        ¦  «        }||z
  | j        z   }|||z  z
  }d| j        z
  |                      |¦  «        z  } |j        g |¢d‘R Ž }|                      |¦  «        }||fS )	Néÿÿÿÿr   r   c              3   óF   K  — | ]}|                      d dd d ¦  «        V — ŒdS )r   r   N)Úrepeat)Ú.0Úvs     r"   ú	<genexpr>z-DiffLlamaAttention.forward.<locals>.<genexpr>x   s4   è è € Ð'jÐ'jÀ¨¯ª°°A°q¸!Ñ(<Ô(<Ð'jÐ'jÐ'jÐ'jÐ'jÐ'jr!   )Údimr/   )ÚdropoutÚscaling)rU   Údtype) Úshaper>   Úq_projÚviewÚ	transposeÚk_projÚv_projr   Úupdater'   r;   Úchunkr   Úget_interfacer-   Ú_attn_implementationr   rW   Úcatr&   Úsumr?   r@   Úfloat32ÚtorX   rA   rB   r9   rE   ÚreshapeÚo_proj)rG   rI   rJ   rK   rL   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚvalue_states1Úvalue_states2Úattention_interfaceÚattn_output1Úattn_weightsÚattn_output2Ú_Úattn_outputÚlambda_1Úlambda_2Úlambda_fulls                           r"   ÚforwardzDiffLlamaAttention.forwarda   s  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜ð (kÐ'jÅeÄkÐR^Ð`aÐghÐFiÑFiÔFiÐ'jÑ'jÔ'jÑ$ˆ�}å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð &9Ð%8ØØØØØð	&
ð Ø”Lð	&
ð 	&
ð ð	&
ð 	&
Ñ"ˆ�lð .Ð-ØØØØØð	
ð Ø”Lð	
ð 	
ð ð	
ð 	
‰ˆ�aõ ”i ¨|Ð <À"ÐEÑEÔEˆõ &+¤[°¸aÀQÐ%GÑ%GÔ%GÑ"ˆ�lÝ”9�UœY t¤~¸¼Ñ'FÈBÕV[ÔVcÐdÑdÔdÑeÔe×hÒhØÔñ
ô 
ˆõ ”9�UœY t¤~¸¼Ñ'FÈBÕV[ÔVcÐdÑdÔdÑeÔe×hÒhØÔñ
ô 
ˆð  Ñ)¨DÔ,<Ñ<ˆØ" [°<Ñ%?Ñ?ˆØ˜4Ô+Ñ+¨t¯~ª~¸kÑ/JÔ/JÑJˆØ)�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r!   )N)NN)r   r   r   Ú__doc__r   Úintr5   r;   ÚTensorÚtupler   r|   Ú__classcell__©rH   s   @r"   r,   r,   >   sï   ø€ € € € € ðð ðjð j˜ð j¸3À¹:ð jð jð jð jð jð jð: /3Ø(,ðC)ð C)à”|ðC)ð # 5¤<°´Ð#=Ô>ðC)ð œ tÑ+ð	C)ð
  ™ðC)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ðC)ð C)ð C)ð C)ð C)ð C)ð C)ð C)r!   r,   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚDiffLlamaDecoderLayerr-   r'   c                 óx   •— t          ¦   «                              ||¦  «         t          ||¬¦  «        | _        d S )N)r-   r'   )r4   r5   r,   Ú	self_attnrF   s      €r"   r5   zDiffLlamaDecoderLayer.__init__¨   s5   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+å+°6ÀYÐOÑOÔOˆŒˆˆr!   )r   r   r   r   r~   r5   r�   r‚   s   @r"   r„   r„   §   sW   ø€ € € € € ðP˜ð P¸3ð Pð Pð Pð Pð Pð Pð Pð Pð Pð Pr!   r„   c                   ó>   — e Zd Z ej        ¦   «         d„ ¦   «         ZdS )ÚDiffLlamaPreTrainedModelc                 ó†  — t          j        | |¦  «         t          |t          ¦  «        r–t	          j        |j        d| j        j        ¦  «         t	          j        |j	        d| j        j        ¦  «         t	          j        |j
        d| j        j        ¦  «         t	          j        |j        d| j        j        ¦  «         d S d S )Nr   )r	   Ú_init_weightsÚ
isinstancer,   ÚinitÚnormal_r?   r-   r=   r@   rA   rB   )rG   Úmodules     r"   rŠ   z&DiffLlamaPreTrainedModel._init_weights¯   s«   € åÔ% d¨FÑ3Ô3Ð3Ý�fÕ0Ñ1Ô1ð 	JÝŒL˜Ô)¨1¨d¬kÔ.HÑIÔIÐIÝŒL˜Ô)¨1¨d¬kÔ.HÑIÔIÐIÝŒL˜Ô)¨1¨d¬kÔ.HÑIÔIÐIÝŒL˜Ô)¨1¨d¬kÔ.HÑIÔIÐIÐIÐIð		Jð 	Jr!   N)r   r   r   r;   Úno_gradrŠ   r    r!   r"   rˆ   rˆ   ®   s:   € € € € € Ø€U„]�_„_ðJð Jñ „_ðJð Jð Jr!   rˆ   c                   ó   — e Zd ZdS )ÚDiffLlamaModelNr   r    r!   r"   r‘   r‘   ¹   r#   r!   r‘   c                   ó   — e Zd ZdS )ÚDiffLlamaForCausalLMNr   r    r!   r"   r“   r“   ½   r#   r!   r“   c                   ó   — e Zd ZdS )Ú"DiffLlamaForSequenceClassificationNr   r    r!   r"   r•   r•   Á   r#   r!   r•   c                   ó   — e Zd ZdS )ÚDiffLlamaForQuestionAnsweringNr   r    r!   r"   r—   r—   Å   r#   r!   r—   c                   ó   — e Zd ZdS )ÚDiffLlamaForTokenClassificationNr   r    r!   r"   r™   r™   É   r#   r!   r™   )rˆ   r‘   r“   r•   r—   r™   )1r%   Úcollections.abcr   r;   r   Ú r   rŒ   Úcache_utilsr   Úmodeling_utilsr   r	   Úutilsr
   Úgemma.modeling_gemmar   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_diffllamar   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr   r(   r*   r,   r„   rˆ   r‘   r“   r•   r—   r™   Ú__all__r    r!   r"   ú<module>r¨      sB  ðð" €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø  Ð  Ð  Ð  Ð  Ð  Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ Ð Ð Ð Ð Ð Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 2Ð 1Ð 1Ð 1Ð 1Ð 1Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ð 
ˆÔ	˜HÑ	%Ô	%€à5Ð Ø#€ð	ð 	ð 	ð 	ð 	�:ñ 	ô 	ð 	ð2ð 2ð 2ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ðf)ð f)ð f)ð f)ð f)˜ñ f)ô f)ð f)ðRPð Pð Pð Pð PÐ-ñ Pô Pð PðJð Jð Jð Jð JÐ3ñ Jô Jð Jð	ð 	ð 	ð 	ð 	�Zñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$=ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Añ 	ô 	ð 	ðð ð €€€r!   