§
    ‚ŠtjŸ(  ã                   óæ  — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ  ej        e ¦  «        Z!dZ"dZ# G d„ dej$        ¦  «        Z% G d„ de¦  «        Z&d&d„Z' G d„ dej$        ¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ d e¦  «        Z+ G d!„ d"e¦  «        Z, G d#„ d$e¦  «        Z-g d%¢Z.dS )'zPyTorch Phi-3 model.é    )ÚCallableN)Únné   )ÚACT2FN)ÚCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )ÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralPreTrainedModelÚeager_attention_forwardÚrotate_half)ÚPhiRotaryEmbeddingé   )Ú
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚPhi3MLPc                 ó"  •— t          ¦   «                              ¦   «          || _        t          j        |j        d|j        z  d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          |j
                 | _        d S )Nr   F©Úbias)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn)Úselfr   Ú	__class__s     €úc/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/phi3/modular_phi3.pyr   zPhi3MLP.__init__1   sz   ø€ Ý‰Œ×ÒÑÔÐàˆŒÝœI fÔ&8¸!¸fÔ>VÑ:VÐ]bÐcÑcÔcˆÔÝœ 6Ô#;¸VÔ=OÐV[Ð\Ñ\Ô\ˆŒÝ# FÔ$5Ô6ˆÔÐÐó    Úhidden_statesÚreturnc                 óº   — |                       |¦  «        }|                     dd¬¦  «        \  }}||                      |¦  «        z  }|                      |¦  «        S )Nr   éÿÿÿÿ©Údim)r#   Úchunkr&   r$   )r'   r+   Ú	up_statesÚgates       r)   ÚforwardzPhi3MLP.forward9   sX   € Ø×%Ò% mÑ4Ô4ˆ	à#Ÿ/š/¨!°˜/Ñ4Ô4‰ˆˆiØ × 2Ò 2°4Ñ 8Ô 8Ñ8ˆ	à�~Š~˜iÑ(Ô(Ð(r*   )Ú__name__Ú
__module__Ú__qualname__r   ÚtorchÚFloatTensorr4   Ú__classcell__©r(   s   @r)   r   r   0   s`   ø€ € € € € ð7ð 7ð 7ð 7ð 7ð) UÔ%6ð )¸5Ô;Lð )ð )ð )ð )ð )ð )ð )ð )r*   r   c                   ó   — e Zd ZdS )ÚPhi3RotaryEmbeddingN©r5   r6   r7   © r*   r)   r=   r=   B   ó   € € € € € Ø€Dr*   r=   c                 ó�  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}t          j        ||z  t	          |¦  «        |z  z   |gd¬¦  «        }
t          j        ||z  t	          |¦  «        |z  z   |	gd¬¦  «        }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r.   .Nr/   )Ú	unsqueezeÚshaper8   Úcatr   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r)   Úapply_rotary_pos_embrQ   F   sæ   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cà”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EåŒi˜% #™+­+°eÑ*<Ô*<¸sÑ*BÑCÀVÐLÐRTÐUÑUÔU€GÝŒi˜% #™+­+°eÑ*<Ô*<¸sÑ*BÑCÀVÐLÐRTÐUÑUÔU€GØ�GÐÐr*   c                   óð   ‡ — e Zd ZdZddededz  fˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	e
dz  d
ee         de	ej        ej        dz  e	ej                 dz  f         fd„Zˆ xZS )ÚPhi3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNr   Ú	layer_idxc                 ó  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j        | _        | j        dz  | _
        |j        | _        d| _        |j        | j        z  d|j        | j        z  z  z   }t          j        |j        | j        z  |j        d¬¦  «        | _        t          j        |j        |d¬¦  «        | _        d S )NÚhead_dimg      à¿Tr   Fr   )r   r   r   rT   Úgetattrr!   Únum_attention_headsrV   Únum_key_value_headsÚnum_key_value_groupsÚscalingÚattention_dropoutÚ	is_causalr   r    Úo_projÚqkv_proj)r'   r   rT   Úop_sizer(   s       €r)   r   zPhi3Attention.__init__g   sí   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø#)Ô#=ˆÔ Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒàÔ,¨t¬}Ñ<¸qÀFÔD^ÐaeÔanÑDnÑ?oÑoˆÝ”i Ô :¸T¼]Ñ JÈFÔL^ÐejÐkÑkÔkˆŒÝœ	 &Ô"4°gÀEÐJÑJÔJˆŒˆˆr*   r+   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr,   c           
      ó´  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }| j        j        | j        z  }	|dd |	…f         }
|d|	|	| j        | j        z  z   …f         }|d|	| j        | j        z  z   d …f         }|
                     |¦  «                             dd¦  «        }
|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�| 	                    ||| j
        ¦  «        \  }}t          j        | j        j        t          ¦  «        } || |
|||f| j        sdn| j        | j        t%          | j        dd ¦  «        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr.   .r   r   g        Úsliding_window)Údropoutr[   rf   )rC   rV   r_   r   rX   rY   ÚviewÚ	transposerQ   ÚupdaterT   r
   Úget_interfaceÚ_attn_implementationr   Útrainingr\   r[   rW   ÚreshapeÚ
contiguousr^   )r'   r+   ra   rb   rc   rd   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesÚ
key_statesÚvalue_statesrG   rH   Úattention_interfaceÚattn_outputÚattn_weightss                     r)   r4   zPhi3Attention.forwardv   s&  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà�mŠm˜MÑ*Ô*ˆØ”KÔ3°d´mÑCˆ	Ø˜3 
  
˜?Ô+ˆØ˜˜i¨)°dÔ6NÐQUÔQ^Ñ6^Ñ*^Ð^Ð^Ô_ˆ
Ø˜3 	¨DÔ,DÀtÄ}Ñ,TÑ TÐ VÐ VÐVÔWˆà#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LÝ" 4¤;Ð0@À$ÑGÔGð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r*   )N)r5   r6   r7   Ú__doc__r   Úintr   r8   ÚTensorÚtupler   r   r	   r4   r:   r;   s   @r)   rS   rS   d   sý   ø€ € € € € ØGÐGðKð K˜zð K°c¸D±jð Kð Kð Kð Kð Kð Kð( )-ð-)ð -)à”|ð-)ð # 5¤<°´Ð#=Ô>ð-)ð œ tÑ+ð	-)ð
  ™ð-)ð Ð-Ô.ð-)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð-)ð -)ð -)ð -)ð -)ð -)ð -)ð -)r*   rS   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚPhi3DecoderLayerr   rT   c                 ó&  •— t          ¦   «                              ||¦  «         || _        t          ||¬¦  «        | _        t          |¦  «        | _        t          j        |j	        ¦  «        | _
        t          j        |j	        ¦  «        | _        d S )N)r   rT   )r   r   r   rS   Ú	self_attnr   Úmlpr   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout)r'   r   rT   r(   s      €r)   r   zPhi3DecoderLayer.__init__§   su   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+ØˆŒÝ&¨fÀ	ÐJÑJÔJˆŒÝ˜6‘?”?ˆŒÝ"$¤*¨VÔ-?Ñ"@Ô"@ˆÔÝ!#¤¨FÔ,>Ñ!?Ô!?ˆÔÐÐr*   NFr+   rb   Úposition_idsrc   Ú	use_cachera   rd   r,   c           
      ó  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	||                      |¦  «        z   }|}|                      |¦  «        }|                      |¦  «        }||                      |¦  «        z   }|S )N)r+   rb   r‡   rc   rˆ   ra   r?   )Úinput_layernormr�   r…   Úpost_attention_layernormr‚   r†   )
r'   r+   rb   r‡   rc   rˆ   ra   rd   ÚresidualÚself_attn_weightss
             r)   r4   zPhi3DecoderLayer.forward¯   s¼   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà+9¨4¬>ð ,
Ø'Ø)Ø%Ø+ØØ 3ð,
ð ,
ð ð,
ð ,
Ñ(ˆÐ(ð ! 4×#:Ò#:¸=Ñ#IÔ#IÑIˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  4×#9Ò#9¸-Ñ#HÔ#HÑHˆØÐr*   )NNNFN)r5   r6   r7   r   r{   r   r8   r|   Ú
LongTensorr   Úboolr}   r   r	   r9   r4   r:   r;   s   @r)   r   r   ¦   s   ø€ € € € € ð@˜zð @°cð @ð @ð @ð @ð @ð @ð /3Ø04Ø(,Ø!&ØHLðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5¤<°´Ð#=Ô>ÀÑEðð Ð-Ô.ðð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uðð ð ð ð ð ð ð r*   r   c                   ó   — e Zd ZdZdS )ÚPhi3PreTrainedModelz0.0.5N)r5   r6   r7   Ú_versionr?   r*   r)   r‘   r‘   Î   s   € € € € € Ø€H€H€Hr*   r‘   c                   ó"   — e Zd Z	 	 	 	 	 	 dd„ZdS )ÚPhi3ForCausalLMNTc                 óð   — |rYt          | j        d¦  «        rD|j        d         | j        j        dz   k    r&|                     ¦   «         }	|	| j        j        k    rd }t          j        | f|||||||dœ|¤Ž}
|
S )NÚ original_max_position_embeddingsr   )Ú	input_idsrc   rb   Úinputs_embedsr‡   rˆ   Úlogits_to_keep)Úhasattrr   rC   r–   Úget_seq_lengthr   Úprepare_inputs_for_generation)r'   r—   rc   rb   r˜   r‡   rˆ   r™   rd   Úpast_lengthÚmodel_inputss              r)   rœ   z-Phi3ForCausalLM.prepare_inputs_for_generationÓ   s¬   € ð" ð	'å˜œÐ%GÑHÔHð	'ð ” Ô" d¤kÔ&RÐUVÑ&VÒVÐVà)×8Ò8Ñ:Ô:ˆKØ˜dœkÔJÒJÐJØ"&�å&ÔDØð

àØ+Ø)Ø'Ø%ØØ)ð

ð 

ð ð

ð 

ˆð Ðr*   )NNNNTN)r5   r6   r7   rœ   r?   r*   r)   r”   r”   Ò   s<   € € € € € ð ØØØØØð$ð $ð $ð $ð $ð $r*   r”   c                   ó   — e Zd ZdS )ÚPhi3ForSequenceClassificationNr>   r?   r*   r)   r    r    ú   r@   r*   r    c                   ó   — e Zd ZdS )ÚPhi3ForTokenClassificationNr>   r?   r*   r)   r¢   r¢   þ   r@   r*   r¢   )r‘   Ú	Phi3Modelr”   r    r¢   )r   )/rz   Úcollections.abcr   r8   r   Úactivationsr   Úcache_utilsr   Ú
generationr   Úmodeling_flash_attention_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   Úmistral.modeling_mistralr   r   r   r   r   r   r   Úphi.modeling_phir   Úconfiguration_phi3r   Ú
get_loggerr5   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚModuler   r=   rQ   rS   r   r‘   r”   r    r¢   Ú__all__r?   r*   r)   ú<module>rµ      sÝ  ðð Ð à $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø  Ð  Ð  Ð  Ð  Ð  Ø )Ð )Ð )Ð )Ð )Ð )Ø BÐ BÐ BÐ BÐ BÐ BØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð 2Ð 1Ð 1Ð 1Ð 1Ð 1Ø *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€à8Ð Ø€ð)ð )ð )ð )ð )ˆbŒiñ )ô )ð )ð$	ð 	ð 	ð 	ð 	Ð,ñ 	ô 	ð 	ðð ð ð ð<?)ð ?)ð ?)ð ?)ð ?)�B”Iñ ?)ô ?)ð ?)ðD%ð %ð %ð %ð %Ð*ñ %ô %ð %ðPð ð ð ð Ð0ñ ô ð ð%ð %ð %ð %ð %Ð(ñ %ô %ð %ðP	ð 	ð 	ð 	ð 	Ð$Dñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð!>ñ 	ô 	ð 	ðð ð €€€r*   