§
    ‚ŠtjÊ  ã                   ó.  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ ddlmZm Z   ej!        e"¦  «        Z# ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z$ G d„ de ¦  «        Z% G d„ de¦  «        Z& G d„ de¦  «        Z' G d„ de¦  «        Z( G d„ de¦  «        Z) G d„ de¦  «        Z* G d„ d e¦  «        Z+ G d!„ d"e¦  «        Z, G d#„ d$e¦  «        Z-g d%¢Z.dS )&é    )ÚCallableN)Ústricté   )ÚCache)ÚPreTrainedConfig)ÚFlashAttentionKwargs)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚloggingé   )	ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forward)Ú
Qwen2ModelÚQwen2RotaryEmbeddingzHuggingFaceTB/SmolLM3-3B)Ú
checkpointc                   ó"  ‡ — e Zd ZU dZdZdgZdZddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
dz  ed<   dZeed<   dZe
ed<   dZeed<   d Zeed!<   d"Zeed#<   d$Ze
dz  ed%<   d&Ze
dz  ed'<   d(Ze
ee
         z  dz  ed)<   dZeez  dz  ed*<   d+Z eed,<   dZ!e
dz  ed-<   dZ"ee
         dz  ed.<   dZ#e
ed/<   dZ$ee         dz  ed0<   d+Z%eed1<   d2Z&ee
z  ed3<   d+Z'eed4<   d"Z(eed5<   ˆ fd6„Z)ˆ xZ*S )7ÚSmolLM3Configa>  
    no_rope_layers (`List[int]`, *optional*):
        List with at least the same length as the number of layers in the model.
        A `1` at an index position indicates that the corresponding layer will use RoPE,
        while a `0` indicates that it's a NoPE layer.
    no_rope_layer_interval (`int`, *optional*, defaults to 4):
        If `no_rope_layers` is `None`, it will be created using a NoPE layer every
        `no_rope_layer_interval` layers.

    ```python
    >>> from transformers import SmolLM3Model, SmolLM3Config

    >>> # Initializing a SmolLM3 style configuration
    >>> configuration = SmolLM3Config()

    >>> # Initializing a model from the SmolLM3 style configuration
    >>> model = SmolLM3Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úsmollm3Úpast_key_valuesg    €„>AÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi õ Ú
vocab_sizei   Úhidden_sizei +  Úintermediate_sizeé$   Únum_hidden_layersé   Únum_attention_headsé   NÚnum_key_value_headsÚsiluÚ
hidden_acti €  Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheiô Úpad_token_idi ô Úbos_token_idiô Úeos_token_idÚrope_parametersFÚuse_sliding_windowÚsliding_windowÚno_rope_layersÚno_rope_layer_intervalÚlayer_typesÚattention_biasç        Úattention_dropoutÚmlp_biasÚtie_word_embeddingsc                 ó¨  •‡ — ‰ j         €‰ j        ‰ _         ‰ j        €%ˆ fd„t          ‰ j        ¦  «        D ¦   «         ‰ _        ‰ j        €pg ‰ _        t          ‰ j        ¦  «        D ]T}‰ j        |         }‰ j        r$‰ j        �|s‰ j                             d¦  «         Œ:‰ j                             d¦  «         ŒU t          ¦   «         j
        di |¤Ž d S )Nc                 óL   •— g | ] }t          |d z   ‰j        z  dk    ¦  «        ‘Œ!S )é   r   )Úintr>   )Ú.0Ú	layer_idxÚselfs     €úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/smollm3/modular_smollm3.pyú
<listcomp>z/SmolLM3Config.__post_init__.<locals>.<listcomp>v   s?   ø€ ð #ð #ð #ØLU•�Y ‘] dÔ&AÑAÀQÒFÑGÔGð#ð #ð #ó    Úsliding_attentionÚfull_attention© )r0   r.   r=   Úranger,   r?   r;   r<   ÚappendÚsuperÚ__post_init__)rK   ÚkwargsrJ   Úhas_ropeÚ	__class__s   `   €rL   rU   zSmolLM3Config.__post_init__q   s  øø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔÐ&ð#ð #ð #ð #ÝY^Ð_cÔ_uÑYvÔYvð#ñ #ô #ˆDÔð ÔÐ#Ø!ˆDÔÝ" 4Ô#9Ñ:Ô:ð >ð >�	ØÔ.¨yÔ9�ØÔ*ð >¨tÔ/BÐ/NÐW_Ð/NØÔ$×+Ò+Ð,?Ñ@Ô@Ð@Ð@àÔ$×+Ò+Ð,<Ñ=Ô=Ð=Ð=à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rN   )+Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planr(   rH   Ú__annotations__r)   r*   r,   r.   r0   r2   Ústrr3   r4   Úfloatr5   r6   Úboolr7   r8   r9   Úlistr:   r	   Údictr;   r<   r=   r>   r?   r@   rB   rC   rD   rU   Ú__classcell__©rX   s   @rL   r   r   ,   s�  ø€ € € € € € ðð ð, €JØ#4Ð"5ÐØ€Mð &/Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ðð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&'Ð˜˜t™Ð'Ð'Ñ'Ø€J�ÐÐÑØ#(Ð˜SÐ(Ð(Ñ(Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ%€L�#˜‘*Ð%Ð%Ñ%Ø%€L�#˜‘*Ð%Ð%Ñ%Ø+1€L�#˜˜Sœ	‘/ DÑ(Ð1Ð1Ñ1Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø$Ð˜Ð$Ð$Ñ$Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø'+€N�D˜”I Ñ$Ð+Ð+Ñ+Ø"#Ð˜CÐ#Ð#Ñ#Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø €N�DÐ Ð Ñ Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€HˆdÐÐÑØ $Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð (ð (ð (ð (rN   r   c                   ó   — e Zd ZdS )ÚSmolLM3RotaryEmbeddingN©rY   rZ   r[   rQ   rN   rL   rk   rk   †   ó   € € € € € Ø€DrN   rk   c                   óÆ   ‡ — e Zd Zdedefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  f         fd„Zˆ xZS )ÚSmolLM3AttentionÚconfigrJ   c                 ó¼   •— t          ¦   «                              ||¦  «         |j        |         | _        |j        r|j        |         dk    r|j        nd | _        d S )NrO   )rT   Ú__init__r=   Úuse_roper;   r?   r<   )rK   rp   rJ   rX   s      €rL   rr   zSmolLM3Attention.__init__‹   si   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+àÔ-¨iÔ8ˆŒð Ô(ðØ-3Ô-?À	Ô-JÐNaÒ-aÐ-að Ô!Ð!àð 	ÔÐÐrN   Nr#   Úposition_embeddingsr$   r   rV   Úreturnc                 ó<  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
| j        r|\  }}t          ||	||¦  «        \  }}	|�| 	                    |	|
| j
        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NéÿÿÿÿrG   r   rA   )ÚdropoutÚscalingr<   )ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projrs   r   ÚupdaterJ   r
   Úget_interfacerp   Ú_attn_implementationr   ÚtrainingrB   ry   r<   ÚreshapeÚ
contiguousÚo_proj)rK   r#   rt   r$   r   rV   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rL   ÚforwardzSmolLM3Attention.forward•   sÑ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàŒ=ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(rN   )N)rY   rZ   r[   r   rH   rr   ÚtorchÚTensorÚtupler   r   r   r’   rh   ri   s   @rL   ro   ro   Š   s×   ø€ € € € € ð
˜}ð 
¸ð 
ð 
ð 
ð 
ð 
ð 
ð )-ð()ð ()à”|ð()ð # 5¤<°´Ð#=Ô>ð()ð œ tÑ+ð	()ð
  ™ð()ð Ð-Ô.ð()ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð()ð ()ð ()ð ()ð ()ð ()ð ()ð ()rN   ro   c                   ó   — e Zd ZdS )ÚSmolLM3DecoderLayerNrl   rQ   rN   rL   r—   r—   À   rm   rN   r—   c                   ó   — e Zd ZdS )ÚSmolLM3PreTrainedModelNrl   rQ   rN   rL   r™   r™   Ä   rm   rN   r™   c                   ó   — e Zd ZdS )ÚSmolLM3ModelNrl   rQ   rN   rL   r›   r›   È   rm   rN   r›   c                   ó   — e Zd ZdS )ÚSmolLM3ForCausalLMNrl   rQ   rN   rL   r�   r�   Ì   rm   rN   r�   c                   ó   — e Zd ZdS )Ú SmolLM3ForSequenceClassificationNrl   rQ   rN   rL   rŸ   rŸ   Ð   rm   rN   rŸ   c                   ó   — e Zd ZdS )ÚSmolLM3ForTokenClassificationNrl   rQ   rN   rL   r¡   r¡   Ô   rm   rN   r¡   c                   ó   — e Zd ZdS )ÚSmolLM3ForQuestionAnsweringNrl   rQ   rN   rL   r£   r£   Ø   rm   rN   r£   )r   r™   r›   r�   rŸ   r¡   r£   )/Úcollections.abcr   r“   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úconfiguration_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_rope_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úqwen2.modeling_qwen2r   r   Ú
get_loggerrY   Úloggerr   rk   ro   r—   r™   r›   r�   rŸ   r¡   r£   Ú__all__rQ   rN   rL   ú<module>r²      sE  ðð %Ð $Ð $Ð $Ð $Ð $à €€€Ø .Ð .Ð .Ð .Ð .Ð .à  Ð  Ð  Ð  Ð  Ð  Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø BÐ BÐ BÐ BÐ BÐ BØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð DÐ CÐ CÐ CÐ CÐ CÐ CÐ Cð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð5Ð6Ñ6Ô6ØðU(ð U(ð U(ð U(ð U(Ð$ñ U(ô U(ñ „ñ 7Ô6ðU(ðp	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð3)ð 3)ð 3)ð 3)ð 3)�~ñ 3)ô 3)ð 3)ðl	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð1ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	�:ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð'Eñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$?ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð";ñ 	ô 	ð 	ðð ð €€€rN   