§
    ‚Štj>+  ã                   óR  — d dl mZ ddlmZ ddlmZ ddlmZmZ  ej	        e
¦  «        Z ed¬¦  «        e G d„ d	e¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d
„ de¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Zg d¢ZdS )é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringÚloggingz meta-llama/Llama-4-Scout-17B-16E)Ú
checkpointc                   ó¢  — e Zd ZU dZddddddddœZdZdZdZee	d	<   d
Z
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZeee         z  eeef         z  e	d<   dZeee         z  eeef         z  e	d<   dZee	d<   dZee	d<   dZee	d<   d Zee	d!<   d"Zee	d#<   d"Zee	d$<   d%Zee	d&<   d'Zeez  e	d(<   d'Zeez  e	d)<   d*Z e!e"z  d*z  e	d+<   d*S ),ÚLlama4VisionConfigaw  
    vision_output_dim (`int`, *optional*, defaults to 7680):
        Dimensionality of the vision model output. Includes output of transformer
        encoder with intermediate layers and global transformer encoder.
    pixel_shuffle_ratio (`float`, *optional*, defaults to 0.5):
        Pixel-shuffle ratio for downsampling patch tokens. Smaller values produce fewer tokens (more downsampling).
    projector_input_dim (`int`, *optional*, defaults to 4096):
        Width of the vision adapter MLP before pixel shuffle. Larger value increases capacity and compute.
    projector_output_dim (`int`, *optional*, defaults to 4096):
        Output width of the vision adapter. Larger value yields higher-dimensional image features.
    projector_dropout (`float`, *optional*, defaults to 0.0):
        Dropout rate inside the vision adapter MLP. Higher value adds more regularization.
    ÚcolwiseÚrowwiseÚcolwise_gather_output)zmodel.layers.*.self_attn.q_projzmodel.layers.*.self_attn.k_projzmodel.layers.*.self_attn.v_projzmodel.layers.*.self_attn.o_projzvision_adapter.mlp.fc1zvision_adapter.mlp.fc2zpatch_embedding.linearÚllama4_vision_modelÚvision_configi   Úhidden_sizeÚgeluÚ
hidden_acté"   Únum_hidden_layersé   Únum_attention_headsr   Únum_channelsi   Úintermediate_sizei   Úvision_output_dimiÀ  Ú
image_sizeé   Ú
patch_sizeçñhãˆµøä>Únorm_epsÚdefaultÚvision_feature_select_strategyç{®Gáz”?Úinitializer_rangeg      à?Úpixel_shuffle_ratioi   Úprojector_input_dimÚprojector_output_dimFÚmulti_modal_projector_biasç        Úprojector_dropoutÚattention_dropoutNÚrope_parameters)#Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úbase_model_tp_planÚ
model_typeÚbase_config_keyr   ÚintÚ__annotations__r   Ústrr   r   r   r   r   r   ÚlistÚtupler   r   Úfloatr!   r#   r$   r%   r&   r'   Úboolr)   r*   r+   r   Údict© ó    úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/llama4/configuration_llama4.pyr   r      sÄ  € € € € € € ðð ð ,5Ø+4Ø+4Ø+4Ø"+Ø"+Ø"9ðð Ðð '€JØ%€Oà€K�ÐÐÑØ€J�ÐÐÑØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø€L�#ÐÐÑØ!Ð�sÐ!Ð!Ñ!Ø!Ð�sÐ!Ð!Ñ!Ø47€J��d˜3”i‘ %¨¨S¨¤/Ñ1Ð7Ð7Ñ7Ø46€J��d˜3”i‘ %¨¨S¨¤/Ñ1Ð6Ð6Ñ6Ø€HˆeÐÐÑØ*3Ð" CÐ3Ð3Ñ3Ø#Ð�uÐ#Ð#Ñ#Ø!$Ð˜Ð$Ð$Ñ$Ø#Ð˜Ð#Ð#Ñ#Ø $Ð˜#Ð$Ð$Ñ$Ø',Ð Ð,Ð,Ñ,Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r<   r   c                   óÆ  ‡ — e Zd ZU dZdZdgZdZdddddddddddddœZddddd	d	dddd
dœ
ZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZeed<   dZe
ed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Ze
d&z  ed'<   d(Ze
d&z  ed)<   d*Ze
ee
         z  d&z  ed+<   d,Zeed-<   d.Z ee
z  ed/<   d(Z!e
ed0<   d1Z"e
ed2<   d&Z#ee
         d&z  ed3<   d(Z$e
ed4<   d$Z%eed5<   d,Z&eed6<   d7Z'eed8<   d.Z(eed9<   d&Z)e*e+z  d&z  ed:<   d&Z,ee
         d&z  ed;<   d<Z-e
ed=<   dZ.e
d&z  ed><   d&Z/ee         d&z  ed?<   d$Z0eed@<   dZ1e
edA<   dBZ2eedC<   d,Z3eedD<   ˆ fdE„Z4ˆ xZ5S )FÚLlama4TextConfigaO  
    intermediate_size_mlp (`int`, *optional*, defaults to 16384):
        Intermediate size of dense MLP layers. Larger value increases FFN capacity and compute.
    moe_layers (`list[int]`, *optional*):
        List of layer indices that use MoE. Overrides `interleave_moe_layer_step` when set.
    interleave_moe_layer_step (`int`, *optional*, defaults to 1):
        Spacing between MoE layers when `moe_layers` is `None`. Larger value means fewer MoE layers.
    use_qk_norm (`bool`, *optional*, defaults to `True`):
        Whether to L2-normalize queries/keys on RoPE layers. Can stabilize attention when enabled.
    no_rope_layers (`list[int]`, *optional*):
        List with at least the same length as the number of layers in the model.
        A `1` at an index position indicates that the corresponding layer will use RoPE,
        while a `0` indicates that it's a NoPE layer.
    no_rope_layer_interval (`int`, *optional*, defaults to 4):
        If `no_rope_layers` is `None`, it will be created using a NoPE layer every
        `no_rope_layer_interval` layers.
    attention_chunk_size (`int`, *optional*, defaults to 8192):
        Chunk size for the attention computation. Smaller value enforces more local attention and lowers memory.
    attn_temperature_tuning (`bool`, *optional*, defaults to `True`):
        Whether to dynamically scale the attention temperature for each query token based on sequence length.
        Recommended for long sequences (e.g., >32k tokens) to maintain stable output results.
    floor_scale (`int`, *optional*, defaults to 8192):
        Base scale (in tokens) for attention temperature tuning. Larger value delays scaling to longer positions.
    attn_scale (`float`, *optional*, defaults to 0.1):
        Strength of attention temperature tuning. Larger value increases scaling at long positions.

    Example:
    Úllama4_textÚpast_key_valuesg    €„Ar   r   Úpacked_rowwise)úlayers.*.self_attn.q_projúlayers.*.self_attn.k_projúlayers.*.self_attn.v_projúlayers.*.self_attn.o_projz-layers.*.feed_forward.shared_expert.gate_projz+layers.*.feed_forward.shared_expert.up_projz-layers.*.feed_forward.shared_expert.down_projú*layers.*.feed_forward.experts.gate_up_projú'layers.*.feed_forward.experts.down_projúlayers.*.feed_forward.gate_projúlayers.*.feed_forward.up_projúlayers.*.feed_forward.down_projÚgrouped_gemmÚ	ep_router)
rC   rD   rE   rF   rG   rH   rI   rJ   rK   zlayers.*.feed_forward.routeri@ Ú
vocab_sizei   r   i    r   i @  Úintermediate_size_mlpé0   r   é(   r   é   Únum_key_value_headsé€   Úhead_dimÚsilur   i   Úmax_position_embeddingsr"   r#   r   Úrms_norm_epsTÚ	use_cacheNÚpad_token_idé   Úbos_token_idé   Úeos_token_idFÚtie_word_embeddingsr(   r*   Únum_experts_per_tokr   Únum_local_expertsÚ
moe_layersÚinterleave_moe_layer_stepÚuse_qk_normÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiser+   Úno_rope_layersé   Úno_rope_layer_intervalÚattention_chunk_sizeÚlayer_typesÚattn_temperature_tuningÚfloor_scalegš™™™™™¹?Ú
attn_scaleÚattention_biasc                 óÐ  •‡ — ‰ j         €‰ j        ‰ _         ˆ fd„t          ‰ j        ¦  «        D ¦   «         }‰ j        r‰ j        n|‰ _        ‰ j        �‰ j        n‰ j        ‰ j        z  ‰ _        ‰ j        �‰ j        n/t          t          ‰ j	        dz
  ‰ j        ‰ j	        ¦  «        ¦  «        ‰ _        ‰ j
        €d„ ‰ j        D ¦   «         ‰ _
         t          ¦   «         j        di |¤Ž d S )Nc                 óL   •— g | ] }t          |d z   ‰j        z  dk    ¦  «        ‘Œ!S )r[   r   )r3   rj   )Ú.0Ú	layer_idxÚselfs     €r=   ú
<listcomp>z2Llama4TextConfig.__post_init__.<locals>.<listcomp>³   s?   ø€ ð "
ð "
ð "
ØHQ�C�˜Q‘ $Ô"=Ñ=ÀÒBÑCÔCð"
ð "
ð "
r<   r[   c                 ó   — g | ]}|rd nd‘Œ	S )Úchunked_attentionÚfull_attentionr;   )rs   Úno_ropes     r=   rv   z2Llama4TextConfig.__post_init__.<locals>.<listcomp>Æ   s1   € ð  ð  ð  ØIP wÐDÐ#Ð#Ð4Dð ð  ð  r<   r;   )rS   r   Úranger   rh   rU   r   rb   r6   rc   rl   ÚsuperÚ__post_init__)ru   ÚkwargsÚdefault_no_rope_layersÚ	__class__s   `  €r=   r}   zLlama4TextConfig.__post_init__¯   s'  øø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$ð"
ð "
ð "
ð "
ÝUZÐ[_Ô[qÑUrÔUrð"
ñ "
ô "
Ðð 6:Ô5HÐd˜dÔ1Ð1ÐNdˆÔØ)-¬Ð)B˜œ˜ÈÔHXÐ\`Ô\tÑHtˆŒð ŒÐ*ð ŒOˆOåÝØÔ2°QÑ6ØÔ*ØÔ2ñô ñô ð 	Œð ÔÐ#ð ð  ØTXÔTgð ñ  ô  ˆDÔð 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r<   )6r,   r-   r.   r/   r1   Úkeys_to_ignore_at_inferenceÚdefault_thetar0   Úbase_model_ep_planrN   r3   r4   r   r   rO   r   r   rS   rU   r   r5   rW   r#   r8   rX   rY   r9   rZ   r\   r^   r6   r_   r*   r`   ra   rb   rc   rd   re   rf   rg   r+   r   r:   rh   rj   rk   rl   rm   rn   ro   rp   r}   Ú__classcell__©r€   s   @r=   r?   r?   M   s=  ø€ € € € € € ðð ð: €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø9BØ7@Ø9BØ6FØ3<Ø+4Ø)2Ø+4ðð Ðð &/Ø%.Ø%.Ø%.Ø6DØ3AØ+4Ø)2Ø+4Ø(3ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!Ø!&Ð˜3Ð&Ð&Ñ&ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€HˆcÐÐÑØ€J�ÐÐÑØ#,Ð˜SÐ,Ð,Ñ,Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ#€L�#˜‘*Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø %Ð˜Ð%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØ#'€J��S”	˜DÑ Ð'Ð'Ñ'Ø%&Ð˜sÐ&Ð&Ñ&Ø€K�ÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø!$Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø'+€N�D˜”I Ñ$Ð+Ð+Ñ+Ø"#Ð˜CÐ#Ð#Ñ#Ø'+Ð˜# ™*Ð+Ð+Ñ+Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø$(Ð˜TÐ(Ð(Ñ(Ø€K�ÐÐÑØ€J�ÐÐÑØ €N�DÐ Ð Ñ ð(ð (ð (ð (ð (ð (ð (ð (ð (r<   r?   c                   ó²   ‡ — e Zd ZU dZdZddddœZeedœZdd	iZ	d
Z
eez  d
z  ed<   d
Zeez  d
z  ed<   dZeed<   dZeed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚLlama4Configat  
    boi_token_index (`int`, *optional*, defaults to 200080):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 200081):
        The end-of-image token index to wrap the image prompt.

    ```python
    >>> from transformers import Llama4Model, Llama4Config

    >>> # Initializing a Llama4 7B style configuration
    >>> configuration = Llama4Config()

    >>> # Initializing a model from the Llama4 7B style configuration
    >>> model = Llama4Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úllama4Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configr   zmulti_modal_projector.linear_1Úcolwise_repNr   r�   i� i‘ iœ Fr_   c                 óÎ  •— | j         €.t          ¦   «         | _         t                               d¦  «         n0t	          | j         t
          ¦  «        rt          di | j         ¤Ž| _         | j        €.t          ¦   «         | _        t                               d¦  «         n0t	          | j        t
          ¦  «        rt          di | j        ¤Ž| _         t          ¦   «         j	        di |¤Ž d S )Nz9vision_config is None, using default llama4 vision configz5text_config is None, using default llama4 text configr;   )
r   r   ÚloggerÚinfoÚ
isinstancer:   r�   r?   r|   r}   )ru   r~   r€   s     €r=   r}   zLlama4Config.__post_init__ö   sÞ   ø€ ØÔÐ%Ý!3Ñ!5Ô!5ˆDÔÝ�KŠKÐSÑTÔTÐTÐTÝ˜Ô*­DÑ1Ô1ð 	JÝ!3Ð!IÐ!I°dÔ6HÐ!IÐ!IˆDÔàÔÐ#Ý/Ñ1Ô1ˆDÔÝ�KŠKÐOÑPÔPÐPÐPÝ˜Ô(­$Ñ/Ô/ð 	DÝ/ÐCÐC°$Ô2BÐCÐCˆDÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'r<   )r,   r-   r.   r/   r1   Úattribute_mapr?   r   Úsub_configsr0   r   r:   r   r4   r�   rŠ   r3   r‹   r‰   r_   r9   r}   r„   r…   s   @r=   r‡   r‡   Í   s÷   ø€ € € € € € ðð ð( €Jà-Ø)Ø)ðð €Mð
 #3ÐEWÐXÐX€Kà(¨-ðÐð 59€M�4Ð*Ñ*¨TÑ1Ð8Ð8Ñ8Ø26€K�Ð(Ñ(¨4Ñ/Ð6Ð6Ñ6Ø!€O�SÐ!Ð!Ñ!Ø!€O�SÐ!Ð!Ñ!Ø#Ð�sÐ#Ð#Ñ#Ø %Ð˜Ð%Ð%Ñ%ð(ð (ð (ð (ð (ð (ð (ð (ð (r<   r‡   )r‡   r?   r   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú
get_loggerr,   r’   r   r?   r‡   Ú__all__r;   r<   r=   ú<module>r�      sv  ðð" /Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 
ˆÔ	˜HÑ	%Ô	%€ð €Ð=Ð>Ñ>Ô>Øð-9ð -9ð -9ð -9ð -9Ð)ñ -9ô -9ñ „ñ ?Ô>ð-9ð` €Ð=Ð>Ñ>Ô>Øð{(ð {(ð {(ð {(ð {(Ð'ñ {(ô {(ñ „ñ ?Ô>ð{(ð| €Ð=Ð>Ñ>Ô>Øð3(ð 3(ð 3(ð 3(ð 3(Ð#ñ 3(ô 3(ñ „ñ ?Ô>ð3(ðl EÐ
DÐ
D€€€r<   