§
    ‚Štjúç  ã                   óê  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	c mZ d dlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;  e2j<        e=¦  «        Z> G d„ de	j?        ¦  «        Z@ G d„ de	j?        ¦  «        ZA G d„ dej	        j?        ¦  «        ZB G d„ de	j?        ¦  «        ZC G d„ d e	jD        ¦  «        ZE G d!„ d"e	j?        ¦  «        ZF G d#„ d$e	j?        ¦  «        ZGd%ejH        d&ejH        d'ejH        d(eIejH        ejH        f         fd)„ZJd*ejH        d+eKd(ejH        fd,„ZL	 d`d.e	j?        d/ejH        d0ejH        d1ejH        d2ejH        dz  d3eMd4eMfd5„ZN	 d`d.e	j?        d/ejH        d0ejH        d1ejH        d2ejH        dz  d3eMd4eMfd6„ZO G d7„ d8e	j?        ¦  «        ZP G d9„ d:e¦  «        ZQe0 G d;„ d<e+¦  «        ¦   «         ZRe0 G d=„ d>eR¦  «        ¦   «         ZS G d?„ d@eRe¦  «        ZT e0dA¬B¦  «        e G dC„ dDe%¦  «        ¦   «         ¦   «         ZU G dE„ dFej	        j?        ¦  «        ZV G dG„ dHe	j?        ¦  «        ZWdI„ ZX G dJ„ dKe	j?        ¦  «        ZYdLejH        d/ejH        fdM„ZZd/ejH        d0ejH        dLejH        d(eIejH        ejH        f         fdN„Z[ G dO„ dPe	j?        ¦  «        Z\ G dQ„ dRe	j?        ¦  «        Z] G dS„ dTe¦  «        Z^ G dU„ dVe	j?        ¦  «        Z_ G dW„ dXe	j?        ¦  «        Z` G dY„ dZe	j?        ¦  «        Za G d[„ d\eR¦  «        Zb G d]„ d^eRe¦  «        Zcg d_¢ZddS )aé    N)ÚCallable)Ú	dataclass)ÚOptional)ÚLlama4VisionConfigé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚcreate_chunked_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚLlama4ConfigÚLlama4TextConfigc                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4TextExpertsÚconfigc                 óÎ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        | j        | _        t          j        t          j
        | j        | j        d| j        z  ¦  «        ¦  «        | _        t          j        t          j        | j        | j        | j        f¦  «        ¦  «        | _        t          |j                 | _        d S ©Né   )ÚsuperÚ__init__Únum_local_expertsÚnum_expertsÚintermediate_sizeÚhidden_sizeÚ
expert_dimÚnnÚ	ParameterÚtorchÚzerosÚgate_up_projÚemptyÚ	down_projr	   Ú
hidden_actÚact_fn©Úselfr(   Ú	__class__s     €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/llama4/modeling_llama4.pyr-   zLlama4TextExperts.__init__8   s²   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ!'Ô!9ˆÔØ!Ô-ˆÔØÔ0ˆŒÝœL­¬°TÔ5EÀtÔGWÐYZÐ]aÔ]lÑYlÑ)mÔ)mÑnÔnˆÔÝœ¥e¤k°4Ô3CÀTÄ_ÐVZÔVfÐ2gÑ&hÔ&hÑiÔiˆŒÝ˜VÔ.Ô/ˆŒˆˆó    Úhidden_statesÚreturnc                 ó\  — |                      | j        j        d         d| j        ¦  «        }t	          j        || j        ¦  «        }|                     dd¬¦  «        \  }}t	          j        ||                      |¦  «        z  | j        ¦  «        }|                      d| j        ¦  «        }|S )a2  
        This should really not be run on a single machine, as we are reaching compute bound:
        - the inputs are expected to be "sorted" per expert already.
        - the weights are viewed with another dim, to match num_expert, 1, shape * num_tokens, shape

        Args:
            hidden_states (torch.Tensor): (batch_size * token_num, hidden_size)
            selected_experts (torch.Tensor): (batch_size * token_num, top_k)
            routing_weights (torch.Tensor): (batch_size * token_num, top_k)
        Returns:
            torch.Tensor
        r   éÿÿÿÿr+   ©Údim)	Úviewr7   Úshaper1   r5   ÚbmmÚchunkr;   r9   )r=   rA   Úgate_upÚgateÚupÚnext_statess         r?   ÚforwardzLlama4TextExperts.forwardB   s˜   € ð &×*Ò*¨4Ô+<Ô+BÀ1Ô+EÀrÈ4ÔK[Ñ\Ô\ˆÝ”)˜M¨4Ô+<Ñ=Ô=ˆØ—=’= ¨�=Ñ+Ô+‰ˆˆbÝ”i  d§k¢k°$Ñ&7Ô&7Ñ!7¸$¼.ÑIÔIˆØ!×&Ò& r¨4Ô+;Ñ<Ô<ˆØÐr@   )	Ú__name__Ú
__module__Ú__qualname__r%   r-   r5   ÚTensorrO   Ú__classcell__©r>   s   @r?   r'   r'   7   sk   ø€ € € € € ð0Ð/ð 0ð 0ð 0ð 0ð 0ð 0ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   r'   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLlama4TextMLPNc                 ó\  •— t          ¦   «                              ¦   «          |€|j        }|| _        t	          j        |j        |d¬¦  «        | _        t	          j        |j        |d¬¦  «        | _        t	          j        ||j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)r,   r-   r0   r(   r3   ÚLinearr1   Ú	gate_projÚup_projr9   r	   r:   Úactivation_fn)r=   r(   r0   r>   s      €r?   r-   zLlama4TextMLP.__init__Y   s™   ø€ Ý‰Œ×ÒÑÔÐàÐ$Ø &Ô 8ÐàˆŒÝœ 6Ô#5Ð7HÈuÐUÑUÔUˆŒÝ”y Ô!3Ð5FÈUÐSÑSÔSˆŒÝœÐ#4°fÔ6HÈuÐUÑUÔUˆŒÝ# FÔ$5Ô6ˆÔÐÐr@   c                 ó¨   — |                       |                      |¦  «        ¦  «        |                      |¦  «        z  }|                      |¦  «        S ©N)r_   r]   r^   r9   )r=   Úxr9   s      r?   rO   zLlama4TextMLP.forwarde   sB   € Ø×&Ò& t§~¢~°aÑ'8Ô'8Ñ9Ô9¸D¿LºLÈ¹O¼OÑKˆ	Ø�~Š~˜iÑ(Ô(Ð(r@   ra   ©rP   rQ   rR   r-   rO   rT   rU   s   @r?   rW   rW   X   sL   ø€ € € € € ð
7ð 
7ð 
7ð 
7ð 
7ð 
7ð)ð )ð )ð )ð )ð )ð )r@   rW   c                   ó8   ‡ — e Zd Zddefˆ fd„Zd„ Zd„ Zd„ Zˆ xZS )ÚLlama4TextL2Normç�íµ ÷Æ°>Úepsc                 óV   •— t          ¦   «                              ¦   «          || _        d S ra   )r,   r-   rg   )r=   rg   r>   s     €r?   r-   zLlama4TextL2Norm.__init__k   s$   ø€ Ý‰Œ×ÒÑÔÐØˆŒˆˆr@   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S ©Nr+   rD   T)Úkeepdim©r5   ÚrsqrtÚpowÚmeanrg   ©r=   rb   s     r?   Ú_normzLlama4TextL2Norm._normo   ó8   € Ø•5”;˜qŸušu Q™xœxŸ}š}¨R¸˜}Ñ>Ô>ÀÄÑIÑJÔJÑJÐJr@   c                 óv   — |                       |                     ¦   «         ¦  «                             |¦  «        S ra   )rq   ÚfloatÚtype_asrp   s     r?   rO   zLlama4TextL2Norm.forwardr   s*   € Ø�zŠz˜!Ÿ'š'™)œ)Ñ$Ô$×,Ò,¨QÑ/Ô/Ð/r@   c                 ó   — d| j         › �S )Nzeps=©rg   ©r=   s    r?   Ú
extra_reprzLlama4TextL2Norm.extra_repru   s   € Ø �d”hÐ Ð Ð r@   )rf   )	rP   rQ   rR   rt   r-   rq   rO   ry   rT   rU   s   @r?   re   re   j   sy   ø€ € € € € ðð ˜Eð ð ð ð ð ð ðKð Kð Kð0ð 0ð 0ð!ð !ð !ð !ð !ð !ð !r@   re   c                   ó2   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zd„ Zˆ xZS )ÚLlama4TextRMSNormçñhãˆµøä>c                 ó¬   •— t          ¦   «                              ¦   «          || _        t          j        t          j        |¦  «        ¦  «        | _        dS )z<
        Llama4RMSNorm is equivalent to T5LayerNorm
        N)r,   r-   rg   r3   r4   r5   ÚonesÚweight)r=   r1   rg   r>   s      €r?   r-   zLlama4TextRMSNorm.__init__z   sA   ø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒˆˆr@   c                 ó�   — |t          j        |                     d¦  «                             dd¬¦  «        | j        z   ¦  «        z  S rj   rl   rp   s     r?   rq   zLlama4TextRMSNorm._norm‚   rr   r@   c                 óŠ   — |                       |                     ¦   «         ¦  «                             |¦  «        }|| j        z  S ra   )rq   rt   ru   r   )r=   rb   Úoutputs      r?   rO   zLlama4TextRMSNorm.forward…   s6   € Ø—’˜AŸGšG™IœIÑ&Ô&×.Ò.¨qÑ1Ô1ˆØ˜œÑ#Ð#r@   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)Útupler   rH   rg   rx   s    r?   ry   zLlama4TextRMSNorm.extra_repr‰   s%   € Ý˜œÔ)Ñ*Ô*Ð<Ð<°$´(Ð<Ð<Ð<r@   )r|   )rP   rQ   rR   r-   rq   rO   ry   rT   rU   s   @r?   r{   r{   y   sm   ø€ € € € € ð<ð <ð <ð <ð <ð <ðKð Kð Kð$ð $ð $ð=ð =ð =ð =ð =ð =ð =r@   r{   c                   ó(   ‡ — e Zd Zˆ fd„Zˆ fd„Zˆ xZS )ÚLlama4Routerc                 ó”   •— t          ¦   «                              |j        |j        d¬¦  «         |j        | _        |j        | _        d S rY   )r,   r-   r1   r.   r/   Únum_experts_per_tokÚtop_kr<   s     €r?   r-   zLlama4Router.__init__Ž   sA   ø€ Ý‰Œ×Ò˜Ô+¨VÔ-EÈEÐÑRÔRÐRØ!Ô3ˆÔØÔ/ˆŒ
ˆ
ˆ
r@   c                 ó–  •— t          ¦   «                              |¦  «        }t          j        || j        d¬¦  «        \  }}t          j        |t          d¦  «        ¦  «                             d||¦  «        }t          j        j	         
                    |                     ¦   «         ¦  «                             |j        ¦  «        }||fS )Nr#   rE   z-inf)r,   rO   r5   Útopkr‰   Ú	full_likert   Úscatter_r3   Ú
functionalÚsigmoidÚtoÚdtype)r=   rA   Úrouter_logitsÚrouter_top_valueÚrouter_indicesÚrouter_scoresr>   s         €r?   rO   zLlama4Router.forward“   s¢   ø€ Ý™œŸš¨Ñ6Ô6ˆÝ+0¬:°mÀTÄZÐUVÐ+WÑ+WÔ+WÑ(Ð˜.Ýœ¨µu¸V±}´}ÑEÔE×NÒNÈqÐR`ÐbrÑsÔsˆÝœÔ+×3Ò3°M×4GÒ4GÑ4IÔ4IÑJÔJ×MÒMÈmÔNaÑbÔbˆØ˜mÐ+Ð+r@   rc   rU   s   @r?   r†   r†   �   sQ   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð
,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r@   r†   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4TextMoec                 ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          |¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        d S ra   )r,   r-   rˆ   r‰   r1   Ú
hidden_dimr.   r/   r'   Úexpertsr†   ÚrouterrW   Úshared_expertr<   s     €r?   r-   zLlama4TextMoe.__init__�   sl   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø Ô,ˆŒØ!Ô3ˆÔÝ(¨Ñ0Ô0ˆŒÝ" 6Ñ*Ô*ˆŒÝ*¨6Ñ2Ô2ˆÔÐÐr@   c                 ó  — |                      d| j        ¦  «        }|                      |¦  «        \  }}|                     |j        d         d¦  «        }||                     dd¦  «                              dd¦  «        z  }|                      |¦  «        }|                      |¦  «        }|                     |                      |j        d         d|j        d         ¦  «         	                    d¬¦  «        ¦  «         ||fS )NrD   r#   r   rE   )
Úreshaper™   r›   ÚrepeatrH   Ú	transposerš   rœ   Úadd_Úsum)r=   rA   r•   r’   Ú	routed_inÚ
routed_outÚouts          r?   rO   zLlama4TextMoe.forward¦   sì   € Ø%×-Ò-¨b°$´/ÑBÔBˆØ'+§{¢{°=Ñ'AÔ'AÑ$ˆ�}Ø!×(Ò(¨Ô)<¸QÔ)?ÀÑCÔCˆ	Ø × 7Ò 7¸¸1Ñ =Ô =× EÒ EÀbÈ!Ñ LÔ LÑLˆ	Ø—\’\ )Ñ,Ô,ˆ
Ø× Ò  Ñ/Ô/ˆØ�Š�×#Ò# MÔ$7¸Ô$:¸BÀ
Ô@PÐQSÔ@TÑUÔU×YÒYÐ^_ÐYÑ`Ô`ÑaÔaÐaØ�MÐ!Ð!r@   rc   rU   s   @r?   r—   r—   œ   sG   ø€ € € € € ð3ð 3ð 3ð 3ð 3ð"ð "ð "ð "ð "ð "ð "r@   r—   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚLlama4TextRotaryEmbeddingÚinv_freqNr(   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr¨   F©Ú
persistentÚoriginal_inv_freq)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr(   Úrope_parametersrª   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r=   r(   ÚdeviceÚrope_init_fnr¨   r>   s        €r?   r-   z"Llama4TextRotaryEmbedding.__init__¶   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr@   r·   ztorch.deviceÚseq_lenrB   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   r+   ©r‘   )r·   r‘   )	r²   Úgetattrr1   Únum_attention_headsr5   ÚarangeÚint64r�   rt   )r(   r·   r¹   ÚbaserF   Úattention_factorr¨   s          r?   r³   z9Llama4TextRotaryEmbedding.compute_default_rope_parametersÆ   sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r@   c                 ó`  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «        }|d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  | 	                    |j        ¦  «        |z   
                    dd¦  «        }t          j        t          j        |¦  «        |¦  «        }|| j        z  }d d d ¦  «         n# 1 swxY w Y   |S )	Nr   rD   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr+   )r¨   rt   ÚexpandrH   Ú
isinstancer·   ÚtypeÚstrr    r�   r    r5   ÚpolarÚ	ones_liker´   )r=   rb   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrÈ   ÚfreqsÚ	freqs_ciss           r?   rO   z!Llama4TextRotaryEmbedding.forwardå   s^  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔeÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	;ð 	;Ø&×)Ò)¨!¬(Ñ3Ô3Ð6KÑK×VÒVÐWXÐZ[Ñ\Ô\ˆEÝœ¥E¤O°EÑ$:Ô$:¸EÑBÔBˆIØ! DÔ$:Ñ:ˆIð	;ð 	;ð 	;ñ 	;ô 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;øøøð 	;ð 	;ð 	;ð 	;ð
 Ðs   Â4A#D#Ä#D'Ä*D'ra   )NNN)rP   rQ   rR   r5   rS   Ú__annotations__r%   r-   Ústaticmethodr   Úintr„   rt   r³   Úno_gradr   rO   rT   rU   s   @r?   r§   r§   ²   sú   ø€ € € € € € ØŒlÐÐÑðVð VÐ/ð Vð Vð Vð Vð Vð Vð  à*.Ø+/Ø"ð*ð *Ø  4Ñ'ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð< €U„]�_„_Øð
ð 
ñ Ôñ „_ð
ð 
ð 
ð 
ð 
r@   r§   ÚxqÚxkrÔ   rB   c           	      óF  — t          j         |                      ¦   «         j        g | j        d d…         ¢d‘d‘R Ž ¦  «        }t          j         |                     ¦   «         j        g |j        d d…         ¢d‘d‘R Ž ¦  «        }t          j        ||d d …d d …d d d …f         z  ¦  «                             d¦  «        }t          j        ||d d …d d …d d d …f         z  ¦  «                             d¦  «        }|                     | ¦  «        |                     |¦  «        fS )NrD   r+   r   )r5   Úview_as_complexrt   rž   rH   Úview_as_realÚflattenru   )rÙ   rÚ   rÔ   Úxq_Úxk_Úxq_outÚxk_outs          r?   Úapply_rotary_embrã   ô   s  € õ
 Ô
Ð 2 §¢¡
¤
Ô 2Ð I°B´H¸S¸b¸S´MÐ IÀ2Ð IÀqÐ IÐ IÐ IÑ
JÔ
J€CÝ
Ô
Ð 2 §¢¡
¤
Ô 2Ð I°B´H¸S¸b¸S´MÐ IÀ2Ð IÀqÐ IÐ IÐ IÑ
JÔ
J€CÝÔ  i°°°°1°1°1°d¸A¸A¸A°Ô&>Ñ >Ñ?Ô?×GÒGÈÑJÔJ€FÝÔ  i°°°°1°1°1°d¸A¸A¸A°Ô&>Ñ >Ñ?Ô?×GÒGÈÑJÔJ€FØ�>Š>˜"ÑÔ˜vŸ~š~¨bÑ1Ô1Ð1Ð1r@   rA   Ún_repc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rH   rÊ   rž   )rA   rä   ÚbatchÚnum_key_value_headsÚslenr¼   s         r?   Ú	repeat_kvré      s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr@   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÎ  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
d¬¦  «        }
t
          j                             |
|| j	        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «         
                    ¦   «         }||
fS )Nr+   r   rD   rE   ©ÚpÚtrainingr#   )ré   Únum_key_value_groupsr5   Úmatmulr    r3   rŽ   Úsoftmaxrñ   rõ   Ú
contiguous©rë   rì   rí   rî   rï   rð   rñ   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r?   Úeager_attention_forwardr     sÓ   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                 óÞ  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        | j        dz  z  }
|�|
|z   }
t          j                             |
d¬¦  «        }
t          j         	                    |
|| j
        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr+   r   ç      à¿rD   rE   ró   r#   )ré   rö   r5   r÷   r    r¼   r3   rŽ   rø   rñ   rõ   rù   rú   s               r?   Úvision_eager_attention_forwardr  '  sÛ   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀVÄ_ÐVZÑEZÑZ€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                   óä   ‡ — e Zd ZdZdefˆ fd„Z	 ddej        deej        ej        f         dej        dz  de	dz  d	e
e         d
eej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚLlama4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr(   c                 óª  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        | _        |j        |j        z  | _	        |j        | _        | j        dz  | _
        |j        | _        |j        | _        |j        | _        |j        | _        d| _        |j        |         | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        |j        | j        z  |j        ¬¦  «        | _        t%          j        |j        | j        z  |j        |j        ¬¦  «        | _        | j        j        r"| j        rt5          |j        ¦  «        | _        d S d S d S )Nr¼   r  TrZ   )r,   r-   r(   Ú	layer_idxr¿   r1   rÀ   r¼   rç   rö   rð   Ú
attn_scaleÚfloor_scaleÚattn_temperature_tuningÚattention_dropoutÚ	is_causalÚno_rope_layersÚuse_roper3   r\   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚuse_qk_normre   Úrms_norm_epsÚqk_norm©r=   r(   r  r>   s      €r?   r-   zLlama4TextAttention.__init__C  sË  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ#)Ô#=ˆÔ Ø$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø#)Ô#=ˆÔ Ø”} dÑ*ˆŒØ Ô+ˆŒØ!Ô-ˆÔØ'-Ô'EˆÔ$Ø!'Ô!9ˆÔØˆŒØÔ-¨iÔ8ˆŒÝ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð Œ;Ô"ð 	A t¤}ð 	AÝ+¨FÔ,?Ñ@Ô@ˆDŒLˆLˆLð	Að 	Að 	Að 	Ar@   NrA   Úposition_embeddingsrï   Úpast_key_valuesrû   rB   c                 óÄ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        } |                      |¦  «        j        g |¢d‘| j        ‘R Ž }	|                      |¦  «                             |¦  «                             dd¦  «        }
| j        r,t          ||	| 	                    |j
        ¦  «        ¦  «        \  }}	t          | d¦  «        r*|                      |¦  «        }|                      |	¦  «        }	| j        rò| j        së|�|                     | j        ¦  «        nd}t!          j        |j         d         |j
        ¬¦  «        |z   }t!          j        t!          j        |                     ¦   «         dz   | j        z  ¦  «        ¦  «        | j        z  dz   }|                     d|d         ddf¦  «                             g |¢d‘d‘R ¦  «        }||z   	                    |j        ¦  «        }|                     dd¦  «        }|	                     dd¦  «        }	|�|                     |	|
| j        ¦  «        \  }	}
t5          j        | j        j        t<          ¦  «        } || ||	|
|f| j        sdn| j         | j!        d	œ|¤Ž\  }} |j"        g |¢d‘R Ž  #                    ¦   «         }|  $                    |¦  «        }||fS )
NrD   r#   r+   r  r   ©r·   r½   rê   )rñ   rð   )%rH   r¼   r  rG   r  r  r    r  rã   r�   r·   Úhasattrr  r
  Úget_seq_lengthr  r5   rÁ   Úlog1pÚfloorrt   r	  r  rÊ   r‘   Úupdater   Úget_interfacer(   Ú_attn_implementationr   rõ   r  rð   rž   rù   r  )r=   rA   r  rï   r  rû   Úinput_shapeÚhidden_shapeÚquery_statesrü   rý   Úpast_seen_tokensÚ	positionsÚattn_scalesÚattention_interfacerÿ   rþ   s                    r?   rO   zLlama4TextAttention.forwarda  s$  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ4�T—[’[ Ñ/Ô/Ô4ÐU°kÐUÀ2ÐUÀtÄ}ÐUÐUÐUˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàŒ=ð 	Ý'7Ø˜jÐ*=×*@Ò*@ÀÔATÑ*UÔ*Uñ(ô (Ñ$ˆL˜*õ �4˜Ñ#Ô#ð 	2ØŸ<š<¨Ñ5Ô5ˆLØŸš jÑ1Ô1ˆJð Ô'ð 	O°´ð 	OØQ`ÐQl˜×=Ò=¸d¼nÑMÔMÐMÐrsÐÝœ ]Ô%8¸Ô%;ÀMÔDXÐYÑYÔYÐ\lÑlˆIå”�EœK¨¯ªÑ):Ô):¸SÑ)@ÀDÔDTÑ(TÑUÔUÑVÔVÐY]ÔYhÑhÐknÑnð ð &×*Ò*¨A¨{¸2¬ÀÀ1Ð+EÑFÔF×MÒMÐNbÐP[ÐNbÐ]^ÐNbÐ`aÐNbÐNbÑcÔcˆKØ(¨;Ñ6×:Ò:¸<Ô;MÑNÔNˆLà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r@   ra   )rP   rQ   rR   Ú__doc__r%   r-   r5   rS   r„   r
   r   r   rO   rT   rU   s   @r?   r  r  @  sí   ø€ € € € € ØGÐGðAÐ/ð Að Að Að Að Að AðF )-ð8)ð 8)à”|ð8)ð # 5¤<°´Ð#=Ô>ð8)ð œ tÑ+ð	8)ð
  ™ð8)ð Ð-Ô.ð8)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð8)ð 8)ð 8)ð 8)ð 8)ð 8)ð 8)ð 8)r@   r  c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  dedz  dedz  d	e	ej        ej        f         dz  d
e
e         de	ej        e	ej        ej        f         dz  f         fd„Zˆ xZS )ÚLlama4TextDecoderLayerc                 ó¢  •— t          ¦   «                              ¦   «          |j        | _        || _        t	          ||¦  «        | _        ||j        v | _        | j        rt          |¦  «        | _	        nt          ||j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _        d S )N)r0   rw   )r,   r-   r1   r  r  Ú	self_attnÚ
moe_layersÚis_moe_layerr—   Úfeed_forwardrW   Úintermediate_size_mlpr{   r  Úinput_layernormÚpost_attention_layernormr  s      €r?   r-   zLlama4TextDecoderLayer.__init__�  sÀ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ"ˆŒÝ,¨V°YÑ?Ô?ˆŒØ%¨Ô):Ð:ˆÔØÔð 	fÝ -¨fÑ 5Ô 5ˆDÔÐå -¨fÈÔHdÐ eÑ eÔ eˆDÔå0°Ô1CÈÔI\Ð]Ñ]Ô]ˆÔÝ(9¸&Ô:LÐRXÔReÐ(fÑ(fÔ(fˆÔ%Ð%Ð%r@   NFrA   rï   rÐ   r  Ú	use_cacher  rû   rB   c           	      ó  — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }	}
||	z   }|}|                      |¦  «        }|                      |¦  «        }| j        r|\  }}
||                     |j        ¦  «        z   }|S )N)rA   r  rï   r  r5  © )r3  r.  r4  r1  r0  rG   rH   )r=   rA   rï   rÐ   r  r5  r  rû   ÚresidualÚattention_statesÚ_s              r?   rO   zLlama4TextDecoderLayer.forward«  sÉ   € ð !ˆà×,Ò,¨]Ñ;Ô;ˆð -˜dœnð 
Ø'Ø 3Ø)Ø+Øð
ð 
ð ð
ð 
ÑÐ˜!ð !Ð#3Ñ3ˆð !ˆØ×5Ò5°mÑDÔDˆØ×)Ò)¨-Ñ8Ô8ˆØÔð 	-Ø,ÑˆM˜1Ø  =×#5Ò#5°h´nÑ#EÔ#EÑEˆØÐr@   )NNNFN)rP   rQ   rR   r-   r5   rS   Ú
LongTensorr
   Úboolr„   r   r   ÚFloatTensorrO   rT   rU   s   @r?   r,  r,  œ  s
  ø€ € € € € ðgð gð gð gð gð" /3Ø04Ø(,Ø!&ØHLð ð  à”|ð ð œ tÑ+ð ð Ô&¨Ñ-ð	 ð
  ™ð ð ˜$‘;ð ð # 5¤<°´Ð#=Ô>ÀÑEð ð Ð-Ô.ð ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð ð  ð  ð  ð  ð  ð  ð  r@   r,  c                   óv   ‡ — e Zd ZU eed<   dZdZdgZdZdZ	dZ
dZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚLlama4PreTrainedModelr(   )ÚimageÚtextTr  Fc                 óŒ  •— t          ¦   «                              |¦  «         t          | j        d¦  «        r| j        j        n| j        j        j        }t          |t          ¦  «        r:t          j	        |j
        d|¬¦  «         t          j	        |j        d|¬¦  «         d S t          |t          ¦  «        r4t          j        |j        |                     |j        ¦  «        ¦  «         d S t          |t           ¦  «        rBt          j	        |j        |j        ¬¦  «         t          j	        |j        |j        ¬¦  «         d S d S )NÚinitializer_rangerê   )ro   Ústd)rD  )r,   Ú_init_weightsr  r(   rC  Útext_configrË   r'   ÚinitÚnormal_r7   r9   ÚLlama4VisionRotaryEmbeddingÚcopy_Úfreqs_ciÚ_compute_freqs_ciÚLlama4VisionModelÚclass_embeddingÚscaleÚpositional_embedding_vlm)r=   rë   rD  r>   s      €r?   rE  z#Llama4PreTrainedModel._init_weightsÛ  s4  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%õ �t”{Ð$7Ñ8Ô8ð;ˆDŒKÔ)Ð)à”Ô(Ô:ð 	õ
 �fÕ/Ñ0Ô0ð 	LÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ ;Ñ<Ô<ð 	LÝŒJ�v”¨×(@Ò(@ÀÄÑ(OÔ(OÑPÔPÐPÐPÐPÝ˜Õ 1Ñ2Ô2ð 	LÝŒL˜Ô/°V´\ÐBÑBÔBÐBÝŒL˜Ô8¸f¼lÐKÑKÔKÐKÐKÐKð	Lð 	Lr@   )rP   rQ   rR   r$   rÕ   Úinput_modalitiesÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr5   rØ   rE  rT   rU   s   @r?   r?  r?  Î  s“   ø€ € € € € € àÐÐÑØ(ÐØ&*Ð#Ø#4Ð"5ÐØ ÐØ€NØÐà!ÐØ"&Ðà€U„]�_„_ðLð Lð Lð Lñ „_ðLð Lð Lð Lð Lr@   r?  c                   ó  ‡ — e Zd ZU dgZdZdZeed<   ee	e
dœZdefˆ fd„Zeeee	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dedz  dej        dz  dedz  dee         deez  fd„¦   «         ¦   «         ¦   «         ¦   «         Zˆ xZS )ÚLlama4TextModelr,  Úmodel)rA  r(   )Ú
attentionsrA   r’   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r7  )r,  )Ú.0r  r(   s     €r?   ú
<listcomp>z,Llama4TextModel.__init__.<locals>.<listcomp>   s$   ø€ ÐhÐhÐh¸9Õ# F¨IÑ6Ô6ÐhÐhÐhr@   rw   ©r(   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer3   Ú	Embeddingr1   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersr{   r  Únormr§   Ú
rotary_embÚgradient_checkpointingÚ	post_initr<   s    `€r?   r-   zLlama4TextModel.__init__ù  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhñ
ô 
ˆŒõ & fÔ&8¸fÔ>QÐRÑRÔRˆŒ	Ý3¸6ÐBÑBÔBˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr@   NÚ	input_idsrï   rÐ   r  Úinputs_embedsr5  rû   rB   c           
      ó&  — |d u |d uz  rt          d¦  «        ‚|€7|                      |                     | j        j        j        ¦  «        ¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j	        |j
        d         |j        ¬¦  «        |z   }|                     d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}|                      ||¦  «        }t#          | j        d | j        j        …         ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||dœ|¤Ž}Œ+|                      |¦  «        }t-          ||r|nd ¬	¦  «        S )Nú:You must specify exactly one of input_ids or inputs_embedsra  r   r#   r  )r(   rp  rï   r  rÐ   )Úfull_attentionÚchunked_attention)rï   rÐ   r  r5  r  )Úlast_hidden_stater  r7  )Ú
ValueErrorrf  r�   r   r·   r   r(   r  r5   rÁ   rH   Ú	unsqueezerË   Údictr   r   rl  Ú	enumeraterj  ri  Úlayer_typesrk  r   )r=   ro  rï   rÐ   r  rp  r5  rû   r&  Úcausal_mask_mappingÚmask_kwargsrA   Úfreq_cisÚiÚdecoder_layers                  r?   rO   zLlama4TextModel.forward	  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨i¯lªl¸4Ô;LÔ;SÔ;ZÑ.[Ô.[Ñ\Ô\ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%?Ð%NÐ%NÀ+Ð%NÐ%Nð#ð #Ðð
 &ˆð —?’? =°,Ñ?Ô?ˆå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#Ø$,ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r@   )NNNNNN)rP   rQ   rR   Ú_no_split_modulesÚbase_model_prefixrQ  r%   rÕ   r  r,  r—   Ú_can_record_outputsr-   r   r!   r"   r   r5   r;  rS   r
   r=  r<  r   r   r„   r   rO   rT   rU   s   @r?   rZ  rZ  í  sX  ø€ € € € € € à1Ð2ÐØÐØ ÐØÐÐÑà)Ø/Ø&ðð ÐðÐ/ð ð ð ð ð ð ð  ØØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
Ð(Ñ	(ð<
ð <
ð <
ñ „^ñ „_ñ  Ôñ Ôð<
ð <
ð <
ð <
ð <
r@   rZ  c                   ó*  ‡ — e Zd ZU dgZdZddiZddiZeed<   defˆ fd„Z	e
e	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dej        d	z  ded	z  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚLlama4ForCausalLMr,  Úlanguage_modelzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr(   c                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rY   )
r,   r-   rZ  r[  rd  r3   r\   r1   r†  rn  r<   s     €r?   r-   zLlama4ForCausalLM.__init__S  sj   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒð 	�ŠÑÔÐÐÐr@   Nr   ro  rï   rÐ   r  rp  Úlabelsr5  Úlogits_to_keeprû   rB   c	           
      óR  —  | j         d||||||dœ|	¤Ž}
|
d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        ¬¦  «        S )az  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Llama4ForCausalLM

        >>> model = Llama4ForCausalLM.from_pretrained("meta-llama4/Llama4-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama4/Llama4-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)ro  rï   rÐ   r  rp  r5  r   N)Úlogitsr‰  rd  )ÚlossrŒ  r  rA   r\  r7  )r[  rË   r×   Úslicer†  Úloss_functionr(   rd  r   r  rA   r\  )r=   ro  rï   rÐ   r  rp  r‰  r5  rŠ  rû   ÚoutputsrA   Úslice_indicesrŒ  r�  s                  r?   rO   zLlama4ForCausalLM.forward\  sö   € ðH �$”*ð 
ØØ)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð   œ
ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r@   )NNNNNNNr   )rP   rQ   rR   r€  r�  Ú_tied_weights_keysÚ_tp_planr%   rÕ   r-   r   r   r5   r;  rS   r
   r=  r<  r×   r   r   r„   r   rO   rT   rU   s   @r?   r„  r„  L  sg  ø€ € € € € € Ø1Ð2ÐØ(ÐØ*Ð,GÐHÐØÐ2Ð3€HØÐÐÑðÐ/ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ð:
ð :
àÔ# dÑ*ð:
ð œ tÑ+ð:
ð Ô&¨Ñ-ð	:
ð
  ™ð:
ð Ô(¨4Ñ/ð:
ð Ô  4Ñ'ð:
ð ˜$‘;ð:
ð ˜eœlÑ*ð:
ð Ð+Ô,ð:
ð 
Ð'Ñ	'ð:
ð :
ð :
ñ „^ñ Ôð:
ð :
ð :
ð :
ð :
r@   r„  zQ
    Base class for Llava causal language model (or autoregressive) outputs.
    ©Úcustom_introc                   óÖ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZej        dz  ed<   dS )	ÚLlama4CausalLMOutputWithPasta3  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size (batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nr�  rŒ  r  rA   r\  Úimage_hidden_states)rP   rQ   rR   r*  r�  r5   r=  rÕ   rŒ  r  r
   rA   r„   r\  r˜  r7  r@   r?   r—  r—  ›  sµ   € € € € € € ðð ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ø48Ð˜Ô*¨TÑ1Ð8Ð8Ñ8Ð8Ð8r@   r—  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4VisionMLP2c                 óX  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t	          j        | j        |j        d¬¦  «        | _        t	          j        |j        |j        d¬¦  «        | _	        t	          j
        ¦   «         | _        |j        | _        d S rY   )r,   r-   r1   r0   r3   r\   Úprojector_input_dimÚfc1Úprojector_output_dimÚfc2ÚGELUr_   Úprojector_dropoutrñ   r<   s     €r?   r-   zLlama4VisionMLP2.__init__º  s‰   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ!'Ô!9ˆÔÝ”9˜TÔ3°VÔ5OÐV[Ð\Ñ\Ô\ˆŒÝ”9˜VÔ8¸&Ô:UÐ\aÐbÑbÔbˆŒÝœW™YœYˆÔØÔ/ˆŒˆˆr@   c                 óè   — |                       |¦  «        }|                      |¦  «        }t          j        || j        | j        ¬¦  «        }|                      |                      |¦  «        ¦  «        S )Nró   )r�  r_   ÚFrñ   rõ   rŸ  ©r=   rA   s     r?   rO   zLlama4VisionMLP2.forwardÃ  s`   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆÝœ	 -°4´<È$Ì-ÐXÑXÔXˆØ×!Ò! $§(¢(¨=Ñ"9Ô"9Ñ:Ô:Ð:r@   rc   rU   s   @r?   rš  rš  ¹  sG   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð;ð ;ð ;ð ;ð ;ð ;ð ;r@   rš  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLlama4MultiModalProjectorc                 ó¨   •— t          ¦   «                              ¦   «          t          j        |j        j        |j        j        d¬¦  «        | _        d S rY   )	r,   r-   r3   r\   Úvision_configÚvision_output_dimrF  r1   Úlinear_1r<   s     €r?   r-   z"Llama4MultiModalProjector.__init__Ë  sJ   ø€ Ý‰Œ×ÒÑÔÐÝœ	ØÔ Ô2ØÔÔ*Øð
ñ 
ô 
ˆŒˆˆr@   c                 ó0   — |                       |¦  «        }|S ra   )rª  )r=   Úimage_featuresrA   s      r?   rO   z!Llama4MultiModalProjector.forwardÓ  s   € ØŸš nÑ5Ô5ˆØÐr@   rc   rU   s   @r?   r¦  r¦  Ê  sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð r@   r¦  c           
      ó²  — | j         \  }}}t          t          j        |¦  «        ¦  «        }|                      |||d¦  «        } |                      ¦   «         \  }}}}|                      ||t          ||z  ¦  «        t          ||z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|                     |t          ||z  ¦  «        t          ||z  ¦  «        t          ||dz  z  ¦  «        ¦  «        }|                     dddd¦  «                             ¦   «         }|                     |d|j         d         ¦  «        }	|	S )NrD   r   r+   r#   r   )rH   r×   ÚmathÚsqrtrG   ÚsizeÚpermuterù   )
Úinput_tensorÚshuffle_ratioÚ
batch_sizeÚnum_patchesÚchannelsÚ
patch_sizeÚheightÚwidthÚreshaped_tensorÚoutput_tensors
             r?   Úpixel_shuffler¼  Ø  sO  € à(4Ô(:Ñ%€J�˜XÝ•T”Y˜{Ñ+Ô+Ñ,Ô,€Jà×$Ò$ Z°¸ZÈÑLÔL€LØ*6×*;Ò*;Ñ*=Ô*=Ñ'€J�˜˜xà"×'Ò'¨
°F½CÀÈÑ@UÑ<VÔ<VÕX[Ð\dÐgtÑ\tÑXuÔXuÑvÔv€OØ%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF€Oà%×*Ò*Ø•C˜ Ñ.Ñ/Ô/µ°U¸]Ñ5JÑ1KÔ1KÍSÐQYÐ]jÐlmÑ]mÑQnÑMoÔMoñô €Oð &×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔF€Oà#×(Ò(¨°R¸Ô9NÈrÔ9RÑSÔS€MØÐr@   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4VisionPixelShuffleMLPc                 óè   •— t          ¦   «                              ¦   «          |j        | _        t          |j        | j        dz  z  ¦  «        | _        |j        | _        t          |¦  «        | _	        d S r*   )
r,   r-   Úpixel_shuffle_ratior×   rœ  Ú	inner_dimrž  Ú
output_dimrš  Úmlpr<   s     €r?   r-   z$Llama4VisionPixelShuffleMLP.__init__í  sa   ø€ Ý‰Œ×ÒÑÔÐØ#)Ô#=ˆÔ Ý˜VÔ7¸DÔ<TÐVWÑ<WÑXÑYÔYˆŒØ Ô5ˆŒÝ# FÑ+Ô+ˆŒˆˆr@   Úencoded_patchesrB   c                 óV   — t          || j        ¦  «        }|                      |¦  «        S ra   )r¼  rÀ  rÃ  )r=   rÄ  s     r?   rO   z#Llama4VisionPixelShuffleMLP.forwardô  s&   € Ý'¨¸Ô9QÑRÔRˆØ�xŠx˜Ñ(Ô(Ð(r@   ©rP   rQ   rR   r-   r5   rS   rO   rT   rU   s   @r?   r¾  r¾  ì  s^   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð) u¤|ð )¸¼ð )ð )ð )ð )ð )ð )ð )ð )r@   r¾  rK  c                 óf   ‡— |j         Šˆfd„t          |j        ¦  «        D ¦   «         } | j        |Ž S )Nc                 ó<   •— g | ]\  }}|d k    s	|‰d z
  k    r|nd ‘ŒS )r#   r7  )r_  r~  ÚdÚndims      €r?   r`  z)reshape_for_broadcast.<locals>.<listcomp>ü  s5   ø€ ÐTÐTÐT±T°Q¸�!�q’&�&˜A ¨¡šM˜MˆQˆQ¨qÐTÐTÐTr@   )rÊ  ry  rH   rG   )rK  rì   rH   rÊ  s      @r?   Úreshape_for_broadcastrË  ú  s<   ø€ ØŒ:€DØTÐTÐTÐT½YÀuÄ{Ñ=SÔ=SÐTÑTÔT€EØˆ8Œ=˜%Ð Ð r@   c                 ó\  — t          j         |                      ¦   «         j        g | j        d d…         ¢d‘d‘R Ž ¦  «        }t          j         |                     ¦   «         j        g |j        d d…         ¢d‘d‘R Ž ¦  «        }t          ||¬¦  «        }|                     |j        ¦  «        }t          j        ||z  ¦  «         	                    d¦  «        }t          j        ||z  ¦  «         	                    d¦  «        }| 
                    | ¦  «        | 
                    |¦  «        fS )NrD   r+   )rK  rì   r   )r5   rÜ   rt   rž   rH   rË  r�   r·   rÝ   rÞ   ru   )rì   rí   rK  Úquery_Úkey_Ú	query_outÚkey_outs          r?   Úvision_apply_rotary_embrÑ     s  € õ
 Ô"Ð#8 5§;¢;¡=¤=Ô#8Ð#R¸%¼+ÀcÀrÀcÔ:JÐ#RÈBÐ#RÐPQÐ#RÐ#RÐ#RÑSÔS€FÝÔ Ð!4 §¢¡¤Ô!4Ð!L°c´iÀÀÀ´nÐ!LÀbÐ!LÈ!Ð!LÐ!LÐ!LÑMÔM€DÝ$¨h¸fÐEÑEÔE€HØ�{Š{˜6œ=Ñ)Ô)€HÝÔ" 6¨HÑ#4Ñ5Ô5×=Ò=¸aÑ@Ô@€IÝÔ  ¨¡Ñ1Ô1×9Ò9¸!Ñ<Ô<€GØ×Ò˜UÑ#Ô# W§_¢_°SÑ%9Ô%9Ð9Ð9r@   c                   óÈ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej        dej        dej        dz  dedz  dee	         d	e
ej        ej        dz  e
ej                 dz  f         fd
„Zˆ xZS )ÚLlama4VisionAttentionr(   c                 ób  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        |j        z  | _        d| _        |j	        | _	        | j        dz  | _
        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nr#   r  TrZ   )r,   r-   r(   r1   Ú	embed_dimrÀ   Ú	num_headsr¼   rö   r  rð   r3   r\   r  r  r  r  r<   s     €r?   r-   zLlama4VisionAttention.__init__  sû   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØÔ*¨fÔ.HÑHˆŒØ$%ˆÔ!Ø!'Ô!9ˆÔØ”} dÑ*ˆŒå”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´ÀÄÑ0NÐUYÐZÑZÔZˆŒÝ”i ¤°´Ñ >ÀÄÐUYÐZÑZÔZˆŒˆˆr@   NrA   rK  rï   r  rû   rB   c                 óÚ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
t          ||	|¬¦  «        \  }}	|                     dd¦  «        }|	                     dd¦  «        }	|
                     dd¦  «        }
t          j	        | j
        j        t          ¦  «        } || ||	|
d f| j        sdn| j        d ddœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )NrD   )rK  r#   r+   rê   F)rñ   rð   r  )rH   r¼   r  rG   r  r  rÑ  r    r   r!  r(   r"  r  rõ   r  rž   rù   r  )r=   rA   rK  rï   r  rû   r#  r$  r%  rü   rý   r)  rÿ   rþ   s                 r?   rO   zLlama4VisionAttention.forward  sž  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆå#:¸<ÈÐ^fÐ#gÑ#gÔ#gÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆå(?Ô(MØŒKÔ,Õ.Lñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØØð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r@   ©NN)rP   rQ   rR   r   r-   r5   rS   r
   r   r   r„   rO   rT   rU   s   @r?   rÓ  rÓ    sÝ   ø€ € € € € ð[Ð1ð [ð [ð [ð [ð [ð [ð& /3Ø(,ð')ð ')à”|ð')ð ”,ð')ð œ tÑ+ð	')ð
  ™ð')ð Ð-Ô.ð')ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r@   rÓ  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4VisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          j        ¦   «         | _        t          j        |j        |j        d¬¦  «        | _	        t          j        |j        |j        d¬¦  «        | _
        d S )NTrZ   )r,   r-   r(   r3   r   r_   r\   r1   r0   r�  rŸ  r<   s     €r?   r-   zLlama4VisionMLP.__init__I  sp   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝœW™YœYˆÔÝ”9˜VÔ/°Ô1IÐPTÐUÑUÔUˆŒÝ”9˜VÔ5°vÔ7IÐPTÐUÑUÔUˆŒˆˆr@   rA   rB   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ra   )r�  r_   rŸ  r¤  s     r?   rO   zLlama4VisionMLP.forwardP  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr@   rÆ  rU   s   @r?   rÚ  rÚ  H  sc   ø€ € € € € ðVð Vð Vð Vð Vð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rÚ  c            
       ól   ‡ — e Zd Zdefˆ fd„Z	 	 d	dej        dej        dej        dz  dedz  fd„Zˆ xZ	S )
ÚLlama4VisionEncoderLayerr(   c                 ó(  •— t          ¦   «                              ¦   «          |j        | _        t          |¦  «        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _	        t          j        |j        ¦  «        | _
        d S ra   )r,   r-   r1   rÓ  r.  rÚ  rÃ  r3   Ú	LayerNormr3  r4  r<   s     €r?   r-   z!Llama4VisionEncoderLayer.__init__X  sr   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå.¨vÑ6Ô6ˆŒÝ" 6Ñ*Ô*ˆŒå!œ|¨FÔ,>Ñ?Ô?ˆÔÝ(*¬°VÔ5GÑ(HÔ(HˆÔ%Ð%Ð%r@   NÚhidden_staterK  rï   Úoutput_attentionsc                 óì   — |}|                       |¦  «        }|                      |||¬¦  «        \  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|f}|r||fz  }|S )N)rK  rï   )r3  r.  r4  rÃ  )r=   rá  rK  rï   râ  r8  rþ   r�  s           r?   rO   z Llama4VisionEncoderLayer.forwardb  s£   € ð  ˆà×+Ò+¨LÑ9Ô9ˆà%)§^¢^ØØØ)ð &4ñ &
ô &
Ñ"ˆ�lð
   ,Ñ.ˆð  ˆØ×4Ò4°\ÑBÔBˆØ—x’x Ñ-Ô-ˆØ ,Ñ.ˆà�/ˆàð 	'Ø˜�Ñ&ˆGàˆr@   rØ  )
rP   rQ   rR   r   r-   r5   rS   r<  rO   rT   rU   s   @r?   rÞ  rÞ  W  s¢   ø€ € € € € ðIÐ1ð Ið Ið Ið Ið Ið Ið /3Ø)-ðð à”lðð ”,ðð œ tÑ+ð	ð
   $™;ðð ð ð ð ð ð ð r@   rÞ  c                   ó’   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 ddej        dej        dej        dz  dedz  d	edz  d
edz  de	e
z  fd„Zˆ xZS )ÚLlama4VisionEncoderz½
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Llama4VisionEncoderLayer`].

    Args:
        config: Llama4VisionConfig
    r(   c                 óâ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        ‰| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7  )rÞ  )r_  r:  r(   s     €r?   r`  z0Llama4VisionEncoder.__init__.<locals>.<listcomp>�  s"   ø€ Ð$oÐ$oÐ$oÈ!Õ%=¸fÑ%EÔ%EÐ$oÐ$oÐ$or@   F)	r,   r-   r(   r3   rg  rh  ri  rj  rm  r<   s    `€r?   r-   zLlama4VisionEncoder.__init__Œ  sf   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$oÐ$oÐ$oÐ$oÍuÐU[ÔUmÑOnÔOnÐ$oÑ$oÔ$oÑpÔpˆŒØ&+ˆÔ#ØˆŒˆˆr@   NrA   rK  rï   râ  Úoutput_hidden_statesÚreturn_dictrB   c                 óX  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|rdnd}|rdnd}| j        D ]/}	|r||fz   } |	||||¬¦  «        }
|r||
d         fz   }|
d         }Œ0|r||fz   }|st          d„ |||fD ¦   «         ¦  «        S t          |||¬¦  «        S )ad  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr7  )rá  rï   râ  rK  r#   r   c              3   ó   K  — | ]}|®|V — Œ	d S ra   r7  ©r_  Úvs     r?   ú	<genexpr>z.Llama4VisionEncoder.forward.<locals>.<genexpr>Ï  s(   è è € ÐeÐe˜qÐWXÐWd˜ÐWdÐWdÐWdÐWdÐeÐer@   ©ru  rA   r\  )r(   râ  rè  ré  rj  r„   r   )r=   rA   rK  rï   râ  rè  ré  Úencoder_statesÚall_attentionsÚencoder_layerÚlayer_outputss              r?   rO   zLlama4VisionEncoder.forward“  s9  € ð> 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà3Ð=˜˜¸ˆØ0Ð:˜˜°dˆà!œ[ð 	-ð 	-ˆMØ#ð CØ!/°=Ð2BÑ!B�à)˜MØ*Ø-Ø"3Ø!ð	ñ ô ˆMð !ð FØ!/°=ÀÔ3CÐ2EÑ!E�à)¨!Ô,ˆMˆMàð 	?Ø+¨}Ð.>Ñ>ˆNàð 	fÝÐeÐe ]°NÀNÐ$SÐeÑeÔeÑeÔeÐeÝØ+¸>ÐVdð
ñ 
ô 
ð 	
r@   ©NNNN)rP   rQ   rR   r*  r   r-   r5   rS   r<  r„   r   rO   rT   rU   s   @r?   rå  rå  ƒ  sÖ   ø€ € € € € ðð ðÐ1ð ð ð ð ð ð ð /3Ø)-Ø,0Ø#'ð?
ð ?
à”|ð?
ð ”,ð?
ð œ tÑ+ð	?
ð
   $™;ð?
ð # T™kð?
ð ˜D‘[ð?
ð 
�Ñ	 ð?
ð ?
ð ?
ð ?
ð ?
ð ?
ð ?
ð ?
r@   rå  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚLlama4UnfoldConvolutionc                 óN  •— t          ¦   «                              ¦   «          |j        }t          |t          ¦  «        r||f}t
          j                             ||j        ¬¦  «        | _        t          j	        |j
        |d         z  |d         z  |j        d¬¦  «        | _        d S )N)Úkernel_sizeÚstrider   r#   FrZ   )r,   r-   r·  rË   r×   r5   r3   ÚUnfoldÚunfoldr\   Únum_channelsr1   Úlinear)r=   r(   rø  r>   s      €r?   r-   z Llama4UnfoldConvolution.__init__Ö  s–   ø€ Ý‰Œ×ÒÑÔÐØÔ'ˆÝ�k¥3Ñ'Ô'ð 	5Ø&¨Ð4ˆKÝ”h—o’o°+ÀfÔFW�oÑXÔXˆŒÝ”iØÔ +¨a¤.Ñ0°;¸q´>ÑAØÔØð
ñ 
ô 
ˆŒˆˆr@   rA   rB   c                 óˆ   — |                       |¦  «        }|                     ddd¦  «        }|                      |¦  «        }|S )Nr   r+   r#   )rû  r±  rý  r¤  s     r?   rO   zLlama4UnfoldConvolution.forwardâ  sA   € ØŸš MÑ2Ô2ˆØ%×-Ò-¨a°°AÑ6Ô6ˆØŸš MÑ2Ô2ˆØÐr@   rÆ  rU   s   @r?   rö  rö  Õ  s^   ø€ € € € € ð

ð 

ð 

ð 

ð 

ð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rö  c                   ó@   ‡ — e Zd Zdefˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )rI  r(   c                 ó¬   •— t          ¦   «                              ¦   «          || _        |                      d|                      |¦  «        d¬¦  «         d S )NrK  Fr¬   )r,   r-   r(   rµ   rL  r<   s     €r?   r-   z$Llama4VisionRotaryEmbedding.__init__ê  sP   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ×Ò˜Z¨×)?Ò)?ÀÑ)GÔ)GÐTYÐÑZÔZÐZÐZÐZr@   c                 óè  — | j         | j        z  }t          j        |dz  t          j        ¬¦  «                             |dz  d¦  «        }t          j        ||d d…         gd¬¦  «        }d|d<   ||z  }||z  }| j        | j        z  dz  }d| j	        d	         t          j        d|d¦  «        d |dz  …          
                    ¦   «         |z  z  z  }|dz   d
         |d d d d …f         z                       dd¬¦  «        }|dz   d
         |d d d d …f         z                       dd¬¦  «        }t          j        ||gd¬¦  «         
                    ¦   «                              ¦   «         dd d d…f         }	|	                     |                     ddd¦  «        dk     d¦  «        }	t          j        t          j        t          j        |	¦  «        t          j        |	¦  «        gd¬¦  «        ¦  «        }
|
S )Nr+   r¾   r#   r   rE   éþÿÿÿ)rD   rD   r½   r»   ).NrD   .)Ú
image_sizer·  r5   rÁ   Úint32rž   Úcatr1   rÀ   r²   rt   Úrepeat_interleaverù   Úmasked_fillrÜ   ÚstackÚcosÚsin)r(   ÚidxÚimg_idxÚfrequencies_xÚfrequencies_yÚfreq_dimÚ	rope_freqÚfreqs_xÚfreqs_yrÓ   r}  s              r?   rL  z-Llama4VisionRotaryEmbedding._compute_freqs_ciï  sü  € àÔ 6Ô#4Ñ4ˆÝ”,˜s A™v­U¬[Ð9Ñ9Ô9×AÒAÀ#ÀqÁ&È!ÑLÔLˆÝ”)˜W g¨b¨q¨b¤kÐ2¸Ð:Ñ:Ô:ˆØˆ�‰Ø #™ˆØ 3™ˆØÔ%¨Ô)CÑCÀqÑHˆØØÔ" <Ô0Ý”˜Q ¨!Ñ,Ô,Ð->°¸A±Ð->Ô?×EÒEÑGÔGÈ(ÑRñTñ
ˆ	ð " AÑ% yÔ1°I¸dÀDÈ!È!È!¸mÔ4LÑL×_Ò_Ð`aÐgiÐ_ÑjÔjˆØ! AÑ% yÔ1°I¸dÀDÈ!È!È!¸mÔ4LÑL×_Ò_Ð`aÐgiÐ_ÑjÔjˆÝ”	˜7 GÐ,°"Ð5Ñ5Ô5×;Ò;Ñ=Ô=×HÒHÑJÔJÈ3ÐPSÐPSÐRSÐPSÈ8ÔTˆØ×!Ò! '§/¢/°"°a¸Ñ";Ô";¸aÒ"?ÀÑCÔCˆÝÔ(­¬µe´iÀÑ6FÔ6FÍÌ	ÐRWÑHXÔHXÐ5YÐ_aÐ)bÑ)bÔ)bÑcÔcˆØˆr@   c                 ó@   — | j                              |j        ¦  «        S ra   )rK  r�   r·   r¤  s     r?   rO   z#Llama4VisionRotaryEmbedding.forward  s   € ØŒ}×Ò Ô 4Ñ5Ô5Ð5r@   )	rP   rQ   rR   r   r-   rÖ   rL  rO   rT   rU   s   @r?   rI  rI  é  sv   ø€ € € € € ð[Ð1ð [ð [ð [ð [ð [ð [ð
 ðð ñ „\ðð&6ð 6ð 6ð 6ð 6ð 6ð 6r@   rI  c                   óº   ‡ — e Zd ZU dZdZdgZeed<   defˆ fd„Zd„ Z		 	 	 	 dde
j        d	e
j        dz  d
edz  dedz  dedz  deee
j        df         z  fd„Zˆ xZS )rM  Úvision_model)r@  rÞ  r(   c                 ó(  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        |j        | _        |j        | _        | j        | j        z  dz  dz   | _        |j        dz  | _        t          |¦  «        | _	        t          j        | j        t          j        | j        ¦  «        z  ¦  «        | _        t          j        | j        t          j        | j        | j        ¦  «        z  ¦  «        | _        t!          |¦  «        | _        t          j        | j        ¦  «        | _        t          j        | j        ¦  «        | _        t+          |¦  «        | _        t/          |¦  «        | _        |                      ¦   «          d S )Nr+   r#   r  )r,   r-   r  r·  r1   rü  rµ  rO  rö  Úpatch_embeddingr3   r4   r5   ÚrandnrN  rP  rI  Úrotary_embeddingrà  Úlayernorm_preÚlayernorm_postrå  r[  r¾  Úvision_adapterrn  r<   s     €r?   r-   zLlama4VisionModel.__init__  sD  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ Ô+ˆŒØ!Ô-ˆÔØ"Ô/ˆÔà œO¨t¬Ñ>À1ÑDÀqÑHˆÔØÔ'¨Ñ-ˆŒ
å6°vÑ>Ô>ˆÔå!œ|¨D¬J½¼ÀTÔEUÑ9VÔ9VÑ,VÑWÔWˆÔÝ(*¬°T´ZÅ%Ä+ÈdÔN^Ð`dÔ`pÑBqÔBqÑ5qÑ(rÔ(rˆÔ%Ý ;¸FÑ CÔ CˆÔõ  œ\¨$Ô*:Ñ;Ô;ˆÔÝ œl¨4Ô+;Ñ<Ô<ˆÔõ )¨Ñ0Ô0ˆŒ
Ý9¸&ÑAÔAˆÔØ�ŠÑÔÐÐÐr@   c                 ó   — | j         S )zg
        This function is used to fetch the first embedding layer to activate grads on inputs.
        )r  rx   s    r?   Úget_input_embeddingsz&Llama4VisionModel.get_input_embeddings&  s   € ð Ô#Ð#r@   NÚpixel_valuesrï   râ  rè  ré  rB   .c                 óð  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|j        \  }}}	}
d}d}|                      |¦  «        }|j        \  }}}|                     ||z  |z  ||¦  «        }| j                             |j        d         d|j        d         ¦  «        }t          j
        ||gd¬¦  «        }|dz  }|                     ||z  |||¦  «        }| j                             |j        |j        ¬¦  «        }||z   }|                      |¦  «        }|                     |d|¦  «        }|                      |¦  «        }|                      |d|||¬¦  «        }|j        }|                      |¦  «        }|dd…dd…dd…f         }|                      |¦  «        }|r|j        nd}|r	|d         }nd}|st/          d	„ |||fD ¦   «         ¦  «        S t1          |||¬
¦  «        S )a  

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, MllamaVisionModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaVisionModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 1, 4, 1025, 7680])
        ```
        Nr#   r   rD   rE   ©r‘   r·   )rï   rè  râ  rK  r+   c              3   ó   K  — | ]}|®|V — Œ	d S ra   r7  rì  s     r?   rî  z,Llama4VisionModel.forward.<locals>.<genexpr>ˆ  s(   è è € Ð_Ð_˜qÐQRÐQ^˜ÐQ^ÐQ^ÐQ^ÐQ^Ð_Ð_r@   rï  )r(   râ  rè  ré  rH   r  rž   rN  rÊ   r5   r  rP  r�   r‘   r·   r  rG   r  r[  ru  r  r  rA   r„   r   )r=   r  rï   râ  rè  ré  rû   Úbatch_size_times_num_tilesrü  r¸  r¹  Únum_concurrent_mediaÚ
num_chunksrá  r:  rµ  r™   rN  Úpositional_embeddingrK  r‚   rA   r\  s                          r?   rO   zLlama4VisionModel.forward,  sŠ  € ðD 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆð COÔBTÑ?Ð" L°&¸%Ø ÐØˆ
Ø×+Ò+¨LÑ9Ô9ˆØ%1Ô%7Ñ"ˆˆ;˜
ð $×+Ò+Ø&Ð)=Ñ=À
ÑJÈKÐYcñ
ô 
ˆð Ô.×5Ò5°lÔ6HÈÔ6KÈQÐP\ÔPbÐceÔPfÑgÔgˆÝ”y ,°Ð!@ÀaÐHÑHÔHˆØ�qÑˆð $×+Ò+Ø&Ð)=Ñ=¸zÈ;ÐXbñ
ô 
ˆð  $Ô<×?Ò?ÀlÔFXÐamÔatÐ?ÑuÔuÐØ#Ð&:Ñ:ˆà×)Ò)¨,Ñ7Ô7ˆà#×(Ò(Ð)CÀRÈÑTÔTˆØ×(Ò(¨Ñ6Ô6ˆà—’ØØØ!5Ø/Øð ñ 
ô 
ˆð Ô/ˆà×*Ò*¨<Ñ8Ô8ˆà# A A A s¨ s¨A¨A¨A IÔ.ˆð ×*Ò*¨<Ñ8Ô8ˆà0DÐN˜Ô,Ð,È$ˆàð 	Ø œˆJˆJàˆJàð 	`ÝÐ_Ð_ \°=À*Ð$MÐ_Ñ_Ô_Ñ_Ô_Ð_å)Ø*Ø'Ø!ð
ñ 
ô 
ð 	
r@   rô  )rP   rQ   rR   r�  rQ  r€  r   rÕ   r-   r  r5   rS   r<  r   r„   rO   rT   rU   s   @r?   rM  rM    s  ø€ € € € € € Ø&ÐØ!ÐØ3Ð4ÐØÐÐÑðÐ1ð ð ð ð ð ð ð2$ð $ð $ð /3Ø)-Ø,0Ø#'ðb
ð b
à”lðb
ð œ tÑ+ðb
ð   $™;ð	b
ð
 # T™kðb
ð ˜D‘[ðb
ð 
$ e¨E¬L¸#Ð,=Ô&>Ñ	>ðb
ð b
ð b
ð b
ð b
ð b
ð b
ð b
r@   rM  c            #       ót  ‡ — e Zd ZU ddgZi ZdZeed<   defˆ fd„Zd„ Z	d„ Z
d„ Zd	„ Ze ed
¬¦  «         ed¬¦  «        dej        dedee         deez  fd„¦   «         ¦   «         ¦   «         Zdej        dej        dej        fd„Ze ed
¬¦  «        e	 	 	 	 	 	 	 	 	 	 	 	 	 d$dej        dz  dej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  dej        dz  dedz  dedz  dedz  d edz  d!eej        z  dee         deez  fd"„¦   «         ¦   «         ¦   «         Z	 	 	 	 	 	 d%d#„Z ˆ xZ!S )&ÚLlama4ForConditionalGenerationr,  rÞ  r[  r(   c                 óž  •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          |¦  «        | _        t          |j        ¦  «        | _	        |j        j
        | _
        t          | j        d¦  «        r| j        j        | _        n| j        j        j        pd| _        |                      ¦   «          d S )Nrb  rD   )r,   r-   rM  r¨  r  r¦  Úmulti_modal_projectorr„  rF  r…  rd  r  r(   rb  rn  r<   s     €r?   r-   z'Llama4ForConditionalGeneration.__init__—  s®   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý-¨fÔ.BÑCÔCˆÔå%>¸vÑ%FÔ%FˆÔ"Ý/°Ô0BÑCÔCˆÔØ Ô,Ô7ˆŒÝ�4”; Ñ/Ô/ð 	KØ $¤Ô 8ˆDÔÐà $¤Ô 7Ô DÐ JÈˆDÔà�ŠÑÔÐÐÐr@   c                 ó4   — | j                              ¦   «         S ra   )r…  Úget_output_embeddingsrx   s    r?   r,  z4Llama4ForConditionalGeneration.get_output_embeddings¥  s   € ØÔ"×8Ò8Ñ:Ô:Ð:r@   c                 ó:   — | j                              |¦  «         d S ra   )r…  Úset_output_embeddings)r=   Únew_embeddingss     r?   r.  z4Llama4ForConditionalGeneration.set_output_embeddings¨  s   € ØÔ×1Ò1°.ÑAÔAÐAÐAÐAr@   c                 ó:   — | j                              |¦  «         d S ra   )r…  Úset_decoder)r=   Údecoders     r?   r1  z*Llama4ForConditionalGeneration.set_decoder«  s   € ØÔ×'Ò'¨Ñ0Ô0Ð0Ð0Ð0r@   c                 ó4   — | j                              ¦   «         S ra   )r…  Úget_decoderrx   s    r?   r4  z*Llama4ForConditionalGeneration.get_decoder®  s   € ØÔ"×.Ò.Ñ0Ô0Ð0r@   F)Útie_last_hidden_stateszOObtains image last hidden states from the vision tower and apply al projection.r”  r  Úvision_feature_select_strategyrû   rB   c                 óZ   — d„ |                      ¦   «         D ¦   «         } | j        |fi |¤ŽS )aj  
        pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
            The tensors corresponding to the input images.
        vision_feature_select_strategy (`str`):
            The feature selection strategy used to select the vision feature from the vision backbone.
            Can be one of `"default"` or `"full"`
        c                 ó   — i | ]
\  }}|®||“ŒS ra   r7  )r_  Úkrí  s      r?   ú
<dictcomp>zELlama4ForConditionalGeneration.get_image_features.<locals>.<dictcomp>Á  s   € ÐCÐCÐC™4˜1˜a°Q°]�!�Q°]°]°]r@   )Úitemsr  )r=   r  r6  rû   s       r?   Úget_image_featuresz1Llama4ForConditionalGeneration.get_image_features±  s;   € ð  DÐC 6§<¢<¡>¤>ÐCÑCÔCˆØ ˆtÔ  Ð8Ð8°Ð8Ð8Ð8r@   ro  rp  r¬  c                 ó   — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     ¦   «         }| 	                    d¦  «         
                    |j        ¦  «        }t          ||j        d         z  |                     ¦   «         k    d|› d|j        d         › �¦  «         |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        Nr!  rD   z6Image features and image tokens do not match, tokens: z, features: r   )r  r5   Útensorr(   Úimage_token_idÚlongr·   Úallr¢   rw  r�   r   rH   Únumel)r=   ro  rp  r¬  Úspecial_image_maskÚn_image_tokenss         r?   Úget_placeholder_maskz3Llama4ForConditionalGeneration.get_placeholder_maskÄ  s  € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà+×/Ò/Ñ1Ô1ˆØ/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐÝØ˜]Ô0°Ô4Ñ4¸×8LÒ8LÑ8NÔ8NÒNØzÀ^ÐzÐzÐaoÔauÐvwÔaxÐzÐzñ	
ô 	
ð 	
ð "Ð!r@   Nr   rï   rÐ   r  r‰  r5  râ  rè  ré  rŠ  c                 óê  — |
�|
n| j         j        }
|�|n| j         j        }|�|n| j         j        }|du |duz  rt	          d¦  «        ‚|�|�t	          d¦  «        ‚|€ |                      ¦   «         |¦  «        }|�§|                      ||d¬¦  «        j        }|                     d| 	                    d¦  «        ¦  «        }|  
                    |¦  «                             |j        |j        ¦  «        }|                      |||¬¦  «        }|                     ||¦  «        } | j        d|||||	|
|||dœ	|¤Ž}|d	         }d}|��k|�¹|dd…|j        d
         d
z
   d…f                              |j        ¦  «        }|ddd…dd…f         |                     |j        ¦  «        d	k                                  ¦   «         }|dd
d…f         |                     |j        ¦  «        d	k                                  ¦   «         }n?|ddd…dd…f                              ¦   «         }|dd
d…f                              ¦   «         }t'          j        ¦   «         } ||                     d| 	                    d¦  «        ¦  «        |                     d¦  «                             |j        ¦  «        ¦  «        }|s|f|d
d…         z   }|�|f|z   n|S t+          |||j        |j        |j        |�|nd¬¦  «        S )añ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration

        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

        >>> prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "USER:  \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed"
        ```Nrr  zdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either oneT)r  r6  ré  rD   )rp  r¬  )	rï   rÐ   r  rp  r5  râ  rè  ré  rŠ  r   r#   .)r�  rŒ  r  rA   r\  r˜  r7  )r(   râ  rè  ré  rv  r  r<  ru  rG   r°  r*  r�   r·   r‘   rE  Úmasked_scatterr…  rH   rù   r3   ÚCrossEntropyLossr—  r  rA   r\  )r=   ro  r  rï   rÐ   r  rp  r6  r‰  r5  râ  rè  ré  rŠ  rû   r¬  Úvision_flatÚprojected_vision_flatrC  r�  rŒ  r�  Úshift_attention_maskÚshift_logitsÚshift_labelsÚloss_fctr‚   s                              r?   rO   z&Llama4ForConditionalGeneration.forwardÛ  s¤  € ðd 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ#¨Ð(AÝØvñô ð ð Ð Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ#Ø!×4Ò4Ø)Ø/MØ ð 5ñ ô ô  ð	 ð )×-Ò-¨b°.×2EÒ2EÀbÑ2IÔ2IÑJÔJˆKØ$(×$>Ò$>¸{Ñ$KÔ$K×$NÒ$NØÔ$ mÔ&9ñ%ô %Ð!ð "&×!:Ò!:Ø¨ÐG\ð ";ñ "ô "Ðð *×8Ò8Ð9KÐMbÑcÔcˆMà%�$Ô%ð 
Ø)Ø%Ø+Ø'ØØ/Ø!5Ø#Ø)ð
ð 
ð ð
ð 
ˆð ˜”ˆàˆØÑàÐ)ð (6°a°a°a¸6¼<È¼?ÈQÑ;NÐ9OÐ9QÐ9QÐ6QÔ'R×'UÒ'UÐV\ÔVcÑ'dÔ'dÐ$Ø% c¨3¨B¨3°°° kÔ2Ð3G×3JÒ3JÈ6Ì=Ñ3YÔ3YÐ]^Ò3^Ô_×jÒjÑlÔl�Ø% c¨1¨2¨2 gœÐ/C×/FÒ/FÀvÄ}Ñ/UÔ/UÐYZÒ/ZÔ[×fÒfÑhÔh��à% c¨3¨B¨3°°° kÔ2×=Ò=Ñ?Ô?�Ø% c¨1¨2¨2 gœ×9Ò9Ñ;Ô;�åÔ*Ñ,Ô,ˆHØ�8Ø×!Ò! " l×&7Ò&7¸Ñ&;Ô&;Ñ<Ô<¸l×>OÒ>OÐPRÑ>SÔ>S×>VÒ>VÐWcÔWjÑ>kÔ>kñô ˆDð ð 	DØ�Y ¨¨¨¤Ñ,ˆFØ'+Ð'7�D�7˜VÑ#Ð#¸VÐCå+ØØØ#Ô3Ø!Ô/ØÔ)Ø2>Ð2J  ÐPTð
ñ 
ô 
ð 	
r@   c           	      ór   —  | j         j        |f|||||dœ|¤Ž}	|s|                     dd¦  «        s||	d<   |	S )N)r  rp  rï   rŠ  Úis_first_iterationr5  Tr  )r…  Úprepare_inputs_for_generationÚget)
r=   ro  r  rp  r  rï   rŠ  rP  rû   Úmodel_inputss
             r?   rQ  z<Llama4ForConditionalGeneration.prepare_inputs_for_generation\  ss   € ð I�tÔ*ÔHØð
à+Ø'Ø)Ø)Ø1ð
ð 
ð ð
ð 
ˆð ð 	8 V§Z¢Z°¸TÑ%BÔ%Bð 	8ð
 ,8ˆL˜Ñ(àÐr@   )NNNNNNNNNNNNr   )NNNNNF)"rP   rQ   rR   r€  r“  r�  r$   rÕ   r-   r,  r.  r1  r4  r!   r"   r   r5   r=  rÍ   r   r   r„   r   r<  r;  rE  rS   r
   r<  r×   r—  rO   rQ  rT   rU   s   @r?   r(  r(  ‘  s  ø€ € € € € € Ø1Ð3MÐNÐØ€HØÐØÐÐÑð˜|ð ð ð ð ð ð ð;ð ;ð ;ðBð Bð Bð1ð 1ð 1ð1ð 1ð 1ð  Ø€_¨EÐ2Ñ2Ô2Ø€^Ð!rÐsÑsÔsð9àÔ'ð9ð ),ð9ð Ð+Ô,ð	9ð
 
Ð+Ñ	+ð9ð 9ð 9ñ tÔsñ 3Ô2ñ  Ôð9ð "ØÔ)ð"Ø:?Ô:Kð"Ø]bÔ]nð"ð "ð "ð "ð.  Ø€_¨EÐ2Ñ2Ô2Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø59Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ð|
ð |
àÔ# dÑ*ð|
ð Ô'¨$Ñ.ð|
ð œ tÑ+ð	|
ð
 Ô&¨Ñ-ð|
ð  ™ð|
ð Ô(¨4Ñ/ð|
ð ),¨d©
ð|
ð Ô  4Ñ'ð|
ð ˜$‘;ð|
ð   $™;ð|
ð # T™kð|
ð ˜D‘[ð|
ð ˜eœlÑ*ð|
ð Ð+Ô,ð|
ð  
Ð-Ñ	-ð!|
ð |
ð |
ñ „^ñ 3Ô2ñ  Ôð|
ðB ØØØØØ ðð ð ð ð ð ð ð r@   r(  )r?  rZ  rM  r„  r(  )rê   )er®  Úcollections.abcr   Údataclassesr   Útypingr   r5   Útorch.nnr3   Útorch.nn.functionalrŽ   r£  Ú/transformers.models.llama4.configuration_llama4r   Ú r   rG  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   Úconfiguration_llama4r$   r%   Ú
get_loggerrP   ÚloggerÚModuler'   rW   re   r{   r\   r†   r—   r§   rS   r„   rã   r×   ré   rt   r   r  r  r,  r?  rZ  r„  r—  rš  r¦  r¼  r¾  rË  rÑ  rÓ  rÚ  rÞ  rå  rö  rI  rM  r(  Ú__all__r7  r@   r?   ú<module>rm     sÒ  ðð €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à NÐ NÐ NÐ NÐ NÐ Nà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð ˜œ	ñ ô ð ðB)ð )ð )ð )ð )�B”Iñ )ô )ð )ð$!ð !ð !ð !ð !�u”x”ñ !ô !ð !ð=ð =ð =ð =ð =˜œ	ñ =ô =ð =ð(,ð ,ð ,ð ,ð ,�2”9ñ ,ô ,ð ,ð"ð "ð "ð "ð "�B”Iñ "ô "ð "ð,?ð ?ð ?ð ?ð ? ¤	ñ ?ô ?ð ?ðD	2ØŒð	2àŒð	2ð Œ|ð	2ð ˆ5Œ<˜œÐ%Ô&ð		2ð 	2ð 	2ð 	2ð	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð( ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ðB ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð2Y)ð Y)ð Y)ð Y)ð Y)˜"œ)ñ Y)ô Y)ð Y)ðx/ð /ð /ð /ð /Ð7ñ /ô /ð /ðd ðLð Lð Lð Lð L˜Oñ Lô Lñ „ðLð< ð[
ð [
ð [
ð [
ð [
Ð+ñ [
ô [
ñ „ð[
ð|L
ð L
ð L
ð L
ð L
Ð-¨ñ L
ô L
ð L
ð^ €ððñ ô ð
 ð9ð 9ð 9ð 9ð 9 ;ñ 9ô 9ñ „ñô ð9ð0;ð ;ð ;ð ;ð ;�u”x”ñ ;ô ;ð ;ð"ð ð ð ð  ¤	ñ ô ð ðð ð ð(
)ð 
)ð 
)ð 
)ð 
) "¤)ñ 
)ô 
)ð 
)ð! E¤Lð !¸¼ð !ð !ð !ð !ð:ØŒ<ð:à	Œð:ð Œlð:ð ˆ5Œ<˜œÐ%Ô&ð	:ð :ð :ð :ð7)ð 7)ð 7)ð 7)ð 7)˜BœIñ 7)ô 7)ð 7)ðtð ð ð ð �b”iñ ô ð ð)ð )ð )ð )ð )Ð9ñ )ô )ð )ðXO
ð O
ð O
ð O
ð O
˜"œ)ñ O
ô O
ð O
ðdð ð ð ð ˜bœiñ ô ð ð(6ð 6ð 6ð 6ð 6 "¤)ñ 6ô 6ð 6ð<G
ð G
ð G
ð G
ð G
Ð-ñ G
ô G
ð G
ðTið ið ið ið iÐ%:¸Oñ iô ið iðXð ð €€€r@   