§
    ‚Štjî ã                   óÔ  — d Z ddlZddlmZ ddlmZ ddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6  e+j7        e8¦  «        Z9dej:        de;de<de=ej:        ej:        f         fd„Z>dej:        de;de;dej?        dej:        f
d „Z@ G d!„ d"ejA        ¦  «        ZB G d#„ d$ejA        ¦  «        ZC G d%„ d&ejA        ¦  «        ZDd'ej:        d(e;dej:        fd)„ZE	 d]d+ejA        d,ej:        d-ej:        d.ej:        d/ej:        dz  d0eFd1eFd2e&e(         fd3„ZG G d4„ d5ejA        ¦  «        ZH G d6„ d7ejA        ¦  «        ZI G d8„ d9ejA        ¦  «        ZJ G d:„ d;ejA        ¦  «        ZK G d<„ d=ejA        ¦  «        ZLd>„ ZMd^d?„ZN G d@„ dAejA        ¦  «        ZO G dB„ dCejA        ¦  «        ZP G dD„ dEe¦  «        ZQ G dF„ dGe¦  «        ZR G dH„ dIejA        ¦  «        ZSe) G dJ„ dKe$¦  «        ¦   «         ZT e)dL¬M¦  «         G dN„ dOeT¦  «        ¦   «         ZU e)dP¬M¦  «         G dQ„ dReT¦  «        ¦   «         ZV e)dS¬M¦  «         G dT„ dUeTe¦  «        ¦   «         ZW e)dV¬M¦  «         G dW„ dXeT¦  «        ¦   «         ZX e)dY¬M¦  «         G dZ„ d[eTe¦  «        ¦   «         ZYg d\¢ZZdS )_zPyTorch Mllama model.é    N)ÚCallable)ÚOptional)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚMllamaConfigÚMllamaTextConfigÚMllamaVisionConfigÚcross_attention_maskÚnum_vision_tokensÚdtypeÚreturnc                 ó
  — | j         ^}}}|                      |d¬¦  «        } |                      ||d¦  «        } |                      d¦  «        } d| z
                       |¦  «        }|                     |                     t          j        ¦  «        t          j        |¦  «        j	        ¦  «        } t          j        |¦  «        j	        }| |k     
                    d¬¦  «                             | ¦  «        d         }| |z  } | |fS )Nr   ©Údiméÿÿÿÿr   ç      ð?).N)ÚshapeÚrepeat_interleaveÚviewÚ	unsqueezeÚtoÚmasked_fillÚtorchÚboolÚfinfoÚminÚanyÚtype_as)	r#   r$   r%   Ú
batch_sizeÚtext_total_lengthÚ_Úinverted_cross_attn_maskÚnegative_inf_valueÚfull_text_row_masked_out_masks	            úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mllama/modeling_mllama.pyÚ_prepare_cross_attention_maskr?   0   s  € ð )=Ô(BÐ%€JÐ! AØ/×AÒAÐBSÐYZÐAÑ[Ô[ÐØ/×4Ò4°ZÐARÐTVÑWÔWÐØ/×9Ò9¸!Ñ<Ô<Ðð !$Ð&:Ñ :×>Ò>¸uÑEÔEÐØ3×?Ò?Ø ×#Ò#¥E¤JÑ/Ô/µ´¸UÑ1CÔ1CÔ1Gñô Ðõ œ UÑ+Ô+Ô/Ðà	Ð!3Ò	3×8Ò8¸RÐ8Ñ@Ô@×HÒHÐI]Ñ^Ô^Ð_hÔið "ð Ð9Ñ9ÐàÐ!>Ð>Ð>ó    Úaspect_ratio_maskÚnum_patchesÚtarget_lengthc                 ó–  — | j         \  }}|                      ||dd¦  «                             |¦  «        }|                     dd|d¦  «        }||z
  }d|d d …d d …| d …f<   d|z
  }|                     |||z  d¦  «        }||                     dd¦  «        z  t          j        |¦  «        j        z  }| 	                    d¦  «        }|S )Nr   r   r*   éþÿÿÿ)
r,   r.   r0   ÚrepeatÚreshapeÚ	transposer2   r4   r5   r/   )rA   rB   rC   r%   r8   Úmax_num_tilesÚattention_maskÚpad_patchess           r>   Ú$_prepare_aspect_ratio_attention_maskrL   L   sî   € ð !2Ô 7Ñ€J�Ø&×+Ò+¨J¸ÀqÈ!ÑLÔL×OÒOÐPUÑVÔV€NØ#×*Ò*¨1¨a°ÀÑBÔB€Nð   +Ñ-€KØ*+€N�1�1�1�a�a�a˜+˜˜˜Ð&Ñ'ð ˜Ñ'€Nð $×+Ò+¨J¸ÈÑ8UÐWXÑYÔY€NØ# n×&>Ò&>¸rÀ2Ñ&FÔ&FÑFÍÌÐUZÑI[ÔI[ÔI_Ñ_€NØ#×-Ò-¨aÑ0Ô0€NàÐr@   c                   ó\   ‡ — e Zd Zd	dedefˆ fd„Zdej        dej        dej        fd„Zˆ xZ	S )
Ú%MllamaPrecomputedAspectRatioEmbeddingTÚconfigÚis_gatedc                 óZ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        || _        t          j        | j        dz   | j        | j        z  ¦  «        | _        |r-t          j	        t          j        d¦  «        ¦  «        | _        d S d S )Nr   )ÚsuperÚ__init__rI   Úhidden_sizeÚmax_aspect_ratio_idrP   r   Ú	EmbeddingÚ	embeddingÚ	Parameterr2   ÚzerosÚgate©ÚselfrO   rP   Ú	__class__s      €r>   rS   z.MllamaPrecomputedAspectRatioEmbedding.__init__h   s—   ø€ Ý‰Œ×ÒÑÔÐØ#Ô1ˆÔØ!Ô-ˆÔØ#)Ô#=ˆÔ Ø ˆŒåœ dÔ&>ÀÑ&BÀDÔDVÐY]ÔYiÑDiÑjÔjˆŒØð 	5Ýœ¥U¤[°¡^¤^Ñ4Ô4ˆDŒIˆIˆIð	5ð 	5r@   Úhidden_stateÚaspect_ratio_idsr&   c                 óÄ   — |                       |¦  «        }|                     d| j        d| j        ¦  «        }| j        r|| j                             ¦   «         z  }||z   }|S )Nr*   r   )rW   rG   rI   rT   rP   rZ   Útanh)r\   r^   r_   Ú
embeddingss       r>   Úforwardz-MllamaPrecomputedAspectRatioEmbedding.forwards   sc   € Ø—^’^Ð$4Ñ5Ô5ˆ
Ø×'Ò'¨¨DÔ,>ÀÀ4ÔCSÑTÔTˆ
àŒ=ð 	7Ø# d¤i§n¢nÑ&6Ô&6Ñ6ˆJà# jÑ0ˆØÐr@   )T)
Ú__name__Ú
__module__Ú__qualname__r"   r3   rS   r2   ÚTensorrc   Ú__classcell__©r]   s   @r>   rN   rN   g   s‚   ø€ € € € € ð	5ð 	5Ð1ð 	5¸Tð 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð E¤Lð ÀEÄLð ÐUZÔUað ð ð ð ð ð ð ð r@   rN   c                   óV   ‡ — e Zd Zdefˆ fd„Zdej        dej        dej        fd„Zˆ xZS )Ú"MllamaPrecomputedPositionEmbeddingrO   c                 ó&  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        |j        z  dz  dz   | _        |j        | _        |j        dz  | _        t          j
        t          j        d¦  «        ¦  «        | _        t          j        | j        | j        ¦  «        }t          j
        | j        |z  ¦  «        | _        t          j        | j        dz   | j        | j        z  | j        z  ¦  «        | _        d S )Né   r   ç      à¿)rR   rS   rI   rU   Ú
image_sizeÚ
patch_sizerB   rT   Úscaler   rX   r2   rY   rZ   ÚrandnrW   rV   Útile_embedding)r\   rO   Úposition_embeddingr]   s      €r>   rS   z+MllamaPrecomputedPositionEmbedding.__init__   sï   ø€ Ý‰Œ×ÒÑÔÐØ#Ô1ˆÔØ#)Ô#=ˆÔ Ø"Ô-°Ô1BÑBÀqÑHÈ1ÑLˆÔØ!Ô-ˆÔØÔ'¨Ñ-ˆŒ
å”L¥¤¨Q¡¤Ñ0Ô0ˆŒ	õ #œ[¨Ô)9¸4Ô;KÑLÔLÐÝœ d¤jÐ3EÑ&EÑFÔFˆŒõ !œlØÔ$ qÑ(¨$Ô*<¸tÔ?OÑ*OÐRVÔRbÑ*bñ
ô 
ˆÔÐÐr@   r^   r_   r&   c                 ól  — d| j                              ¦   «         z
  | j        z  }||                     dd| j        | j        ¦  «        z   }|                      |¦  «        }|j        d         }|                     || j	        | j        | j        ¦  «        }| j                              ¦   «         |z  }||z   }|S )Nr   r   )
rZ   ra   rW   r.   rB   rT   rs   r,   rG   rI   )r\   r^   r_   Úgated_position_embeddingÚtile_position_embeddingr8   Úgated_tile_position_embeddings          r>   rc   z*MllamaPrecomputedPositionEmbedding.forward’   s½   € à$%¨¬	¯ªÑ(8Ô(8Ñ$8¸D¼NÑ#JÐ Ø#Ð&>×&CÒ&CÀAÀqÈ$ÔJZÐ\`Ô\lÑ&mÔ&mÑmˆð #'×"5Ò"5Ð6FÑ"GÔ"GÐØ!Ô'¨Ô*ˆ
Ø"9×"AÒ"AØ˜Ô*¨DÔ,<¸dÔ>Nñ#
ô #
Ðð )-¬	¯ªÑ(8Ô(8Ð;RÑ(RÐ%Ø#Ð&CÑCˆàÐr@   )	rd   re   rf   r"   rS   r2   rg   rc   rh   ri   s   @r>   rk   rk   ~   sv   ø€ € € € € ð
Ð1ð 
ð 
ð 
ð 
ð 
ð 
ð& E¤Lð ÀEÄLð ÐUZÔUað ð ð ð ð ð ð ð r@   rk   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚMllamaVisionMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S ©N)rR   rS   rO   r   Ú
hidden_actÚactivation_fnr   ÚLinearrT   Úintermediate_sizeÚfc1Úfc2©r\   rO   r]   s     €r>   rS   zMllamaVisionMLP.__init__¥   sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆr@   Úhidden_statesr&   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r|   )r�   r~   r‚   )r\   r„   s     r>   rc   zMllamaVisionMLP.forward¬   s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐr@   )rd   re   rf   rS   r2   rg   rc   rh   ri   s   @r>   rz   rz   ¤   sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð r@   rz   r„   Ún_repc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r,   ÚexpandrG   )r„   r†   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r>   Ú	repeat_kvr�   ´   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr@   ç        ÚmoduleÚqueryÚkeyÚvaluerJ   ÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nrm   r   r*   )r)   r%   )ÚpÚtrainingr   )r�   Únum_key_value_groupsr2   ÚmatmulrH   r   Ú
functionalÚsoftmaxÚfloat32r0   r%   r”   r˜   Ú
contiguous)r�   r�   r‘   r’   rJ   r“   r”   r•   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r>   Úeager_attention_forwardr£   Á   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r@   c                   ó¨   ‡ — e Zd Zdefˆ fd„Z eddd¬¦  «        	 ddej        dej        dz  d	eej        ej        dz  f         fd
„¦   «         Z	ˆ xZ
S )ÚMllamaVisionAttentionrO   c                 óJ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        |j        z  | _        | j        dz  | _        d| _	        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nrn   r   F©Úbias)rR   rS   rO   rT   Ú	embed_dimÚattention_headsÚ	num_headsrŒ   r“   r™   r   r   Úq_projÚk_projÚv_projÚo_projrƒ   s     €r>   rS   zMllamaVisionAttention.__init__Û   sð   ø€ Ý‰Œ×ÒÑÔÐàˆŒØÔ+ˆŒØÔ/ˆŒØÔ*¨fÔ.DÑDˆŒØ”} dÑ*ˆŒØ$%ˆÔ!å”i ¤°´ÀÄÑ0NÐUZÐ[Ñ[Ô[ˆŒÝ”i ¤°´ÀÄÑ0NÐUZÐ[Ñ[Ô[ˆŒÝ”i ¤°´ÀÄÑ0NÐUZÐ[Ñ[Ô[ˆŒÝ”i ¤°´Ñ >ÀÄÐUZÐ[Ñ[Ô[ˆŒˆˆr@   r^   r„   úv5.20©Únew_nameÚversionNrJ   r&   c                 óì  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|j        \  }}}	|j        \  }	}
}	|                     ||| j        | j        ¦  «                             dd¦  «        }|                     ||
| j        | j        ¦  «                             dd¦  «        }|                     ||
| j        | j        ¦  «                             dd¦  «        }t          j	        | j
        j        t          ¦  «        } || ||||fd| j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS ©Nr   rm   rŽ   ©r”   r“   r*   )r¬   r­   r®   r,   r.   r«   rŒ   rH   r   Úget_interfacerO   Ú_attn_implementationr£   r“   rG   rž   r¯   )r\   r„   rJ   r•   r�   r‘   r’   r8   Ú	q_seq_lenr:   Ú
kv_seq_lenÚattention_interfacer¢   r¡   s                 r>   rc   zMllamaVisionAttention.forwardê   s  € ð —’˜MÑ*Ô*ˆØ�kŠk˜-Ñ(Ô(ˆØ—’˜MÑ*Ô*ˆà#(¤;Ñ ˆ
�I˜qØœ9Ñˆˆ:�qà—
’
˜: y°$´.À$Ä-ÑPÔP×ZÒZÐ[\Ð^_Ñ`Ô`ˆØ�hŠh�z :¨t¬~¸t¼}ÑMÔM×WÒWÐXYÐ[\Ñ]Ô]ˆØ—
’
˜: z°4´>À4Ä=ÑQÔQ×[Ò[Ð\]Ð_`ÑaÔaˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨*°iÀÑDÔD×OÒOÑQÔQˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r@   r|   )rd   re   rf   r"   rS   r   r2   rg   Útuplerc   rh   ri   s   @r>   r¥   r¥   Ú   sÁ   ø€ € € € € ð\Ð1ð \ð \ð \ð \ð \ð \ð €_�^¨oÀwÐOÑOÔOð /3ð#)ð #)à”|ð#)ð œ tÑ+ð#)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð#)ð #)ð #)ñ PÔOð#)ð #)ð #)ð #)ð #)r@   r¥   c                   ó€   ‡ — e Zd Zddedefˆ fd„Z eddd¬¦  «        	 ddej        d
ej        d	z  fd„¦   «         Z	ˆ xZ
S )ÚMllamaVisionEncoderLayerFrO   rP   c                 óv  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        |j        | _        t          |¦  «        | _        t          |¦  «        | _
        t          j        | j        |j        ¬¦  «        | _        t          j        | j        |j        ¬¦  «        | _        |rxt          j        t#          j        d¦  «        t&          j        z  dz  ¦  «        | _        t          j        t#          j        d¦  «        t&          j        z  dz  ¦  «        | _        d S d S )N©Úepsr   é   )rR   rS   rT   rª   Únum_attention_headsrP   r€   r¥   Ú	self_attnrz   Úmlpr   Ú	LayerNormÚnorm_epsÚinput_layernormÚpost_attention_layernormrX   r2   ÚonesÚmathÚpiÚ	gate_attnÚgate_ffnr[   s      €r>   rS   z!MllamaVisionEncoderLayer.__init__  sú   ø€ Ý‰Œ×ÒÑÔÐà!Ô-ˆÔØ#)Ô#9ˆÔ Ø ˆŒØ!'Ô!9ˆÔå.¨vÑ6Ô6ˆŒÝ" 6Ñ*Ô*ˆŒå!œ|¨DÔ,<À&Ä/ÐRÑRÔRˆÔÝ(*¬°TÔ5EÈ6Ì?Ð([Ñ([Ô([ˆÔ%àð 	FÝœ\­%¬*°Q©-¬-½$¼'Ñ*AÀAÑ*EÑFÔFˆDŒNÝœL­¬°A©¬½¼Ñ)@À1Ñ)DÑEÔEˆDŒMˆMˆMð	Fð 	Fr@   r^   r„   r°   r±   NrJ   c                 ó`  — |}|                       |¦  «        }|                      ||¬¦  «        \  }}| j        r| j                             ¦   «         |z  }||z   }|}|                      |¦  «        }|                      |¦  «        }| j        r| j                             ¦   «         |z  }||z   }|S )N©rJ   )rÈ   rÄ   rP   rÍ   ra   rÉ   rÅ   rÎ   )r\   r„   rJ   Úresidualr¡   s        r>   rc   z MllamaVisionEncoderLayer.forward$  sÂ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ&*§n¢n°]ÐSa nÑ&bÔ&bÑ#ˆ�|ØŒ=ð 	BØ œN×/Ò/Ñ1Ô1°MÑAˆMØ  =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØŒ=ð 	AØ œM×.Ò.Ñ0Ô0°=Ñ@ˆMØ  =Ñ0ˆàÐr@   )Fr|   )rd   re   rf   r"   r3   rS   r   r2   rg   rc   rh   ri   s   @r>   r¾   r¾     s²   ø€ € € € € ðFð FÐ1ð F¸Tð Fð Fð Fð Fð Fð Fð$ €_�^¨oÀwÐOÑOÔOð /3ðð à”|ðð œ tÑ+ðð ð ñ PÔOðð ð ð ð r@   r¾   c                   ó\   ‡ — e Zd ZdZddefˆ fd„Z	 ddej        dej        dz  d	efd
„Z	ˆ xZ
S )ÚMllamaVisionEncoderz±
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`MllamaEncoderLayer`].

    Args:
        config: MllamaConfig
    é    FrO   c                 óÜ   •‡‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆˆfd„t          |¦  «        D ¦   «         ¦  «        | _        d| _        ‰| _        d S )Nc                 ó0   •— g | ]}t          ‰‰¦  «        ‘ŒS © )r¾   )Ú.0r:   rO   rP   s     €€r>   ú
<listcomp>z0MllamaVisionEncoder.__init__.<locals>.<listcomp>I  s%   ø€ Ð$kÐ$kÐ$kÐTUÕ%=¸fÀhÑ%OÔ%OÐ$kÐ$kÐ$kr@   F)rR   rS   rO   r   Ú
ModuleListÚrangeÚlayersÚgradient_checkpointing)r\   rO   Ú
num_layersrP   r]   s    ` `€r>   rS   zMllamaVisionEncoder.__init__F  sh   øøø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$kÐ$kÐ$kÐ$kÐ$kÕY^Ð_iÑYjÔYjÐ$kÑ$kÔ$kÑlÔlˆŒØ&+ˆÔ#ØˆŒˆˆr@   Nr„   rJ   r&   c                 ób   — d}| j         D ]} |||¬¦  «        }||fz   }Œt          ||¬¦  «        S )a8  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)

        r×   )r^   rJ   )Úlast_hidden_stater„   )rÜ   r   )r\   r„   rJ   Úencoder_statesÚencoder_layers        r>   rc   zMllamaVisionEncoder.forwardM  sZ   € ð( ˆØ!œ[ð 	?ð 	?ˆMØ)˜MØ*Ø-ðñ ô ˆMð ,¨}Ð.>Ñ>ˆNˆNå°ÈnÐ]Ñ]Ô]Ð]r@   )rÔ   Fr|   )rd   re   rf   Ú__doc__r"   rS   r2   rg   r   rc   rh   ri   s   @r>   rÓ   rÓ   =  s£   ø€ € € € € ðð ðð Ð1ð ð ð ð ð ð ð /3ð^ð ^à”|ð^ð œ tÑ+ð^ð 
ð	^ð ^ð ^ð ^ð ^ð ^ð ^ð ^r@   rÓ   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚMllamaTextRMSNormç�íµ ÷Æ°>rÁ   r&   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z@
        MllamaTextRMSNorm is equivalent to T5LayerNorm
        N)rR   rS   r   rX   r2   rÊ   ÚweightÚvariance_epsilon)r\   rT   rÁ   r]   s      €r>   rS   zMllamaTextRMSNorm.__init__n  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr@   r„   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Nrm   r*   T)Úkeepdim)	r%   r0   r2   r�   ÚpowÚmeanÚrsqrtré   rè   )r\   r„   Úinput_dtypeÚvariances       r>   rc   zMllamaTextRMSNorm.forwardv  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r@   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r¼   rè   r,   ré   ©r\   s    r>   Ú
extra_reprzMllamaTextRMSNorm.extra_repr}  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr@   )ræ   )
rd   re   rf   ÚfloatrS   r2   rg   rc   ró   rh   ri   s   @r>   rå   rå   m  sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr@   rå   c                   óæ   ‡ — e Zd ZdZ	 	 ddedz  dedz  fˆ fd„Z	 	 	 	 ddej        dej        dz  de	dz  d	ej        dz  d
e
dz  deej        ej        dz  eej                 dz  f         fd„Zˆ xZS )ÚMllamaTextCrossAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNrO   Ú	layer_idxc                 ó6  •— t          ¦   «                              ¦   «          || _        | j        j        | _        | j        j        | _        |j        | _        |j        | _        |j        | j        z  | _        || _	        | j        | j        z  | _
        | j        dz  | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        d S )Nrn   Fr§   rÀ   )rR   rS   rO   rÃ   r«   rŠ   r”   rT   rŒ   r÷   r™   r“   r   r   r¬   r­   r®   r¯   rå   Úrms_norm_epsÚq_normÚk_norm©r\   rO   r÷   r]   s      €r>   rS   z!MllamaTextCrossAttention.__init__„  sS  ø€ õ
 	‰Œ×ÒÑÔÐØˆŒØœÔ8ˆŒØ#'¤;Ô#BˆÔ Ø”~ˆŒØ!Ô-ˆÔØÔ*¨d¬nÑ<ˆŒØ"ˆŒØ$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒå'¨¬¸6Ô;NÐOÑOÔOˆŒÝ'¨¬¸6Ô;NÐOÑOÔOˆŒˆˆr@   r„   Úcross_attention_statesÚpast_key_valuesrJ   Ú	use_cacher&   c                 óX  — |                      ¦   «         \  }}}	|                      |¦  «        }
|
                     ||| j        | j        ¦  «                             dd¦  «        }
|                      |
¦  «        }
|�Í|                      |¦  «        }|                      |¦  «        }|                     |d| j	        | j        ¦  «                             dd¦  «        }|                     |d| j	        | j        ¦  «                             dd¦  «        }|  
                    |¦  «        }|�|                     ||| j        ¦  «        \  }}nX|�G|                     ¦   «         dk    r/|j        | j                 j        |j        | j                 j        }}nt#          d¦  «        ‚t%          j        | j        j        t,          ¦  «        } || |
|||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )	z#Input shape: Batch x Time x Channelr   rm   Nr*   r   z^Cross attention layer can't find neither `cross_attn_states` nor cached values for key/values!rŽ   r¶   )Úsizer¬   r.   r«   rŒ   rH   rú   r­   r®   rŠ   rû   Úupdater÷   Úget_seq_lengthrÜ   ÚkeysÚvaluesÚ
ValueErrorr   r·   rO   r¸   r£   r˜   r”   r“   rG   rž   r¯   )r\   r„   rý   rþ   rJ   rÿ   r•   ÚbszÚq_lenr:   Úquery_statesrŸ   r    r»   r¢   r¡   s                   r>   rc   z MllamaTextCrossAttention.forwardœ  sB  € ð &×*Ò*Ñ,Ô,‰ˆˆU�AØ—{’{ =Ñ1Ô1ˆØ#×(Ò(¨¨e°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆØ—{’{ <Ñ0Ô0ˆà!Ð-ØŸšÐ%;Ñ<Ô<ˆJØŸ;š;Ð'=Ñ>Ô>ˆLØ#Ÿš¨¨b°$Ô2JÈDÌMÑZÔZ×dÒdÐefÐhiÑjÔjˆJØ'×,Ò,¨S°"°dÔ6NÐPTÔP]Ñ^Ô^×hÒhÐijÐlmÑnÔnˆLàŸš ZÑ0Ô0ˆJØÐ*ð ,;×+AÒ+AÀ*ÈlÐ\`Ô\jÑ+kÔ+kÑ(�
˜LøØÐ(¨_×-KÒ-KÑ-MÔ-MÐPQÒ-QÐ-QàÔ& t¤~Ô6Ô;ØÔ& t¤~Ô6Ô=ð %ˆJˆJõ
 Øpñô ð õ )@Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r@   )NN)NNNN)rd   re   rf   rã   r!   ÚintrS   r2   rg   r	   r3   r¼   rc   rh   ri   s   @r>   rö   rö   �  s  ø€ € € € € ØGÐGð +/Ø $ðPð Pà  4Ñ'ðPð ˜‘:ðPð Pð Pð Pð Pð Pð6 7;Ø(,Ø.2Ø!%ð6)ð 6)à”|ð6)ð !&¤¨tÑ 3ð6)ð  ™ð	6)ð
 œ tÑ+ð6)ð ˜$‘;ð6)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð6)ð 6)ð 6)ð 6)ð 6)ð 6)ð 6)ð 6)r@   rö   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr*   rm   r(   )r,   r2   Úcat)ÚxÚx1Úx2s      r>   Úrotate_halfr  Ö  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r@   c                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r/   r  )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r>   Úapply_rotary_pos_embr  Þ  sc   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr@   c                   ó^   ‡ — e Zd Zdedefˆ fd„Z	 d	dej        dej        dej        fd„Zˆ xZ	S )
ÚMllamaTextSelfAttentionrO   r÷   c                 ó°  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | j        z  | _        | j        | j        z  | _	        | j        dz  | _
        || _        d| _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nrn   TFr§   )rR   rS   rO   rÃ   r«   r”   rT   rŠ   rŒ   r™   r“   r÷   Ú	is_causalr   r   r¬   r­   r®   r¯   rü   s      €r>   rS   z MllamaTextSelfAttention.__init__ø  s"  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ3ˆŒØ”~ˆŒØ!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ*¨d¬nÑ<ˆŒØ$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒà"ˆŒØˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆr@   Nr„   rJ   Úposition_embeddingsc                 ó‚  — |                      ¦   «         \  }}}|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }|	                     ||| j        | j        ¦  «                             dd¦  «        }	|
                     ||| j        | j        ¦  «                             dd¦  «        }
|                     ||| j        | j        ¦  «                             dd¦  «        }|\  }}t          |	|
||¦  «        \  }	}
|�| 
                    |
|| j        ¦  «        \  }
}t          j        | j        j        t           ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS rµ   )r  r¬   r­   r®   r.   r«   rŒ   rH   rŠ   r  r  r÷   r   r·   rO   r¸   r£   r˜   r”   r“   rG   rž   r¯   )r\   r„   rJ   r  rþ   r•   r  r  r:   r	  rŸ   r    r  r  r»   r¢   r¡   s                    r>   rc   zMllamaTextSelfAttention.forward  sÖ  € ð &×*Ò*Ñ,Ô,‰ˆˆU�Aà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨¨e°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆØ—_’_ S¨%°Ô1IÈ4Ì=ÑYÔY×cÒcÐdeÐghÑiÔiˆ
Ø#×(Ò(¨¨e°TÔ5MÈtÌ}Ñ]Ô]×gÒgÐhiÐklÑmÔmˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r@   r|   )
rd   re   rf   r!   r
  rS   r2   rg   rc   rh   ri   s   @r>   r  r  ÷  s”   ø€ € € € € ð^Ð/ð ^¸Cð ^ð ^ð ^ð ^ð ^ð ^ð0 ð*)ð *)à”|ð*)ð œð*)ð #œ\ð	*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r@   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMllamaTextMLPc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NFr§   )rR   rS   rO   rT   r€   r   r   Ú	gate_projÚup_projÚ	down_projr   r}   Úact_fnrƒ   s     €r>   rS   zMllamaTextMLP.__init__:  s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒå˜VÔ.Ô/ˆŒˆˆr@   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S r|   )r&  r'  r$  r%  )r\   r  r&  s      r>   rc   zMllamaTextMLP.forwardE  sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr@   )rd   re   rf   rS   rc   rh   ri   s   @r>   r!  r!  9  sG   ø€ € € € € ð	0ð 	0ð 	0ð 	0ð 	0ðð ð ð ð ð ð r@   r!  c                   óh  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dej        dz  d	ej        dz  d
eej        ej        f         dz  dej	        dz  de
dz  dedz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚMllamaSelfAttentionDecoderLayerrO   r÷   c                 óB  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        || _        d S )N)rO   r÷   rÀ   )rR   rS   rT   r  rÄ   r!  rÅ   rå   rù   rÈ   rÉ   r÷   rü   s      €r>   rS   z(MllamaSelfAttentionDecoderLayer.__init__L  s‰   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå0¸È)ÐTÑTÔTˆŒå  Ñ(Ô(ˆŒÝ0°Ô1CÈÔI\Ð]Ñ]Ô]ˆÔÝ(9¸&Ô:LÐRXÔReÐ(fÑ(fÔ(fˆÔ%à"ˆŒˆˆr@   NFr„   rý   r#   rJ   r=   Úposition_idsrþ   rÿ   r  r•   r&   c
           
      óÎ   — |}|                       |¦  «        } | j        d||||||	dœ|
¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )a‘  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.

            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )r„   rJ   r,  rþ   rÿ   r  r×   )rÈ   rÄ   rÉ   rÅ   )r\   r„   rý   r#   rJ   r=   r,  rþ   rÿ   r  r•   rÑ   Úself_attn_weightss                r>   rc   z'MllamaSelfAttentionDecoderLayer.forwardX  s¤   € ð> !ˆà×,Ò,¨]Ñ;Ô;ˆð ,:¨4¬>ð ,
Ø'Ø)Ø%Ø+ØØ 3ð,
ð ,
ð ð,
ð ,
Ñ(ˆÐ(ð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr@   )NNNNNNFN)rd   re   rf   r!   r
  rS   r2   rg   r¼   Ú
LongTensorr	   r3   r   r   ÚFloatTensorrc   rh   ri   s   @r>   r*  r*  K  sa  ø€ € € € € ð
#Ð/ð 
#¸Cð 
#ð 
#ð 
#ð 
#ð 
#ð 
#ð 7;Ø48Ø.2ØRVØ04Ø(,Ø!&ØHLð5ð 5à”|ð5ð !&¤¨tÑ 3ð5ð $œl¨TÑ1ð	5ð
 œ tÑ+ð5ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'Oð5ð Ô&¨Ñ-ð5ð  ™ð5ð ˜$‘;ð5ð # 5¤<°´Ð#=Ô>ÀÑEð5ð Ð-Ô.ð5ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r@   r*  c                   ó  ‡ — e Zd ZdZdededdfˆ fd„Z	 	 	 	 ddej        d	ej        d
ej        dej        de	ej        ej        f         dej
        dz  dedz  dedz  dej        dz  dee         de	ej                 fd„Zˆ xZS )Ú MllamaCrossAttentionDecoderLayerzLCross-attention transformer block with tanh-gated attention and feedforward.rO   r÷   r&   Nc                 ó  •— t          ¦   «                              ¦   «          || _        t          ||¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
                             t          j        d¦  «        ¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _        t          j
                             t          j        d¦  «        ¦  «        | _        d S )N)r÷   rÀ   r   )rR   rS   r÷   rö   Ú
cross_attnrå   rT   rù   rÈ   r2   r   rX   rY   Úcross_attn_attn_gater!  rÅ   rÉ   Úcross_attn_mlp_gaterü   s      €r>   rS   z)MllamaCrossAttentionDecoderLayer.__init__“  sÀ   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒÝ2°6ÀYÐOÑOÔOˆŒå0°Ô1CÈÔI\Ð]Ñ]Ô]ˆÔÝ$)¤H×$6Ò$6µu´{À1±~´~Ñ$FÔ$FˆÔ!å  Ñ(Ô(ˆŒÝ(9¸&Ô:LÐRXÔReÐ(fÑ(fÔ(fˆÔ%Ý#(¤8×#5Ò#5µe´kÀ!±n´nÑ#EÔ#EˆÔ Ð Ð r@   Fr„   rý   r#   rJ   r=   r,  rþ   rÿ   r  r•   c
                 óT  — |}|                       |¦  «        } | j        d||||dœ|
¤Ž\  }}|| j                             ¦   «         |z  z   }|}|                      |¦  «        }|                      |¦  «        }|�|d d …df         |z  }|| j                             ¦   «         |z  z   }|S )N)r„   rJ   rý   rþ   r   r×   )rÈ   r4  r5  ra   rÉ   rÅ   r6  )r\   r„   rý   r#   rJ   r=   r,  rþ   rÿ   r  r•   rÑ   r¡   s                r>   rc   z(MllamaCrossAttentionDecoderLayer.forwardŸ  sÞ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆà&5 d¤oð '
Ø'Ø/Ø#9Ø+ð	'
ð '
ð
 ð'
ð '
Ñ#ˆ�|ð ! 4Ô#<×#AÒ#AÑ#CÔ#CÀmÑ#SÑSˆà ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ(Ð4Ø9¸!¸!¸!¸Q¸$Ô?À-ÑOˆMØ  4Ô#;×#@Ò#@Ñ#BÔ#BÀ]Ñ#RÑRˆàÐr@   )NNFN)rd   re   rf   rã   r!   r
  rS   r2   rg   r¼   r/  r	   r3   r   r   rc   rh   ri   s   @r>   r2  r2  �  s/  ø€ € € € € ØVÐVð
FÐ/ð 
F¸Cð 
FÀDð 
Fð 
Fð 
Fð 
Fð 
Fð 
Fð& 15Ø(,Ø!&Ø37ð ð  à”|ð ð !&¤ð ð $œlð	 ð
 œð ð (-¨U¬\¸5¼<Ð-GÔ'Hð ð Ô&¨Ñ-ð ð  ™ð ð ˜$‘;ð ð #œ\¨DÑ0ð ð Ð-Ô.ð ð 
ˆuŒ|Ô	ð ð  ð  ð  ð  ð  ð  ð  r@   r2  c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚMllamaRotaryEmbeddingÚinv_freqNrO   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr:  F)Ú
persistentÚoriginal_inv_freq)rR   rS   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrO   Úrope_parametersr<  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r\   rO   ÚdeviceÚrope_init_fnr:  r]   s        €r>   rS   zMllamaRotaryEmbedding.__init__Æ  sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr@   rH  ztorch.deviceÚseq_lenr&   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarŒ   Nr+   r   rm   ©r%   )rH  r%   )	rC  ÚgetattrrT   rÃ   r2   ÚarangeÚint64r0   rô   )rO   rH  rJ  Úbaser)   Úattention_factorr:  s          r>   rD  z5MllamaRotaryEmbedding.compute_default_rope_parametersÖ  sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r@   c                 ó  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «        }|d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   	                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r*   r   ÚmpsÚcpuF)Údevice_typeÚenabledrm   r(   rM  )r:  rô   rˆ   r,   Ú
isinstancerH  ÚtypeÚstrr   rH   r2   r  r  rE  r  r0   r%   )
r\   r  r,  Úinv_freq_expandedÚposition_ids_expandedrV  ÚfreqsÚembr  r  s
             r>   rc   zMllamaRotaryEmbedding.forwardõ  s¥  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔeÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Â4BEÅEÅEr|   )NNN)rd   re   rf   r2   rg   Ú__annotations__r!   rS   Ústaticmethodr   r
  r¼   rô   rD  Úno_gradr   rc   rh   ri   s   @r>   r9  r9  Ã  sú   ø€ € € € € € ØŒlÐÐÑðVð VÐ/ð Vð Vð Vð Vð Vð Vð  à*.Ø+/Ø"ð*ð *Ø  4Ñ'ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð< €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r@   r9  c                   óŽ   ‡ — e Zd ZU eed<   dZdZdZg d¢ZdZ	dZ
dZdZdZeegeegdœZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )	ÚMllamaPreTrainedModelrO   Úmodel)ÚimageÚtextT)r¾   r2  r*  F)r„   Ú
attentionsc                 óT  •— t          ¦   «                              |¦  «         t          | j        d| j                             ¦   «         j        ¦  «        }t          |t          ¦  «        rt          j	        |j
        |¬¦  «         d S t          |t          ¦  «        r6t          j	        |j        |¬¦  «         t          j        |j        ¦  «         d S t          |t          ¦  «        r?|j        r8t          j	        |j        |¬¦  «         t          j	        |j        |¬¦  «         d S t          |t&          ¦  «        r4t          j        |j        ¦  «         t          j        |j        ¦  «         d S t          |t,          ¦  «        r"|j        rt          j        |j        ¦  «         d S d S d S )NÚinitializer_range)Ústd)rR   Ú_init_weightsrN  rO   Úget_text_configri  rX  ÚMllamaVisionModelÚinitÚnormal_Úclass_embeddingrk   rW   Úzeros_rZ   r¾   rP   rÍ   rÎ   r2  r5  r6  rN   )r\   r�   rj  r]   s      €r>   rk  z#MllamaPreTrainedModel._init_weights  s�  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�d”kÐ#6¸¼×8SÒ8SÑ8UÔ8UÔ8gÑhÔhˆå�fÕ/Ñ0Ô0ð 	)ÝŒL˜Ô/°SÐ9Ñ9Ô9Ð9Ð9Ð9Ý˜Õ BÑCÔCð 	)ÝŒL˜Ô)¨sÐ3Ñ3Ô3Ð3ÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜Õ 8Ñ9Ô9ð 	)¸f¼oð 	)ÝŒL˜Ô)¨sÐ3Ñ3Ô3Ð3ÝŒL˜œ¨cÐ2Ñ2Ô2Ð2Ð2Ð2Ý˜Õ @ÑAÔAð 	)ÝŒK˜Ô3Ñ4Ô4Ð4ÝŒK˜Ô2Ñ3Ô3Ð3Ð3Ð3Ý˜Õ EÑFÔFð 	)ØŒð )Ý”˜FœKÑ(Ô(Ð(Ð(Ð(ð	)ð 	)ð)ð )r@   )rd   re   rf   r    r_  Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendr*  r2  r  rö   Ú_can_record_outputsr2   ra  rk  rh   ri   s   @r>   rc  rc    s¾   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#ðð ð Ðð
 #ÐØ€NØÐØÐØ"&Ðà9Ð;[Ð\ð
 /Ð0HÐIðð Ðð €U„]�_„_ð)ð )ð )ð )ñ „_ð)ð )ð )ð )ð )r@   rc  zH
    The Mllama Vision Model which consists of two vision encoders.
    )Úcustom_introc                   óÈ   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zd„ Zde	j
        de	j
        fd„Zeeed	e	j
        d
e	j
        de	j
        defd„¦   «         ¦   «         ¦   «         Zˆ xZS )rm  rO   Úvision_model)re  c                 ó’  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j        | j        z  dz  dz   | _        |j        dz  | _	        t          j        |j        | j        | j        | j        dd¬¦  «        | _        t          j        | j	        t          j        | j        ¦  «        z  ¦  «        | _        t#          |¦  «        | _        t'          |d¬¦  «        | _        t'          |d¬¦  «        | _        t          j        | j        ¦  «        | _        t          j        | j        ¦  «        | _        t3          ||j        d¬¦  «        | _        t3          ||j        d¬¦  «        | _        |                      ¦   «          d S )	Nrm   r   rn   ÚvalidF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr¨   T)rP   )rR   rS   ro   rp   rI   rT   Únum_channelsÚintermediate_layers_indicesrB   rq   r   ÚConv2dÚpatch_embeddingrX   r2   rr   rp  rk   Úgated_positional_embeddingrN   Úpre_tile_positional_embeddingÚpost_tile_positional_embeddingrÆ   Úlayernorm_preÚlayernorm_postrÓ   Únum_hidden_layersÚtransformerÚnum_global_layersÚglobal_transformerÚ	post_initrƒ   s     €r>   rS   zMllamaVisionModel.__init__=  sš  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØ Ô+ˆŒØ#Ô1ˆÔØ!Ô-ˆÔØ"Ô/ˆÔØ+1Ô+MˆÔ(à œO¨t¬Ñ>À1ÑDÀqÑHˆÔØÔ'¨Ñ-ˆŒ
å!œyØÔ+ØÔ)ØœØ”?ØØð 
ñ  
ô  
ˆÔõ  "œ|¨D¬J½¼ÀTÔEUÑ9VÔ9VÑ,VÑWÔWˆÔÝ*LÈVÑ*TÔ*TˆÔ'å-RÐSYÐdhÐ-iÑ-iÔ-iˆÔ*Ý.SÐTZÐeiÐ.jÑ.jÔ.jˆÔ+õ  œ\¨$Ô*:Ñ;Ô;ˆÔÝ œl¨4Ô+;Ñ<Ô<ˆÔõ /¨v°vÔ7OÐZ_Ð`Ñ`Ô`ˆÔÝ"5°f¸fÔ>VÐaeÐ"fÑ"fÔ"fˆÔà�ŠÑÔÐÐÐr@   c                 ó   — | j         S )zg
        This function is used to fetch the first embedding layer to activate grads on inputs.
        )r‰  rò   s    r>   Úget_input_embeddingsz&MllamaVisionModel.get_input_embeddingsb  s   € ð Ô#Ð#r@   r^   r&   c                 ó„   — |j         \  }}}| j                             |d|¦  «        }t          j        ||gd¬¦  «        }|S )Nr   r(   )r,   rp  rˆ   r2   r  )r\   r^   r8   r:   rT   rp  s         r>   Úapply_class_embeddingz'MllamaVisionModel.apply_class_embeddingh  sJ   € Ø%1Ô%7Ñ"ˆ
�A�{ØÔ.×5Ò5°jÀ!À[ÑQÔQˆÝ”y /°<Ð!@ÀaÐHÑHÔHˆØÐr@   Úpixel_valuesr_   rA   c                 ó(  ‡— |j         \  }}}}}	}
|                     ||z  |z  ||	|
¦  «        }|                     ||z  d¦  «        }| j        j        j        }| j        j        j        }|                      |                     ||¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }|j         \  }}}|                     ||z  |d|¦  «        }|  	                    ||¦  «        }|                     ||z  |z  ||¦  «        }|  
                    |¦  «        }|dz  }|                     ||z  |||¦  «        }|                      ||¦  «        }|                      |¦  «        }d|j         d         dz  z
  dz  }ddd|f}t          j        ||dd¬¦  «        }|dk    r| nd	}|                     ||z  d¦  «        }t          || j        |j         d         | j        ¬
¦  «        }|                     ||z  d|¦  «        }|                      ||¬¦  «        Š‰j        }|                      |¦  «        }|                     ||z  |||z   |¦  «        }|                      ||¦  «        }|                     ||z  |||z   z  |¦  «        }|                      ||¬¦  «        }|j        }|                     ||z  |||z   |¦  «        }|d	d	…d	d	…d	|…f         }|                     |||||¦  «        }ˆfd„| j        D ¦   «         }t1          j        |d¬¦  «        }|                     ||z  |||z   d¦  «        }|d	d	…d	d	…d	|…f         }|                     ||||d¦  «        }t1          j        ||gd¬¦  «        }t7          |¬¦  «        S )a†  
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, MllamaVisionModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaVisionModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> inputs = processor(images=image, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 1, 4, 1025, 7680])
        ```
        r*   rm   r   é   rE   r   Úconstant)Úmoder’   N)rA   rB   rC   r%   rÐ   c                 ó*   •— g | ]}‰j         |         ‘ŒS r×   )r„   )rØ   ÚiÚoutputs     €r>   rÙ   z-MllamaVisionModel.forward.<locals>.<listcomp>ê  s!   ø€ Ð)lÐ)lÐ)lÀa¨&Ô*>¸qÔ*AÐ)lÐ)lÐ)lr@   r(   )rà   )r,   rG   r‰  rè   r%   rH  r0   ÚflattenrH   r‹  r—  rŠ  r�  ÚFÚpadrL   rB   r.   r�  rà   rŽ  rŒ  r’  r‡  r2   Ústackr  r   )r\   r˜  r_   rA   r•   r8   Únum_concurrent_mediaÚ	num_tilesr†  ÚheightÚwidthÚtarget_dtypeÚtarget_deviceÚpatch_embedsr^   r:   rB   r)   Únum_padding_patchesr…  Úslice_indexrJ   Úglobal_outputÚall_intermediate_hidden_statesÚintermediate_hidden_statesrŸ  s                            @r>   rc   zMllamaVisionModel.forwardn  sƒ  ø€ ð` T`ÔSeÑPˆ
Ð(¨)°\À6È5à#×+Ò+¨JÐ9MÑ,MÐPYÑ,YÐ[gÐioÐqvÑwÔwˆØ+×3Ò3°JÐAUÑ4UÐWYÑZÔZÐð Ô+Ô2Ô8ˆØÔ,Ô3Ô:ˆØ×+Ò+¨L¯OªO¸MÈ<Ñ,XÔ,XÑYÔYˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆð +Ô0Ñˆˆ;˜Ø#×+Ò+¨JÐ9MÑ,MÈyÐZ\Ð^aÑbÔbˆØ×9Ò9¸,ÐHXÑYÔYˆð $×+Ò+¨JÐ9MÑ,MÐPYÑ,YÐ[fÐhkÑlÔlˆØ×1Ò1°,Ñ?Ô?ˆØ�qÑˆð $×+Ò+¨JÐ9MÑ,MÈyÐZeÐgjÑkÔkˆØ×6Ò6°|ÐEUÑVÔVˆà×)Ò)¨,Ñ7Ô7ˆð  ! LÔ$6°rÔ$:¸QÑ$>Ñ?À1ÑDÐà�a˜Ð/Ð0ˆå”u˜\¨7¸È1ÐMÑMÔMˆØ.AÀAÒ.EÐ.EÐ*Ð*Ð*È4ˆð +×2Ò2°:Ð@TÑ3TÐVXÑYÔYˆÝ=Ø,ØÔ(Ø&Ô,¨QÔ/Ø”*ð	
ñ 
ô 
ˆð $×(Ò(¨Ð6JÑ)JÈBÐPSÑTÔTˆØ×!Ò!ØØ)ð "ñ 
ô 
ˆð Ô/ˆà×*Ò*¨<Ñ8Ô8ˆð $×+Ò+ØÐ-Ñ-¨y¸+ÐH[Ñ:[Ð]`ñ
ô 
ˆð ×:Ò:¸<ÐIYÑZÔZˆØ#×+Ò+ØÐ-Ñ-¨y¸KÐJ]Ñ<]Ñ/^Ð`cñ
ô 
ˆð ×/Ò/ØØ)ð 0ñ 
ô 
ˆð %Ô6ˆð $×+Ò+ØÐ-Ñ-¨y¸+ÐH[Ñ:[Ð]`ñ
ô 
ˆð $ A A A q q q¨,¨;¨,Ð$6Ô7ˆØ#×+Ò+¨JÐ8LÈiÐYdÐfiÑjÔjˆð *mÐ)lÐ)lÐ)lÈ4ÔKkÐ)lÑ)lÔ)lÐ&Ý%*¤[Ð1OÐUWÐ%XÑ%XÔ%XÐ"ð &@×%GÒ%GØÐ-Ñ-¨y¸+ÐH[Ñ:[Ð]_ñ&
ô &
Ð"ð &@ÀÀÀÀ1À1À1ÀlÀ{ÀlÐ@RÔ%SÐ"Ø%?×%GÒ%GØÐ,¨i¸Àbñ&
ô &
Ð"õ
 ”y ,Ð0JÐ!KÐQSÐTÑTÔTˆå°Ð>Ñ>Ô>Ð>r@   )rd   re   rf   r"   r_  rr  rs  rS   r•  r2   rg   r—  r   r   r   r   rc   rh   ri   s   @r>   rm  rm  3  sÿ   ø€ € € € € € ð ÐÐÑØ&ÐØ!Ðð#Ð1ð #ð #ð #ð #ð #ð #ðJ$ð $ð $ð°%´,ð À5Ä<ð ð ð ð ð  ØØðH?Ø!œLðH?Ø<A¼LðH?Ø]bÔ]iðH?à	ðH?ð H?ð H?ñ „^ñ „_ñ  ÔðH?ð H?ð H?ð H?ð H?r@   rm  zc
    The Mllama Text Model which consists of transformer with self and cross attention layers.
    c                   ób  ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zee	e
e	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  deej        ej        f         dz  dedz  dej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         ¦   «         Zˆ xZS )ÚMllamaTextModelrO   zlanguage_model.model)rf  c                 ó�  •— t          ¦   «                              |¦  «         |j        | _        |j        | _        t          j        |j        dz   |j        | j        ¦  «        | _        |j	        | _	        g }t          |j        ¦  «        D ]R}|| j	        v r$|                     t          ||¦  «        ¦  «         Œ/|                     t          ||¦  «        ¦  «         ŒSt          j        |¦  «        | _        t#          |j        |j        ¬¦  «        | _        t)          |¬¦  «        | _        d| _        |                      ¦   «          d S )Nrš  rÀ   ©rO   F)rR   rS   Úpad_token_idÚpadding_idxÚ
vocab_sizer   rV   rT   Úembed_tokensÚcross_attention_layersrÛ   r�  Úappendr2  r*  rÚ   rÜ   rå   rù   Únormr9  Ú
rotary_embrÝ   r“  )r\   rO   rÜ   r÷   r]   s       €r>   rS   zMllamaTextModel.__init__  s*  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒÝœL¨Ô):¸QÑ)>ÀÔ@RÐTXÔTdÑeÔeˆÔØ&,Ô&CˆÔ#àˆÝ˜vÔ7Ñ8Ô8ð 	Rð 	RˆIØ˜DÔ7Ð7Ð7Ø—’Õ>¸vÀyÑQÔQÑRÔRÐRÐRà—’Õ=¸fÀiÑPÔPÑQÔQÐQÐQå”m FÑ+Ô+ˆŒÝ% fÔ&8¸fÔ>QÐRÑRÔRˆŒ	Ý/°vÐ>Ñ>Ô>ˆŒà&+ˆÔ#Ø�ŠÑÔÐÐÐr@   NÚ	input_idsrJ   r,  rý   r#   r=   rþ   Úinputs_embedsrÿ   r•   r&   c
                 óÌ  — |	�|	n| j         j        }	|du |duz  rt          d¦  «        ‚|€|                      |¦  «        }|}|	r|€t	          | j         ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }t          | j         ||||¬¦  «        }|                      ||¬¦  «        }t          | j        ¦  «        D ]H\  }}|| j        v }|du p|duo|                     |¦  «        dk    }|r|€|rŒ6 ||f|||||||	|d	œ|
¤Ž}ŒI|                      |¦  «        }t#          ||¬
¦  «        S )aQ  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.

        Example:

        ```python
        >>> from transformers import AutoProcessor, MllamaTextModel

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaTextModel.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> text = "<|image|>If I had to write a haiku for this one"
        >>> inputs = processor(text=text, return_tensors="pt")

        >>> output = model(**inputs)

        >>> print(output.last_hidden_state.shape)
        torch.Size([1, 13, 4096])
        ```
        Nú:You must specify exactly one of input_ids or inputs_embedsr³  r   r   ©rH  )rO   r½  rJ   rþ   r,  )r,  )rý   r#   rJ   r=   r,  rþ   rÿ   r  )rà   rþ   )rO   rÿ   r  r·  r
   r  r2   rO  r,   rH  r/   r   r»  Ú	enumeraterÜ   r¸  rº  r   )r\   r¼  rJ   r,  rý   r#   r=   rþ   r½  rÿ   r•   r„   Úpast_seen_tokensÚcausal_maskr  ÚidxÚdecoder_layerÚis_cross_attention_layerÚis_cross_attention_cache_emptys                      r>   rc   zMllamaTextModel.forward  s  € ðp "+Ð!6�I�I¸D¼KÔ<Qˆ	à˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà%ˆàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð #Ÿošo¨mÈ,˜oÑWÔWÐõ #,¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�ð (+¨dÔ.IÐ'IÐ$Ø-<ÀÐ-Dð .Ø tÐ+ÐX°×0NÒ0NÈsÑ0SÔ0SÐWXÒ0Xð +ð (ð Ð,BÐ,JÐOmÐ,JØà)˜MØðà'=Ø%9Ø*Ø.KØ)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå&Ø+Ø+ð
ñ 
ô 
ð 	
r@   )	NNNNNNNNN)rd   re   rf   r!   r_  rr  rs  rS   r   r   r   r   r2   r/  rg   r0  r¼   r	   r3   r   r   r   rc   rh   ri   s   @r>   r±  r±  ü  s’  ø€ € € € € € ð ÐÐÑØ.ÐØ ÐðÐ/ð ð ð ð ð ð ð*  ØØØð .2Ø.2Ø04Ø;?Ø48ØRVØ(,Ø26Ø!%ðn
ð n
àÔ# dÑ*ðn
ð œ tÑ+ðn
ð Ô&¨Ñ-ð	n
ð
 !&Ô 1°DÑ 8ðn
ð $œl¨TÑ1ðn
ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'Oðn
ð  ™ðn
ð Ô(¨4Ñ/ðn
ð ˜$‘;ðn
ð Ð-Ô.ðn
ð 
!ðn
ð n
ð n
ñ „^ñ Ôñ „_ñ  Ôðn
ð n
ð n
ð n
ð n
r@   r±  zE
    The Mllama Text Model with a language modeling head on top.
    c                   ón  ‡ — e Zd ZU eed<   dZdZˆ fd„Zee		 	 	 	 	 	 	 	 	 	 	 dde
j        dz  de
j        dz  d	e
j        dz  d
e
j        dz  de
j        dz  dee
j        e
j        f         dz  dedz  de
j        dz  de
j        dz  dedz  dee
j        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚMllamaForCausalLMrO   TÚlanguage_modelc                 óˆ  •— t          ¦   «                              |                     ¦   «         ¦  «         |                     ¦   «         | _        | j        j        | _        t
                               | j        ¦  «        | _        t          j	        | j        j
        | j        d¬¦  «        | _        |                      ¦   «          d S r#  )rR   rS   rl  Útext_configr¶  r±  Ú_from_configrd  r   r   rT   Úlm_headr“  rƒ   s     €r>   rS   zMllamaForCausalLM.__init__š  s—   ø€ Ý‰Œ×Ò˜×/Ò/Ñ1Ô1Ñ2Ô2Ð2Ø!×1Ò1Ñ3Ô3ˆÔØÔ*Ô5ˆŒÝ$×1Ò1°$Ô2BÑCÔCˆŒ
Ý”y Ô!1Ô!=¸t¼ÐUZÐ[Ñ[Ô[ˆŒà�ŠÑÔÐÐÐr@   Nr   r¼  rJ   r,  rý   r#   r=   rþ   r½  Úlabelsrÿ   Úlogits_to_keepr•   r&   c                 ón  —  | j         d|||||||||
dœ	|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «                             ¦   «         }d}|	� | j        ||	| j        fi |¤Ž}t          |||j
        |j        |j        ¬¦  «        S )a  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MllamaForCausalLM

        >>> model = MllamaForCausalLM.from_pretrained("Llama-3.2-11B-Vision")
        >>> tokenizer = AutoTokenizer.from_pretrained("Llama-3.2-11B-Vision")

        >>> prompt = "If I had to write a haiku, it would be:"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
        >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        >>> print(result)
        If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
        I love the idea of snowflakes gently falling, each one
        ```
        )	r¼  rý   rJ   r,  r#   r=   rþ   r½  rÿ   N©ÚlossÚlogitsrþ   r„   rg  r×   )rd  rà   rX  r
  ÚslicerÎ  rô   Úloss_functionr¶  r   rþ   r„   rg  )r\   r¼  rJ   r,  rý   r#   r=   rþ   r½  rÏ  rÿ   rÐ  r•   Úoutputsr„   Úslice_indicesrÔ  rÓ  s                     r>   rc   zMllamaForCausalLM.forward£  s  € ð| �$”*ð 
ØØ#9Ø)Ø%Ø!5Ø*GØ+Ø'Øð
ð 
ð ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔA×GÒGÑIÔIˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r@   )NNNNNNNNNNr   )rd   re   rf   r!   r_  rv  rr  rS   r   r   r2   r/  rg   r¼   r	   r0  r3   r
  r   r   r   rc   rh   ri   s   @r>   rÉ  rÉ  �  s¢  ø€ € € € € € ð ÐÐÑØ!ÐØ(Ððð ð ð ð ð Øð .2Ø.2Ø04Ø:>Ø8<ØRVØ(,Ø26Ø*.Ø!%Ø-.ðW
ð W
àÔ# dÑ*ðW
ð œ tÑ+ðW
ð Ô&¨Ñ-ð	W
ð
 !&Ô 0°4Ñ 7ðW
ð $Ô.°Ñ5ðW
ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'OðW
ð  ™ðW
ð Ô(¨4Ñ/ðW
ð Ô  4Ñ'ðW
ð ˜$‘;ðW
ð ˜eœlÑ*ðW
ð Ð+Ô,ðW
ð 
Ð'Ñ	'ðW
ð W
ð W
ñ „^ñ ÔðW
ð W
ð W
ð W
ð W
r@   rÉ  zr
    The Mllama model which consists of a vision encoder and a language model without language modeling head.
    c                   ó@  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej
        dz  dej
        dz  dej
        dz  d	ej
        dz  d
ej
        dz  dej        dz  dedz  dej	        dz  dedz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚMllamaModelrO   c                 óê  •— t          ¦   «                              |¦  «         |j        j        | _        |j        j        | _        |j        j        | _        |j        j        | _        t           	                    |j        ¦  «        | _
        t           	                    |j        ¦  «        | _        t          j        |j        j        |j        j        d¬¦  «        | _        |                      ¦   «          d S )NTr§   )rR   rS   rÌ  r¶  rT   Úvision_configrI   Úvision_output_dimrm  rÍ  r~  r±  rÊ  r   r   Úmulti_modal_projectorr“  rƒ   s     €r>   rS   zMllamaModel.__init__  sÊ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô,Ô7ˆŒØ!Ô-Ô9ˆÔØ#Ô1Ô?ˆÔØ!'Ô!5Ô!GˆÔå-×:Ò:¸6Ô;OÑPÔPˆÔÝ-×:Ò:¸6Ô;MÑNÔNˆÔÝ%'¤YØÔ Ô2ØÔÔ*Øð&
ñ &
ô &
ˆÔ"ð
 	�ŠÑÔÐÐÐr@   Nr¼  r˜  rA   r_   rJ   r#   rý   r,  rþ   r½  rÿ   r•   r&   c                 óø  — |du |
duz  rt          d¦  «        ‚|�|�t          d¦  «        ‚|�j|€t          d¦  «        ‚|                      |||¬¦  «        }|j        }|                      |¦  «                             d|j        d         | j        ¦  «        }|�%t          || j        j        | j	        ¬¦  «        \  }}nd}|�{|	�|	 
                    ¦   «         nd	}|�|j        d
         n|
j        d
         }|�|j        n|
j        }t          j        ||¬¦  «        |z   }|dd…dd…|f         }|dd…dd…|f         } | j        d|||||||	||
dœ	|¤Ž}t          |j        |j        |j        |j        ¬¦  «        S )ar  
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        Nr¿  zM`pixel_values` and `cross_attention_states` cannot be provided simultaneouslyzA`aspect_ratio_ids` must be provided if `pixel_values` is provided)r˜  r_   rA   r*   rE   )r$   r%   r   r   rÀ  )	r¼  rJ   r,  rý   r#   r=   rþ   rÿ   r½  )rà   rþ   r„   rg  r×   )r  r~  rà   rÞ  rG   r,   rT   r?   rB   r%   r  rH  r2   rO  rÊ  r   rþ   r„   rg  )r\   r¼  r˜  rA   r_   rJ   r#   rý   r,  rþ   r½  rÿ   r•   Úvision_outputsr=   rÂ  rJ  rH  Úcurrent_posr×  s                       r>   rc   zMllamaModel.forward  s$  € ð\ ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ#Ð(>Ð(JÝÐlÑmÔmÐmàÐ#ØÐ'Ý Ð!dÑeÔeÐeà!×.Ò.Ø)Ø!1Ø"3ð /ñ ô ˆNð
 &4Ô%EÐ"Ø%)×%?Ò%?Ð@VÑ%WÔ%W×%_Ò%_ØÐ*Ô0°Ô4°dÔ6Fñ&ô &Ð"ð  Ð+ÝB_Ø$Ø"&Ô"3Ô"?Ø”jðCñ Cô CÑ?Ð Ð"?Ð"?ð -1Ð)àÐ+ØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆGØ)2Ð)>�YÔ%Ð%ÀMÔDXˆFÝœ, w°vÐ>Ñ>Ô>ÐAQÑQˆKà#7¸¸¸¸1¸1¸1¸kÐ8IÔ#JÐ Ø,IÈ!È!È!ÈQÈQÈQÐP[ÐJ[Ô,\Ð)à%�$Ô%ð 
ØØ)Ø%Ø#9Ø!5Ø*GØ+ØØ'ð
ð 
ð ð
ð 
ˆõ 'Ø%Ô7Ø#Ô3Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r@   )NNNNNNNNNNN)rd   re   rf   r    rS   r   r   r2   r/  r0  rg   r	   r3   r   r   r   rc   rh   ri   s   @r>   rÚ  rÚ  ÿ  s~  ø€ € € € € ð˜|ð ð ð ð ð ð ð  Øð .2Ø15Ø15Ø04Ø.2Ø48Ø6:Ø04Ø(,Ø26Ø!%ðd
ð d
àÔ# dÑ*ðd
ð Ô'¨$Ñ.ðd
ð !œ<¨$Ñ.ð	d
ð
  œ,¨Ñ-ðd
ð œ tÑ+ðd
ð $œl¨TÑ1ðd
ð !&¤¨tÑ 3ðd
ð Ô&¨Ñ-ðd
ð  ™ðd
ð Ô(¨4Ñ/ðd
ð ˜$‘;ðd
ð Ð-Ô.ðd
ð 
!ðd
ð d
ð d
ñ „^ñ Ôðd
ð d
ð d
ð d
ð d
r@   rÚ  zS
    The Mllama model which consists of a vision encoder and a language model.
    c            "       ó   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej
        dz  dej
        dz  d	ej
        dz  d
ej
        dz  dej
        dz  dej        dz  dedz  dej	        dz  dej        dz  dedz  deej
        z  dee         deez  fd„¦   «         ¦   «         Z	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ fd„Zˆ xZS )ÚMllamaForConditionalGenerationrO   c                 óú   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        j        |j        j        d¬¦  «        | _	        |  
                    ¦   «          d S r#  )rR   rS   rÚ  rd  r   r   rÌ  rT   r¶  rÎ  r“  rƒ   s     €r>   rS   z'MllamaForConditionalGeneration.__init__†  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý  Ñ(Ô(ˆŒ
Ý”y Ô!3Ô!?ÀÔASÔA^ÐejÐkÑkÔkˆŒØ�ŠÑÔÐÐÐr@   Nr   r¼  r˜  rA   r_   rJ   r#   rý   r,  rþ   r½  rÏ  rÿ   rÐ  r•   r&   c                 ób  —  | j         d|||||||||	|
|dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j        j        j	        fi |¤Ž}t          |||j        |j        |j        ¬¦  «        S )a·  
        aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*):
            Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`:

            - 1 for tiles that are **not masked**,
            - 0 for tiles that are **masked**.
        aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*):
            Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image.
            These ids correspond to indices in the model's list of supported aspect ratios, offset by 1.

            For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]:
            - An image with aspect ratio [1, 1] would have ID 1
            - An image with aspect ratio [1, 2] would have ID 2
            - An image with aspect ratio [2, 1] would have ID 3

            The id 0 is reserved for padding (i.e., no image).

            If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, MllamaForConditionalGeneration

        >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision"
        >>> model = MllamaForConditionalGeneration.from_pretrained(checkpoint)
        >>> processor = AutoProcessor.from_pretrained(checkpoint)

        >>> prompt = "<|image|>If I had to write a haiku for this one"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")

        >>> # Generate
        >>> output = model.generate(**inputs, max_new_tokens=15)

        >>> prompt_len = inputs.input_ids.shape[-1]
        >>> generated_ids = output[:, prompt_len:]
        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
        >>> print(generated_text)
        [', it would be:.\\nA stop sign in Chinatown.\\n']
        ```
        )r¼  r˜  rA   r_   r#   rý   rJ   r,  rþ   r½  rÿ   NrÒ  r×   )rd  rà   rX  r
  rÕ  rÎ  rÖ  rO   rÌ  r¶  r   rþ   r„   rg  )r\   r¼  r˜  rA   r_   rJ   r#   rý   r,  rþ   r½  rÏ  rÿ   rÐ  r•   r×  r„   rØ  rÔ  rÓ  s                       r>   rc   z&MllamaForConditionalGeneration.forwardŒ  s  € ðb �$”*ð 
ØØ%Ø/Ø-Ø!5Ø#9Ø)Ø%Ø+Ø'Øð
ð 
ð ð
ð 
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´kÔ6MÔ6XÐcÐcÐ\bÐcÐcˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r@   Fc                 óz   •—  t          ¦   «         j        |f|	|
|||||||||dœ|¤Ž}|s|
rd |d<   d |d<   d |d<   |S )N)rþ   rÿ   r½  r,  rJ   r˜  r_   rA   r#   rÐ  Úis_first_iterationr˜  r_   rA   )rR   Úprepare_inputs_for_generation)r\   r¼  r½  rJ   r,  r˜  r_   rA   r#   rþ   rÿ   rÐ  rç  r•   Úmodel_inputsr]   s                  €r>   rè  z<MllamaForConditionalGeneration.prepare_inputs_for_generationý  sŒ   ø€ ð$ =•u‘w”wÔ<Øð
à+ØØ'Ø%Ø)Ø%Ø-Ø/Ø!5Ø)Ø1ð
ð 
ð ð
ð 
ˆð$ "ð 	5 ið 	5Ø+/ˆL˜Ñ(Ø/3ˆLÐ+Ñ,Ø04ˆLÐ,Ñ-àÐr@   c                 óÂ   •— |                      dd ¦  «        } t          ¦   «         j        d|||dœ|¤Ž}|�(t          j        ||d d …dd …df         gd¬¦  «        |d<   |S )Nr#   )r×  Úmodel_kwargsÚis_encoder_decoderr*   .r   r(   r×   )ÚgetrR   Ú#_update_model_kwargs_for_generationr2   r  )r\   r×  rë  rì  r•   Úcross_attention_mask_prevr]   s         €r>   rî  zBMllamaForConditionalGeneration._update_model_kwargs_for_generation(  sš   ø€ Ø$0×$4Ò$4Ð5KÈTÑ$RÔ$RÐ!ØB•u‘w”wÔBð 
ØØ%Ø1ð
ð 
ð ð	
ð 
ˆð %Ð0Ý38´9Ø*Ð,EÀaÀaÀaÈÈÈÈcÀkÔ,RÐSÐYZð4ñ 4ô 4ˆLÐ/Ñ0ð Ðr@   )NNNNNNNNNNNNr   )NNNNNNNNNFNF)rd   re   rf   r    rS   r   r   r2   r/  r0  rg   r	   r3   r
  r   r   r¼   r   rc   rè  rî  rh   ri   s   @r>   rã  rã  ~  s
  ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð .2Ø15Ø15Ø04Ø.2Ø48Ø6:Ø04Ø(,Ø26Ø*.Ø!%Ø-.ðm
ð m
àÔ# dÑ*ðm
ð Ô'¨$Ñ.ðm
ð !œ<¨$Ñ.ð	m
ð
  œ,¨Ñ-ðm
ð œ tÑ+ðm
ð $œl¨TÑ1ðm
ð !&¤¨tÑ 3ðm
ð Ô&¨Ñ-ðm
ð  ™ðm
ð Ô(¨4Ñ/ðm
ð Ô  4Ñ'ðm
ð ˜$‘;ðm
ð ˜eœlÑ*ðm
ð Ð+Ô,ðm
ð  
Ð'Ñ	'ð!m
ð m
ð m
ñ „^ñ Ôðm
ðb ØØØØØØØ!ØØØØ ð)ð )ð )ð )ð )ð )ðVð ð ð ð ð ð ð ð r@   rã  )rã  rÉ  r±  rm  rc  rÚ  )rŽ   )r   )[rã   rË   Úcollections.abcr   Útypingr   r2   Útorch.nn.functionalr   r›   r¡  Ú r   rn  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mllamar    r!   r"   Ú
get_loggerrd   Úloggerrg   r
  rZ  r¼   r?   r%   rL   ÚModulerN   rk   rz   r�   rô   r£   r¥   r¾   rÓ   rå   rö   r  r  r  r!  r*  r2  r9  rc  rm  r±  rÉ  rÚ  rã  Ú__all__r×   r@   r>   ú<module>r     sÏ  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ð `ðð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ TÐ Tð 
ˆÔ	˜HÑ	%Ô	%€ð?Øœ,ð?àð?ð ð?ð ˆ5Œ<˜œÐ%Ô&ð	?ð ?ð ?ð ?ð8Ø”|ðàðð ðð Œ;ð	ð
 „\ðð ð ð ð6ð ð ð ð ¨B¬Iñ ô ð ð."ð "ð "ð "ð "¨¬ñ "ô "ð "ðLð ð ð ð �b”iñ ô ð ð 	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð( ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð24)ð 4)ð 4)ð 4)ð 4)˜BœIñ 4)ô 4)ð 4)ðn)ð )ð )ð )ð )˜rœyñ )ô )ð )ðX,^ð ,^ð ,^ð ,^ð ,^˜"œ)ñ ,^ô ,^ð ,^ð`Jð Jð Jð Jð J˜œ	ñ Jô Jð Jð(Q)ð Q)ð Q)ð Q)ð Q)˜rœyñ Q)ô Q)ð Q)ðj(ð (ð (ðð ð ð ð2>)ð >)ð >)ð >)ð >)˜bœiñ >)ô >)ð >)ðDð ð ð ð �B”Iñ ô ð ð$Bð Bð Bð Bð BÐ&@ñ Bô Bð BðJ/ð /ð /ð /ð /Ð'Añ /ô /ð /ðf?<ð ?<ð ?<ð ?<ð ?<˜BœIñ ?<ô ?<ð ?<ðD ð*)ð *)ð *)ð *)ð *)˜Oñ *)ô *)ñ „ð*)ðZ €ððñ ô ð
A?ð A?ð A?ð A?ð A?Ð-ñ A?ô A?ñô ð
A?ðH €ððñ ô ð
L
ð L
ð L
ð L
ð L
Ð+ñ L
ô L
ñô ð
L
ð^ €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð-¨ñ g
ô g
ñô ð
g
ðT €ððñ ô ð
w
ð w
ð w
ð w
ð w
Ð'ñ w
ô w
ñô ð
w
ðt €ððñ ô ð
sð sð sð sð sÐ%:¸Oñ sô sñô ð
sðlð ð €€€r@   