§
    ‚Štj:ô  ã                   óÄ  — d dl mZ d dlmZ d dlZd dlZd dlmZ d dlmc m	Z
 ddlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5m6Z6  G d„ dej7        ¦  «        Z8 G d„ dej7        ¦  «        Z9 G d„ dej7        ¦  «        Z: G d„ de¦  «        Z;dej<        de=dej<        fd „Z>	 d\d"ej7        d#ej<        d$ej<        d%ej<        d&ej<        dz  d'e?d(e?d)e&e(         fd*„Z@d+„ ZAd]d,„ZB G d-„ d.ej7        ¦  «        ZC G d/„ d0ej7        ¦  «        ZDe) G d1„ d2e$¦  «        ¦   «         ZE G d3„ d4eE¦  «        ZF G d5„ d6eE¦  «        ZG G d7„ d8eE¦  «        ZHd9ej<        d:e=dz  dej<        fd;„ZI G d<„ d=eE¦  «        ZJd^d?e=fd@„ZK	 d_dCej<        dDe=d?e=dEe=fdF„ZLdGej<        dHejM        dIe=dJeNdKe=dej<        fdL„ZOdMdejP        fdNej<        dOe=dPe=dQeQdRe=dSejR        deSej<        ej<        f         fdT„ZT G dU„ dVeE¦  «        ZU e)dW¬X¦  «         G dY„ dZeEe¦  «        ¦   «         ZVg d[¢ZWdS )`é    )ÚCallable)ÚOptionalNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	BltConfigÚBltGlobalTransformerConfigÚBltLocalDecoderConfigÚBltLocalEncoderConfigÚBltPatcherConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBltMLPc                 ó˜  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚintermediate_sizeÚnnÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr+   Ú	__class__s     €úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/blt/modeling_blt.pyr*   zBltMLP.__init__5   s¦   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ!'Ô!9ˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒå˜VÔ.Ô/ˆŒˆˆó    c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S ©N)r2   r4   r0   r1   )r6   Úxr2   s      r8   ÚforwardzBltMLP.forward@   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr9   )Ú__name__Ú
__module__Ú__qualname__r*   r=   Ú__classcell__©r7   s   @r8   r$   r$   4   sG   ø€ € € € € ð	0ð 	0ð 	0ð 	0ð 	0ðð ð ð ð ð ð r9   r$   c                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
Ú
BltRMSNormç�íµ ÷Æ°>ÚepsÚreturnNc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z9
        BltRMSNorm is equivalent to T5LayerNorm
        N)r)   r*   r.   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)r6   r,   rF   r7   s      €r8   r*   zBltRMSNorm.__init__F   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr9   Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtorJ   Úfloat32ÚpowÚmeanÚrsqrtrM   rL   )r6   rN   Úinput_dtypeÚvariances       r8   r=   zBltRMSNorm.forwardN   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r9   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)ÚtuplerL   ÚshaperM   ©r6   s    r8   Ú
extra_reprzBltRMSNorm.extra_reprU   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr9   )rE   )
r>   r?   r@   Úfloatr*   rJ   ÚTensorr=   r_   rA   rB   s   @r8   rD   rD   E   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr9   rD   c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚBltRotaryEmbeddingÚinv_freqNr+   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultrd   F)Ú
persistentÚoriginal_inv_freq)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr+   Úrope_parametersrf   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r6   r+   ÚdeviceÚrope_init_fnrd   r7   s        €r8   r*   zBltRotaryEmbedding.__init__\   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr9   rr   ztorch.deviceÚseq_lenrG   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   rP   ©rS   )rr   rS   )	rm   Úgetattrr,   Únum_attention_headsrJ   ÚarangeÚint64rT   r`   )r+   rr   rt   ÚbaseÚdimÚattention_factorrd   s          r8   rn   z2BltRotaryEmbedding.compute_default_rope_parametersl   sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r9   c                 ó  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «        }|d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   	                    dd¦  «        }t          j        |dd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   rQ   r   ÚmpsÚcpuF)Údevice_typeÚenabledrP   ©r   ry   )rd   r`   Úexpandr]   Ú
isinstancerr   ÚtypeÚstrr   Ú	transposerJ   Úrepeat_interleaveÚcosro   ÚsinrT   rS   )
r6   r<   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr„   ÚfreqsÚembr�   rŽ   s
             r8   r=   zBltRotaryEmbedding.forwardŠ   s¤  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔeÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝÔ)¨%°¸Ð;Ñ;Ô;ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Â4BEÅEÅEr;   ©NNN)r>   r?   r@   rJ   ra   Ú__annotations__r   r*   Ústaticmethodr   Úintr\   r`   rn   Úno_gradr   r=   rA   rB   s   @r8   rc   rc   Y   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜yð Vð Vð Vð Vð Vð Vð  à#'Ø+/Ø"ð*ð *Ø˜DÑ ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r9   rc   c                   ód  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  d	eej        ej        f         dz  d
ej        dz  de	dz  de
dz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚBltTransformerLayerÚ	layer_idxc                 óB  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        || _        d S )N)r+   r›   ©rF   )r)   r*   r,   ÚBltSelfAttentionÚ	self_attnr$   ÚmlprD   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr›   ©r6   r+   r›   r7   s      €r8   r*   zBltTransformerLayer.__init__œ   s†   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå)°À9ÐMÑMÔMˆŒÝ˜&‘>”>ˆŒÝ)¨&Ô*<À&ÔBUÐVÑVÔVˆÔÝ(2°6Ô3EÈ6ÔK^Ð(_Ñ(_Ô(_ˆÔ%à"ˆŒˆˆr9   NFrN   Úcross_attention_statesÚcross_attention_maskÚattention_maskÚfull_text_row_masked_out_maskr�   Úpast_key_valuesÚ	use_cacheÚposition_embeddingsÚkwargsrG   c
           
      óÎ   — |}|                       |¦  «        } | j        d||||||	dœ|
¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )a‘  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.

            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        )rN   r§   r�   r©   rª   r«   © )r¢   rŸ   r£   r    )r6   rN   r¥   r¦   r§   r¨   r�   r©   rª   r«   r¬   ÚresidualÚself_attn_weightss                r8   r=   zBltTransformerLayer.forward§   s¤   € ð> !ˆà×,Ò,¨]Ñ;Ô;ˆð ,:¨4¬>ð ,
Ø'Ø)Ø%Ø+ØØ 3ð,
ð ,
ð ð,
ð ,
Ñ(ˆÐ(ð ! =Ñ0ˆð !ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆàÐr9   )NNNNNNFN)r>   r?   r@   r—   r*   rJ   ra   r\   Ú
LongTensorr   Úboolr   r   ÚFloatTensorr=   rA   rB   s   @r8   rš   rš   ›   sY  ø€ € € € € ð	#¨#ð 	#ð 	#ð 	#ð 	#ð 	#ð 	#ð 7;Ø48Ø.2ØRVØ04Ø(,Ø!&ØHLð5ð 5à”|ð5ð !&¤¨tÑ 3ð5ð $œl¨TÑ1ð	5ð
 œ tÑ+ð5ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'Oð5ð Ô&¨Ñ-ð5ð  ™ð5ð ˜$‘;ð5ð # 5¤<°´Ð#=Ô>ÀÑEð5ð Ð-Ô.ð5ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð5ð 5ð 5ð 5ð 5ð 5ð 5ð 5r9   rš   rN   Ún_reprG   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)r]   r‡   Úreshape)rN   r´   ÚbatchÚnum_key_value_headsÚslenrw   s         r8   Ú	repeat_kvrº   ß   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr9   ç        ÚmoduleÚqueryÚkeyÚvaluer§   ÚscalingÚdropoutr¬   c                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrP   r   rQ   )r   rS   ©ÚpÚtrainingr   )rº   Únum_key_value_groupsrJ   Úmatmulr‹   r.   Ú
functionalÚsoftmaxrU   rT   rS   rÁ   rÅ   Ú
contiguous)r¼   r½   r¾   r¿   r§   rÀ   rÁ   r¬   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r8   Úeager_attention_forwardrÏ   ë   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r9   c                 ó’   — | dd d d…f         }| ddd d…f         }t          j        | |gd¬¦  «                             d¦  «        }|S )N.rP   r   rQ   r†   éþÿÿÿ)rJ   ÚstackÚflatten)r<   Úx1Úx2Úrot_xs       r8   Úrotate_halfr×     sU   € à	
ˆ3���!�ˆ8Œ€BØ	
ˆ3���1�ˆ9Œ€BÝŒK˜"˜˜b˜	 rÐ*Ñ*Ô*×2Ò2°2Ñ6Ô6€EØ€Lr9   c                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer×   )ÚqÚkr�   rŽ   Úunsqueeze_dimÚq_embedÚk_embeds          r8   Úapply_rotary_pos_embrß     sc   € ð$ �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr9   c                   ó^   ‡ — e Zd Zdedefˆ fd„Z	 d	dej        dej        dej        fd„Zˆ xZ	S )
rž   r+   r›   c                 ó°  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | j        z  | _        | j        | j        z  | _	        | j        dz  | _
        || _        d| _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        d S )Nç      à¿TFr'   )r)   r*   r+   r{   Ú	num_headsrÁ   r,   r¸   rw   rÆ   rÀ   r›   Ú	is_causalr.   r/   Úq_projÚk_projÚv_projÚo_projr¤   s      €r8   r*   zBltSelfAttention.__init__&  s"  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ3ˆŒØ”~ˆŒØ!Ô-ˆÔØ#)Ô#=ˆÔ ØÔ*¨d¬nÑ<ˆŒØ$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒà"ˆŒØˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒˆˆr9   NrN   r§   r«   c                 ó‚  — |                      ¦   «         \  }}}|                      |¦  «        }	|                      |¦  «        }
|                      |¦  «        }|	                     ||| j        | j        ¦  «                             dd¦  «        }	|
                     ||| j        | j        ¦  «                             dd¦  «        }
|                     ||| j        | j        ¦  «                             dd¦  «        }|\  }}t          |	|
||¦  «        \  }	}
|�| 
                    |
|| j        ¦  «        \  }
}t          j        | j        j        t           ¦  «        } || |	|
||f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||fS )Nr   rP   r»   ©rÁ   rÀ   rQ   )Úsizerå   ræ   rç   Úviewrã   rw   r‹   r¸   rß   Úupdater›   r   Úget_interfacer+   Ú_attn_implementationrÏ   rÅ   rÁ   rÀ   r¶   rÊ   rè   )r6   rN   r§   r«   r©   r¬   ÚbszÚq_lenÚ_Úquery_statesrË   rÌ   r�   rŽ   Úattention_interfacerÎ   rÍ   s                    r8   r=   zBltSelfAttention.forward9  sÖ  € ð &×*Ò*Ñ,Ô,‰ˆˆU�Aà—{’{ =Ñ1Ô1ˆØ—[’[ Ñ/Ô/ˆ
Ø—{’{ =Ñ1Ô1ˆà#×(Ò(¨¨e°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆØ—_’_ S¨%°Ô1IÈ4Ì=ÑYÔY×cÒcÐdeÐghÑiÔiˆ
Ø#×(Ò(¨¨e°TÔ5MÈtÌ}Ñ]Ô]×gÒgÐhiÐklÑmÔmˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(r9   r;   )
r>   r?   r@   r   r—   r*   rJ   ra   r=   rA   rB   s   @r8   rž   rž   %  s“   ø€ € € € € ð^˜yð ^°Sð ^ð ^ð ^ð ^ð ^ð ^ð0 ð*)ð *)à”|ð*)ð œð*)ð #œ\ð	*)ð *)ð *)ð *)ð *)ð *)ð *)ð *)r9   rž   c                   óØ   ‡ — e Zd ZdZddedededz  fˆ fd„Z	 	 ddej        dej        dz  d	ej        dz  d
e	e
         deej        ej        dz  eej                 dz  f         f
d„Zˆ xZS )ÚBltCrossAttentionz<Cross-attention module for Blt, following transformers styleNr+   r›   r,   c                 óD  •— t          ¦   «                              ¦   «          || _        | j        j        | _        | j        j        | _        |j        | _        |j        | _        |j        | j        z  | _        || _	        | j        | j        z  | _
        | j        dz  | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        | j        z  d¬¦  «        | _        t          j        | j        | j        z  | j        d¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        t%          | j        |j        ¬¦  «        | _        d| _        d S )Nrâ   Fr'   r�   )r)   r*   r+   r{   rã   r¸   rÁ   r,   rw   r›   rÆ   rÀ   r.   r/   rå   ræ   rç   rè   rD   r¡   Úq_normÚk_normrä   )r6   r+   r›   r,   r7   s       €r8   r*   zBltCrossAttention.__init__i  sZ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØœÔ8ˆŒØ#'¤;Ô#BˆÔ Ø”~ˆŒØ!Ô-ˆÔØÔ*¨d¬nÑ<ˆŒØ"ˆŒØ$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒå”i Ô 0°$´.À4Ä=Ñ2PÐW\Ð]Ñ]Ô]ˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i Ô 0°$Ô2JÈTÌ]Ñ2ZÐafÐgÑgÔgˆŒÝ”i ¤°´Ñ >ÀÔ@PÐW\Ð]Ñ]Ô]ˆŒÝ  Ô!1°vÔ7JÐKÑKÔKˆŒÝ  Ô!1°vÔ7JÐKÑKÔKˆŒØˆŒˆˆr9   rN   r¥   r§   r¬   rG   c                 ój  — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }	|  	                    |¦  «        }
|	                     |d| j
        | j        ¦  «                             dd¦  «        }	|
                     |d| j
        | j        ¦  «                             dd¦  «        }
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||z   }||fS )z#Input shape: Batch x Time x Channelr   rP   rQ   r»   rê   )rë   rø   rå   rì   rã   rw   r‹   rù   ræ   rç   r¸   r   rî   r+   rï   rÏ   rÅ   rÁ   rÀ   r¶   rÊ   rè   )r6   rN   r¥   r§   r¬   rð   rñ   rò   ró   rË   rÌ   rô   rÎ   rÍ   s                 r8   r=   zBltCrossAttention.forward}  s½  € ð &×*Ò*Ñ,Ô,‰ˆˆU�AØ—{’{ =Ñ1Ô1ˆØ—{’{ <Ñ0Ô0ˆØ#×(Ò(¨¨e°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆà!%§¢Ð-CÑ!DÔ!DÐØ—[’[Ð!7Ñ8Ô8ˆ
Ø—{’{Ð#9Ñ:Ô:ˆØ—_’_ S¨"¨dÔ.FÈÌÑVÔV×`Ò`ÐabÐdeÑfÔfˆ
Ø#×(Ò(¨¨b°$Ô2JÈDÌMÑZÔZ×dÒdÐefÐhiÑjÔjˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ! MÑ1ˆØ˜LÐ(Ð(r9   r;   ©NN)r>   r?   r@   Ú__doc__r   r—   r*   rJ   ra   r   r   r\   r=   rA   rB   s   @r8   rö   rö   f  sé   ø€ € € € € ØFÐFðð ˜yð °Sð ÀsÈTÁzð ð ð ð ð ð ð. 7;Ø.2ð	$)ð $)à”|ð$)ð !&¤¨tÑ 3ð$)ð œ tÑ+ð	$)ð
 Ð+Ô,ð$)ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð$)ð $)ð $)ð $)ð $)ð $)ð $)ð $)r9   rö   c                   ó°   ‡ — e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZdZ eed¬¦  «         eed	¬¦  «        d
œZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚBltPreTrainedModelr+   Úmodel)ÚimageÚtextTrš   Fr   )Úindexr   )rN   Ú
attentionsc           	      ó¨	  •— t          ¦   «                              |¦  «         |j        j        }t	          |t
          j        ¦  «        r§t          | j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j
        dd¦  «        }|€|j        }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         |j        �$t          j        |j        |j                 ¦  «         dS t	          |t"          t$          f¦  «        s|d	v �rnt          | j        dd¦  «        }|€t          |d¦  «        r|j        }|€<d
D ]9}t          ||d¦  «        }|�$t          |d¦  «        r|j        j        d         } nŒ:|€dS |dz  }dD ]s}t          ||d¦  «        }|�^t          |d¦  «        rNt          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         Œtt          |dt          |dd¦  «        ¦  «        }	|	�^t          |	d¦  «        rNt          j        |	j        d|d|z  d|z  ¬¦  «         t          |	dd¦  «        �t          j        |	j        ¦  «         dS t	          |t,          ¦  «        s|dk    �rËt          | j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j
        dd¦  «        }d}
|�|dz  }
t          |dt          |dd¦  «        ¦  «        }t          |dd¦  «        }t          |dt          |dd¦  «        ¦  «        }||fD ]y}|�ut          |d¦  «        re|
p|j        j        d         dz  }t          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         Œz|�ut          |d¦  «        re|j        j        d         }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         dS t	          |t
          j        ¦  «        rR|j        }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         |j        �t          j        |j        ¦  «         dS dS )a  
        Initialize BLT weights following the original ByteLatentTransformer:

        - Most weights are drawn from a truncated normal.
        - Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs).
        - Norm layers are set to weight = 1, bias = 0.
        r,   NÚencoder_configrâ   r»   éýÿÿÿr   )rW   ÚstdÚaÚb)ÚMllamaTextSelfAttentionÚMllamaTextCrossAttention)rå   ræ   rç   rè   ÚdenserL   rQ   )rå   ræ   rç   r(   rè   r  ÚMllamaTextMLPÚdecoder_configr0   Úfc1r1   r2   Úfc2r   )r)   Ú_init_weightsr7   r>   rˆ   r.   Ú	Embeddingrz   r+   Úhasattrr  Úembedding_dimÚinitÚtrunc_normal_rL   Úpadding_idxÚzeros_rž   rö   r,   r]   r(   r$   r  r/   Úin_features)r6   r¼   Ú
class_namer,   r  r   ÚnameÚprojÚ	proj_namerè   Úin_stdr0   r1   r2   Ú
hidden_dimÚout_stdÚfan_inr7   s                    €r8   r  z BltPreTrainedModel._init_weightsµ  s}  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%àÔ%Ô.ˆ
õ �f�bœlÑ+Ô+ð 	Ý! $¤+¨}¸dÑCÔCˆKØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ØÐ"Ø$Ô2�à˜tÑ#ˆCÝÔØ”ØØØ�s‘(Ø�c‘'ðñ ô ð ð Ô!Ð-Ý”˜FœM¨&Ô*<Ô=Ñ>Ô>Ð>ØˆFõ �fÕ/Õ1BÐCÑDÔDð ,	È
ð W
ð I
ñ I
õ ˜$œ+ }°dÑ;Ô;ˆCØˆ{�w v¨}Ñ=Ô=ˆ{ØÔ(�Øˆ{ØMð ð �DÝ" 6¨4°Ñ6Ô6�DØÐ'­G°D¸(Ñ,CÔ,CÐ'Ø"œkÔ/°Ô3˜Ø˜øØˆ{Ø�à�t‘)ˆCð <ð /ð /�	Ý˜v y°$Ñ7Ô7�ØÐ#­°°hÑ(?Ô(?Ð#ÝÔ&ØœØ ØØ˜s™(Ø˜c™'ðñ ô ð õ ˜t V¨TÑ2Ô2Ð>Ýœ D¤IÑ.Ô.Ð.øõ ˜V X­w°v¸wÈÑ/MÔ/MÑNÔNˆFØÐ!¥g¨f°hÑ&?Ô&?Ð!ÝÔ"Ø”MØØØ˜3‘hØ˜#‘gðñ ô ð õ ˜6 6¨4Ñ0Ô0Ð<Ý”K ¤Ñ,Ô,Ð,ØˆFõ �f�fÑ%Ô%ð +	¨°Ò)FÑ)FÝ! $¤+¨}¸dÑCÔCˆKØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ð ˆFØÐ&Ø$ dÑ*�å ¨µW¸VÀUÈDÑ5QÔ5QÑRÔRˆIÝ˜f i°Ñ6Ô6ˆGÝ ¨µW¸VÀUÈDÑ5QÔ5QÑRÔRˆIð # GÐ,ð /ð /�ØÐ#­°°hÑ(?Ô(?Ð#Ø ÐB T¤[Ô%6°qÔ%9¸TÑ%A�CÝÔ&ØœØ ØØ˜s™(Ø˜c™'ðñ ô ð õ ˜t V¨TÑ2Ô2Ð>Ýœ D¤IÑ.Ô.Ð.øð Ð$­°¸HÑ)EÔ)EÐ$Ø&Ô-Ô3°AÔ6�
Ø$ dÑ*�ÝÔ"ØÔ$ØØØ˜7‘lØ˜'‘kðñ ô ð õ ˜9 f¨dÑ3Ô3Ð?Ý”K 	¤Ñ/Ô/Ð/ØˆFõ �f�bœiÑ(Ô(ð 	ØÔ'ˆFØ˜$‘,ˆCÝÔØ”ØØØ�s‘(Ø�c‘'ðñ ô ð ð Œ{Ð&Ý”˜FœKÑ(Ô(Ð(ØˆFð	ð 	r9   )r>   r?   r@   r   r•   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendr   rš   rž   Ú_can_record_outputsrJ   r˜   r  rA   rB   s   @r8   rþ   rþ   ¤  sÎ   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#Ø.Ð/ÐØ"ÐØ€NØ ÐØÐØ"'Ðà'˜Ð(;À1ÐEÑEÔEØ$�nÐ%5¸QÐ?Ñ?Ô?ðð Ðð
 €U„]�_„_ðJð Jð Jð Jñ „_ðJð Jð Jð Jð Jr9   rþ   c                   ó"  ‡ — e Zd ZU eed<   d eedd¬¦  «        iZdefˆ fd„Z	 	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j        dz  d
e	j        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  dedz  de	j        dz  dee         fd„Zd„ Zˆ xZS )ÚBltLocalEncoderr+   Úencoder_attentionsr   Úlocal_encoder©r  Ú
layer_namec                 óž  •‡— t          ¦   «                              ‰¦  «         d| _        ‰| _        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰¬¦  «        | _
        t	          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        t	          j        ‰j        ‰j        ¦  «        | _        t	          j        ¦   «         | _        ‰j        r‰j        nd}t          |¦  «        D ]1}| j                             t+          ‰|‰j        ¬¦  «        ¦  «         Œ2|                      ¦   «          d S )NFc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r®   ©rš   ©Ú.0r›   r+   s     €r8   ú
<listcomp>z,BltLocalEncoder.__init__.<locals>.<listcomp>N  ó$   ø€ ÐeÐeÐe¸	Õ  ¨Ñ3Ô3ÐeÐeÐer9   ©r+   ©r  Úout_featuresr(   r   ©r+   r›   r,   )r)   r*   Úgradient_checkpointingr+   r.   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersrc   Ú
rotary_embr/   r,   Úcross_attn_kÚpatch_embedding_projectionr  Ú
vocab_sizeÚembed_tokensÚcross_attn_layersÚcross_attn_all_layersÚappendrö   Ú	post_init©r6   r+   Úlayers_to_addr›   r7   s    `  €r8   r*   zBltLocalEncoder.__init__I  sI  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#ØˆŒÝ”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ -°FÐ;Ñ;Ô;ˆŒÝ*,¬)ØÔ*ØÔ+¨fÔ.AÑAØð+
ñ +
ô +
ˆÔ'õ
 œL¨Ô):¸FÔ<NÑOÔOˆÔÝ!#¤¡¤ˆÔØ4:Ô4PÐW˜Ô0Ð0ÐVWˆÝ˜}Ñ-Ô-ð 	ð 	ˆIØÔ"×)Ò)Ý!¨¸9ÐRXÔRdÐeÑeÔeñô ð ð ð 	�ŠÑÔÐÐÐr9   NÚ	input_idsÚinputs_embedsÚpatch_embedsr§   r�   r©   Úencoder_attention_maskÚnum_patchesÚ	patch_idsr¬   c
                 ó¬  — |€|                       |¦  «        }|j        d         }t          j        || j        j        | j        ¬¦  «        }|€Mt          j        |j        d         |j        ¬¦  «         	                    d¦  «         
                    |d¦  «        }|                      ||¦  «        }t          j        || j        j        | j        ¬¦  «        }t          | j        ¦  «        D ]å\  }} ||f|||dœ|
¤Ž}|t          | j        ¦  «        dz
  k    s| j        j        r¬|                      |||	¦  «        }|                      |¦  «        }|                     ||j        d         | j        j        z  | j        j        ¦  «        }| j        j        r|nd} | j        |         d|||dœ|
¤Ž\  }}||                     |j        ¦  «        z   }Œæ|}||fS )	Nr   rÃ   r   ©rr   rQ   ©r«   r§   r©   ©rN   r¥   r§   r®   )rF  r]   ÚFrÁ   r+   rÅ   rJ   r|   rr   rÙ   r‡   rB  Ú	enumeraterA  ÚlenrH  Úpatch_reducerD  r¶   rC  r,   rG  rT   )r6   rM  rN  rO  r§   r�   r©   rP  rQ  rR  r¬   Ú
batch_sizerN   r«   ÚidxÚlayerr›   Úcross_attention_outputrò   Úencoder_cross_statess                       r8   r=   zBltLocalEncoder.forward`  s  € ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà"Ô(¨Ô+ˆ
Ýœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆàÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆå# D¤KÑ0Ô0ð 	]ð 	]‰JˆC�Ø!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMð •c˜$œ+Ñ&Ô&¨Ñ*Ò*Ð*¨d¬kÔ.OÐ*Ø#×0Ò0°ÀÈYÑWÔW�Ø#×>Ò>¸|ÑLÔL�Ø+×3Ò3Ø Ô 2°1Ô 5¸¼Ô8PÑ PÐRVÔR]ÔRiñ ô  �ð $(¤;Ô#DÐK˜C˜CÈ!�	Ø,M¨DÔ,BÀ9Ô,Mð -Ø".Ø+8Ø#9ð-ð -ð ð	-ð -Ñ)Ð&¨ð  ,Ð.D×.GÒ.GÈÔH[Ñ.\Ô.\Ñ\�øØ+ÐØÐ2Ð2Ð2r9   c                 óB  — |j         d         }|j         d         }|                     d¦  «                             dd|j         d         ¦  «        }t          j        |||f|j        |j        ¬¦  «        }|                     |d|dd¬¦  «        }|dd…d|…dd…f         }|S )	a†  
        Reduce variable length patches to single embedding per patch
        Note: this works with variable number of patches for different sequences in the batch
        It handles variable length patches by assuming that patch_lengths will be 0 for any
        extra patches on the *right*. Since there can be a variable number of patches
        this function also return the number of patches for each sequence in the batch.
        Any embeddings on the right that are not allocated to a patch
        (i.e. if the sum(patch_lengths[i]) < seq_len for any i)
        will be sent to a dummy patch, which is trimmed before returning.
        r   rQ   ©rS   rr   r   ÚamaxF)Úsrcr   r  ÚreduceÚinclude_selfN)r]   rÙ   r‡   rJ   ÚzerosrS   rr   Úscatter_reduce)r6   rN   Úmax_num_patchesrR  r[  r  Úreduced_embeddingss          r8   rZ  zBltLocalEncoder.patch_reduce”  sË   € ð #Ô(¨Ô+ˆ
Ø%Ô+¨BÔ/ˆà×'Ò'¨Ñ+Ô+×2Ò2°2°r¸=Ô;NÈrÔ;RÑSÔSˆ	å"œ[Ø˜¨-Ð8ÀÔ@SÐ\iÔ\pð
ñ 
ô 
Ðð 0×>Ò>ØØØØØð ?ñ 
ô 
Ðð 0°°°Ð3C°OÐ3CÀQÀQÀQÐ0FÔGÐà!Ð!r9   ©	NNNNNNNNN)r>   r?   r@   r!   r•   r   rž   r+  r*   rJ   r±   ra   r   r—   r   r   r=   rZ  rA   rB   s   @r8   r-  r-  C  s\  ø€ € € € € € Ø!Ð!Ð!Ñ!à˜n˜nÐ-=ÀQÐSbÐcÑcÔcðÐðÐ4ð ð ð ð ð ð ð2 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:Ø"&Ø)-ð23ð 23àÔ# dÑ*ð23ð ”| dÑ*ð23ð ”l TÑ)ð	23ð
 œ tÑ+ð23ð Ô&¨Ñ-ð23ð  ™ð23ð !&¤¨tÑ 3ð23ð ˜4‘Zð23ð ”< $Ñ&ð23ð Ð+Ô,ð23ð 23ð 23ð 23ðh"ð "ð "ð "ð "ð "ð "r9   r-  c                   óÚ   ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	dz  d
ej        dz  de
e         fd„Zˆ xZS )ÚBltLocalDecoderr+   c                 ó¤  •‡— t          ¦   «                              ‰¦  «         d| _        ‰| _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          ‰¬¦  «        | _        t          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        t#          ‰j        ‰j        ¬¦  «        | _        t          j        ¦   «         | _        ‰j        r‰j        nd}t          |¦  «        D ]1}| j                             t/          ‰|‰j        ¬¦  «        ¦  «         Œ2|                      ¦   «          d S )	NFTc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r®   r4  r5  s     €r8   r7  z,BltLocalDecoder.__init__.<locals>.<listcomp>½  r8  r9   r9  r:  r�   r   r<  )r)   r*   r=  r+   Úcross_attn_decoderr.   r>  r?  r@  rA  rc   rB  r/   Úhidden_size_globalr,   rC  rD  rD   r¡   ÚnormrG  rH  rI  rö   rJ  rK  s    `  €r8   r*   zBltLocalDecoder.__init__·  sQ  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#ØˆŒØ"&ˆÔÝ”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ -°FÐ;Ñ;Ô;ˆŒÝ*,¬)ØÔ1ØÔ+¨fÔ.AÑAØð+
ñ +
ô +
ˆÔ'õ
 ˜vÔ1°vÔ7JÐKÑKÔKˆŒ	Ý!#¤¡¤ˆÔØ4:Ô4PÐW˜Ô0Ð0ÐVWˆÝ˜}Ñ-Ô-ð 	ð 	ˆIØÔ"×)Ò)Ý!¨¸9ÐRXÔRdÐeÑeÔeñô ð ð ð 	�ŠÑÔÐÐÐr9   NrM  rN  rO  r§   r�   r©   rP  r¬   c                 óÌ  — |j         d         }	|}
|                      |¦  «        }|                     |	|j         d         | j        j        z  | j        j        ¦  «        }|�| j        s|
|z   }
|€Mt          j        |j         d         |j	        ¬¦  «         
                    d¦  «                             |	d¦  «        }|                      |
|¦  «        }t          j        |
| j        j        | j        ¬¦  «        }
t!          | j        ¦  «        D ]C\  }}|dk    s| j        j        r | j        |         d|
||dœ|¤Ž\  }}|
|z   }
 ||
f|||dœ|¤Ž}
ŒD|                      |
¦  «        }|S )	Nr   r   rT  rQ   rÃ   rV  rU  r®   )r]   rD  r¶   r+   rC  r,   ro  rJ   r|   rr   rÙ   r‡   rB  rW  rÁ   rÅ   rX  rA  rH  rG  rq  )r6   rM  rN  rO  r§   r�   r©   rP  r¬   r[  rN   r«   Úir]  r^  rò   Úlogitss                    r8   r=   zBltLocalDecoder.forwardÏ  s·  € ð #Ô(¨Ô+ˆ
Ø%ˆØ×6Ò6°|ÑDÔDˆØ#×+Ò+Ø˜Ô*¨1Ô-°´Ô0HÑHÈ$Ì+ÔJañ
ô 
ˆð Ð#¨DÔ,CÐ#Ø)¨LÑ8ˆMàÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆå! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØ�AŠvˆv˜œÔ:ˆvØ,E¨DÔ,BÀ1Ô,Eð -Ø"/Ø+7Ø#9ð-ð -ð ð	-ð -Ñ)Ð&¨ð !.Ð0FÑ F�Ø!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMˆMð —’˜=Ñ)Ô)ˆØˆr9   ©NNNNNNN)r>   r?   r@   r    r•   r*   rJ   r±   ra   r   r   r   r=   rA   rB   s   @r8   rl  rl  ´  s  ø€ € € € € € Ø!Ð!Ð!Ñ!ðÐ4ð ð ð ð ð ð ð4 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:ð.ð .àÔ# dÑ*ð.ð ”| dÑ*ð.ð ”l TÑ)ð	.ð
 œ tÑ+ð.ð Ô&¨Ñ-ð.ð  ™ð.ð !&¤¨tÑ 3ð.ð Ð+Ô,ð.ð .ð .ð .ð .ð .ð .ð .r9   rl  c                   ó°   ‡ — e Zd ZU eed<   d eedd¬¦  «        iZdefˆ fd„Z	 	 	 dde	j
        d	e	j
        dz  d
e	j        dz  dedz  dee         f
d„Zˆ xZS )ÚBltGlobalTransformerr+   Úglobal_attentionsr   Úglobal_transformerr0  c                 óø  •— t          ¦   «                              |¦  «         || _        t          j        ¦   «         | _        t          |j        ¦  «        D ]*}| j                             t          ||¦  «        ¦  «         Œ+t          |¬¦  «        | _        t          |dd ¦  «        �'t          j        |j        |j        d¬¦  «        | _        nt          j        ¦   «         | _        |                      ¦   «          d S )Nr9  Úencoder_cross_output_sizeFr'   )r)   r*   r+   r.   r>  rA  r?  r@  rI  rš   rc   rB  rz   r/   r{  r,   Útoken_embedding_projectionÚIdentityrJ  r¤   s      €r8   r*   zBltGlobalTransformer.__init__  så   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ”m‘o”oˆŒÝ˜vÔ7Ñ8Ô8ð 	Gð 	GˆIØŒK×ÒÕ2°6¸9ÑEÔEÑFÔFÐFÐFÝ,°FÐ;Ñ;Ô;ˆŒõ �6Ð6¸Ñ=Ô=ÐIÝ.0¬iØÔ0°&Ô2DÈ5ð/ñ /ô /ˆDÔ+Ð+õ /1¬k©m¬mˆDÔ+à�ŠÑÔÐÐÐr9   NrN  r§   r�   r©   r¬   c                 óª  — |j         \  }}}|                      |¦  «        }	t          j        |	| j        j        | j        ¬¦  «        }	|€Mt          j        |j         d         |j        ¬¦  «         	                    d¦  «         
                    |d¦  «        }|                      |	|¦  «        }
t          | j        ¦  «        D ]\  }} ||	f|
||dœ|¤Ž}	Œ|	S )NrÃ   r   rT  r   rQ   rU  )r]   r|  rW  rÁ   r+   rÅ   rJ   r|   rr   rÙ   r‡   rB  rX  rA  )r6   rN  r§   r�   r©   r¬   r[  rt   rò   rN   r«   rs  r]  s                r8   r=   zBltGlobalTransformer.forward  sü   € ð "/Ô!4Ñˆ
�G˜QØ×7Ò7¸ÑFÔFˆÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆØÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝ! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØ!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMˆMð Ðr9   r”   )r>   r?   r@   r   r•   r   rž   r+  r*   rJ   ra   r±   r   r   r   r=   rA   rB   s   @r8   rw  rw     sâ   ø€ € € € € € Ø&Ð&Ð&Ñ&à˜^˜^Ð,<ÀAÐRfÐgÑgÔgðÐðÐ9ð ð ð ð ð ð ð* /3Ø04Ø(,ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð Ð+Ô,ðð ð ð ð ð ð ð r9   rw  Úpatch_lengthsÚmax_patch_lengthc                 óˆ  — |€| S |                       d¦  «        }g }| D ]}g }||dk             D ]Y}|                     ¦   «         }t          ||¦  «        \  }}|                     |g|z  ¦  «         |r|                     |¦  «         ŒZ|                     |¦  «         Œ€t          d„ |D ¦   «         ¦  «        }	t          j        ||	f| j        | j	        ¬¦  «        }
t          |¦  «        D ]<\  }}|r5t          j        || j        | j	        ¬¦  «        |
|dt          |¦  «        …f<   Œ=|
dk                         d¬¦  «                             ¦   «         |
j        d         k     ra|
dk                         d¬¦  «                             ¦   «                              ¦   «                              ¦   «         dz   }|
dd…d|…f         }
|
S )a£  
    Splits patch lengths into smaller segments if they exceed `max_patch_length`.
    Pads the result to uniform length across the batch.

    Args:
        patch_lengths (torch.Tensor): [batch_size, num_patches] tensor of patch lengths.
        max_patch_length (int, optional): Maximum allowed length per patch.

    Returns:
        torch.Tensor: [batch_size, max_len] tensor of split and padded patch lengths.
    Nr   c              3   ó4   K  — | ]}t          |¦  «        V — Œd S r;   )rY  )r6  Úsplitss     r8   ú	<genexpr>z(process_patch_lengths.<locals>.<genexpr>P  s(   è è € Ð6Ð6 &•#�f‘+”+Ð6Ð6Ð6Ð6Ð6Ð6r9   ra  r†   r   )rë   ÚitemÚdivmodÚextendrI  ÚmaxrJ   rf  rS   rr   rX  ÚtensorrY  ÚanyÚsumr]   Únonzero)r  r€  r[  Ú	processedÚseqrƒ  ÚlengthÚfull_chunksÚ	remainderÚmax_lenÚpaddedrs  Úlast_nonzeros                r8   Úprocess_patch_lengthsr•  3  sí  € ð ÐØÐà×#Ò# AÑ&Ô&€JØ€Iàð !ð !ˆØˆØ˜# š'”lð 	)ð 	)ˆFØ—[’[‘]”]ˆFÝ%+¨FÐ4DÑ%EÔ%EÑ"ˆK˜Ø�MŠMÐ+Ð,¨{Ñ:Ñ;Ô;Ð;Øð )Ø—’˜iÑ(Ô(Ð(øØ×Ò˜Ñ Ô Ð Ð õ Ð6Ð6¨IÐ6Ñ6Ô6Ñ6Ô6€GÝŒ[˜* gÐ.°mÔ6IÐR_ÔRfÐgÑgÔg€Få˜yÑ)Ô)ð tð t‰	ˆˆ6Øð 	tÝ',¤|°FÀ-ÔBUÐ^kÔ^rÐ'sÑ'sÔ'sˆF�1�m�˜F™œ�mÐ#Ñ$øð 	�!Š×Ò˜QÐÑÔ×#Ò#Ñ%Ô%¨¬°Q¬Ò7Ð7Ø !š×(Ò(¨QÐ(Ñ/Ô/×7Ò7Ñ9Ô9×=Ò=Ñ?Ô?×DÒDÑFÔFÈÑJˆØ˜˜˜˜=˜L˜=Ð(Ô)ˆà€Mr9   c                   óú   ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	edz  d
edz  dedz  dedz  dee         fd„Ze	 	 dd„¦   «         Zˆ xZS )Ú
BltPatcherr+   c                 óx  •— t          ¦   «                              |¦  «         t          | j        ¬¦  «        | _        t          j        ¦   «         | _        t          | j        j	        ¦  «        D ]/}| j         
                    t          | j        |¦  «        ¦  «         Œ0t          j        | j        j        | j        j        ¦  «        | _        t!          | j        j        | j        j        ¬¦  «        | _        t          j        | j        j        | j        j        d¬¦  «        | _        |                      ¦   «          d S )Nr9  r�   Fr'   )r)   r*   rc   r+   rB  r.   r>  rA  r?  r@  rI  rš   r  rE  r,   rF  rD   r¡   rq  r/   Úlm_headrJ  r¤   s      €r8   r*   zBltPatcher.__init__b  sÿ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý,°D´KÐ@Ñ@Ô@ˆŒÝ”m‘o”oˆŒÝ˜tœ{Ô<Ñ=Ô=ð 	Lð 	LˆIØŒK×ÒÕ2°4´;À	ÑJÔJÑKÔKÐKÐKÝœL¨¬Ô)?ÀÄÔAXÑYÔYˆÔÝ˜tœ{Ô6¸D¼KÔ<TÐUÑUÔUˆŒ	Ý”yØŒKÔ#ØŒKÔ"Øð
ñ 
ô 
ˆŒð 	�ŠÑÔÐÐÐr9   NrM  r§   r�   r©   rN  rª   Ú
patch_sizeÚ	thresholdr€  r¬   c
                 óT  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }|}|                      ||¦  «        }| j        D ]} ||||¬¦  «        }Œ|                      |                      |¦  «        ¦  «        }t
          j                             |¬¦  «                             ¦   «         }|j        d d	…         \  }}|�|                      ||||¬
¦  «        }n#t          j        ||f|j        |j        ¬¦  «        }t+          ||	¦  «        }|||fS )Nú:You must specify exactly one of input_ids or inputs_embedsr9  r   r   rT  ©r+   rN  r§   r©   r�   )r«   r§   )rt  rP   )Ú	entropiesÚsequence_lengthrš  r›  ra  )Ú
ValueErrorrF  r	   r+   Úget_seq_lengthrJ   r|   r]   rr   rÙ   r   rB  rA  r™  rq  ÚdistributionsÚCategoricalÚentropyÚpatch_lengths_from_entropiesrK   rS   r•  )r6   rM  r§   r�   r©   rN  rª   rš  r›  r€  r¬   Úpast_seen_tokensÚcausal_maskrN   r«   r]  rt  Úprediction_entropiesr[  r   r  s                        r8   r=   zBltPatcher.forwardr  sü  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨m¸\ÑJÔJÐà”[ð 	vð 	vˆEØ!˜E -ÐEXÐitÐuÑuÔuˆMˆMà—’˜dŸiši¨Ñ6Ô6Ñ7Ô7ˆÝ$Ô2×>Ò>ÀfÐ>ÑMÔM×UÒUÑWÔWÐà&3Ô&9¸"¸1¸"Ô&=Ñ#ˆ
�OØÐ!Ø ×=Ò=Ø.Ø /Ø%Ø#ð	 >ñ ô ˆMˆMõ "œJØ˜_Ð-°]Ô5HÐQ^ÔQeðñ ô ˆMõ .¨mÐ=MÑNÔNˆØ# ]°FÐ:Ð:r9   c                 óª  — | j         d         }t          j        ddgt          j        | j        ¬¦  «                             d¦  «                             |d¦  «        }|j         d         }| dd…dd…f         } | |k    }|j         d         }t          j        || j        ¬¦  «                             d¦  «                             |d¦  «        }	t          j	        |	|¦  «        }
t          j
        |	|
gd¬¦  «        }t          j
        || gd¬¦  «        }||                              ||¦  «        }|                     d¬¦  «                             ¦   «         }|dd…d|…f         }t          j
        |||z   fd¬¦  «        }t          j	        |dd…dd…f         |dz
  ¦  «        }t          j
        |dd…dd…f         dz
  |fd¬¦  «        }||z
  dz   }|S )zÔ
        Computes patch lengths from token entropies.

        Depending on whether a threshold is provided, the function uses either:
        - Thresholding the entropy values (when `threshold` is set).
        r   r   ra  NrT  rQ   r†   )r]   rJ   r‰  Úlongrr   rÙ   Úrepeatr|   r‡   Ú	full_likeÚcatr¶   r‹  rˆ  )rŸ  r   rš  r›  r[  Úinit_tokensÚoffsetÚ
patch_maskrt   Útoken_indicesÚsentinelÚpadded_indicesÚpadded_maskÚpatch_startsÚmax_valid_patchesÚpatch_start_idsÚ
last_tokenÚ
patch_endsr  s                      r8   r¦  z'BltPatcher.patch_lengths_from_entropies­  sù  € ð ”_ QÔ'ˆ
õ ŒL˜!˜Q˜¥u¤z¸)Ô:JÐKÑKÔK×UÒUÐVWÑXÔX×_Ò_Ð`jÐlmÑnÔnð 	ð Ô" 1Ô%ˆð ˜a˜a˜a   ˜eÔ$ˆ	ð  Ò*ˆ
àÔ" 1Ô%ˆõ œ W°YÔ5EÐFÑFÔF×PÒPÐQRÑSÔS×ZÒZÐ[eÐgiÑjÔjˆÝ”? =°'Ñ:Ô:ˆÝœ M°8Ð#<À!ÐDÑDÔDˆõ ”i ¨j¨[Ð 9¸qÐAÑAÔAˆð & kÔ2×:Ò:¸:ÀwÑOÔOˆØ&ŸNšN¨q˜NÑ1Ô1×5Ò5Ñ7Ô7ÐØ# A A AÐ'9Ð(9Ð'9Ð$9Ô:ˆõ  œ) [°,ÀÑ2GÐ$HÈaÐPÑPÔPˆõ ”_ _°Q°Q°Q¸¸¸°UÔ%;¸_ÈqÑ=PÑQÔQˆ
Ý”Y °°°°1°2°2°Ô 6¸Ñ :¸JÐGÈQÐOÑOÔOˆ
à" _Ñ4°qÑ8ˆàÐr9   rj  rû   )r>   r?   r@   r"   r•   r*   rJ   r±   ra   r   r³   r²   r—   r`   r   r   r=   r–   r¦  rA   rB   s   @r8   r—  r—  _  sM  ø€ € € € € € ØÐÐÑðÐ/ð ð ð ð ð ð ð$ .2Ø.2Ø04Ø(,Ø26Ø!%Ø!%Ø"&Ø'+ð9;ð 9;àÔ# dÑ*ð9;ð œ tÑ+ð9;ð Ô&¨Ñ-ð	9;ð
  ™ð9;ð Ô(¨4Ñ/ð9;ð ˜$‘;ð9;ð ˜$‘Jð9;ð ˜4‘<ð9;ð  ™*ð9;ð Ð+Ô,ð9;ð 9;ð 9;ð 9;ðv ð Øð	3ð 3ð 3ñ „\ð3ð 3ð 3ð 3ð 3r9   r—  éÊš;Úprimec                 óÖ   — t          j        |t           j        | j        ¬¦  «        }t          j        | j        d         | j        ¬¦  «        }||z  }t          j        | |z  d¬¦  «        S )aœ  
    A polynomial rolling hash algorithm that converts sequences
    of tokens into hash values. The hash is computed as:
        hash = (token_0 * prime^0 + token_1 * prime^1 + ... + token_n * prime^n)

    The rolling hash allows the model to efficiently
    identify and encode recurring byte-level patterns in the input text.

    Args:
        token_tensor (torch.Tensor): [batch_size, seq_len, group_size] containing token IDs to hash
        prime (int): Prime number used as the base for the polynomial hash.

    Returns:
        torch.Tensor: Hash values of shape [batch_size, seq_len] where each value
                     represents the hash of the corresponding token group

    Example:
        >>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]])
        >>> hashes = rolling_polynomial_hash(tokens, prime=31)
        >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2
        >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2
    ra  rQ   rT  r†   )rJ   r‰  r}   rr   r|   r]   r‹  )Útoken_tensorr¼  Úprime_tensorÚpowersÚprime_powerss        r8   Úrolling_polynomial_hashrÂ  ä  sb   € õ. ”< ­U¬[ÀÔATÐUÑUÔU€LÝŒ\˜,Ô,¨RÔ0¸Ô9LÐMÑMÔM€FØ Ñ'€LÝŒ9�\ LÑ0°bÐ9Ñ9Ô9Ð9r9   rP   é0u  Ú	token_idsÚ
group_sizeÚmax_hashc                 óL  — t          j        ¦   «         5  | j        \  }}t          j        ||dz
  t           j        | j        ¬¦  «        }t          j        || gd¬¦  «        }|                     d|d¦  «        }t          ||¦  «        }	|	|z  }
ddd¦  «         n# 1 swxY w Y   |
S )z1Hash token groups and map to range [0, max_hash].r   ra  r†   N)	rJ   r˜   r]   rf  r}   rr   r®  ÚunfoldrÂ  )rÄ  rÅ  r¼  rÆ  r[  rt   ÚpaddingÚpadded_tokensÚwindowsÚhashesÚhash_valuess              r8   Úbyte_group_hash_functionrÎ    sæ   € õ 
Œ‰Œð 	(ð 	(Ø'œoÑˆ
�Gå”+˜j¨*°q©.ÅÄÐT]ÔTdÐeÑeÔeˆÝœ	 7¨IÐ"6¸AÐ>Ñ>Ô>ˆð  ×&Ò& q¨*°aÑ8Ô8ˆÝ(¨°%Ñ8Ô8ˆØ˜xÑ'ˆð	(ð 	(ð 	(ñ 	(ô 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(øøøð 	(ð 	(ð 	(ð 	(ð Ðs   ”A9BÂBÂ BÚlocal_encoder_tokensÚencoder_hash_tok_embeddingÚ$encoder_hash_byte_group_nb_functionsÚencoder_hash_byte_group_sizeÚencoder_hash_byte_group_vocabc                 ó$  — g d¢}|                      | ¦  «        }d}t          |¦  «        D ]d}	||	t          |¦  «        z           }
|D ]G}t          | ||
|¦  «        }|||z  z   }| ||¦  «                             |j        ¦  «        z  }|dz  }ŒHŒe|S )z=Compute token embeddings enhanced with hash-based embeddings.)r»  l   Í21A ißoYl   Ívt l   ß. l   }îg l   �A§u l   í†0 l   ©ÿT l   AK l   ™| r   r   )rF  r?  rY  rÎ  rT   rr   )rÏ  r/  rÐ  rÑ  rÒ  rÓ  ÚprimesÚ
embeddingsÚembedding_idxÚfunc_nbr¼  rÅ  Úhash_idsÚoffset_hash_idss                 r8   Úcompute_hash_embeddingsrÛ    sÐ   € ðð ð €Fð ×+Ò+Ð,@ÑAÔA€JØ€MÝÐ=Ñ>Ô>ð ð ˆØ�w¥ V¡¤Ñ,Ô-ˆØ6ð 	ð 	ˆJÝ/Ð0DÀjÐRWÐYvÑwÔwˆHà&¨Ð9VÑ)VÑVˆOØÐ4Ð4°_ÑEÔE×HÒHÈÔIZÑ[Ô[Ñ[ˆJØ˜QÑˆMˆMð	ð Ðr9   FrR  rQ  r   Úpatches_as_queriesrC  rS   c                 ó¾  — | j         \  }}| j        }|rƒ||z  }	|}
t          j        ||¬¦  «                             d¦  «                             d¦  «                             |||¦  «        }|                      d¦  «                             |||¦  «        }n‚|}	||z  }
|                      d¦  «                             |||¦  «        }t          j        ||¬¦  «                             d¦  «                             d¦  «                             |||¦  «        }||k    }|rdnd}|                     ||¬¦  «        }||	|
f}|j         |k    rt          d|j         › d|› �¦  «        ‚|                     d¦  «        }d|                     |¦  «        z
  }| 	                    |                     t          j
        ¦  «        t          j        |¦  «        j        ¦  «        }|S )	aR  
    Prepare cross-attention mask for patch-based attention, following mllama's robust approach.

    This function creates masks that control which patches can attend to which other patches,
    with support for query/key role swapping and cross-attention multipliers.

    Args:
        patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids.
        num_patches (int): Total number of patches.
        sequence_length (int): Length of the sequence.
        patches_as_queries (bool): If True, patches are used as queries, otherwise as keys.
        cross_attn_k (int): Cross-attention multiplier for repeating patches.
        dtype (torch.dtype): Data type for the output mask.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]:
            - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len]
    rT  r   rQ   r   r†   zCross attention mask shape z doesn't match expected rx   )r]   rr   rJ   r|   rÙ   r‡   rŒ   r¡  rT   Úmasked_fillr²   ÚfinfoÚmin)rR  rQ  r   rÜ  rC  rS   r[  rt   rr   rñ   Úkv_lenÚq_patch_idsÚkv_patch_idsr¦   Ú
repeat_dimÚexpected_shapeÚinverted_cross_attn_masks                    r8   Ú#_prepare_patch_cross_attention_maskrç  9  s  € ð4 $œ/Ñ€J�ØÔ€Fð ð 
Ø˜lÑ*ˆØ ˆõ ŒL˜¨VÐ4Ñ4Ô4ßŠY�q‰\Œ\ßŠY�r‰]Œ]ßŠV�J ¨WÑ5Ô5ð	 	ð !×*Ò*¨1Ñ-Ô-×4Ò4°ZÀÈgÑVÔVˆˆàˆØ˜|Ñ+ˆà×)Ò)¨"Ñ-Ô-×4Ò4°ZÀÈ+ÑVÔVˆåŒL˜¨VÐ4Ñ4Ô4×>Ò>¸qÑAÔA×KÒKÈAÑNÔN×UÒUÐV`ÐbiÐkvÑwÔwð 	ð '¨,Ò6Ðð )Ð0��¨b€JØ/×AÒAÀ,ÐT^ÐAÑ_Ô_Ðð ! %¨Ð0€NØÔ! ^Ò3Ð3ÝØnÐ*>Ô*DÐnÐnÐ^lÐnÐnñ
ô 
ð 	
ð
 0×9Ò9¸!Ñ<Ô<Ðð  #Ð%9×%<Ò%<¸UÑ%CÔ%CÑCÐØ3×?Ò?Ø ×#Ò#¥E¤JÑ/Ô/µ´¸UÑ1CÔ1CÔ1Gñô Ðð  Ðr9   c                   ó"  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
edz  dee         deez  fd„¦   «         ¦   «         Zd„ Zd„ Zdej	        dedej	        fd„Zˆ xZS )ÚBltModelr+   c                 óŠ  •— t          ¦   «                              |¦  «         d| _        || _        t	          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          |j        ¦  «        | _        |j        t          |j        ¦  «        z  }|j        |z  }t#          j        ||j        j        ¦  «        | _        | j        j        rVt-          |j        ¦  «        | _        | j                             ¦   «          | j                             ¦   «         D ]	}d|_        Œ
nd | _        |                      ¦   «          d S )NF)r)   r*   r=  r+   r-  r  r/  rw  Úglobal_configry  rl  r  Úlocal_decoderrÑ  rY  rÒ  rÓ  r.   r  r,   rÐ  Úpatch_in_forwardr—  Úpatcher_configÚpatcherÚevalÚ
parametersÚrequires_gradrJ  )r6   r+   Únum_embeddingsÚtotal_vocab_sizeÚparamr7   s        €r8   r*   zBltModel.__init__ˆ  s#  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#àˆŒÝ,¨VÔ-BÑCÔCˆÔÝ"6°vÔ7KÑ"LÔ"LˆÔÝ,¨VÔ-BÑCÔCˆÔØÔDÅsÈ6ÔKnÑGoÔGoÑoˆØ!Ô?À.ÑPÐÝ*,¬,Ð7GÈÔI^ÔIjÑ*kÔ*kˆÔ'ØŒ;Ô'ð 	 Ý% fÔ&;Ñ<Ô<ˆDŒLØŒL×ÒÑÔÐØœ×0Ò0Ñ2Ô2ð ,ð ,�Ø&+�Ô#Ð#ð,ð  ˆDŒLØ�ŠÑÔÐÐÐr9   NrM  r  r§   r�   r©   rN  rª   r¬   rG   c                 óB  — |d u |d uz  rt          d¦  «        ‚|rq|€7t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }n8t	          |t          ¦  «        s#t          |t          | j        ¬¦  «        ¦  «        }|�|}	|j        \  }
}}nF|j        \  }
}t          || j        | j        | j        j	        | j        j
        | j        j        ¦  «        }	|€É| j        j        dk    re| j        �^|€t          d¦  «        ‚|                      || j        j        | j        j        | j        j        | j        j        |j        ¬¦  «        \  }}}nT|�|j        n|j        }|�|j        n|j        }t)          t+          j        |
|dz   f||¬¦  «        | j        j        ¦  «        }|                      ||¦  «        }|€V|�|                     ¦   «         nd}t+          j        |	j        d         |	j        ¬	¦  «        |z   }|                     d¦  «        }t7          | j        |	||�|j        nd |¬
¦  «        }t;          ||j        d         |d| j        j        |	j        ¬¦  «        } | j        d||	||||j        d         ||�|j        nd dœ|¤Ž\  }}|                     |
|j        d         d¦  «        }t+          j        d|j        d         |j        ¬	¦  «        }|                     d¦  «        }t7          | j        |d d d ¬
¦  «        } | j         d|||dœ|¤Ž}|                      |d d …dd …f         |¦  «        }t;          ||j        d         |d| j        j        |	j        ¬¦  «        } | j!        d||||||�|j"        nd |dœ|¤Ž}tG          ||¬¦  «        S )Nr�  r9  r¥  z0input_ids is required for entropy-based patching)rš  r›  r€  Úpatching_batch_sizerr   r   ra  r   rT  rž  T)rR  rQ  r   rÜ  rC  rS   )rM  rN  r§   r�   rP  rQ  rR  r©   rQ   )rN  r§   r�   F)rM  rN  rO  r§   r�   r©   rP  )Úlast_hidden_stater©   r®   )$r¡  r
   r	   r+   rˆ   r]   rÛ  r/  rÐ  rÑ  rÒ  rÓ  Úpatching_moderï  rš  Úpatching_thresholdr€  r÷  rr   rS   r•  rJ   rK   Ú_patch_ids_from_lengthsr¢  r|   rÙ   r   Úself_attention_cacherç  rC  rì   ry  rì  Úcross_attention_cacher   )r6   rM  r  r§   r�   r©   rN  rª   r¬   Úencoder_embedsr[  r   rò   rr   rS   rR  r§  r¨  Úcross_attn_mask_encÚencoder_hidden_statesr_  Úglobal_position_idsÚglobal_causal_maskÚglobal_hidden_statesÚdecoder_patch_idsÚcross_attn_mask_decÚoutputs                              r8   r=   zBltModel.forwardœ  sŸ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	iØÐ&Ý"5Ý ¨¬Ð4Ñ4Ô4µlÈ$Ì+Ð6VÑ6VÔ6Vñ#ô #��õ   Õ1DÑEÔEð iõ #6°oÅ|Ð[_Ô[fÐGgÑGgÔGgÑ"hÔ"h�ð Ð$Ø*ˆNØ-:Ô-@Ñ*ˆJ˜¨¨à*3¬/Ñ'ˆJ˜Ý4ØØÔ"ØÔ/Ø”Ô@Ø”Ô8Ø”Ô9ñô ˆNð Ð ØŒ{Ô(¨IÒ5Ð5¸$¼,Ð:RØÐ$Ý$Ð%WÑXÔXÐXØ&*§l¢lØØ#œ{Ô5Ø"œkÔ<Ø%)¤[Ô%AØ(,¬Ô(GØ$Ô+ð '3ñ 'ô 'Ñ#��= ! !ð .7Ð-B˜Ô)Ð)ÈÔH\�Ø+4Ð+@˜	œ˜ÀmÔFY�Ý 5Ý”J 
¨O¸aÑ,?Ð@ÈÐV\Ð]Ñ]Ô]Ø”KÔ0ñ!ô !�ð ×0Ò0°ÀÑPÔPˆ	àÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(<¸QÔ(?ÈÔH]Ð^Ñ^Ô^ÐaqÑqˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø(Ø)ØDSÐD_˜OÔ@Ð@ÐeiØ%ð
ñ 
ô 
ˆõ BØØ%Ô+¨AÔ.Ø+Ø#ØœÔ1Ø Ô&ð
ñ 
ô 
Ðð 7I°dÔ6Hð 
7
ØØ(Ø&Ø%Ø#6Ø%Ô+¨AÔ.ØØDSÐD_˜OÔ@Ð@Ðeið
7
ð 
7
ð ð
7
ð 
7
Ñ3ÐÐ3ð  4×8Ò8¸À]ÔEXÐYZÔE[Ð]_Ñ`Ô`ÐÝ#œl¨1Ð.BÔ.HÈÔ.KÐThÔToÐpÑpÔpÐØ1×;Ò;¸AÑ>Ô>ÐÝ/Ø”;Ø.ØØ Øð
ñ 
ô 
Ðð  7˜tÔ6ð  
Ø.Ø-Ø,ð 
ð  
ð ð	 
ð  
Ðð !×8Ò8¸ÀqÀqÀqÈ!È"È"ÀuÔ9MÈÑ_Ô_ÐÝAØ'Ø%Ô+¨AÔ.Ø+Ø$ØœÔ1Ø Ô&ð
ñ 
ô 
Ðð $�Ô#ð 	
ØØ/Ø-Ø&Ø%ØETÐE`˜OÔAÐAÐfjØ#6ð	
ð 	
ð ð	
ð 	
ˆõ 'Ø$Ø+ð
ñ 
ô 
ð 	
r9   c                 ó   — | j         j        S r;   ©r/  rF  r^   s    r8   Úget_input_embeddingszBltModel.get_input_embeddings#  s   € ØÔ!Ô.Ð.r9   c                 ó   — || j         _        d S r;   r  )r6   r¿   s     r8   Úset_input_embeddingszBltModel.set_input_embeddings&  s   € Ø*/ˆÔÔ'Ð'Ð'r9   rt   c                 ó®  — |j         d         }t          j        t          j        |d|j        |j        ¬¦  «        |                     d¬¦  «        d d …d d…f         gd¬¦  «        }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «                             d¦  «        k     	                    d¬¦  «        dz
  S )Nr   r   ra  rQ   r†   rT  )
r]   rJ   r®  rf  rS   rr   Úcumsumr|   rÙ   r‹  )r6   r  rt   r[  r¶  Útoken_positionss         r8   rû  z BltModel._patch_ids_from_lengths)  s×   € Ø"Ô(¨Ô+ˆ
Ý”yå”˜J¨°Ô1DÈ]ÔMaÐbÑbÔbØ×$Ò$¨Ð$Ñ,Ô,¨Q¨Q¨Q°°°¨VÔ4ðð ð
ñ 
ô 
ˆõ  œ, w°}Ô7KÐLÑLÔLˆØ×&Ò& qÑ)Ô)¨_×-FÒ-FÀqÑ-IÔ-I×-SÒ-SÐTVÑ-WÔ-WÒW×\Ò\ÐacÐ\ÑdÔdÐghÑhÐhr9   ru  )r>   r?   r@   r   r*   r   r   rJ   r±   ra   r   r³   r²   r   r   r\   r   r=   r	  r  r—   rû  rA   rB   s   @r8   ré  ré  ‡  s„  ø€ € € € € ð˜yð ð ð ð ð ð ð(  Øð .2Ø-1Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð ”| dÑ*ðC
ð œ tÑ+ð	C
ð
 Ô&¨Ñ-ðC
ð  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
Ð(Ñ	(ðC
ð C
ð C
ñ „_ñ  ÔðC
ðJ/ð /ð /ð0ð 0ð 0ð
i°U´\ð 
iÈCð 
iÐTYÔT`ð 
ið 
ið 
ið 
ið 
ið 
ið 
ið 
ir9   ré  zB
    The Blt Text Model with a language modeling head on top.
    )Úcustom_introc                   ó|  ‡ — e Zd ZU eed<   dZdZddiZdefˆ fd„Ze	e
	 	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  deej        ej        f         dz  dedz  dej        dz  dej        dz  dedz  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚBltForCausalLMr+   Frÿ   z'model.local_encoder.embed_tokens.weightzlm_head.weightc                 ó:  •— t          ¦   «                              |¦  «         |                     ¦   «         | _        |j        | _        t          |¦  «        | _        t          j        |j	        j
        |j        d¬¦  «        | _        |                      ¦   «          d S r&   )r)   r*   Úget_text_configÚtext_configrE  ré  rÿ   r.   r/   r  r,   r™  rJ  r5   s     €r8   r*   zBltForCausalLM.__init__A  s€   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!×1Ò1Ñ3Ô3ˆÔØ Ô+ˆŒÝ˜fÑ%Ô%ˆŒ
Ý”y Ô!6Ô!BÀFÔDUÐ\aÐbÑbÔbˆŒà�ŠÑÔÐÐÐr9   Nr   rM  r§   r�   r¥   r¦   r¨   r©   rN  Úlabelsrª   Úlogits_to_keepr¬   rG   c                 ól  —  | j         d||||||||
dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «                             ¦   «         }d}|	� | j        ||	| j        fi |¤Ž}t          |||j
        |j        |j        ¬¦  «        S )aý
  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BltForCausalLM

        >>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf")

        >>> prompt = "If I had to write a haiku, it would be:"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
        >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        >>> print(result)
        If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
        I love the idea of snowflakes gently falling, each one
        ```
        )rM  r§   r�   r¦   r¨   r©   rN  rª   N)Úlossrt  r©   rN   r  r®   )rÿ   rø  rˆ   r—   Úslicer™  r`   Úloss_functionrE  r   r©   rN   r  )r6   rM  r§   r�   r¥   r¦   r¨   r©   rN  r  rª   r  r¬   ÚoutputsrN   Úslice_indicesrt  r  s                     r8   r=   zBltForCausalLM.forwardJ  s   € ð| �$”*ð 

ØØ)Ø%Ø!5Ø*GØ+Ø'Øð

ð 

ð ð

ð 

ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔA×GÒGÑIÔIˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   )NNNNNNNNNNr   )r>   r?   r@   r   r•   r&  r"  Ú_tied_weights_keysr*   r   r   rJ   r±   ra   r\   r   r³   r²   r—   r   r   r   r=   rA   rB   s   @r8   r  r  6  sº  ø€ € € € € € ð ÐÐÑØ"ÐØÐØCÐEUÐVÐð˜yð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø:>Ø8<ØRVØ(,Ø26Ø*.Ø!%Ø-.ðV
ð V
àÔ# dÑ*ðV
ð œ tÑ+ðV
ð Ô&¨Ñ-ð	V
ð
 !&Ô 0°4Ñ 7ðV
ð $Ô.°Ñ5ðV
ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'OðV
ð  ™ðV
ð Ô(¨4Ñ/ðV
ð Ô  4Ñ'ðV
ð ˜$‘;ðV
ð ˜eœlÑ*ðV
ð Ð+Ô,ðV
ð 
Ð'Ñ	'ðV
ð V
ð V
ñ „^ñ ÔðV
ð V
ð V
ð V
ð V
r9   r  )rþ   ré  r—  r  )r»   )r   )r»  )rP   r»  rÃ  )XÚcollections.abcr   Útypingr   rJ   Útorch.distributionsÚtorch.nnr.   Útorch.nn.functionalrÈ   rW  Ú r   r  Úactivationsr   Úcache_utilsr   r	   r
   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úconfiguration_bltr   r   r    r!   r"   ÚModuler$   rD   rc   rš   ra   r—   rº   r`   rÏ   r×   rß   rž   rö   rþ   r-  rl  rw  r•  r—  rÂ  rÎ  r  ÚlistrÛ  rU   r²   rS   r\   rç  ré  r  Ú__all__r®   r9   r8   ú<module>r5     sø  ðð* %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eðð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ˆRŒYñ ô ð ð"Jð Jð Jð Jð J�”ñ Jô Jð Jð(><ð ><ð ><ð ><ð ><˜œñ ><ô ><ð ><ðDAð Að Að Að AÐ4ñ Aô Að AðH	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2ð ð ðð ð ð ð2>)ð >)ð >)ð >)ð >)�r”yñ >)ô >)ð >)ðB;)ð ;)ð ;)ð ;)ð ;)˜œ	ñ ;)ô ;)ð ;)ð| ð[ð [ð [ð [ð [˜ñ [ô [ñ „ð[ð|n"ð n"ð n"ð n"ð n"Ð(ñ n"ô n"ð n"ðbIð Ið Ið Ið IÐ(ñ Iô Ið IðX0ð 0ð 0ð 0ð 0Ð-ñ 0ô 0ð 0ðf)¨¬ð )ÈÈtÉð )ÐX]ÔXdð )ð )ð )ð )ðXBð Bð Bð Bð BÐ#ñ Bô Bð BðJ:ð :°ð :ð :ð :ð :ð< \aðð ØŒ|ðØ),ðØ9<ðØUXðð ð ð ð$#Øœ,ð#ð !#¤ð#ð +.ð	#ð
 #'ð#ð $'ð#ð „\ð#ð #ð #ð #ðT  %ØØœðK ð K ØŒ|ðK àðK ð ðK ð ð	K ð
 ðK ð Œ;ðK ð ˆ5Œ<˜œÐ%Ô&ðK ð K ð K ð K ð\lið lið lið lið liÐ!ñ liô lið lið^ €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð'¨ñ g
ô g
ñô ð
g
ðT MÐ
LÐ
L€€€r9   