§
    ‚ŠtjÁ  ã                   ó  — d Z ddlmZ ddlZddlZddlmZ ddlmc mZ	 ddl
mZ ddlmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z;  e#j<        e=¦  «        Z>dMde?fd„Z@	 dNdejA        de?de?de?fd„ZBdejA        dejC        d e?d!eDd"e?d#ejA        fd$„ZEd%dejF        fd&ejA        d'e?d(e?d)eGd*e?d+ejH        d#eIejA        ejA        f         fd,„ZJd-ejA        d.e?dz  d#ejA        fd/„ZK G d0„ d1e2¦  «        ZL G d2„ d3e3¦  «        ZM G d4„ d5e-¦  «        ZN G d6„ d7e0¦  «        ZO G d8„ d9e4¦  «        ZP G d:„ d;e1¦  «        ZQe! G d<„ d=e/¦  «        ¦   «         ZR G d>„ d?eR¦  «        ZS G d@„ dAeR¦  «        ZT G dB„ dCeR¦  «        ZU G dD„ dEeR¦  «        ZV G dF„ dGeR¦  «        ZW e!dH¬I¦  «         G dJ„ dKeRe¦  «        ¦   «         ZXg dL¢ZYdS )Oz<Blt modular model, inheriting from Mllama where appropriate.é    )ÚCallableNé   )Úinitialization)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Úrotate_half)ÚLlamaRotaryEmbedding)ÚMllamaPreTrainedModelÚMllamaSelfAttentionDecoderLayerÚMllamaTextCrossAttentionÚMllamaTextMLPÚMllamaTextRMSNormÚMllamaTextSelfAttentionÚeager_attention_forwardé   )Ú	BltConfigÚBltGlobalTransformerConfigÚBltLocalDecoderConfigÚBltLocalEncoderConfigÚBltPatcherConfigéÊš;Úprimec                 óÖ   — t          j        |t           j        | j        ¬¦  «        }t          j        | j        d         | j        ¬¦  «        }||z  }t          j        | |z  d¬¦  «        S )aœ  
    A polynomial rolling hash algorithm that converts sequences
    of tokens into hash values. The hash is computed as:
        hash = (token_0 * prime^0 + token_1 * prime^1 + ... + token_n * prime^n)

    The rolling hash allows the model to efficiently
    identify and encode recurring byte-level patterns in the input text.

    Args:
        token_tensor (torch.Tensor): [batch_size, seq_len, group_size] containing token IDs to hash
        prime (int): Prime number used as the base for the polynomial hash.

    Returns:
        torch.Tensor: Hash values of shape [batch_size, seq_len] where each value
                     represents the hash of the corresponding token group

    Example:
        >>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]])
        >>> hashes = rolling_polynomial_hash(tokens, prime=31)
        >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2
        >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2
    ©ÚdtypeÚdeviceéÿÿÿÿ©r.   ©Údim)ÚtorchÚtensorÚint64r.   ÚarangeÚshapeÚsum)Útoken_tensorr*   Úprime_tensorÚpowersÚprime_powerss        úa/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/blt/modular_blt.pyÚrolling_polynomial_hashr>   9   sb   € õ. ”< ­U¬[ÀÔATÐUÑUÔU€LÝŒ\˜,Ô,¨RÔ0¸Ô9LÐMÑMÔM€FØ Ñ'€LÝŒ9�\ LÑ0°bÐ9Ñ9Ô9Ð9ó    é0u  Ú	token_idsÚ
group_sizeÚmax_hashc                 óL  — t          j        ¦   «         5  | j        \  }}t          j        ||dz
  t           j        | j        ¬¦  «        }t          j        || gd¬¦  «        }|                     d|d¦  «        }t          ||¦  «        }	|	|z  }
ddd¦  «         n# 1 swxY w Y   |
S )z1Hash token groups and map to range [0, max_hash].r#   r,   r1   N)	r3   Úno_gradr7   Úzerosr5   r.   ÚcatÚunfoldr>   )rA   rB   r*   rC   Ú
batch_sizeÚseq_lenÚpaddingÚpadded_tokensÚwindowsÚhashesÚhash_valuess              r=   Úbyte_group_hash_functionrP   V   sæ   € õ 
Œ‰Œð 	(ð 	(Ø'œoÑˆ
�Gå”+˜j¨*°q©.ÅÄÐT]ÔTdÐeÑeÔeˆÝœ	 7¨IÐ"6¸AÐ>Ñ>Ô>ˆð  ×&Ò& q¨*°aÑ8Ô8ˆÝ(¨°%Ñ8Ô8ˆØ˜xÑ'ˆð	(ð 	(ð 	(ñ 	(ô 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(øøøð 	(ð 	(ð 	(ð 	(ð Ðs   ”A9BÂBÂ BÚlocal_encoder_tokensÚencoder_hash_tok_embeddingÚ$encoder_hash_byte_group_nb_functionsÚencoder_hash_byte_group_sizeÚencoder_hash_byte_group_vocabÚreturnc                 ó$  — g d¢}|                      | ¦  «        }d}t          |¦  «        D ]d}	||	t          |¦  «        z           }
|D ]G}t          | ||
|¦  «        }|||z  z   }| ||¦  «                             |j        ¦  «        z  }|dz  }ŒHŒe|S )z=Compute token embeddings enhanced with hash-based embeddings.)r)   l   Í21A ißoYl   Ívt l   ß. l   }îg l   �A§u l   í†0 l   ©ÿT l   AK l   ™| r   r#   )Úembed_tokensÚrangeÚlenrP   Útor.   )rQ   Úlocal_encoderrR   rS   rT   rU   ÚprimesÚ
embeddingsÚembedding_idxÚfunc_nbr*   rB   Úhash_idsÚoffset_hash_idss                 r=   Úcompute_hash_embeddingsrc   h   sÐ   € ðð ð €Fð ×+Ò+Ð,@ÑAÔA€JØ€MÝÐ=Ñ>Ô>ð ð ˆØ�w¥ V¡¤Ñ,Ô-ˆØ6ð 	ð 	ˆJÝ/Ð0DÀjÐRWÐYvÑwÔwˆHà&¨Ð9VÑ)VÑVˆOØÐ4Ð4°_ÑEÔE×HÒHÈÔIZÑ[Ô[Ñ[ˆJØ˜QÑˆMˆMð	ð Ðr?   FÚ	patch_idsÚnum_patchesÚsequence_lengthÚpatches_as_queriesÚcross_attn_kr-   c                 ó¾  — | j         \  }}| j        }|rƒ||z  }	|}
t          j        ||¬¦  «                             d¦  «                             d¦  «                             |||¦  «        }|                      d¦  «                             |||¦  «        }n‚|}	||z  }
|                      d¦  «                             |||¦  «        }t          j        ||¬¦  «                             d¦  «                             d¦  «                             |||¦  «        }||k    }|rdnd}|                     ||¬¦  «        }||	|
f}|j         |k    rt          d|j         › d|› �¦  «        ‚|                     d¦  «        }d|                     |¦  «        z
  }| 	                    |                     t          j
        ¦  «        t          j        |¦  «        j        ¦  «        }|S )	aR  
    Prepare cross-attention mask for patch-based attention, following mllama's robust approach.

    This function creates masks that control which patches can attend to which other patches,
    with support for query/key role swapping and cross-attention multipliers.

    Args:
        patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids.
        num_patches (int): Total number of patches.
        sequence_length (int): Length of the sequence.
        patches_as_queries (bool): If True, patches are used as queries, otherwise as keys.
        cross_attn_k (int): Cross-attention multiplier for repeating patches.
        dtype (torch.dtype): Data type for the output mask.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]:
            - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len]
    r0   r   r/   r#   r1   zCross attention mask shape z doesn't match expected g      ð?)r7   r.   r3   r6   Ú	unsqueezeÚexpandÚrepeat_interleaveÚ
ValueErrorr[   Úmasked_fillÚboolÚfinfoÚmin)rd   re   rf   rg   rh   r-   rI   rJ   r.   Úq_lenÚkv_lenÚq_patch_idsÚkv_patch_idsÚcross_attention_maskÚ
repeat_dimÚexpected_shapeÚinverted_cross_attn_masks                    r=   Ú#_prepare_patch_cross_attention_maskrz   Ž   s  € ð4 $œ/Ñ€J�ØÔ€Fð ð 
Ø˜lÑ*ˆØ ˆõ ŒL˜¨VÐ4Ñ4Ô4ßŠY�q‰\Œ\ßŠY�r‰]Œ]ßŠV�J ¨WÑ5Ô5ð	 	ð !×*Ò*¨1Ñ-Ô-×4Ò4°ZÀÈgÑVÔVˆˆàˆØ˜|Ñ+ˆà×)Ò)¨"Ñ-Ô-×4Ò4°ZÀÈ+ÑVÔVˆåŒL˜¨VÐ4Ñ4Ô4×>Ò>¸qÑAÔA×KÒKÈAÑNÔN×UÒUÐV`ÐbiÐkvÑwÔwð 	ð '¨,Ò6Ðð )Ð0��¨b€JØ/×AÒAÀ,ÐT^ÐAÑ_Ô_Ðð ! %¨Ð0€NØÔ! ^Ò3Ð3ÝØnÐ*>Ô*DÐnÐnÐ^lÐnÐnñ
ô 
ð 	
ð
 0×9Ò9¸!Ñ<Ô<Ðð  #Ð%9×%<Ò%<¸UÑ%CÔ%CÑCÐØ3×?Ò?Ø ×#Ò#¥E¤JÑ/Ô/µ´¸UÑ1CÔ1CÔ1Gñô Ðð  Ðr?   Úpatch_lengthsÚmax_patch_lengthc                 óˆ  — |€| S |                       d¦  «        }g }| D ]}g }||dk             D ]Y}|                     ¦   «         }t          ||¦  «        \  }}|                     |g|z  ¦  «         |r|                     |¦  «         ŒZ|                     |¦  «         Œ€t          d„ |D ¦   «         ¦  «        }	t          j        ||	f| j        | j	        ¬¦  «        }
t          |¦  «        D ]<\  }}|r5t          j        || j        | j	        ¬¦  «        |
|dt          |¦  «        …f<   Œ=|
dk                         d¬¦  «                             ¦   «         |
j        d         k     ra|
dk                         d¬¦  «                             ¦   «                              ¦   «                              ¦   «         dz   }|
dd…d|…f         }
|
S )a£  
    Splits patch lengths into smaller segments if they exceed `max_patch_length`.
    Pads the result to uniform length across the batch.

    Args:
        patch_lengths (torch.Tensor): [batch_size, num_patches] tensor of patch lengths.
        max_patch_length (int, optional): Maximum allowed length per patch.

    Returns:
        torch.Tensor: [batch_size, max_len] tensor of split and padded patch lengths.
    Nr   c              3   ó4   K  — | ]}t          |¦  «        V — Œd S ©N)rZ   )Ú.0Úsplitss     r=   ú	<genexpr>z(process_patch_lengths.<locals>.<genexpr>ù   s(   è è € Ð6Ð6 &•#�f‘+”+Ð6Ð6Ð6Ð6Ð6Ð6r?   r,   r1   r#   )ÚsizeÚitemÚdivmodÚextendÚappendÚmaxr3   rF   r-   r.   Ú	enumerater4   rZ   Úanyr8   r7   Únonzero)r{   r|   rI   Ú	processedÚseqr�   ÚlengthÚfull_chunksÚ	remainderÚmax_lenÚpaddedÚiÚlast_nonzeros                r=   Úprocess_patch_lengthsr•   Ü   sí  € ð ÐØÐà×#Ò# AÑ&Ô&€JØ€Iàð !ð !ˆØˆØ˜# š'”lð 	)ð 	)ˆFØ—[’[‘]”]ˆFÝ%+¨FÐ4DÑ%EÔ%EÑ"ˆK˜Ø�MŠMÐ+Ð,¨{Ñ:Ñ;Ô;Ð;Øð )Ø—’˜iÑ(Ô(Ð(øØ×Ò˜Ñ Ô Ð Ð õ Ð6Ð6¨IÐ6Ñ6Ô6Ñ6Ô6€GÝŒ[˜* gÐ.°mÔ6IÐR_ÔRfÐgÑgÔg€Få˜yÑ)Ô)ð tð t‰	ˆˆ6Øð 	tÝ',¤|°FÀ-ÔBUÐ^kÔ^rÐ'sÑ'sÔ'sˆF�1�m�˜F™œ�mÐ#Ñ$øð 	�!Š×Ò˜QÐÑÔ×#Ò#Ñ%Ô%¨¬°Q¬Ò7Ð7Ø !š×(Ò(¨QÐ(Ñ/Ô/×7Ò7Ñ9Ô9×=Ò=Ñ?Ô?×DÒDÑFÔFÈÑJˆØ˜˜˜˜=˜L˜=Ð(Ô)ˆà€Mr?   c                   ó   — e Zd ZdS )ÚBltMLPN©Ú__name__Ú
__module__Ú__qualname__© r?   r=   r—   r—     ó   € € € € € Ø€Dr?   r—   c                   ó   — e Zd ZdS )Ú
BltRMSNormNr˜   rœ   r?   r=   rŸ   rŸ     r�   r?   rŸ   c                   óN   — e Zd Z ej        ¦   «         ed„ ¦   «         ¦   «         ZdS )ÚBltRotaryEmbeddingc                 ó  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «        }|d d …d d d …f                              ¦   «         }t	          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   	                    dd¦  «        }t          j        |dd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r/   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr   r1   )r-   )Úinv_freqÚfloatrk   r7   Ú
isinstancer.   ÚtypeÚstrr   Ú	transposer3   rl   ÚcosÚattention_scalingÚsinr[   r-   )
ÚselfÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr¥   ÚfreqsÚembr­   r¯   s
             r=   ÚforwardzBltRotaryEmbedding.forward  s¤  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔeÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝÔ)¨%°¸Ð;Ñ;Ô;ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Â4BEÅEÅEN)r™   rš   r›   r3   rE   r   r·   rœ   r?   r=   r¡   r¡     s@   € € € € € Ø€U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <r?   r¡   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚBltTransformerLayerÚ	layer_idxc                 ó  •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        d S )N)Úconfigrº   ©Úeps)ÚsuperÚ__init__ÚBltSelfAttentionÚ	self_attnr—   ÚmlprŸ   Úhidden_sizeÚrms_norm_epsÚinput_layernormÚpost_attention_layernorm©r°   r¼   rº   Ú	__class__s      €r=   rÀ   zBltTransformerLayer.__init__"  sv   ø€ Ý‰Œ×ÒÑÔÐå)°À9ÐMÑMÔMˆŒÝ˜&‘>”>ˆŒÝ)¨&Ô*<À&ÔBUÐVÑVÔVˆÔÝ(2°6Ô3EÈ6ÔK^Ð(_Ñ(_Ô(_ˆÔ%Ð%Ð%r?   )r™   rš   r›   ÚintrÀ   Ú__classcell__©rÉ   s   @r=   r¹   r¹   !  sO   ø€ € € € € ð`¨#ð `ð `ð `ð `ð `ð `ð `ð `ð `ð `r?   r¹   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )rÁ   r¼   rº   c                 óL   •— t          ¦   «                              ||¦  «         d S r   )r¿   rÀ   rÈ   s      €r=   rÀ   zBltSelfAttention.__init__,  s#   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ð+Ð+r?   )r™   rš   r›   r$   rÊ   rÀ   rË   rÌ   s   @r=   rÁ   rÁ   +  sK   ø€ € € € € ð,˜yð ,°Sð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,r?   rÁ   c            
       óŒ   ‡ — e Zd ZdZddedededz  fˆ fd„Z	 	 ddej        dej        dz  d	ej        dz  d
e	e
         fd„Zˆ xZS )ÚBltCrossAttentionz<Cross-attention module for Blt, following transformers styleNr¼   rº   rÄ   c                 óÖ   •— t          ¦   «                              ¦   «          d| _        t          | j        |j        ¬¦  «        | _        t          | j        |j        ¬¦  «        | _        d S )NFr½   )r¿   rÀ   Ú	is_causalrŸ   rÄ   rÅ   Úq_normÚk_norm)r°   r¼   rº   rÄ   rÉ   s       €r=   rÀ   zBltCrossAttention.__init__3  sX   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ  Ô!1°vÔ7JÐKÑKÔKˆŒÝ  Ô!1°vÔ7JÐKÑKÔKˆŒˆˆr?   Úhidden_statesÚcross_attention_statesÚattention_maskÚkwargsc                 ój  — |                      ¦   «         \  }}}|                      |¦  «        }|                      |¦  «        }|                     ||| j        | j        ¦  «                             dd¦  «        }|                      |¦  «        }|                      |¦  «        }	|  	                    |¦  «        }
|	                     |d| j
        | j        ¦  «                             dd¦  «        }	|
                     |d| j
        | j        ¦  «                             dd¦  «        }
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }}|                     ||d¦  «                             ¦   «         }|                      |¦  «        }||z   }||fS )Nr#   r   r/   ç        )ÚdropoutÚscaling)rƒ   rÓ   Úq_projÚviewÚ	num_headsÚhead_dimr¬   rÔ   Úk_projÚv_projÚnum_key_value_headsr   Úget_interfacer¼   Ú_attn_implementationr"   ÚtrainingrÛ   rÜ   ÚreshapeÚ
contiguousÚo_proj)r°   rÕ   rÖ   r×   rØ   Úbszrr   Ú_Úquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                 r=   r·   zBltCrossAttention.forward9  s½  € ð &×*Ò*Ñ,Ô,‰ˆˆU�AØ—{’{ =Ñ1Ô1ˆØ—{’{ <Ñ0Ô0ˆØ#×(Ò(¨¨e°T´^ÀTÄ]ÑSÔS×]Ò]Ð^_ÐabÑcÔcˆà!%§¢Ð-CÑ!DÔ!DÐØ—[’[Ð!7Ñ8Ô8ˆ
Ø—{’{Ð#9Ñ:Ô:ˆØ—_’_ S¨"¨dÔ.FÈÌÑVÔV×`Ò`ÐabÐdeÑfÔfˆ
Ø#×(Ò(¨¨b°$Ô2JÈDÌMÑZÔZ×dÒdÐefÐhiÑjÔjˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨u°bÑ9Ô9×DÒDÑFÔFˆØ—k’k +Ñ.Ô.ˆØ! MÑ1ˆØ˜LÐ(Ð(r?   r   ©NN)r™   rš   r›   Ú__doc__r$   rÊ   rÀ   r3   ÚTensorr   r   r·   rË   rÌ   s   @r=   rÐ   rÐ   0  sÈ   ø€ € € € € ØFÐFðLð L˜yð L°Sð LÀsÈTÁzð Lð Lð Lð Lð Lð Lð 7;Ø.2ð	#)ð #)à”|ð#)ð !&¤¨tÑ 3ð#)ð œ tÑ+ð	#)ð
 Ð+Ô,ð#)ð #)ð #)ð #)ð #)ð #)ð #)ð #)r?   rÐ   c                   ó’   — e Zd ZU eed<   dZdZdZdgZ e	e
d¬¦  «         e	ed¬¦  «        dœZ ej        ¦   «         d„ ¦   «         Zd	S )
ÚBltPreTrainedModelr¼   Fr¹   r   )Úindexr#   )rÕ   Ú
attentionsc           	      óŽ	  — t          j        | |¦  «         |j        j        }t	          |t
          j        ¦  «        r§t          | j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j
        dd¦  «        }|€|j        }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         |j        �$t          j        |j        |j                 ¦  «         dS t	          |t"          t$          f¦  «        s|d	v �rnt          | j        dd¦  «        }|€t          |d¦  «        r|j        }|€<d
D ]9}t          ||d¦  «        }|�$t          |d¦  «        r|j        j        d         } nŒ:|€dS |dz  }dD ]s}t          ||d¦  «        }|�^t          |d¦  «        rNt          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         Œtt          |dt          |dd¦  «        ¦  «        }	|	�^t          |	d¦  «        rNt          j        |	j        d|d|z  d|z  ¬¦  «         t          |	dd¦  «        �t          j        |	j        ¦  «         dS t	          |t,          ¦  «        s|dk    �rËt          | j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j        dd¦  «        }|€0t          | j        d¦  «        rt          | j        j
        dd¦  «        }d}
|�|dz  }
t          |dt          |dd¦  «        ¦  «        }t          |dd¦  «        }t          |dt          |dd¦  «        ¦  «        }||fD ]y}|�ut          |d¦  «        re|
p|j        j        d         dz  }t          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         Œz|�ut          |d¦  «        re|j        j        d         }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         t          |dd¦  «        �t          j        |j        ¦  «         dS t	          |t
          j        ¦  «        rR|j        }|dz  }t          j        |j        d|d|z  d|z  ¬¦  «         |j        �t          j        |j        ¦  «         dS dS )a  
        Initialize BLT weights following the original ByteLatentTransformer:

        - Most weights are drawn from a truncated normal.
        - Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs).
        - Norm layers are set to weight = 1, bias = 0.
        rÄ   NÚencoder_configg      à¿rÚ   éýÿÿÿr   )ÚmeanÚstdÚaÚb)r!   r   )rÝ   rá   râ   ré   ÚdenseÚweightr/   )rÝ   rá   râ   Úbiasré   r   r   Údecoder_configÚ	gate_projÚfc1Úup_projÚ	down_projÚfc2r#   )r   Ú_init_weightsrÉ   r™   r©   ÚnnÚ	EmbeddingÚgetattrr¼   Úhasattrrú   Úembedding_dimÚinitÚtrunc_normal_r  Úpadding_idxÚzeros_rÁ   rÐ   rÄ   r7   r  r—   r  ÚLinearÚin_features)r°   ÚmoduleÚ
class_namerÄ   rý   r2   ÚnameÚprojÚ	proj_nameré   Úin_stdr  r  r  Ú
hidden_dimÚout_stdÚfan_ins                    r=   r	  z BltPreTrainedModel._init_weightss  sw  € õ 	Ô% d¨FÑ3Ô3Ð3àÔ%Ô.ˆ
õ �f�bœlÑ+Ô+ð 	Ý! $¤+¨}¸dÑCÔCˆKØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ØÐ"Ø$Ô2�à˜tÑ#ˆCÝÔØ”ØØØ�s‘(Ø�c‘'ðñ ô ð ð Ô!Ð-Ý”˜FœM¨&Ô*<Ô=Ñ>Ô>Ð>ØˆFõ �fÕ/Õ1BÐCÑDÔDð ,	È
ð W
ð I
ñ I
õ ˜$œ+ }°dÑ;Ô;ˆCØˆ{�w v¨}Ñ=Ô=ˆ{ØÔ(�Øˆ{ØMð ð �DÝ" 6¨4°Ñ6Ô6�DØÐ'­G°D¸(Ñ,CÔ,CÐ'Ø"œkÔ/°Ô3˜Ø˜øØˆ{Ø�à�t‘)ˆCð <ð /ð /�	Ý˜v y°$Ñ7Ô7�ØÐ#­°°hÑ(?Ô(?Ð#ÝÔ&ØœØ ØØ˜s™(Ø˜c™'ðñ ô ð õ ˜t V¨TÑ2Ô2Ð>Ýœ D¤IÑ.Ô.Ð.øõ ˜V X­w°v¸wÈÑ/MÔ/MÑNÔNˆFØÐ!¥g¨f°hÑ&?Ô&?Ð!ÝÔ"Ø”MØØØ˜3‘hØ˜#‘gðñ ô ð õ ˜6 6¨4Ñ0Ô0Ð<Ý”K ¤Ñ,Ô,Ð,ØˆFõ �f�fÑ%Ô%ð +	¨°Ò)FÑ)FÝ! $¤+¨}¸dÑCÔCˆKØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ØÐ"¥w¨t¬{Ð<LÑ'MÔ'MÐ"Ý% d¤kÔ&@À-ÐQUÑVÔV�ð ˆFØÐ&Ø$ dÑ*�å ¨µW¸VÀUÈDÑ5QÔ5QÑRÔRˆIÝ˜f i°Ñ6Ô6ˆGÝ ¨µW¸VÀUÈDÑ5QÔ5QÑRÔRˆIð # GÐ,ð /ð /�ØÐ#­°°hÑ(?Ô(?Ð#Ø ÐB T¤[Ô%6°qÔ%9¸TÑ%A�CÝÔ&ØœØ ØØ˜s™(Ø˜c™'ðñ ô ð õ ˜t V¨TÑ2Ô2Ð>Ýœ D¤IÑ.Ô.Ð.øð Ð$­°¸HÑ)EÔ)EÐ$Ø&Ô-Ô3°AÔ6�
Ø$ dÑ*�ÝÔ"ØÔ$ØØØ˜7‘lØ˜'‘kðñ ô ð õ ˜9 f¨dÑ3Ô3Ð?Ý”K 	¤Ñ/Ô/Ð/ØˆFõ �f�bœiÑ(Ô(ð 	ØÔ'ˆFØ˜$‘,ˆCÝÔØ”ØØØ�s‘(Ø�c‘'ðñ ô ð ð Œ{Ð&Ý”˜FœKÑ(Ô(Ð(ØˆFð	ð 	r?   N)r™   rš   r›   r$   Ú__annotations__Ú_supports_attention_backendÚ_supports_flash_attnÚ_supports_flex_attnÚ_no_split_modulesr   r¹   rÁ   Ú_can_record_outputsr3   rE   r	  rœ   r?   r=   rö   rö   _  s˜   € € € € € € àÐÐÑØ"'ÐØ ÐØÐØ.Ð/Ðà'˜Ð(;À1ÐEÑEÔEØ$�nÐ%5¸QÐ?Ñ?Ô?ðð Ðð €U„]�_„_ðJð Jñ „_ðJð Jð Jr?   rö   c                   ó"  ‡ — e Zd ZU eed<   d eedd¬¦  «        iZdefˆ fd„Z	 	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j        dz  d
e	j        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  dedz  de	j        dz  dee         fd„Zd„ Zˆ xZS )ÚBltLocalEncoderr¼   Úencoder_attentionsr#   r\   ©r÷   Ú
layer_namec                 óž  •‡— t          ¦   «                              ‰¦  «         d| _        ‰| _        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰¬¦  «        | _
        t	          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        t	          j        ‰j        ‰j        ¦  «        | _        t	          j        ¦   «         | _        ‰j        r‰j        nd}t          |¦  «        D ]1}| j                             t+          ‰|‰j        ¬¦  «        ¦  «         Œ2|                      ¦   «          d S )NFc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rœ   ©r¹   ©r€   rº   r¼   s     €r=   ú
<listcomp>z,BltLocalEncoder.__init__.<locals>.<listcomp>  ó$   ø€ ÐeÐeÐe¸	Õ  ¨Ñ3Ô3ÐeÐeÐer?   ©r¼   ©r  Úout_featuresr  r#   ©r¼   rº   rÄ   )r¿   rÀ   Úgradient_checkpointingr¼   r
  Ú
ModuleListrY   Únum_hidden_layersÚlayersr¡   Ú
rotary_embr  rÄ   rh   Úpatch_embedding_projectionr  Ú
vocab_sizerX   Úcross_attn_layersÚcross_attn_all_layersr‡   rÐ   Ú	post_init©r°   r¼   Úlayers_to_addrº   rÉ   s    `  €r=   rÀ   zBltLocalEncoder.__init__  sI  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#ØˆŒÝ”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ -°FÐ;Ñ;Ô;ˆŒÝ*,¬)ØÔ*ØÔ+¨fÔ.AÑAØð+
ñ +
ô +
ˆÔ'õ
 œL¨Ô):¸FÔ<NÑOÔOˆÔÝ!#¤¡¤ˆÔØ4:Ô4PÐW˜Ô0Ð0ÐVWˆÝ˜}Ñ-Ô-ð 	ð 	ˆIØÔ"×)Ò)Ý!¨¸9ÐRXÔRdÐeÑeÔeñô ð ð ð 	�ŠÑÔÐÐÐr?   NÚ	input_idsÚinputs_embedsÚpatch_embedsr×   r²   Úpast_key_valuesÚencoder_attention_maskre   rd   rØ   c
                 ó¬  — |€|                       |¦  «        }|j        d         }t          j        || j        j        | j        ¬¦  «        }|€Mt          j        |j        d         |j        ¬¦  «         	                    d¦  «         
                    |d¦  «        }|                      ||¦  «        }t          j        || j        j        | j        ¬¦  «        }t          | j        ¦  «        D ]å\  }} ||f|||dœ|
¤Ž}|t          | j        ¦  «        dz
  k    s| j        j        r¬|                      |||	¦  «        }|                      |¦  «        }|                     ||j        d         | j        j        z  | j        j        ¦  «        }| j        j        r|nd} | j        |         d|||dœ|
¤Ž\  }}||                     |j        ¦  «        z   }Œæ|}||fS )	Nr   ©Úpræ   r#   r0   r/   ©Úposition_embeddingsr×   rB  ©rÕ   rÖ   r×   rœ   )rX   r7   ÚFrÛ   r¼   ræ   r3   r6   r.   rj   rk   r7  r‰   r6  rZ   r;  Úpatch_reducer8  rç   rh   rÄ   r:  r[   )r°   r?  r@  rA  r×   r²   rB  rC  re   rd   rØ   rI   rÕ   rH  ÚidxÚlayerrº   Úcross_attention_outputrë   Úencoder_cross_statess                       r=   r·   zBltLocalEncoder.forward  s  € ð Ð Ø ×-Ò-¨iÑ8Ô8ˆMà"Ô(¨Ô+ˆ
Ýœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆàÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆå# D¤KÑ0Ô0ð 	]ð 	]‰JˆC�Ø!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMð •c˜$œ+Ñ&Ô&¨Ñ*Ò*Ð*¨d¬kÔ.OÐ*Ø#×0Ò0°ÀÈYÑWÔW�Ø#×>Ò>¸|ÑLÔL�Ø+×3Ò3Ø Ô 2°1Ô 5¸¼Ô8PÑ PÐRVÔR]ÔRiñ ô  �ð $(¤;Ô#DÐK˜C˜CÈ!�	Ø,M¨DÔ,BÀ9Ô,Mð -Ø".Ø+8Ø#9ð-ð -ð ð	-ð -Ñ)Ð&¨ð  ,Ð.D×.GÒ.GÈÔH[Ñ.\Ô.\Ñ\�øØ+ÐØÐ2Ð2Ð2r?   c                 óB  — |j         d         }|j         d         }|                     d¦  «                             dd|j         d         ¦  «        }t          j        |||f|j        |j        ¬¦  «        }|                     |d|dd¬¦  «        }|dd…d|…dd…f         }|S )	a†  
        Reduce variable length patches to single embedding per patch
        Note: this works with variable number of patches for different sequences in the batch
        It handles variable length patches by assuming that patch_lengths will be 0 for any
        extra patches on the *right*. Since there can be a variable number of patches
        this function also return the number of patches for each sequence in the batch.
        Any embeddings on the right that are not allocated to a patch
        (i.e. if the sum(patch_lengths[i]) < seq_len for any i)
        will be sent to a dummy patch, which is trimmed before returning.
        r   r/   r,   r#   ÚamaxF)Úsrcr2   r÷   ÚreduceÚinclude_selfN)r7   rj   rk   r3   rF   r-   r.   Úscatter_reduce)r°   rÕ   Úmax_num_patchesrd   rI   r  Úreduced_embeddingss          r=   rK  zBltLocalEncoder.patch_reduceR  sË   € ð #Ô(¨Ô+ˆ
Ø%Ô+¨BÔ/ˆà×'Ò'¨Ñ+Ô+×2Ò2°2°r¸=Ô;NÈrÔ;RÑSÔSˆ	å"œ[Ø˜¨-Ð8ÀÔ@SÐ\iÔ\pð
ñ 
ô 
Ðð 0×>Ò>ØØØØØð ?ñ 
ô 
Ðð 0°°°Ð3C°OÐ3CÀQÀQÀQÐ0FÔGÐà!Ð!r?   ©	NNNNNNNNN)r™   rš   r›   r'   r  r   rÁ   r#  rÀ   r3   Ú
LongTensorrô   r   rÊ   r   r   r·   rK  rË   rÌ   s   @r=   r%  r%    s\  ø€ € € € € € Ø!Ð!Ð!Ñ!à˜n˜nÐ-=ÀQÐSbÐcÑcÔcðÐðÐ4ð ð ð ð ð ð ð2 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:Ø"&Ø)-ð23ð 23àÔ# dÑ*ð23ð ”| dÑ*ð23ð ”l TÑ)ð	23ð
 œ tÑ+ð23ð Ô&¨Ñ-ð23ð  ™ð23ð !&¤¨tÑ 3ð23ð ˜4‘Zð23ð ”< $Ñ&ð23ð Ð+Ô,ð23ð 23ð 23ð 23ðh"ð "ð "ð "ð "ð "ð "r?   r%  c                   óÚ   ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	e	dz  d
ej        dz  de
e         fd„Zˆ xZS )ÚBltLocalDecoderr¼   c                 ó¤  •‡— t          ¦   «                              ‰¦  «         d| _        ‰| _        d| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          ‰¬¦  «        | _        t          j        ‰j        ‰j        ‰j        z  d¬¦  «        | _        t#          ‰j        ‰j        ¬¦  «        | _        t          j        ¦   «         | _        ‰j        r‰j        nd}t          |¦  «        D ]1}| j                             t/          ‰|‰j        ¬¦  «        ¦  «         Œ2|                      ¦   «          d S )	NFTc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS rœ   r+  r,  s     €r=   r-  z,BltLocalDecoder.__init__.<locals>.<listcomp>{  r.  r?   r/  r0  r½   r#   r2  )r¿   rÀ   r3  r¼   Úcross_attn_decoderr
  r4  rY   r5  r6  r¡   r7  r  Úhidden_size_globalrÄ   rh   r8  rŸ   rÅ   Únormr:  r;  r‡   rÐ   r<  r=  s    `  €r=   rÀ   zBltLocalDecoder.__init__u  sQ  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#ØˆŒØ"&ˆÔÝ”mØeÐeÐeÐeÅUÈ6ÔKcÑEdÔEdÐeÑeÔeñ
ô 
ˆŒõ -°FÐ;Ñ;Ô;ˆŒÝ*,¬)ØÔ1ØÔ+¨fÔ.AÑAØð+
ñ +
ô +
ˆÔ'õ
 ˜vÔ1°vÔ7JÐKÑKÔKˆŒ	Ý!#¤¡¤ˆÔØ4:Ô4PÐW˜Ô0Ð0ÐVWˆÝ˜}Ñ-Ô-ð 	ð 	ˆIØÔ"×)Ò)Ý!¨¸9ÐRXÔRdÐeÑeÔeñô ð ð ð 	�ŠÑÔÐÐÐr?   Nr?  r@  rA  r×   r²   rB  rC  rØ   c                 óÌ  — |j         d         }	|}
|                      |¦  «        }|                     |	|j         d         | j        j        z  | j        j        ¦  «        }|�| j        s|
|z   }
|€Mt          j        |j         d         |j	        ¬¦  «         
                    d¦  «                             |	d¦  «        }|                      |
|¦  «        }t          j        |
| j        j        | j        ¬¦  «        }
t!          | j        ¦  «        D ]C\  }}|dk    s| j        j        r | j        |         d|
||dœ|¤Ž\  }}|
|z   }
 ||
f|||dœ|¤Ž}
ŒD|                      |
¦  «        }|S )	Nr   r#   r0   r/   rE  rI  rG  rœ   )r7   r8  rç   r¼   rh   rÄ   r^  r3   r6   r.   rj   rk   r7  rJ  rÛ   ræ   r‰   r6  r;  r:  r`  )r°   r?  r@  rA  r×   r²   rB  rC  rØ   rI   rÕ   rH  r“   rM  rN  rë   Úlogitss                    r=   r·   zBltLocalDecoder.forward�  s·  € ð #Ô(¨Ô+ˆ
Ø%ˆØ×6Ò6°|ÑDÔDˆØ#×+Ò+Ø˜Ô*¨1Ô-°´Ô0HÑHÈ$Ì+ÔJañ
ô 
ˆð Ð#¨DÔ,CÐ#Ø)¨LÑ8ˆMàÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆå! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØ�AŠvˆv˜œÔ:ˆvØ,E¨DÔ,BÀ1Ô,Eð -Ø"/Ø+7Ø#9ð-ð -ð ð	-ð -Ñ)Ð&¨ð !.Ð0FÑ F�Ø!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMˆMð —’˜=Ñ)Ô)ˆØˆr?   ©NNNNNNN)r™   rš   r›   r&   r  rÀ   r3   rY  rô   r   r   r   r·   rË   rÌ   s   @r=   r[  r[  r  s  ø€ € € € € € Ø!Ð!Ð!Ñ!ðÐ4ð ð ð ð ð ð ð4 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:ð.ð .àÔ# dÑ*ð.ð ”| dÑ*ð.ð ”l TÑ)ð	.ð
 œ tÑ+ð.ð Ô&¨Ñ-ð.ð  ™ð.ð !&¤¨tÑ 3ð.ð Ð+Ô,ð.ð .ð .ð .ð .ð .ð .ð .r?   r[  c                   ó°   ‡ — e Zd ZU eed<   d eedd¬¦  «        iZdefˆ fd„Z	 	 	 dde	j
        d	e	j
        dz  d
e	j        dz  dedz  dee         f
d„Zˆ xZS )ÚBltGlobalTransformerr¼   Úglobal_attentionsr#   Úglobal_transformerr'  c                 óø  •— t          ¦   «                              |¦  «         || _        t          j        ¦   «         | _        t          |j        ¦  «        D ]*}| j                             t          ||¦  «        ¦  «         Œ+t          |¬¦  «        | _        t          |dd ¦  «        �'t          j        |j        |j        d¬¦  «        | _        nt          j        ¦   «         | _        |                      ¦   «          d S )Nr/  Úencoder_cross_output_sizeF©r  )r¿   rÀ   r¼   r
  r4  r6  rY   r5  r‡   r¹   r¡   r7  r  r  ri  rÄ   Útoken_embedding_projectionÚIdentityr<  rÈ   s      €r=   rÀ   zBltGlobalTransformer.__init__Ä  så   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ”m‘o”oˆŒÝ˜vÔ7Ñ8Ô8ð 	Gð 	GˆIØŒK×ÒÕ2°6¸9ÑEÔEÑFÔFÐFÐFÝ,°FÐ;Ñ;Ô;ˆŒõ �6Ð6¸Ñ=Ô=ÐIÝ.0¬iØÔ0°&Ô2DÈ5ð/ñ /ô /ˆDÔ+Ð+õ /1¬k©m¬mˆDÔ+à�ŠÑÔÐÐÐr?   Nr@  r×   r²   rB  rØ   c                 óª  — |j         \  }}}|                      |¦  «        }	t          j        |	| j        j        | j        ¬¦  «        }	|€Mt          j        |j         d         |j        ¬¦  «         	                    d¦  «         
                    |d¦  «        }|                      |	|¦  «        }
t          | j        ¦  «        D ]\  }} ||	f|
||dœ|¤Ž}	Œ|	S )NrE  r#   r0   r   r/   rG  )r7   rk  rJ  rÛ   r¼   ræ   r3   r6   r.   rj   rk   r7  r‰   r6  )r°   r@  r×   r²   rB  rØ   rI   rJ   rë   rÕ   rH  r“   rM  s                r=   r·   zBltGlobalTransformer.forwardÖ  sü   € ð "/Ô!4Ñˆ
�G˜QØ×7Ò7¸ÑFÔFˆÝœ	 -°4´;Ô3FÐQUÔQ^Ð_Ñ_Ô_ˆØÐå”˜]Ô0°Ô3¸MÔ<PÐQÑQÔQ×[Ò[Ð\]Ñ^Ô^×eÒeÐfpÐrtÑuÔuð ð #Ÿošo¨m¸\ÑJÔJÐÝ! $¤+Ñ.Ô.ð 	ð 	‰HˆAˆuØ!˜EØðà$7Ø-Ø /ð	ð ð
 ðð ˆMˆMð Ðr?   )NNN)r™   rš   r›   r%   r  r   rÁ   r#  rÀ   r3   rô   rY  r   r   r   r·   rË   rÌ   s   @r=   re  re  ¾  sâ   ø€ € € € € € Ø&Ð&Ð&Ñ&à˜^˜^Ð,<ÀAÐRfÐgÑgÔgðÐðÐ9ð ð ð ð ð ð ð* /3Ø04Ø(,ðð à”|ðð œ tÑ+ðð Ô&¨Ñ-ð	ð
  ™ðð Ð+Ô,ðð ð ð ð ð ð ð r?   re  c                   óú   ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej
        dz  d	edz  d
edz  dedz  dedz  dee         fd„Ze	 	 dd„¦   «         Zˆ xZS )Ú
BltPatcherr¼   c                 óx  •— t          ¦   «                              |¦  «         t          | j        ¬¦  «        | _        t          j        ¦   «         | _        t          | j        j	        ¦  «        D ]/}| j         
                    t          | j        |¦  «        ¦  «         Œ0t          j        | j        j        | j        j        ¦  «        | _        t!          | j        j        | j        j        ¬¦  «        | _        t          j        | j        j        | j        j        d¬¦  «        | _        |                      ¦   «          d S )Nr/  r½   Frj  )r¿   rÀ   r¡   r¼   r7  r
  r4  r6  rY   r5  r‡   r¹   r  r9  rÄ   rX   rŸ   rÅ   r`  r  Úlm_headr<  rÈ   s      €r=   rÀ   zBltPatcher.__init__ô  sÿ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý,°D´KÐ@Ñ@Ô@ˆŒÝ”m‘o”oˆŒÝ˜tœ{Ô<Ñ=Ô=ð 	Lð 	LˆIØŒK×ÒÕ2°4´;À	ÑJÔJÑKÔKÐKÐKÝœL¨¬Ô)?ÀÄÔAXÑYÔYˆÔÝ˜tœ{Ô6¸D¼KÔ<TÐUÑUÔUˆŒ	Ý”yØŒKÔ#ØŒKÔ"Øð
ñ 
ô 
ˆŒð 	�ŠÑÔÐÐÐr?   Nr?  r×   r²   rB  r@  Ú	use_cacheÚ
patch_sizeÚ	thresholdr|   rØ   c
                 óT  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }|}|                      ||¦  «        }| j        D ]} ||||¬¦  «        }Œ|                      |                      |¦  «        ¦  «        }t
          j                             |¬¦  «                             ¦   «         }|j        d d	…         \  }}|�|                      ||||¬
¦  «        }n#t          j        ||f|j        |j        ¬¦  «        }t+          ||	¦  «        }|||fS )Nú:You must specify exactly one of input_ids or inputs_embedsr/  r   r#   r0   ©r¼   r@  r×   rB  r²   )rH  r×   )rb  r   )Ú	entropiesrf   rs  rt  r,   )rm   rX   r   r¼   Úget_seq_lengthr3   r6   r7   r.   rj   r
   r7  r6  rq  r`  ÚdistributionsÚCategoricalÚentropyÚpatch_lengths_from_entropiesÚonesr-   r•   )r°   r?  r×   r²   rB  r@  rr  rs  rt  r|   rØ   Úpast_seen_tokensÚcausal_maskrÕ   rH  rM  rb  Úprediction_entropiesrI   rf   r{   s                        r=   r·   zBltPatcher.forward  sü  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨m¸\ÑJÔJÐà”[ð 	vð 	vˆEØ!˜E -ÐEXÐitÐuÑuÔuˆMˆMà—’˜dŸiši¨Ñ6Ô6Ñ7Ô7ˆÝ$Ô2×>Ò>ÀfÐ>ÑMÔM×UÒUÑWÔWÐà&3Ô&9¸"¸1¸"Ô&=Ñ#ˆ
�OØÐ!Ø ×=Ò=Ø.Ø /Ø%Ø#ð	 >ñ ô ˆMˆMõ "œJØ˜_Ð-°]Ô5HÐQ^ÔQeðñ ô ˆMõ .¨mÐ=MÑNÔNˆØ# ]°FÐ:Ð:r?   c                 óª  — | j         d         }t          j        ddgt          j        | j        ¬¦  «                             d¦  «                             |d¦  «        }|j         d         }| dd…dd…f         } | |k    }|j         d         }t          j        || j        ¬¦  «                             d¦  «                             |d¦  «        }	t          j	        |	|¦  «        }
t          j
        |	|
gd¬¦  «        }t          j
        || gd¬¦  «        }||                              ||¦  «        }|                     d¬¦  «                             ¦   «         }|dd…d|…f         }t          j
        |||z   fd¬¦  «        }t          j	        |dd…dd…f         |dz
  ¦  «        }t          j
        |dd…dd…f         dz
  |fd¬¦  «        }||z
  dz   }|S )zÔ
        Computes patch lengths from token entropies.

        Depending on whether a threshold is provided, the function uses either:
        - Thresholding the entropy values (when `threshold` is set).
        r   r#   r,   Nr0   r/   r1   )r7   r3   r4   Úlongr.   rj   Úrepeatr6   rk   Ú	full_likerG   rç   r8   rˆ   )rx  rf   rs  rt  rI   Úinit_tokensÚoffsetÚ
patch_maskrJ   Útoken_indicesÚsentinelÚpadded_indicesÚpadded_maskÚpatch_startsÚmax_valid_patchesÚpatch_start_idsÚ
last_tokenÚ
patch_endsr{   s                      r=   r}  z'BltPatcher.patch_lengths_from_entropies?  sù  € ð ”_ QÔ'ˆ
õ ŒL˜!˜Q˜¥u¤z¸)Ô:JÐKÑKÔK×UÒUÐVWÑXÔX×_Ò_Ð`jÐlmÑnÔnð 	ð Ô" 1Ô%ˆð ˜a˜a˜a   ˜eÔ$ˆ	ð  Ò*ˆ
àÔ" 1Ô%ˆõ œ W°YÔ5EÐFÑFÔF×PÒPÐQRÑSÔS×ZÒZÐ[eÐgiÑjÔjˆÝ”? =°'Ñ:Ô:ˆÝœ M°8Ð#<À!ÐDÑDÔDˆõ ”i ¨j¨[Ð 9¸qÐAÑAÔAˆð & kÔ2×:Ò:¸:ÀwÑOÔOˆØ&ŸNšN¨q˜NÑ1Ô1×5Ò5Ñ7Ô7ÐØ# A A AÐ'9Ð(9Ð'9Ð$9Ô:ˆõ  œ) [°,ÀÑ2GÐ$HÈaÐPÑPÔPˆõ ”_ _°Q°Q°Q¸¸¸°UÔ%;¸_ÈqÑ=PÑQÔQˆ
Ý”Y °°°°1°2°2°Ô 6¸Ñ :¸JÐGÈQÐOÑOÔOˆ
à" _Ñ4°qÑ8ˆàÐr?   rX  rò   )r™   rš   r›   r(   r  rÀ   r3   rY  rô   r   ÚFloatTensorro   rÊ   r¨   r   r   r·   Ústaticmethodr}  rË   rÌ   s   @r=   ro  ro  ñ  sM  ø€ € € € € € ØÐÐÑðÐ/ð ð ð ð ð ð ð$ .2Ø.2Ø04Ø(,Ø26Ø!%Ø!%Ø"&Ø'+ð9;ð 9;àÔ# dÑ*ð9;ð œ tÑ+ð9;ð Ô&¨Ñ-ð	9;ð
  ™ð9;ð Ô(¨4Ñ/ð9;ð ˜$‘;ð9;ð ˜$‘Jð9;ð ˜4‘<ð9;ð  ™*ð9;ð Ð+Ô,ð9;ð 9;ð 9;ð 9;ðv ð Øð	3ð 3ð 3ñ „\ð3ð 3ð 3ð 3ð 3r?   ro  c                   ó"  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej        dz  de
dz  d	ej        dz  d
edz  dee         deez  fd„¦   «         ¦   «         Zd„ Zd„ Zdej	        dedej	        fd„Zˆ xZS )ÚBltModelr¼   c                 óŠ  •— t          ¦   «                              |¦  «         d| _        || _        t	          |j        ¦  «        | _        t          |j        ¦  «        | _	        t          |j        ¦  «        | _        |j        t          |j        ¦  «        z  }|j        |z  }t#          j        ||j        j        ¦  «        | _        | j        j        rVt-          |j        ¦  «        | _        | j                             ¦   «          | j                             ¦   «         D ]	}d|_        Œ
nd | _        |                      ¦   «          d S )NF)r¿   rÀ   r3  r¼   r%  rú   r\   re  Úglobal_configrg  r[  r  Úlocal_decoderrS   rZ   rT   rU   r
  r  rÄ   rR   Úpatch_in_forwardro  Úpatcher_configÚpatcherÚevalÚ
parametersÚrequires_gradr<  )r°   r¼   Únum_embeddingsÚtotal_vocab_sizeÚparamrÉ   s        €r=   rÀ   zBltModel.__init__w  s#  ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø&+ˆÔ#àˆŒÝ,¨VÔ-BÑCÔCˆÔÝ"6°vÔ7KÑ"LÔ"LˆÔÝ,¨VÔ-BÑCÔCˆÔØÔDÅsÈ6ÔKnÑGoÔGoÑoˆØ!Ô?À.ÑPÐÝ*,¬,Ð7GÈÔI^ÔIjÑ*kÔ*kˆÔ'ØŒ;Ô'ð 	 Ý% fÔ&;Ñ<Ô<ˆDŒLØŒL×ÒÑÔÐØœ×0Ò0Ñ2Ô2ð ,ð ,�Ø&+�Ô#Ð#ð,ð  ˆDŒLØ�ŠÑÔÐÐÐr?   Nr?  r{   r×   r²   rB  r@  rr  rØ   rV   c                 óB  — |d u |d uz  rt          d¦  «        ‚|rq|€7t          t          | j        ¬¦  «        t          | j        ¬¦  «        ¦  «        }n8t	          |t          ¦  «        s#t          |t          | j        ¬¦  «        ¦  «        }|�|}	|j        \  }
}}nF|j        \  }
}t          || j        | j        | j        j	        | j        j
        | j        j        ¦  «        }	|€É| j        j        dk    re| j        �^|€t          d¦  «        ‚|                      || j        j        | j        j        | j        j        | j        j        |j        ¬¦  «        \  }}}nT|�|j        n|j        }|�|j        n|j        }t)          t+          j        |
|dz   f||¬¦  «        | j        j        ¦  «        }|                      ||¦  «        }|€V|�|                     ¦   «         nd}t+          j        |	j        d         |	j        ¬	¦  «        |z   }|                     d¦  «        }t7          | j        |	||�|j        nd |¬
¦  «        }t;          ||j        d         |d| j        j        |	j        ¬¦  «        } | j        d||	||||j        d         ||�|j        nd dœ|¤Ž\  }}|                     |
|j        d         d¦  «        }t+          j        d|j        d         |j        ¬	¦  «        }|                     d¦  «        }t7          | j        |d d d ¬
¦  «        } | j         d|||dœ|¤Ž}|                      |d d …dd …f         |¦  «        }t;          ||j        d         |d| j        j        |	j        ¬¦  «        } | j!        d||||||�|j"        nd |dœ|¤Ž}tG          ||¬¦  «        S )Nrv  r/  r|  z0input_ids is required for entropy-based patching)rs  rt  r|   Úpatching_batch_sizer.   r#   r,   r   r0   rw  T)rd   re   rf   rg   rh   r-   )r?  r@  r×   r²   rC  re   rd   rB  r/   )r@  r×   r²   F)r?  r@  rA  r×   r²   rB  rC  )Úlast_hidden_staterB  rœ   )$rm   r   r   r¼   r©   r7   rc   r\   rR   rS   rT   rU   Úpatching_moder›  rs  Úpatching_thresholdr|   r£  r.   r-   r•   r3   r~  Ú_patch_ids_from_lengthsry  r6   rj   r
   Úself_attention_cacherz   rh   rÞ   rg  r˜  Úcross_attention_cacher   )r°   r?  r{   r×   r²   rB  r@  rr  rØ   Úencoder_embedsrI   rf   rë   r.   r-   rd   r  r€  Úcross_attn_mask_encÚencoder_hidden_statesrO  Úglobal_position_idsÚglobal_causal_maskÚglobal_hidden_statesÚdecoder_patch_idsÚcross_attn_mask_decÚoutputs                              r=   r·   zBltModel.forward‹  sŸ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	iØÐ&Ý"5Ý ¨¬Ð4Ñ4Ô4µlÈ$Ì+Ð6VÑ6VÔ6Vñ#ô #��õ   Õ1DÑEÔEð iõ #6°oÅ|Ð[_Ô[fÐGgÑGgÔGgÑ"hÔ"h�ð Ð$Ø*ˆNØ-:Ô-@Ñ*ˆJ˜¨¨à*3¬/Ñ'ˆJ˜Ý4ØØÔ"ØÔ/Ø”Ô@Ø”Ô8Ø”Ô9ñô ˆNð Ð ØŒ{Ô(¨IÒ5Ð5¸$¼,Ð:RØÐ$Ý$Ð%WÑXÔXÐXØ&*§l¢lØØ#œ{Ô5Ø"œkÔ<Ø%)¤[Ô%AØ(,¬Ô(GØ$Ô+ð '3ñ 'ô 'Ñ#��= ! !ð .7Ð-B˜Ô)Ð)ÈÔH\�Ø+4Ð+@˜	œ˜ÀmÔFY�Ý 5Ý”J 
¨O¸aÑ,?Ð@ÈÐV\Ð]Ñ]Ô]Ø”KÔ0ñ!ô !�ð ×0Ò0°ÀÑPÔPˆ	àÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(<¸QÔ(?ÈÔH]Ð^Ñ^Ô^ÐaqÑqˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø(Ø)ØDSÐD_˜OÔ@Ð@ÐeiØ%ð
ñ 
ô 
ˆõ BØØ%Ô+¨AÔ.Ø+Ø#ØœÔ1Ø Ô&ð
ñ 
ô 
Ðð 7I°dÔ6Hð 
7
ØØ(Ø&Ø%Ø#6Ø%Ô+¨AÔ.ØØDSÐD_˜OÔ@Ð@Ðeið
7
ð 
7
ð ð
7
ð 
7
Ñ3ÐÐ3ð  4×8Ò8¸À]ÔEXÐYZÔE[Ð]_Ñ`Ô`ÐÝ#œl¨1Ð.BÔ.HÈÔ.KÐThÔToÐpÑpÔpÐØ1×;Ò;¸AÑ>Ô>ÐÝ/Ø”;Ø.ØØ Øð
ñ 
ô 
Ðð  7˜tÔ6ð  
Ø.Ø-Ø,ð 
ð  
ð ð	 
ð  
Ðð !×8Ò8¸ÀqÀqÀqÈ!È"È"ÀuÔ9MÈÑ_Ô_ÐÝAØ'Ø%Ô+¨AÔ.Ø+Ø$ØœÔ1Ø Ô&ð
ñ 
ô 
Ðð $�Ô#ð 	
ØØ/Ø-Ø&Ø%ØETÐE`˜OÔAÐAÐfjØ#6ð	
ð 	
ð ð	
ð 	
ˆõ 'Ø$Ø+ð
ñ 
ô 
ð 	
r?   c                 ó   — | j         j        S r   ©r\   rX   )r°   s    r=   Úget_input_embeddingszBltModel.get_input_embeddings  s   € ØÔ!Ô.Ð.r?   c                 ó   — || j         _        d S r   r´  )r°   Úvalues     r=   Úset_input_embeddingszBltModel.set_input_embeddings  s   € Ø*/ˆÔÔ'Ð'Ð'r?   rJ   c                 ó®  — |j         d         }t          j        t          j        |d|j        |j        ¬¦  «        |                     d¬¦  «        d d …d d…f         gd¬¦  «        }t          j        ||j        ¬¦  «        }|                     d¦  «        |                     d¦  «                             d¦  «        k     	                    d¬¦  «        dz
  S )Nr   r#   r,   r/   r1   r0   )
r7   r3   rG   rF   r-   r.   Úcumsumr6   rj   r8   )r°   r{   rJ   rI   r�  Útoken_positionss         r=   r§  z BltModel._patch_ids_from_lengths  s×   € Ø"Ô(¨Ô+ˆ
Ý”yå”˜J¨°Ô1DÈ]ÔMaÐbÑbÔbØ×$Ò$¨Ð$Ñ,Ô,¨Q¨Q¨Q°°°¨VÔ4ðð ð
ñ 
ô 
ˆõ  œ, w°}Ô7KÐLÑLÔLˆØ×&Ò& qÑ)Ô)¨_×-FÒ-FÀqÑ-IÔ-I×-SÒ-SÐTVÑ-WÔ-WÒW×\Ò\ÐacÐ\ÑdÔdÐghÑhÐhr?   rc  )r™   rš   r›   r$   rÀ   r   r   r3   rY  rô   r   r’  ro   r   r   Útupler   r·   rµ  r¸  rÊ   r§  rË   rÌ   s   @r=   r•  r•  v  s„  ø€ € € € € ð˜yð ð ð ð ð ð ð(  Øð .2Ø-1Ø.2Ø04Ø(,Ø26Ø!%ðC
ð C
àÔ# dÑ*ðC
ð ”| dÑ*ðC
ð œ tÑ+ð	C
ð
 Ô&¨Ñ-ðC
ð  ™ðC
ð Ô(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ô,ðC
ð 
Ð(Ñ	(ðC
ð C
ð C
ñ „_ñ  ÔðC
ðJ/ð /ð /ð0ð 0ð 0ð
i°U´\ð 
iÈCð 
iÐTYÔT`ð 
ið 
ið 
ið 
ið 
ið 
ið 
ið 
ir?   r•  zB
    The Blt Text Model with a language modeling head on top.
    )Úcustom_introc                   ó|  ‡ — e Zd ZU eed<   dZdZddiZdefˆ fd„Ze	e
	 	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  deej        ej        f         dz  dedz  dej        dz  dej        dz  dedz  deej        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚBltForCausalLMr¼   FÚmodelz'model.local_encoder.embed_tokens.weightzlm_head.weightc                 ó:  •— t          ¦   «                              |¦  «         |                     ¦   «         | _        |j        | _        t          |¦  «        | _        t          j        |j	        j
        |j        d¬¦  «        | _        |                      ¦   «          d S )NFrj  )r¿   rÀ   Úget_text_configÚtext_configr9  r•  rÀ  r
  r  r  rÄ   rq  r<  )r°   r¼   rÉ   s     €r=   rÀ   zBltForCausalLM.__init__0  s€   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!×1Ò1Ñ3Ô3ˆÔØ Ô+ˆŒÝ˜fÑ%Ô%ˆŒ
Ý”y Ô!6Ô!BÀFÔDUÐ\aÐbÑbÔbˆŒà�ŠÑÔÐÐÐr?   Nr   r?  r×   r²   rÖ   rv   Úfull_text_row_masked_out_maskrB  r@  Úlabelsrr  Úlogits_to_keeprØ   rV   c                 ól  —  | j         d||||||||
dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «                             ¦   «         }d}|	� | j        ||	| j        fi |¤Ž}t          |||j
        |j        |j        ¬¦  «        S )aý
  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BltForCausalLM

        >>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf")

        >>> prompt = "If I had to write a haiku, it would be:"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
        >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        >>> print(result)
        If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
        I love the idea of snowflakes gently falling, each one
        ```
        )r?  r×   r²   rv   rÄ  rB  r@  rr  N)Úlossrb  rB  rÕ   rø   rœ   )rÀ  r¤  r©   rÊ   Úslicerq  r¨   Úloss_functionr9  r   rB  rÕ   rø   )r°   r?  r×   r²   rÖ   rv   rÄ  rB  r@  rÅ  rr  rÆ  rØ   ÚoutputsrÕ   Úslice_indicesrb  rÈ  s                     r=   r·   zBltForCausalLM.forward9  s   € ð| �$”*ð 

ØØ)Ø%Ø!5Ø*GØ+Ø'Øð

ð 

ð ð

ð 

ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔA×GÒGÑIÔIˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå%ØØØ#Ô3Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r?   )NNNNNNNNNNr   )r™   rš   r›   r$   r  Ú_can_compile_fullgraphÚbase_model_prefixÚ_tied_weights_keysrÀ   r   r   r3   rY  rô   r¼  r   r’  ro   rÊ   r   r   r   r·   rË   rÌ   s   @r=   r¿  r¿  %  sº  ø€ € € € € € ð ÐÐÑØ"ÐØÐØCÐEUÐVÐð˜yð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø:>Ø8<ØRVØ(,Ø26Ø*.Ø!%Ø-.ðV
ð V
àÔ# dÑ*ðV
ð œ tÑ+ðV
ð Ô&¨Ñ-ð	V
ð
 !&Ô 0°4Ñ 7ðV
ð $Ô.°Ñ5ðV
ð (-¨U¬\¸5¼<Ð-GÔ'HÈ4Ñ'OðV
ð  ™ðV
ð Ô(¨4Ñ/ðV
ð Ô  4Ñ'ðV
ð ˜$‘;ðV
ð ˜eœlÑ*ðV
ð Ð+Ô,ðV
ð 
Ð'Ñ	'ðV
ð V
ð V
ñ „^ñ ÔðV
ð V
ð V
ð V
ð V
r?   r¿  )rö   r•  ro  r¿  )r)   )r   r)   r@   )Zró   Úcollections.abcr   r3   Útorch.distributionsÚtorch.nnr
  Útorch.nn.functionalÚ
functionalrJ  Ú r   r  Úcache_utilsr   r   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úcohere2.modeling_cohere2r   Úllama.modeling_llamar   Úmllama.modeling_mllamar   r   r   r   r    r!   r"   Úconfiguration_bltr$   r%   r&   r'   r(   Ú
get_loggerr™   ÚloggerrÊ   r>   rô   rP   r  Úlistrc   Úfloat32ro   r-   r¼  rz   r•   r—   rŸ   r¡   r¹   rÁ   rÐ   rö   r%  r[  re  ro  r•  r¿  Ú__all__rœ   r?   r=   ú<module>ré     sf  ðð CÐ Bà $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ðð ð ð ð ð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ð:ð :°ð :ð :ð :ð :ð< \aðð ØŒ|ðØ),ðØ9<ðØUXðð ð ð ð$#Øœ,ð#ð !#¤ð#ð +.ð	#ð
 #'ð#ð $'ð#ð „\ð#ð #ð #ð #ðT  %ØØœðK ð K ØŒ|ðK àðK ð ðK ð ð	K ð
 ðK ð Œ;ðK ð ˆ5Œ<˜œÐ%Ô&ðK ð K ð K ð K ð\)¨¬ð )ÈÈtÉð )ÐX]ÔXdð )ð )ð )ð )ðX	ð 	ð 	ð 	ð 	ˆ]ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"ñ 	ô 	ð 	ð<ð <ð <ð <ð <Ð-ñ <ô <ð <ð"`ð `ð `ð `ð `Ð9ñ `ô `ð `ð,ð ,ð ,ð ,ð ,Ð.ñ ,ô ,ð ,ð
,)ð ,)ð ,)ð ,)ð ,)Ð0ñ ,)ô ,)ð ,)ð^ ð^ð ^ð ^ð ^ð ^Ð.ñ ^ô ^ñ „ð^ðBn"ð n"ð n"ð n"ð n"Ð(ñ n"ô n"ð n"ðbIð Ið Ið Ið IÐ(ñ Iô Ið IðX0ð 0ð 0ð 0ð 0Ð-ñ 0ô 0ð 0ðfBð Bð Bð Bð BÐ#ñ Bô Bð BðJlið lið lið lið liÐ!ñ liô lið lið^ €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð'¨ñ g
ô g
ñô ð
g
ðTð ð €€€r?   