§
    ‚Štj—  ã                   óŒ   — d Z ddlZddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Zd
gZ	dS )zJamba model configurationé    N)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzai21labs/Jamba-v0.1)Ú
checkpointc                   ó`  ‡ — e Zd ZU dZdZdgZddiZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d <   d!Zedz  e	d"<   d#Zeee         z  dz  e	d$<   d%Zee	d&<   d'Zeez  e	d(<   d#Zee	d)<   d*Z ee	d<   d#Z!ee	d+<   d!Z"ee	d,<   dZ#ee	d-<   d.Z$ee	d/<   dZ%ee	d0<   d*Z&ee	d1<   d.Z'ee	d2<   d#Z(ee	d3<   d4Z)eez  e	d5<   dZ*ee	d6<   dZ+ee	d7<   ˆ fd8„Z,e-d9„ ¦   «         Z.e-d:„ ¦   «         Z/e-d;„ ¦   «         Z0d<„ Z1ˆ xZ2S )=ÚJambaConfiga   
    expert_layer_period (`int`, *optional*, defaults to 2):
        Once in this many layers, we will have an expert layer
    expert_layer_offset (`int`, *optional*, defaults to 1):
        The first layer index that contains an expert mlp layer
    attn_layer_period (`int`, *optional*, defaults to 8):
        Once in this many layers, we will have a vanilla attention layer
    attn_layer_offset (`int`, *optional*, defaults to 4):
        The first layer index that contains a vanilla attention mlp layer
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
        `causal-conv1d` are installed, and the mamba modules are running on a CUDA device. Raises ValueError if
        `True` and kernels are not available
    mamba_dt_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the mamba discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    ÚjambaÚpast_key_valuesÚnum_local_expertsÚnum_expertsi   Ú
vocab_sizeFÚtie_word_embeddingsi   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefr   NÚpad_token_idé   Úbos_token_idé   Úeos_token_idi   Úmax_position_embeddingsg        Úattention_dropoutÚnum_experts_per_toké   Úexpert_layer_periodÚexpert_layer_offsetÚattn_layer_periodé   Úattn_layer_offsetÚuse_mamba_kernelsÚmamba_d_stateÚmamba_d_convÚmamba_expandÚautoÚmamba_dt_rankÚmamba_conv_biasÚmamba_proj_biasc                 óÆ   •— | j         €| j        | _         | j        dk    rt          j        | j        dz  ¦  «        n| j        | _         t          ¦   «         j        di |¤Ž d S )Nr0   r&   © )r   r   r1   ÚmathÚceilr   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/jamba/configuration_jamba.pyr9   zJambaConfig.__post_init__R   sk   ø€ ØÔ#Ð+Ø'+Ô'?ˆDÔ$àAEÔASÐW]ÒA]ÐA]�TœY tÔ'7¸"Ñ'<Ñ=Ô=Ð=ÐcgÔcuˆÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 óD   ‡ — ˆ fd„t          ‰ j        ¦  «        D ¦   «         S )Nc                 ó>   •— g | ]}|‰j         z  ‰j        k    rd nd‘ŒS )Ú	attentionÚmamba)r)   r+   ©Ú.0Úir:   s     €r=   ú
<listcomp>z1JambaConfig.layers_block_type.<locals>.<listcomp>[   sC   ø€ ð 
ð 
ð 
àð ˜tÔ5Ñ5¸Ô9OÒOÐOˆKˆKÐU\ð
ð 
ð 
r>   ©Úranger   ©r:   s   `r=   Úlayers_block_typezJambaConfig.layers_block_typeY   ó9   ø€ ð
ð 
ð 
ð 
å˜4Ô1Ñ2Ô2ð
ñ 
ô 
ð 	
r>   c                 ó$   — d„ | j         D ¦   «         S )Nc                 ó"   — g | ]}|d k    rdnd‘ŒS )rB   Úlinear_attentionÚfull_attentionr5   )rD   Úxs     r=   rF   z+JambaConfig.layer_types.<locals>.<listcomp>c   s'   € ÐiÐiÐiÈQ a¨7¢l lÐ"Ð"Ð8HÐiÐiÐir>   )rJ   rI   s    r=   Úlayer_typeszJambaConfig.layer_types`   s   € ð jÐiÐRVÔRhÐiÑiÔiÐir>   c                 óD   ‡ — ˆ fd„t          ‰ j        ¦  «        D ¦   «         S )Nc                 óH   •— g | ]}|‰j         z  ‰j        k    r‰j        nd ‘ŒS )r   )r'   r(   r   rC   s     €r=   rF   z2JambaConfig.layers_num_experts.<locals>.<listcomp>g   sG   ø€ ð 
ð 
ð 
àð !" DÔ$<Ñ <ÀÔ@XÒ XÐ XˆDÔÐÐ^_ð
ð 
ð 
r>   rG   rI   s   `r=   Úlayers_num_expertszJambaConfig.layers_num_expertse   rK   r>   c                 óÆ   — | j         | j        k    r t          d| j         › d| j        › d�¦  «        ‚| j        | j        k    r t          d| j        › d| j        › d�¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.zattention layer offset (z/) must be smaller than attention layer period (ú)zexpert layer offset (z,) must be smaller than expert layer period (N)r+   r)   Ú
ValueErrorr(   r'   rI   s    r=   Úvalidate_architecturez!JambaConfig.validate_architecturel   s¹   € àÔ! TÔ%;Ò;Ð;Ýð L¨4Ô+Að  Lð  LÐrvô  sIð  Lð  Lð  Lñô ð ð Ô# tÔ'?Ò?Ð?Ýð J¨Ô(@ð  Jð  JÐnrô  oGð  Jð  Jð  Jñô ð ð @Ð?r>   )3Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   Ústrr   Úfloatr   r   r   r   r   r    r"   Úlistr#   r$   r%   r   r'   r(   r)   r+   r,   r-   r.   r/   r1   r2   r3   r9   ÚpropertyrJ   rQ   rT   rX   Ú__classcell__)r<   s   @r=   r	   r	      sâ  ø€ € € € € € ðð ð" €JØ#4Ð"5Ðà˜]ð€Mð €J�ÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø#)Ð˜SÐ)Ð)Ñ)Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø Ð˜Ð Ð Ñ Ø€K�ÐÐÑØ Ð˜Ð Ð Ñ Ø Ð˜Ð Ð Ñ ØÐ�sÐÐÑØÐ�sÐÐÑØ"Ð�tÐ"Ð"Ñ"Ø€M�3ÐÐÑØ€L�#ÐÐÑØ€L�#ÐÐÑØ%€M�3˜‘9Ð%Ð%Ñ%Ø €O�TÐ Ð Ñ Ø!€O�TÐ!Ð!Ñ!ð(ð (ð (ð (ð (ð ð
ð 
ñ „Xð
ð ðjð jñ „Xðjð ð
ð 
ñ „Xð
ð
ð 
ð 
ð 
ð 
ð 
ð 
r>   r	   )
r\   r6   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   Ú__all__r5   r>   r=   ú<module>rl      s²   ðð  Ð à €€€à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð0Ð1Ñ1Ô1Øð\ð \ð \ð \ð \Ð"ñ \ô \ñ „ñ 2Ô1ð\ð~ ˆ/€€€r>   