§
    ‚Štj¦  ã                   óŒ   — d Z ddlZddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Zd
gZ	dS )zZamba model configurationé    N)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzZyphra/Zamba-7B-v1)Ú
checkpointc                   óN  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   d	Zee	d <   d!Zee	d"<   d#Zedz  e	d$<   d!Zedz  e	d%<   dZeee         z  dz  e	d&<   d'Z ee	d(<   d)Z!eez  e	d*<   d+Z"ee	d,<   d-Z#ee	d.<   d	Z$ee	d/<   dZ%ee	d0<   d-Z&ee	d1<   dZ'ee	d2<   d3Z(eez  e	d4<   d5Z)ee	d6<   d7Z*ee	d8<   d9Z+ee	d:<   d	Z,ee	d;<   d<Z-ee	d=<   ˆ fd>„Z.d?„ Z/d@„ Z0ˆ xZ1S )AÚZambaConfiga!  
    attention_hidden_size (`int`, *optional*):
        Dimension of the hidden representations of the inputs to the Attention layer.
    attention_head_dim (`int`, *optional*):
        Dimension of the attention head in the Transformer decoder.
    n_mamba_heads (`int`, *optional*, defaults to 2):
        Number of mamba heads for each mamba layer.
    hidden_mamba_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the mamba layer.
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    attn_layer_period (`int`, *optional*, defaults to 6):
        Once in this many layers, we will have a shared attention layer
    attn_layer_offset (`int`, *optional*, defaults to 4):
        Offset of the shared attention layer
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
        `causal-conv1d` are installed, and the mamba modules are running on a CUDA device. Raises ValueError if
        `True` and kernels are not available
    mamba_dt_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the mamba discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    ÚzambaÚpast_key_valuesÚlayers_block_typeÚattention_head_dim)Úlayer_typesÚhead_dimi }  Ú
vocab_sizeTÚtie_word_embeddingsi€  Úhidden_sizeNÚattention_hidden_sizei :  Úintermediate_sizeéL   Únum_hidden_layersé   Únum_attention_headsÚnum_key_value_headsé   Ún_mamba_headsÚgeluÚ
hidden_actÚsiluÚhidden_mamba_actg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsÚ	use_cacheé   Únum_logits_to_keepr   Úpad_token_idÚbos_token_idÚeos_token_idi   Úmax_position_embeddingsg        Úattention_dropouté   Úattn_layer_periodé   Úattn_layer_offsetÚuse_mamba_kernelsÚmamba_d_stateÚmamba_d_convÚmamba_expandÚautoÚmamba_dt_rankgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxg-Cëâ6?Útime_step_floorÚmamba_conv_biasFÚmamba_proj_biasc                 ó^  •— | j         p	d| j        z  | _         | j        pd| j        z  | j        z  | _        | j        dk    rt          j        | j        dz  ¦  «        n| j        | _        |                      | j        | j	        | j
        ¦  «        | _         t          ¦   «         j        di |¤Ž d S )Nr   r2   r   © )r   r   r   r   r3   ÚmathÚceilÚ_layers_block_typer   r+   r-   r   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zamba/configuration_zamba.pyr?   zZambaConfig.__post_init__\   sº   ø€ Ø%)Ô%?Ð%WÀ1ÀtÔGWÑCWˆÔ"Ø"&Ô"9Ð"m¸QÀÔAQÑ=QÐUYÔUmÑ=mˆÔØAEÔASÐW]ÒA]ÐA]�TœY tÔ'7¸"Ñ'<Ñ=Ô=Ð=ÐcgÔcuˆÔØ!%×!8Ò!8ØÔ" DÔ$:¸DÔ<Rñ"
ô "
ˆÔð 	�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 óZ   — | j         | j        z  | j        z  dk    rt          d¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z;`intermediate_size` should be divisible by `n_mamba_heads`.N)r1   r   r   Ú
ValueError)r@   s    rC   Úvalidate_architecturez!ZambaConfig.validate_architecturee   s8   € àÔ Ô 0Ñ0°DÔ4FÑFÈ!ÒKÐKÝÐZÑ[Ô[Ð[ð LÐKrD   c                 óR   ‡‡— g d¢ˆˆfd„t          |dz
  ¦  «        D ¦   «         z   }|S )N)Úlinear_attentionrI   Úhybridc                 ó*   •— g | ]}|‰z  ‰k    rd nd‘ŒS )rJ   rI   r:   )Ú.0Úir-   r+   s     €€rC   ú
<listcomp>z2ZambaConfig._layers_block_type.<locals>.<listcomp>o   s?   ø€ ð 
ð 
ð 
àð Ð-Ñ-Ð1BÒBÐBˆHˆHÐHZð
ð 
ð 
rD   r   )Úrange)r@   r   r+   r-   Úlayerss     `` rC   r=   zZambaConfig._layers_block_typej   sZ   øø€ ð
ð 
ð 
ð
ð 
ð 
ð 
ð 
åÐ,¨qÑ0Ñ1Ô1ð
ñ 
ô 
ñ	
ˆð ˆrD   )2Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   r   r   r   Ústrr   r    Úfloatr!   r"   r$   r%   r&   r'   Úlistr(   r)   r+   r-   r.   r/   r0   r1   r3   r4   r5   r6   r7   r8   r?   rG   r=   Ú__classcell__)rB   s   @rC   r	   r	      sÇ  ø€ € € € € € ðð ð6 €JØ#4Ð"5ÐØ$7ÐEYÐZÐZ€Mà€J�ÐÐÑØ $Ð˜Ð$Ð$Ñ$Ø€K�ÐÐÑØ(,Ð˜3 ™:Ð,Ð,Ñ,Ø"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø%)Ð˜˜d™
Ð)Ð)Ñ)Ø!Ð˜Ð!Ð!Ñ!Ø€M�3ÐÐÑØ€J�ÐÐÑØ"Ð�cÐ"Ð"Ñ"Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØÐ˜ÐÐÑØ €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø#'Ð˜SÐ'Ð'Ñ'Ø%(Ð�u˜s‘{Ð(Ð(Ñ(ØÐ�sÐÐÑØÐ�sÐÐÑØ"Ð�tÐ"Ð"Ñ"Ø€M�3ÐÐÑØ€L�#ÐÐÑØ€L�#ÐÐÑØ%€M�3˜‘9Ð%Ð%Ñ%Ø €M�5Ð Ð Ñ Ø€M�5ÐÐÑØ!€O�UÐ!Ð!Ñ!Ø €O�TÐ Ð Ñ Ø!€O�TÐ!Ð!Ñ!ð(ð (ð (ð (ð (ð\ð \ð \ð
	ð 	ð 	ð 	ð 	ð 	ð 	rD   r	   )
rT   r;   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   Ú__all__r:   rD   rC   ú<module>rc      s²   ðð  Ð à €€€à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð/Ð0Ñ0Ô0ØðYð Yð Yð Yð YÐ"ñ Yô Yñ „ñ 1Ô0ðYðx ˆ/€€€rD   