§
    ‚Štje  ã                   ó�   — d dl mZ ddlmZmZ ddlmZ ddlmZ  ed¬¦  «        e G d„ d	e¦  «        ¦   «         ¦   «         Z	d	gZ
d
S )é    )Ústricté   )ÚPreTrainedConfigÚremap_legacy_layer_types)ÚRopeParameters)Úauto_docstringzZyphra/Zamba2-2.7B)Ú
checkpointc                   óÈ  ‡ — e Zd ZU dZdZdddœZdgZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee         dz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee         eedf         z  dz  e	d<   d Zee	d!<   d"Zee	d#<   d"Zee	d$<   d%Zee	d&<   d'Zee	d(<   d'Z ee	d)<   dZ!edz  e	d*<   d+Z"ee	d,<   d-Z#ee	d.<   dZ$edz  e	d/<   d0Z%eez  e	d1<   dZ&ee	d2<   d'Z'ee	d3<   d4Z(ee	d5<   d'Z)ee	d6<   dZ*e+e,z  dz  e	d7<   d8Z-ee	d9<   d:Z.ee	d;<   d"Z/ee	d<<   dZ0ee	d=<   d>Z1edz  e	d?<   dZ2edz  e	d@<   dZ3eee         z  dz  e	dA<   d'Z4ee	dB<   d"Z5ee	dC<   ˆ fdD„Z6ˆ xZ7S )EÚZamba2Configað	  
    mamba_ngroups (`int`, *optional*, defaults to 1):
        Number of groups for the evolution matrices of mamba 2.
    n_mamba_heads (`int`, *optional*, defaults to 8):
        Number of heads for the evolution matrices of mamba 2.
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels.
    use_conv_bias (`bool`, *optional*, defaults to `True`):
        Whether or not to use bias in the convolution layer of the mixer block.
    chunk_size (`int`, *optional*, defaults to 256):
        Size of the chunks that will comprise the sequence.
    use_mem_eff_path (`bool`, *optional*, defaults to `False`):
        Whether or not to use the fused conv1d and scan in mamba2 layers.
    add_bias_linear (`bool`, *optional*, defaults to `False`):
        Flag indicating whether or not to use bias in various layers
    num_mem_blocks (`int`, *optional*, defaults to 1):
        Number of unshared transformer blocks.
    use_shared_attention_adapter (`bool`, *optional*, defaults to `False`):
        If True, unshared adapters (formally the same as LoRA but used in the base model) will be added to the q, k, v projectors in the shared attention layers.
    adapter_rank (`int`, *optional*, defaults to 128):
        Rank of the adapter in the shared MLP and shared attention layers.
    use_mem_rope (`bool`, *optional*, defaults to `False`):
        If True, includes RoPE in the shared attention layers.
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    use_long_context (`bool`, *optional*, defaults to `False`):
        Activates the context-extended version of Zamba by modifying RoPE.

    Example:
    ```python
    >>> from transformers import Zamba2Model, Zamba2Config
    >>> # Initializing a Zamba2-2.7B style configuration
    >>> configuration = Zamba2Config()
    >>> # Initializing a model from the Zamba2-2.7B style configuration
    >>> model = Zamba2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úzamba2Úlayers_block_typeÚattention_head_dim)Úlayer_typesÚhead_dimÚpast_key_valuesi }  Ú
vocab_sizei   Úmax_position_embeddingsi 
  Úhidden_sizeé6   Únum_hidden_layersNé@   Úmamba_d_stateé   Úmamba_d_convé   Úmamba_expandé   Úmamba_ngroupsgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxg-Cëâ6?Útime_step_floor.Útime_step_limité   Ún_mamba_headsTÚuse_mamba_kernelsÚuse_conv_biasé   Ú
chunk_sizeFÚuse_mem_eff_pathÚadd_bias_linearÚintermediate_sizeÚgeluÚ
hidden_acté    Únum_attention_headsÚnum_key_value_headsg        Úattention_dropoutÚnum_mem_blocksÚuse_shared_attention_adapteré€   Úadapter_rankÚuse_mem_ropeÚrope_parametersg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsÚ	use_cacheÚnum_logits_to_keepr   Úpad_token_idÚbos_token_idÚeos_token_idÚuse_long_contextÚtie_word_embeddingsc                 ót  •— | j         p	d| j        z  | _         d| j        z  | _        d| j        z  | j        z  | _        t          | j        | j        z  ¦  «        | j        z  | _        | j	        rd| _
        | j        €| j        | _        | j        | j        z  | _        | j        | _        | j        €4dgdgdz  dgz   dz  z   dgdz  z   dgz   dgdz  z   dgz   dgdz  z   | _        nt          | j        ¦  «        | _        d	„ t!          | j        ¦  «        D ¦   «         | _         t%          ¦   «         j        d
i |¤Ž d S )Nr   r   i @  Úlinear_attentioné   Úhybridé   r   c                 ó$   — g | ]\  }}|d k    ¯|‘ŒS )rD   © )Ú.0ÚindexÚtypes      úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zamba2/configuration_zamba2.pyú
<listcomp>z.Zamba2Config.__post_init__.<locals>.<listcomp>�   s(   € Ð pÐ pÐ p©;¨5°$Ð_cÐgoÒ_oÐ_o Ð_oÐ_oÐ_oó    rG   )r+   r   Úattention_hidden_sizer/   r   Úintr   r$   Úmamba_headdimr?   r   r0   Úkv_channelsÚnum_query_groupsr   r   Ú	enumerateÚhybrid_layer_idsÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €rK   rV   zZamba2Config.__post_init__q   sˆ  ø€ Ø!%Ô!7Ð!O¸1¸tÔ?OÑ;OˆÔØ%&¨Ô)9Ñ%9ˆÔ"Ø"# dÔ&6Ñ"6¸$Ô:RÑ"RˆÔÝ  Ô!2°TÔ5EÑ!EÑFÔFÈ$ÔJ\Ñ\ˆÔØÔ ð 	1Ø+0ˆDÔ(àÔ#Ð+Ø'+Ô'?ˆDÔ$àÔ+¨tÔ/GÑGˆÔØ $Ô 8ˆÔð Ô!Ð)à#Ð$Ø&Ð'¨!Ñ+¨x¨jÑ8¸AÑ=ñ>à%Ð&¨Ñ*ñ+ð �*ñð &Ð&¨Ñ*ñ	+ð
 �*ñð &Ð&¨Ñ*ñ+ð Ô"Ð"õ &>¸dÔ>TÑ%UÔ%UˆDÔ"Ø pÐ p½)ÀDÔDZÑ:[Ô:[Ð pÑ pÔ pˆÔØ�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'rM   )8Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚattribute_mapÚkeys_to_ignore_at_inferencer   rO   Ú__annotations__r   r   r   r   ÚlistÚstrr   r   r   r   r   Úfloatr    r!   r"   Útupler$   r%   Úboolr&   r(   r)   r*   r+   r-   r/   r0   r1   r2   r3   r5   r6   r7   r   Údictr8   r9   r:   r;   r<   r=   r>   r?   r@   rV   Ú__classcell__)rY   s   @rK   r   r      s  ø€ € € € € € ð)ð )ðV €JØ$7ÐEYÐZÐZ€MØ#4Ð"5Ðà€J�ÐÐÑØ#'Ð˜SÐ'Ð'Ñ'Ø€K�ÐÐÑØÐ�sÐÐÑØ*.Ð�t˜C”y 4Ñ'Ð.Ð.Ñ.Ø€M�3ÐÐÑØ€L�#ÐÐÑØ€L�#ÐÐÑØ€M�3ÐÐÑØ €M�5Ð Ð Ñ Ø€M�5ÐÐÑØ!€O�UÐ!Ð!Ñ!Ø>B€O�T˜%”[ 5¨°¨Ô#4Ñ4°tÑ;ÐBÐBÑBØ€M�3ÐÐÑØ"Ð�tÐ"Ð"Ñ"Ø€M�4ÐÐÑØ€J�ÐÐÑØ"Ð�dÐ"Ð"Ñ"Ø!€O�TÐ!Ð!Ñ!Ø$(Ð�s˜T‘zÐ(Ð(Ñ(Ø€J�ÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&*Ð˜˜t™Ð*Ð*Ñ*Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø€N�CÐÐÑØ).Ð  $Ð.Ð.Ñ.Ø€L�#ÐÐÑØ€L�$ÐÐÑØ48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØÐ˜ÐÐÑØ €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø"Ð�dÐ"Ð"Ñ"Ø $Ð˜Ð$Ð$Ñ$ð(ð (ð (ð (ð (ð (ð (ð (ð (rM   r   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   r   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__rG   rM   rK   ú<module>rn      s¼   ðð" /Ð .Ð .Ð .Ð .Ð .à MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð/Ð0Ñ0Ô0Øðt(ð t(ð t(ð t(ð t(Ð#ñ t(ô t(ñ „ñ 1Ô0ðt(ðn Ð
€€€rM   