§
    ‚Štjå  ã                   ó’   — d Z ddlmZ ddlmZ ddlmZ ddlmZ e edd¬	¦  «         G d
„ de¦  «        ¦   «         ¦   «         Z	dgZ
dS )zBamba model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz´
    The BambaModel is a hybrid [mamba2](https://github.com/state-spaces/mamba) architecture with SwiGLU.
    The checkpoints are  jointly trained by IBM, Princeton, and UIUC.
    zibm-fms/Bamba-9.8b-2.2T-hf)Úcustom_introÚ
checkpointc                   ó  ‡ — e Zd ZU dZdZddiZdgZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d <   d!Zeee         z  dz  e	d"<   d#Zee	d$<   d%Zeez  dz  e	d&<   dZee         dz  e	d'<   d(Zedz  e	d)<   d*Z eez  dz  e	d+<   dZ!edz  e	d,<   d-Z"edz  e	d.<   d/Z#edz  e	d0<   d!Z$edz  e	d1<   d-Z%edz  e	d2<   dZ&edz  e	d3<   dZ'edz  e	d4<   d5Z(edz  e	d6<   d7Z)edz  e	d8<   d% ed9¦  «        fZ*ee         e+eef         z  dz  e	d:<   d%Z,edz  e	d;<   dZ-e.e/z  dz  e	d<<   dZ0ee	d=<   dZ1ee	d><   ˆ fd?„Z2e3d@„ ¦   «         Z4dA„ Z5ˆ xZ6S )BÚBambaConfigaN  
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    attn_layer_indices (`list`, *optional*):
        Specifies the layer indices that will have full attention. Must contain values at most num_hidden_layers.
    z_loss_coefficient (`float`, *optional*, defaults to 0.0):
        Coefficient for auxiliary z-loss used to control logit growth during training
    ÚbambaÚlayer_typesÚlayers_block_typeÚpast_key_valuesi ô Ú
vocab_sizeFÚtie_word_embeddingsi   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   NÚnum_key_value_headsÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheé   Únum_logits_to_keepr   Úpad_token_idÚbos_token_idé   Úeos_token_idi   Úmax_position_embeddingsg        Úattention_dropoutÚattn_layer_indicesé€   Úmamba_n_headsÚautoÚmamba_d_headÚmamba_n_groupsé   Úmamba_d_stateé   Úmamba_d_convÚmamba_expandÚmamba_chunk_sizeÚmamba_conv_biasÚmamba_proj_biasgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxÚinfÚtime_step_limitÚz_loss_coefficientÚrope_parametersÚattention_biasÚmlp_biasc                 óü   •— | j         €| j        | _         | j        dk    r| j        | j        z  | j        z  | _        | j        �t          | j        ¦  «        nd | _        d|d<    t          ¦   «         j	        di |¤Ž d S )Nr)   g      à?Úpartial_rotary_factor© )
r   r   r*   r0   r   r(   r7   ÚtupleÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/bamba/configuration_bamba.pyrA   zBambaConfig.__post_init__T   s‘   ø€ àÔ#Ð+Ø'+Ô'?ˆDÔ$ð Ô Ò&Ð&Ø $Ô 1°DÔ4DÑ DÈÔHZÑ ZˆDÔà>BÔ>RÐ>^�u TÔ%9Ñ:Ô:Ð:ÐdhˆÔØ*-ˆÐ&Ñ'à�‰ŒÔÐ'Ð' Ð'Ð'Ð'Ð'Ð'ó    c                 óD   ‡ — ˆ fd„t          ‰ j        ¦  «        D ¦   «         S )Nc                 ó8   •— g | ]}‰j         r|‰j         v rd nd‘ŒS )Úfull_attentionÚlinear_attention)r&   )Ú.0ÚirB   s     €rE   ú
<listcomp>z1BambaConfig.layers_block_type.<locals>.<listcomp>d   sE   ø€ ð 
ð 
ð 
àð "&Ô!8Ðr¸QÀ$ÔBYÐ=YÐ=YÐÐÐ`rð
ð 
ð 
rF   )Úranger   )rB   s   `rE   r   zBambaConfig.layers_block_typeb   s9   ø€ ð
ð 
ð 
ð 
å˜4Ô1Ñ2Ô2ð
ñ 
ô 
ð 	
rF   c                 ó¢   — | j         | j        z  }|| j        z  dk    rt          d¦  «        ‚| j        | j        z  |k    rt          d¦  «        ‚dS )zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z4mamba_n_heads must divide mamba_expand * hidden_sizezPThe dimensions for the Mamba head state do not match the model intermediate_sizeN)r0   r   r(   Ú
ValueErrorr*   )rB   Úmamba_intermediates     rE   Úvalidate_architecturez!BambaConfig.validate_architecturei   sd   € à!Ô.°Ô1AÑAÐØ Ô 2Ñ2°aÒ7Ð7ÝÐSÑTÔTÐTàÔ˜tÔ1Ñ1Ð5GÒGÐGÝÐoÑpÔpÐpð HÐGrF   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚattribute_mapÚkeys_to_ignore_at_inferencer   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   Ústrr   Úfloatr   r   r   r    r!   r#   Úlistr$   r%   r&   r(   r*   r+   r-   r/   r0   r1   r2   r3   r4   r5   r7   r?   r8   r9   r   Údictr:   r;   rA   Úpropertyr   rR   Ú__classcell__)rD   s   @rE   r   r      sN  ø€ € € € € € ðð ð €JØ"Ð$7Ð8€MØ#4Ð"5Ðà€J�ÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø€K�ÐÐÑØ"Ð�sÐ"Ð"Ñ"ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø&'Ð˜˜t™Ð'Ð'Ñ'Ø€J�ÐÐÑØ#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ%&Ð˜˜d™
Ð&Ð&Ñ&Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+,€L�#˜˜Sœ	‘/ DÑ(Ð,Ð,Ñ,Ø#)Ð˜SÐ)Ð)Ñ)Ø,/Ð�u˜s‘{ TÑ)Ð/Ð/Ñ/Ø+/Ð˜˜Sœ	 DÑ(Ð/Ð/Ñ/Ø #€M�3˜‘:Ð#Ð#Ñ#Ø%+€L�#˜‘)˜dÑ"Ð+Ð+Ñ+Ø!"€N�C˜$‘JÐ"Ð"Ñ"Ø #€M�3˜‘:Ð#Ð#Ñ#Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø#&Ð�c˜D‘jÐ&Ð&Ñ&Ø#'€O�T˜D‘[Ð'Ð'Ñ'Ø#(€O�T˜D‘[Ð(Ð(Ñ(Ø"'€M�5˜4‘<Ð'Ð'Ñ'Ø"%€M�5˜4‘<Ð%Ð%Ñ%ØADÀeÀeÈEÁlÄlÐ@S€O�T˜%”[ 5¨°¨Ô#6Ñ6¸Ñ=ÐSÐSÑSØ'*Ð˜ ™Ð*Ð*Ñ*Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø €N�DÐ Ð Ñ Ø€HˆdÐÐÑð(ð (ð (ð (ð (ð ð
ð 
ñ „Xð
ðqð qð qð qð qð qð qrF   r   N)rV   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__r>   rF   rE   ú<module>rh      sÒ   ðð  Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð Ø€ðð ,ðñ ô ðQqð Qqð Qqð Qqð QqÐ"ñ Qqô Qqñô ñ „ðQqðh ˆ/€€€rF   