
    <h'                     `    S r SSKJr  SSKJr  \R
                  " \5      r " S S\5      rS/r	g)zBamba model configuration   )PretrainedConfig)loggingc                      ^  \ rS rSrSrSrS/r                            SU 4S jjr\S 5       r	Sr
U =r$ )	BambaConfig   a  
This is the configuration class to store the configuration of a [`BambaModel`]. It is used to instantiate a
BambaModel model according to the specified arguments, defining the model architecture. Instantiating a configuration
with defaults taken from [ibm-fms/Bamba-9.8b-2.2T-hf](https://huggingface.co/ibm-fms/Bamba-9.8b-2.2T-hf).

The BambaModel is a hybrid [mamba2](https://github.com/state-spaces/mamba) architecture with SwiGLU.
The checkpoints are  jointly trained by IBM, Princeton, and UIUC.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    vocab_size (`int`, *optional*, defaults to 128000):
        Vocabulary size of the Bamba model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`BambaModel`]
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
        model has an output word embedding layer.
    hidden_size (`int`, *optional*, defaults to 4096):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 14336):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 32):
        Number of hidden layers in the Transformer encoder.
    num_attention_heads (`int`, *optional*, defaults to 32):
        Number of attention heads for each attention layer in the Transformer encoder.
    num_key_value_heads (`int`, *optional*, defaults to 8):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `8`.
    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    pad_token_id (`int`, *optional*, defaults to 0):
        The id of the padding token.
    bos_token_id (`int`, *optional*, defaults to 1):
        The id of the "beginning-of-sequence" token.
    eos_token_id (`int`, *optional*, defaults to 2):
        The id of the "end-of-sequence" token.
    max_position_embeddings (`int`, *optional*, defaults to 262144):
        Max cached sequence length for the model
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    attn_layer_indices (`list`, *optional*):
        Specifies the layer indices that will have full attention. Must contain values at most num_hidden_layers.
    mamba_n_heads (`int`, *optional*, defaults to 128):
        The number of mamba heads used in the v2 implementation.
    mamba_d_head (`int`, *optional*, defaults to `"auto"`):
        Head embedding dimension size
    mamba_n_groups (`int`, *optional*, defaults to 1):
        The number of the mamba groups used in the v2 implementation.
    mamba_d_state (`int`, *optional*, defaults to 256):
        The dimension the mamba state space latents
    mamba_d_conv (`int`, *optional*, defaults to 4):
        The size of the mamba convolution kernel
    mamba_expand (`int`, *optional*, defaults to 2):
        Expanding factor (relative to hidden_size) used to determine the mamba intermediate size
    mamba_chunk_size (`int`, *optional*, defaults to 256):
        The chunks in which to break the sequence when doing prefill/training
    mamba_conv_bias (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use bias in the convolution layer of the mamba mixer block.
    mamba_proj_bias (`bool`, *optional*, defaults to `False`):
        Flag indicating whether or not to use bias in the input and output projections (["in_proj", "out_proj"]) of the mamba mixer block
    z_loss_coefficient (`float`, *optional*, defaults to 0.0):
        Coefficient for auxiliary z-loss used to control logit growth during training

bambapast_key_valuesc                 6  > Xl         X l        X0l        X@l        XPl        X`l        UU l        UU l        SU l        SU l	        Uc  UnXpl
        Xl        Xl        Xl        Xl        Xl        UU l        SU l        S U l        SU l        UU-  nUU-  S:w  a  [)        S5      eUS:X  a  UU-  nUU-  U:w  a  [)        S5      eUU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        [>        TU ]  " S	UUUUS.UD6  g )
NFg     @g      ?    z4mamba_n_heads must divide mamba_expand * hidden_sizeautozPThe dimensions for the Mamba head state do not match the model intermediate_size)pad_token_idbos_token_ideos_token_idtie_word_embeddings )!
vocab_sizer   hidden_sizeintermediate_sizenum_hidden_layersnum_attention_headsmax_position_embeddingsattention_dropoutattention_biasmlp_biasnum_key_value_heads
hidden_actinitializer_rangerms_norm_eps	use_cachenum_logits_to_keepattn_layer_indices
rope_thetarope_scalingpartial_rotary_factor
ValueErrormamba_n_headsmamba_d_headmamba_n_groupsmamba_d_statemamba_d_convmamba_expandmamba_chunk_sizemamba_conv_biasmamba_proj_biasz_loss_coefficientsuper__init__) selfr   r   r   r   r   r   r   r   r   r   r   r    r   r   r   r   r   r!   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   kwargsmamba_intermediate	__class__s                                   e/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/bamba/configuration_bamba.pyr1   BambaConfig.__init__o   sa   @ %#6 &!2!2#6 '>$!2# &"5#6 $!2(""4"4! %(")K7-2STT 6!->L-'+==opp*(,*(( 0.."4 	
%%% 3		

 	
    c                     [        U R                  5       Vs/ sH'  nU R                  (       a  XR                  ;   a  SOSPM)     sn$ s  snf )N	attentionmamba)ranger   r!   )r2   is     r6   layers_block_typeBambaConfig.layers_block_type   sM     4112
2 !33=T=T8TK[bb2
 	
 
s   -A)r   r   r!   r   r   r   r   r,   r-   r*   r'   r)   r+   r(   r&   r.   r   r   r   r   r   r    r$   r   r#   r"   r   r   r   r/   )i  Fi   i 8      r@      silug{Gz?gh㈵>T   r   rC      i           N   r   rC         rD   rG   TFrE   )__name__
__module____qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferencer1   propertyr>   __static_attributes____classcell__)r5   s   @r6   r   r      s    Qf J#4"5 ! &;Y
v 
 
r8   r   N)
rM   configuration_utilsr   utilsr   
get_loggerrI   loggerr   __all__r   r8   r6   <module>rX      s;      3  
		H	%w
" w
t /r8   