
    <h`'                     `    S SK JrJr  S SKJr  \R
                  " \5      r " S S\5      rS/r	g)   )PretrainedConfiglayer_type_validation)loggingc                      ^  \ rS rSrSrSrS/r0 SS_SS_SS_S	S
_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_rS/S/4SS/S/4S/S/4S.r                            S"U 4S  jjr	S!r
U =r$ )#Dots1Config   a  
This is the configuration class to store the configuration of a [`Dots1Model`]. It is used to instantiate a
`dots.llm1` model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of
[rednote-hilab/dots.llm1.base](https://huggingface.co/rednote-hilab/dots.llm1.base).

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    vocab_size (`int`, *optional*, defaults to 152064):
        Vocabulary size of the model. Defines the number of different tokens that can be represented by the
        `input_ids` passed when calling [`Dots1Model`].
    hidden_size (`int`, *optional*, defaults to 4608):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 10944):
        Dimension of the MLP representations.
    moe_intermediate_size (`int`, *optional*, defaults to 1408):
        Dimension of the MoE representations.
    num_hidden_layers (`int`, *optional*, defaults to 62):
        Number of hidden layers in the Transformer decoder.
    num_attention_heads (`int`, *optional*, defaults to 32):
        Number of attention heads for each attention layer in the Transformer decoder.
    num_key_value_heads (`int`, *optional*, defaults to 32):
        Number of key/value heads for Grouped Query Attention. If `num_key_value_heads=num_attention_heads`, Multi
        Head Attention (MHA) is used. If `num_key_value_heads=1`, Multi Query Attention (MQA) is used. Otherwise,
        Grouped Query Attention (GQA) is used. If not specified, defaults to `num_attention_heads`.
    n_shared_experts (`int`, *optional*, default=None):
        Number of shared experts. None means dense model.
    n_routed_experts (`int`, *optional*, default=None):
        Number of routed experts. None means dense model.
    n_group (`int`, *optional*, defaults to 1):
        Number of groups for routed experts.
    topk_group (`int`, *optional*, defaults to 1):
        Number of selected groups for each token (selected experts only within `topk_group` groups).
    num_experts_per_tok (`int`, *optional*, default=None):
        Number of selected experts. None means dense model.
    first_k_dense_replace (`int`, *optional*, defaults to 0):
        Number of dense layers at the beginning of the model before the first MoE layer.
    norm_topk_prob (`bool`, *optional*, defaults to `False`):
        Whether to normalize the weights of the routed experts.
    hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string).
    max_position_embeddings (`int`, *optional*, defaults to 2048):
        Maximum sequence length the model might ever be used with.
    initializer_range (`float`, *optional*, defaults to 0.02):
        Standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
        Epsilon used by the RMS normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions. Only relevant if `config.is_decoder=True`.
    tie_word_embeddings (`bool`, *optional*, defaults to `False`):
        Whether to tie the input and output word embeddings.
    rope_theta (`float`, *optional*, defaults to 10000.0):
        The base period of the RoPE embeddings.
    rope_scaling (`dict`, *optional*):
        Dictionary for scaling RoPE embeddings. Supports `{"type": strategy name, "factor": scaling factor}`.
    attention_bias (`bool`, *optional*, defaults to `False`):
        Whether to use a bias in the self-attention projections.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        Dropout ratio for the attention probabilities.
    routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scaling factor for routed experts.
    sliding_window (`int`, *optional*, defaults to 4096):
        Size of the sliding window for attention. If not specified, defaults to `4096`.
    max_window_layers (`int`, *optional*, defaults to 62):
        The number of layers using full attention. The first `max_window_layers` layers will use full attention, while any
        additional layer afterwards will use SWA (Sliding Window Attention).
    layer_types (`list`, *optional*):
        Attention pattern for each layer.

Examples:
    ```python
    >>> from transformers import Dots1Model, Dots1Config

    >>> # Initializing a Dots1 style configuration
    >>> configuration = Dots1Config()

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
dots1past_key_valueszlayers.*.self_attn.q_projcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projrowwisez layers.*.mlp.experts.*.gate_projlocal_colwisezlayers.*.mlp.experts.*.up_projz layers.*.mlp.experts.*.down_projlocal_rowwisezlayers.*.mlp.experts.*localz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.shared_expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlpgather	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormc                 p  > Xl         UU l        X l        X0l        X@l        XPl        X`l        Xl        Xl        Xl	        Xl
        Xl        Uc  UnXl        Xl        Xpl        Xl        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        UU l        U R4                  cI  [7        U R
                  5       Vs/ sH$  nU R0                  b  UU R2                  :  a  SOSPM&     snU l        [9        U R4                  5        [:        TU ]x  " SSU0UD6  g s  snf )Nsliding_attentionfull_attentiontie_word_embeddings )
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizemoe_intermediate_sizenum_hidden_layersnum_attention_headsn_shared_expertsn_routed_expertsnum_experts_per_tokfirst_k_dense_replacenorm_topk_probn_group
topk_groupnum_key_value_heads
hidden_actinitializer_rangerms_norm_eps	use_cache
rope_thetarope_scalingattention_biasattention_dropoutrouted_scaling_factorsliding_windowmax_window_layerslayer_typesranger   super__init__) selfr   r   r    r!   r"   r#   r+   r$   r%   r)   r*   r&   r'   r(   r,   r   r-   r.   r/   r   r0   r1   r2   r3   r4   r5   r6   r7   kwargsi	__class__s                                   e/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/dots1/configuration_dots1.pyr:   Dots1Config.__init__   sX   @ %'>$&!2%:"!2#6  0 0#6 %:",&"5$#6 $!2("$(,!2%:",!2&#
 t556	  7A &&2qD<R<R7R $%& 7	 D 	d../ 	
 3	
	
 s   *D3)r2   r3   r'   r,   r   r-   r    r7   r   r6   r!   r)   r%   r$   r(   r#   r&   r"   r+   r.   r1   r0   r4   r5   r*   r/   r   )i R i   i*  i  >       rB   NN   rC   N    Fsilui   g{Gz?gư>TFg     @NFg        g      ?i   rA   N)__name__
__module____qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr:   __static_attributes____classcell__)r>   s   @r?   r   r      sv   Qf J#4"5#Y#Y 	$Y 	$Y	
 	+O 	)/ 	+O 	!' 	0 	. 	0 	&w 	!/ 	 	!/  	!( &(9:#%568IJ!"_$56 "  $!!;J
 J
    r   N)
configuration_utilsr   r   utilsr   
get_loggerrF   loggerr   __all__r   rQ   r?   <module>rW      s9    K  
		H	%z
" z
z /rQ   