§
    ‚ŠtjÙ  ã                   óŒ   — d dl mZ ddlmZ ddlmZ ddlmZ  ed¬¦  «        e G d„ d	e¦  «        ¦   «         ¦   «         Zd	gZ	d
S )é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz JetBrains/Mellum2-12B-A2.5B-Base)Ú
checkpointc                   óv  ‡ — e Zd ZU dZdZdgZddiZddddddd	dd
ddddœZdddd
dœZdgdgfddgdgfdgdgfdœZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d*Zeed+<   d,Zeez  d,z  ed-<   d*Zeed.<   d/Zed,z  ed0<   d1Z eez  ed2<   d3Z!eed4<   d5Z"eed6<   d7Z#eed<   d(Z$eed8<   d*Z%eed9<   d:Z&eed;<   d,Z'ed,z  ed<<   d,Z(ed,z  ed=<   d,Z)ee*e         z  d,z  ed><   d?Z+eed@<   d,Z,e*e         d,z  edA<   d,Z-e*e         d,z  edB<   ˆ fdC„Z.dD„ Z/ˆ xZ0S )EÚMellumConfigu}  
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” `"dense"` or `"sparse"`. Length must equal
        `num_hidden_layers`. Defaults to all sparse.

    ```python
    >>> from transformers import MellumModel, MellumConfig

    >>> configuration = MellumConfig()
    >>> model = MellumModel(configuration)
    >>> configuration = model.config
    ```
    ÚmellumÚpast_key_valuesÚnum_expertsÚnum_local_expertsÚcolwiseÚreplicated_with_grad_allreduceÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Ú	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi € Ú
vocab_sizei 	  Úhidden_sizei   Úintermediate_sizeé   Únum_hidden_layersé    Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsNÚrope_parametersÚattention_biasi   Úsliding_windowg        Úattention_dropouti€  Úmoe_intermediate_sizeé   Únum_experts_per_toké@   Únorm_topk_probÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚpad_token_idÚbos_token_idÚeos_token_idé€   Úhead_dimÚlayer_typesÚmlp_layer_typesc                 óÐ   •— | j         €dg| j        z  | _         | j        €dg| j        z  | _        | j        €dddœdddœdœ| _         t	          ¦   «         j        d
i |¤dd	dhi¤Ž d S )NÚfull_attentionÚsparseÚdefaultg    €„A)Ú	rope_typeÚ
rope_thetag     ˆÃ@)rC   Úsliding_attentionÚignore_keys_at_rope_validationrH   © )r@   r$   rA   r0   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mellum/configuration_mellum.pyrL   zMellumConfig.__post_init__o   s´   ø€ ØÔÐ#Ø 0Ð1°DÔ4JÑJˆDÔØÔÐ'Ø$, :°Ô0FÑ#FˆDÔ àÔÐ'à09ÈÐ"RÐ"RØ3<ÈGÐ%TÐ%Tð$ð $ˆDÔ ð
 	�‰ŒÔð 	
ð 	
Øð	
ð 	
à,?ÐAQÐ+Rð	
ð 	
ð 	
ð 	
ð 	
ð 	
ó    c                 ó   — |S )NrJ   )rM   rN   s     rP   Úconvert_rope_params_to_dictz(MellumConfig.convert_rope_params_to_dict€   s   € àˆrQ   )1Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapÚbase_model_tp_planÚbase_model_ep_planÚbase_model_pp_planr    ÚintÚ__annotations__r!   r"   r$   r&   r(   r*   Ústrr+   r,   Úfloatr-   r.   Úboolr/   r0   Údictr   r1   r2   r3   r4   r6   r   r8   r9   r:   r;   r<   r=   Úlistr?   r@   rA   rL   rS   Ú__classcell__)rO   s   @rP   r
   r
      s  ø€ € € € € € ðð ð €JØ#4Ð"5Ðð 	Ð*ð€Mð &/Ø%.Ø%.Ø%EØ%EØ%.Ø-=Ø*3Ø 0Ø"+Ø )Ø"+ðð Ðð$ )Ø-;Ø*8Ø 0ð	ð Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø48€O�T˜NÑ*¨TÑ1Ð8Ð8Ñ8Ø €N�DÐ Ð Ñ Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø!$Ð˜3Ð$Ð$Ñ$Ø Ð˜Ð Ð Ñ Ø€K�ÐÐÑØ€N�DÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø#€L�#˜‘*Ð#Ð#Ñ#Ø#€L�#˜‘*Ð#Ð#Ñ#Ø+/€L�#˜˜Sœ	‘/ DÑ(Ð/Ð/Ñ/Ø€HˆcÐÐÑØ$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,ð
ð 
ð 
ð 
ð 
ð"ð ð ð ð ð ð rQ   r
   N)
Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rJ   rQ   rP   ú<module>rk      s¶   ðð* /Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð=Ð>Ñ>Ô>Øðdð dð dð dð dÐ#ñ dô dñ „ñ ?Ô>ðdðN Ð
€€€rQ   