§
    ‚Štj!  ã                   óœ   — d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Zd
gZdS )é    )ÚAnyÚLiteral)Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzpoolside/laguna-XS.2)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZi dd“dd“dd“dd“d	d
“dd“dd“dd“dd“dd
“dd“dd
“dd“dd“dd“dd
“ZdgdgfddgdgfdgdgfdœZddddd œZd!Z	e
ed"<   d#Ze
ed$<   d%Ze
ed&<   d'Ze
ed(<   d)Ze
ed*<   d+Ze
ed,<   d-Zeed.<   d/Ze
ed0<   d1Zeed2<   d3Zeed4<   d5Zeed6<   d7Zeed8<   d9Zeez  d9z  ed:<   d;Ze
ed<<   d=Zee
z  ed><   d;Ze
ed?<   d;Z e
ed@<   d+Z!e
edA<   dBZ"e
edC<   d7Z#eedD<   dEZ$eedF<   d9Z%e&e         d9z  edG<   d9Z'e
d9z  edH<   d9Z(e
d9z  edI<   d9Z)e
e&e
         z  d9z  edJ<   dKZ*e
edL<   d7Z+eedM<   d5Z,eez  edN<   d9Z-e&e
         d9z  edO<   d9Z.e&e         d9z  edP<   dQZ/eedR<   d7Z0eedS<   d=Z1eedT<   ˆ fdU„Z2dV„ Z3dW„ Z4ˆ xZ5S )XÚLagunaConfigu¤  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    ÚlagunaÚpast_key_valueszlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projú!layers.*.mlp.experts.gate_up_projÚpacked_colwiseúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   i ˆ Ú
vocab_sizei   Úhidden_sizei    Úintermediate_sizeé(   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsNÚrope_parametersi   Úsliding_windowg        Úattention_dropoutÚmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚnum_experts_per_toké   Únum_expertsÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚlayer_typesÚpad_token_idÚbos_token_idÚeos_token_idé€   Úhead_dimÚattention_biasÚgatingÚnum_attention_heads_per_layerÚmlp_layer_typesç      ð?Úmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputÚmoe_router_logit_softcappingc                 ó  •— | j         €dg| j        z  | _         | j        €dgdg| j        dz
  z  z   | _        | j        €| j        g| j        z  | _        ddddœdd	d
dœdœ}| j        €|| _         t          ¦   «         j        di |¤dddhi¤Ž d S )NÚfull_attentionÚdenseÚsparseé   Údefaultg    €„Ag      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@rD   )rI   Úsliding_attentionÚignore_keys_at_rope_validationrQ   © )r:   r$   rC   rB   r&   r0   ÚsuperÚ__post_init__)ÚselfÚkwargsÚdefault_rope_paramsÚ	__class__s      €úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/laguna/configuration_laguna.pyrU   zLagunaConfig.__post_init__‚   sÝ   ø€ ØÔÐ#Ø 0Ð1°DÔ4JÑJˆDÔØÔÐ'Ø$+ 9°¨z¸TÔ=SÐVWÑ=WÑ/XÑ#XˆDÔ ØÔ-Ð5Ø26Ô2JÐ1KÈdÔNdÑ1dˆDÔ.ð -6ÀXÐhkÐlÐlØ/8ÈÐjmÐ!nÐ!nðe
ð e
Ðð ÔÐ'Ø#6ˆDÔ ð 	�‰ŒÔÐoÐo ÐoÐoÐH[Ð]mÐGnÐoÐoÐoÐoÐoÐoó    c                 ó   — |S )NrS   )rV   rW   s     rZ   Úconvert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict”   s   € àˆr[   c                 óü  — | j         rt          d¦  «        ‚| j        �Jt          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚t          | j        ¦  «        | j        k    r-t          dt          | j        ¦  «        › d| j        › d�¦  «        ‚dS )z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rF   ÚNotImplementedErrorrB   Úlenr$   Ú
ValueErrorr:   rC   )rV   s    rZ   Úvalidate_architecturez"LagunaConfig.validate_architecture˜   sa  € àÔ0ð 	Ý%ð9ñô ð ð
 Ô.Ð:Ý�DÔ6Ñ7Ô7¸4Ô;QÒQÐQåðL½¸TÔ=_Ñ9`Ô9`ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔÑ Ô  DÔ$:Ò:Ð:ÝðL¥s¨4Ô+;Ñ'<Ô'<ð Lð LØ15Ô1GðLð Lð Lñô ð õ ˆtÔ#Ñ$Ô$¨Ô(>Ò>Ð>ÝðL­3¨tÔ/CÑ+DÔ+Dð Lð LØ15Ô1GðLð Lð Lñô ð ð ?Ð>r[   )6Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planr    ÚintÚ__annotations__r!   r"   r$   r&   r(   r*   Ústrr+   r,   Úfloatr-   r.   Úboolr/   r0   r   Údictr1   r2   r3   r4   r5   r7   r8   r9   r:   Úlistr;   r<   r=   r?   r@   rA   rB   rC   rE   rF   rG   rU   r]   rb   Ú__classcell__)rY   s   @rZ   r   r      sü  ø€ € € € € € ðð ð: €JØ#4Ð"5ÐðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ðð Ðð )Ø-;Ø*8Ø 0ð	ð Ðð €J�ÐÐÑØ€K�ÐÐÑØ!Ð�sÐ!Ð!Ñ!ØÐ�sÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ %Ð˜Ð%Ð%Ñ%Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø€N�CÐÐÑØ%(Ð�u˜s‘{Ð(Ð(Ñ(Ø!$Ð˜3Ð$Ð$Ñ$Ø+.Ð# SÐ.Ð.Ñ.Ø Ð˜Ð Ð Ñ Ø€K�ÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø"'Ð˜%Ð'Ð'Ñ'Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø#€L�#˜‘*Ð#Ð#Ñ#Ø#€L�#˜‘*Ð#Ð#Ñ#Ø+/€L�#˜˜Sœ	‘/ DÑ(Ð/Ð/Ñ/ð €HˆcÐÐÑØ €N�DÐ Ð Ñ Ø€FˆD�3‰JÐÐÑØ6:Ð! 4¨¤9¨tÑ#3Ð:Ð:Ñ:à(,€O�T˜#”Y Ñ%Ð,Ð,Ñ,Ø'*Ð˜uÐ*Ð*Ñ*Ø-2Ð$ dÐ2Ð2Ñ2Ø*-Ð  %Ð-Ð-Ñ-ðpð pð pð pð pð$ð ð ðð ð ð ð ð ð r[   r   N)Útypingr   r   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr	   r   Ú__all__rS   r[   rZ   ú<module>rz      sÎ   ðð(  Ð Ð Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð1Ð2Ñ2Ô2ØðQð Qð Qð Qð QÐ#ñ Qô Qñ „ñ 3Ô2ðQðh Ð
€€€r[   