§
    ‚Štj¶  ã                   óœ   — d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d¬¦  «        e G d	„ d
e¦  «        ¦   «         ¦   «         Zd
gZdS )é    )ÚAnyÚLiteral)Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzZyphra/ZAYA1-8B)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdddddœZdZee	d	<   d
Z
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZeez  dz  e	d<   dZedz  e	d<   d Zeez  e	d!<   d
Zee	d"<   d#Zee	d$<   d%Zee	d&<   d'Zee	d(<   dZ e!e         dz  e	d)<   d*Z"edz  e	d+<   dZ#edz  e	d,<   d-Z$ee!e         z  dz  e	d.<   d/Z%ee	d0<   d'Z&ee	d1<   d'Z'ee	d2<   d3Z(ee	d4<   dZ)ee	d5<   dZ*ee	d6<   ˆ fd7„Z+d8„ Z,d9„ Z-ˆ xZ.S ):Ú
ZayaConfiga€  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    ÚzayaÚpast_key_valuesÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.gatez!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsi€  Ú
vocab_sizei   Úhidden_sizeé(   Únum_hidden_layersé   Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚtie_word_embeddingsNÚrope_parametersÚsliding_windowg        Úattention_dropoutÚmoe_intermediate_sizeé   Únum_experts_per_toké   Únum_expertsFÚoutput_router_logitsÚlayer_typesr   Úpad_token_idÚbos_token_idéj   Úeos_token_idé€   Úhead_dimÚattention_biasÚlm_head_biasé   Úrouter_hidden_sizeÚ	cca_time0Ú	cca_time1c                 óÒ   •— | j         €dg| j        z  nt          | j         ¦  «        | _         ddddœddddœdœ}| j        €|| _         t	          ¦   «         j        d
i |¤ddd	hi¤Ž d S )NÚhybridÚdefaultg    ÐSAg      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@)r8   Úhybrid_slidingÚignore_keys_at_rope_validationr=   © )r*   r   Úlistr!   ÚsuperÚ__post_init__)ÚselfÚkwargsÚdefault_rope_paramsÚ	__class__s      €úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zaya/configuration_zaya.pyrB   zZayaConfig.__post_init___   s¯   ø€ ØBFÔBRÐBZ˜H˜:¨Ô(>Ñ>Ð>Õ`dÐeiÔeuÑ`vÔ`vˆÔð 'Ø)Ø),ðð ð 'Ø&Ø),ðð ðZ
ð Z
Ðð ÔÐ'Ø#6ˆDÔ à�‰ŒÔÐdÐd ÐdÐdÈÐRbÐGcÐdÐdÐdÐdÐdÐdó    c                 ó   — |S )Nr?   )rC   rD   s     rG   Úconvert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dicts   s   € àˆrH   c                 óÀ   — | j         dk    rt          d¦  «        ‚| j        | j        z  dk    rt          d¦  «        ‚d| j        v r| j        €t          d¦  «        ‚dS dS )z'Part of ``@strict``-powered validation.r%   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.r=   NzJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r&   Ú
ValueErrorr   r   r*   r"   )rC   s    rG   Úvalidate_architecturez ZayaConfig.validate_architecturew   sw   € àÔ# qÒ(Ð(ÝÐTÑUÔUÐUØÔ# dÔ&>Ñ>À!ÒCÐCÝÐaÑbÔbÐbØ˜tÔ/Ð/Ð/°DÔ4GÐ4OÝÐiÑjÔjÐjð 0Ð/Ð4OÐ4OrH   )/Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_ep_planr   ÚintÚ__annotations__r   r   r   r   r   Ústrr   r   Úfloatr   r   Úboolr    r!   r   Údictr"   r#   r$   r&   r(   r)   r*   r@   r+   r,   r.   r0   r1   r2   r4   r5   r6   rB   rJ   rM   Ú__classcell__)rF   s   @rG   r   r      s–  ø€ € € € € € ðð ð( €JØ#4Ð"5Ðà(Ø-;Ø*8Ø 0ð	ð Ðð €J�ÐÐÑØ€K�ÐÐÑØÐ�sÐÐÑØ Ð˜Ð Ð Ñ Ø Ð˜Ð Ð Ñ Ø€J�ÐÐÑØ#)Ð˜SÐ)Ð)Ñ)Ø#Ð�uÐ#Ð#Ñ#Ø€L�%ÐÐÑØ€IˆtÐÐÑØ $Ð˜Ð$Ð$Ñ$Ø48€O�^ dÑ*¨TÑ1Ð8Ð8Ñ8Ø!%€N�C˜$‘JÐ%Ð%Ñ%Ø%(Ð�u˜s‘{Ð(Ð(Ñ(Ø!%Ð˜3Ð%Ð%Ñ%à Ð˜Ð Ð Ñ Ø€K�ÐÐÑØ!&Ð˜$Ð&Ð&Ñ&Ø$(€K��c”˜TÑ!Ð(Ð(Ñ(Ø €L�#˜‘*Ð Ð Ñ Ø €L�#˜‘*Ð Ð Ñ Ø+.€L�#˜˜Sœ	‘/ DÑ(Ð.Ð.Ñ.ð €HˆcÐÐÑØ €N�DÐ Ð Ñ à€L�$ÐÐÑØ!Ð˜Ð!Ð!Ñ!Ø€IˆsÐÐÑØ€IˆsÐÐÑðeð eð eð eð eð(ð ð ðkð kð kð kð kð kð krH   r   N)Útypingr   r   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr	   r   Ú__all__r?   rH   rG   ú<module>rb      sÕ   ðð*  Ð Ð Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .à 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø #Ð #Ð #Ð #Ð #Ð #ð €Ð,Ð-Ñ-Ô-Øð^kð ^kð ^kð ^kð ^kÐ!ñ ^kô ^kñ „ñ .Ô-ð^kðB ˆ.€€€rH   