§
    ‚ŠtjŠ�  ã                   ó:  — d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ d dl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  G d„ dej3        ¦  «        Z4 ed¦  «         G d„ dej3        ¦  «        ¦   «         Z5dej6        de7dej6        fd„Z8 G d„ d ej3        ¦  «        Z9 G d!„ d"ej3        ¦  «        Z:d#„ Z;	 dFd%ej3        d&ej6        d'ej6        d(ej6        d)ej6        dz  d*e<d+e<d,e&e(         fd-„Z=dGd.„Z> G d/„ d0ej3        ¦  «        Z? G d1„ d2e¦  «        Z@ G d3„ d4ej3        ¦  «        ZA G d5„ d6ej3        ¦  «        ZB G d7„ d8ej3        ¦  «        ZCe G d9„ d:ej3        ¦  «        ¦   «         ZD G d;„ d<ej3        ¦  «        ZEe) G d=„ d>e$¦  «        ¦   «         ZFe) G d?„ d@eF¦  «        ¦   «         ZG e)dA¬B¦  «         G dC„ dDeFe¦  «        ¦   «         ZHg dE¢ZIdS )Hé    )ÚCallable)ÚAnyÚOptionalN)Únn)Úinité   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
ZayaConfigc                   óà   ‡ — e Zd ZU ej        ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d         de
dz  dedz  d	ed
ef         f
d„¦   «         Z ej        ¦   «         edd„¦   «         ¦   «         Zˆ xZS )ÚZayaRotaryEmbeddingÚinv_freqÚconfigc                 ó€  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          t          |j        ¦  «        ¦  «        | _        i | _	        | j        D ]È}| j        j
        |         }|€Œ|d         | j	        |<   | j        }| j	        |         dk    rt          | j	        |                  } || j        |¬¦  «        \  }}|                      |› d�|d¬¦  «         |                      |› d�|                     ¦   «         d¬¦  «         t          | |› d�|¦  «         ŒÉd S )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   ÚlistÚsetÚlayer_typesr(   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)Úselfr&   r+   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingÚ	__class__s          €úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/zaya/modeling_zaya.pyr1   zZayaRotaryEmbedding.__init__1   s^  ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!ØˆŒÝ¥ FÔ$6Ñ 7Ô 7Ñ8Ô8ˆÔØˆŒØÔ*ð 	Uð 	UˆJØœ+Ô5°jÔAˆKØÐ"Øà)4°[Ô)AˆDŒN˜:Ñ&Ø%)Ô%IˆLØŒ~˜jÔ)¨YÒ6Ð6Ý2°4´>À*Ô3MÔN�Ø4@°LÀÄÐYcÐ4dÑ4dÔ4dÑ1ˆMÐ1Ø× Ò  JÐ!9Ð!9Ð!9¸=ÐUZÐ Ñ[Ô[Ð[Ø× Ò  JÐ!BÐ!BÐ!BÀM×DWÒDWÑDYÔDYÐfkÐ ÑlÔlÐlÝ�D˜ZÐ;Ð;Ð;Ð=SÑTÔTÐTÐTð	Uð 	Uó    NÚdeviceztorch.deviceÚseq_lenr+   Úreturnztorch.Tensorc                 ón  — | j         |         d         }| j         |                              dd¦  «        }t          | dd¦  «        p| j        | j        z  }t          ||z  ¦  «        }d}d|t          j        d|dt          j        ¬¦  «         	                    |t          j
        ¬	¦  «        |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   é   ©Údtype)rE   rN   )r8   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚint64ÚtoÚfloat)
r&   rE   rF   r+   ÚbaserJ   rK   ÚdimÚattention_factorr%   s
             rC   r9   z3ZayaRotaryEmbedding.compute_default_rope_parametersF   sÄ   € ð. Ô% jÔ1°,Ô?ˆà &Ô 6°zÔ B× FÒ FÐG^Ð`cÑ dÔ dÐÝ˜6 :¨tÑ4Ô4Ðh¸Ô8JÈfÔNhÑ8hˆÝ�(Ð2Ñ2Ñ3Ô3ˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rD   c                 ó|  — t          | |› d�¦  «        }t          | |› d�¦  «        }|d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬	¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd
¦  «        }	t          j        |	|	fd¬¦  «        }
|
                     ¦   «         |z  }|
                     ¦   «         |z  }d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬¦  «        |                     |j        ¬¦  «        fS )Nr,   r/   r   éÿÿÿÿr!   ÚmpsÚcpuF)Údevice_typeÚenabledrL   ©rZ   rM   )rP   rX   ÚexpandÚshaperW   rE   Ú
isinstanceÚtypeÚstrr   Ú	transposerT   ÚcatÚcosÚsinrN   )r=   ÚxÚposition_idsr+   r%   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr`   ÚfreqsÚembrj   rk   s                rC   ÚforwardzZayaRotaryEmbedding.forwardk   sâ  € õ ˜4 JÐ!9Ð!9Ð!9Ñ:Ô:ˆÝ# D¨ZÐ*KÐ*KÐ*KÑLÔLÐà$ T¨1¨1¨1¨d ]Ô3×9Ò9Ñ;Ô;×BÒBÀ<ÔCUÐVWÔCXÐZ\Ð^_Ñ`Ô`×cÒcÐdeÔdlÑmÔmÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	0ð 	0Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)Ð/Ñ/ˆCØ—'’'‘)”)Ð/Ñ/ˆCð		0ð 	0ð 	0ñ 	0ô 	0ð 	0ð 	0ð 	0ð 	0ð 	0ð 	0øøøð 	0ð 	0ð 	0ð 	0ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   Ã-BE=Å=FÆF©NNNN©N)Ú__name__Ú
__module__Ú__qualname__rT   ÚTensorÚ__annotations__r"   r1   Ústaticmethodr   rS   rg   ÚtuplerX   r9   Úno_gradr   rs   Ú__classcell__©rB   s   @rC   r$   r$   .   s
  ø€ € € € € € ØŒlÐÐÑðU˜zð Uð Uð Uð Uð Uð Uð* à$(Ø+/Ø"Ø!%ð	"*ð "*Ø˜TÑ!ð"*à˜Ô(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ô	%ð"*ð "*ð "*ñ „\ð"*ðH €U„]�_„_Øð<ð <ð <ñ Ôñ „_ð<ð <ð <ð <ð <rD   r$   ÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚZayaRMSNormç�íµ ÷Æ°>ÚepsrG   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z:
        ZayaRMSNorm is equivalent to T5LayerNorm
        N)r0   r1   r   Ú	ParameterrT   ÚonesÚweightÚvariance_epsilon)r=   rQ   r„   rB   s      €rC   r1   zZayaRMSNorm.__init__€   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐrD   Úhidden_statesc                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S )NrL   r]   T)Úkeepdim)	rN   rW   rT   Úfloat32ÚpowÚmeanÚrsqrtr‰   rˆ   )r=   rŠ   Úinput_dtypeÚvariances       rC   rs   zZayaRMSNorm.forwardˆ   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:rD   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r|   rˆ   rd   r‰   )r=   s    rC   Ú
extra_reprzZayaRMSNorm.extra_repr�   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIrD   )rƒ   )
rv   rw   rx   rX   r1   rT   ry   rs   r”   r~   r   s   @rC   r‚   r‚   ~   sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð JrD   r‚   rŠ   Ún_reprG   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rd   rc   Úreshape)rŠ   r•   ÚbatchÚnum_key_value_headsÚslenrK   s         rC   Ú	repeat_kvr›   “   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTrD   c                   ód   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej        de	dz  dej        dz  fd	„Z
ˆ xZS )ÚZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    r&   Ú	layer_idxc                 ó¤  •— t          ¦   «                              ¦   «          || _        || _        |j        | _        |j        | _        |j        | _        | j        dz
  | j        dz
  z   | _	        |j
        | _
        |j        | _        |j        | _        | j        | j
        z  | _        | j        | j        z  }| j
        | j        z  }t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        || j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        t          j        | j        |dz  | j        j        ¬¦  «        | _        ||z   }t          j        ||| j        |dd¬¦  «        | _        t          j        ||| j        | j
        | j        z   dd¬¦  «        | _        d S )Nr!   ©ÚbiasrL   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingÚstride)r0   r1   r&   rž   rQ   Ú	cca_time0Údepthwise_kernel_sizeÚ	cca_time1Úgrouped_kernel_sizeÚconv_kernel_sizer™   rR   rK   Únum_key_value_groupsr   ÚLinearÚattention_biasÚq_projÚk_projÚv_proj_currentÚv_proj_delayedÚConv1dÚconv_qk_depthwiseÚconv_qk_grouped)r=   r&   rž   Úquery_hidden_sizeÚkey_value_hidden_sizeÚconv_channelsrB   s         €rC   r1   zZayaCCAProjection.__init__¨   sÈ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒà!Ô-ˆÔà%+Ô%5ˆÔ"Ø#)Ô#3ˆÔ Ø!%Ô!;¸aÑ!?ÀDÔD\Ð_`ÑD`Ñ aˆÔà#)Ô#=ˆÔ Ø#)Ô#=ˆÔ ØœˆŒØ$(Ô$<ÀÔ@XÑ$XˆÔ!à Ô4°t´}ÑDÐØ $Ô 8¸4¼=Ñ HÐå”i Ô 0Ð2CÈ$Ì+ÔJdÐeÑeÔeˆŒÝ”i Ô 0Ð2GÈdÌkÔNhÐiÑiÔiˆŒÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔÝ œi¨Ô(8Ð:OÐSTÑ:TÐ[_Ô[fÔ[uÐvÑvÔvˆÔà-Ð0AÑAˆÝ!#¤Ø%Ø&ØÔ2Ø ØØð"
ñ "
ô "
ˆÔõ  "œyØ%Ø&ØÔ0ØÔ,¨tÔ/GÑGØØð 
ñ  
ô  
ˆÔÐÐrD   NrŠ   Úpast_key_valuesÚ	conv_maskc                 ó  — |�*||d d …d d …d f                               |j        ¦  «        z  }|j        d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }|                      |¦  «        }t          j        ||gd¬¦  «        } |j        |Ž }	 |j        |Ž  	                    dd¦  «        }
t          |
| j        ¦  «         	                    dd¦  «        }
|	|
z   dz  }	 |	j        g |¢d‘| j        ‘| j        ‘R Ž                      d¬¦  «        }
| 	                    dd¦  «        }|d uo|                     | j        ¦  «        }|r6|j        | j                 j        d         }t          j        ||gd¬¦  «        }nt#          j        || j        df¦  «        }|�W|d| j         d …f         }t#          j        || j        |j        d         z
  df¦  «        }|                     || j        ¦  «         |                      |¦  «        }|                      |¦  «         	                    dd¦  «        }|	j        d         |	j        d         z  } |dd |…f         j        |Ž |	z   } |d|d …f         j        |Ž |
z   }|                      |¦  «        }|                      |¦  «        }|r1|j        | j                 j        d                              d¦  «        }n5|                      |                     |d         d| j        ¦  «        ¦  «        }t          j        ||d d …d d…f         gd¬¦  «        }|�(|                     |d d …dd d …f         | j        ¦  «          t          j        ||gd¬¦  «        j        |Ž }|||fS )	Nr]   rb   r!   rL   ç      à?éþÿÿÿr   .)rW   rN   rd   rK   r°   r±   rT   ri   Úviewrh   r›   r­   r�   Úhas_previous_staterž   ÚlayersÚconv_statesÚFÚpadr¬   Úupdate_conv_staterµ   r¶   r²   r³   Úrecurrent_statesÚ	unsqueezeÚ	new_zerosrQ   Úupdate_recurrent_state)r=   rŠ   rº   r»   Úinput_shapeÚhidden_shapeÚprojected_queriesÚprojected_keysÚ	qk_statesÚquery_residualÚkey_residualÚuse_precomputed_statesÚcached_qk_statesÚnew_conv_stater·   ÚqueryÚkeyÚvalue_currentÚdelayed_v_stateÚrecurrent_v_stateÚvalue_delayedÚvalues                         rC   rs   zZayaCCAProjection.forwardÒ   sä  € ð Ð Ø)¨I°a°a°a¸¸¸¸D°jÔ,A×,DÒ,DÀ]ÔEXÑ,YÔ,YÑYˆMà#Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà ŸKšK¨Ñ6Ô6ÐØŸš ]Ñ3Ô3ˆÝ”IÐ0°.ÐAÀrÐJÑJÔJˆ	à/Ð*Ô/°Ð>ˆØ*�~Ô*¨LÐ9×CÒCÀAÀqÑIÔIˆÝ  ¨tÔ/HÑIÔI×SÒSÐTUÐWXÑYÔYˆØ(¨<Ñ7¸3Ñ>ˆØ*�~Ô*Ðf¨KÐf¸Ðf¸TÔ=VÐfÐX\ÔXeÐfÐfÐf×kÒkÐprÐkÑsÔsˆà×'Ò'¨¨1Ñ-Ô-ˆ	Ø!0¸Ð!<Ð!sÀ×AcÒAcÐdhÔdrÑAsÔAsÐØ!ð 	EØ.Ô5°d´nÔEÔQÐRSÔTÐÝœ	Ð#3°YÐ"?ÀRÐHÑHÔHˆIˆIåœ˜i¨$Ô*?ÀÐ)CÑDÔDˆIàÐ&Ø& s¨TÔ-BÐ,BÐ,DÐ,DÐ'DÔEˆNÝœU >°DÔ4IÈNÔL`ÐacÔLdÑ4dÐfgÐ3hÑiÔiˆNØ×-Ò-¨n¸d¼nÑMÔMÐMà×*Ò*¨9Ñ5Ô5ˆ	Ø×(Ò(¨Ñ3Ô3×=Ò=¸aÀÑCÔCˆ	à*Ô0°Ô4°~Ô7KÈBÔ7OÑOÐØ7�	˜#Ð1Ð 1Ð1Ð1Ô2Ô7¸ÐFÈÑWˆØ5ˆi˜Ð.Ð/Ð/Ð/Ô0Ô5°|ÐDÀ|ÑSˆð ×+Ò+¨MÑ:Ô:ˆØ×-Ò-¨mÑ<Ô<ˆØ!ð 	rØ /Ô 6°t´~Ô FÔ WÐXYÔ Z× dÒ dÐefÑ gÔ gÐÐà $× 3Ò 3°M×4KÒ4KÈKÐXYÌNÐ\]Ð_cÔ_oÑ4pÔ4pÑ qÔ qÐÝœ	Ð#4°oÀaÀaÀaÈÈ"ÈÀfÔ6MÐ"NÐTUÐVÑVÔVˆàÐ&Ø×2Ò2°?À1À1À1ÀbÈ!È!È!À8Ô3LÈdÌnÑ]Ô]Ð]àF•”	˜=¨-Ð8¸bÐAÑAÔAÔFÈÐUˆà�c˜5Ð Ð rD   ru   )rv   rw   rx   Ú__doc__r"   rS   r1   rT   ry   r
   rs   r~   r   s   @rC   r�   r�   Ÿ   s�   ø€ € € € € ðð ð(
˜zð (
°cð (
ð (
ð (
ð (
ð (
ð (
ð\ *.ð	9!ð 9!à”|ð9!ð  ™ð9!ð ”< $Ñ&ð	9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!ð 9!rD   r�   c                   ót   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        deej        ej        f         fd„Z	ˆ xZ
S )Ú
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    r&   c                 óÆ   •— t          ¦   «                              ¦   «          |j        dz  | _        t	          j        t          j        |j        ¦  «        ¦  «        | _	        d S )Nr½   )
r0   r1   rK   Úhead_dim_scaler   r†   rT   Úzerosr™   Útemp©r=   r&   rB   s     €rC   r1   zZayaQKNorm.__init__  sJ   ø€ Ý‰Œ×ÒÑÔÐØ$œo¨sÑ2ˆÔÝ”L¥¤¨VÔ-GÑ!HÔ!HÑIÔIˆŒ	ˆ	ˆ	rD   Úquery_statesÚ
key_statesrG   c                 óJ  — t          j        |j        ¦  «        j        }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        |                     ddd¬¦  «                             |¦  «        z  z  }|| j        d d d d …d f         z  }||fS )NrL   r]   T)ÚprZ   rŒ   )rT   ÚfinforN   r„   rß   ÚnormÚ	clamp_minrá   )r=   rã   rä   Únorm_epss       rC   rs   zZayaQKNorm.forward  s³   € Ý”;˜|Ô1Ñ2Ô2Ô6ˆØ#ØÔ ,×"3Ò"3°a¸RÈÐ"3Ñ"NÔ"N×"XÒ"XÐYaÑ"bÔ"bÑbñ
ˆð  ØÔ *§/¢/°A¸2Àt /Ñ"LÔ"L×"VÒ"VÐW_Ñ"`Ô"`Ñ`ñ
ˆ
ð   $¤)¨D°$¸¸¸¸4Ð,?Ô"@Ñ@ˆ
Ø˜ZÐ'Ð'rD   )rv   rw   rx   rÛ   r"   r1   rT   ry   r|   rs   r~   r   s   @rC   rÝ   rÝ     s”   ø€ € € € € ðð ðJ˜zð Jð Jð Jð Jð Jð Jð
	( E¤Lð 	(¸e¼lð 	(ÈuÐUZÔUaÐchÔcoÐUoÔOpð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(rD   rÝ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Nr]   rL   rb   )rd   rT   ri   )rl   Úx1Úx2s      rC   Úrotate_halfrî   $  s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'rD   ç        ÚmodulerÔ   rÕ   rÚ   Úattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrL   r   r]   )rZ   rN   )ræ   Útrainingr!   )r›   r­   rT   Úmatmulrh   r   Ú
functionalÚsoftmaxr�   rW   rN   ró   rö   Ú
contiguous)rð   rÔ   rÕ   rÚ   rñ   rò   ró   rô   rä   Úvalue_statesÚattn_weightsÚattn_outputs               rC   Úeager_attention_forwardrþ   +  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rD   c                 ó˜  — |                      |¦  «        }|                      |¦  «        }|j        d         }| dd|…f         | d|d…f         }}|dd|…f         |d|d…f         }	}||z  t          |¦  «        |z  z   }
||z  t          |¦  «        |z  z   }t          j        |
|gd¬¦  «        }
t          j        ||	gd¬¦  «        }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r]   .Nrb   )rÇ   rd   rî   rT   ri   )ÚqÚkrj   rk   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               rC   Úapply_rotary_pos_embr
  E  sô   € ð( �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€Cð ”˜2”€JØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€EØ�c˜;˜J˜;Ð&Ô'¨¨3°
°°Ð+;Ô)<ˆ6€Eð �s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€GØ�s‰{�{¨5Ñ1Ô1°CÑ7Ñ8€Gõ Œi˜ &Ð)¨rÐ2Ñ2Ô2€GÝŒi˜ &Ð)¨rÐ2Ñ2Ô2€GØ�GÐÐrD   c                   óÚ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	e
ef         dz  dedz  d	eej        ej        f         dz  d
ee         deej        ej        dz  f         fd„Zˆ xZS )ÚZayaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr&   rž   c                 óx  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j        | _        | j        dz  | _
        |j        | _        d| _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        t#          | j        |¬¦  «        | _        |j        |         | _        | j        dk    r|j        nd | _        |j        | _        |j        | _        t-          |¦  «        | _        d S )NrK   g      à¿Tr    ©r&   rž   Úhybrid_sliding)r0   r1   r&   rž   rP   rQ   rR   rK   r™   r­   rò   Úattention_dropoutÚ	is_causalr   r®   r¯   Úo_projr�   Úqkv_projr7   r+   Úsliding_windowrÝ   Úqk_norm©r=   r&   rž   rB   s      €rC   r1   zZayaAttention.__init__n  s+  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø#)Ô#=ˆÔ Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒõ *Ø”;Øð
ñ 
ô 
ˆŒð !Ô,¨YÔ7ˆŒØ7;´ÐJZÒ7ZÐ7Z˜fÔ3Ð3Ð`dˆÔØ!Ô-ˆÔØ#)Ô#=ˆÔ Ý! &Ñ)Ô)ˆŒˆˆrD   NrŠ   rñ   rº   Úposition_embeddingsrô   rG   c                 óÎ  — |j         d d…         }|pi }|                     d¦  «        }|                     d¦  «        }	|                      |||	¦  «        \  }
}}|                      |
|¦  «        \  }
}|
                     dd¦  «        }
|                     dd¦  «        }|                     dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�|                     ||| j        ¦  «        \  }}t          j	        | j
        j        t          ¦  «        } || |
|||f| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž }|                      |¦  «        }||fS )Nr]   ÚcausalÚconvr!   rL   rï   )ró   rò   r  )rd   rO   r  r  rh   r
  Úupdaterž   r   Úget_interfacer&   Ú_attn_implementationrþ   rö   r  rò   r  r—   r  )r=   rŠ   rñ   rº   r  rô   rÊ   Úmask_mappingÚcausal_maskr»   rã   rä   rû   rj   rk   Úattention_interfacerý   rü   s                     rC   rs   zZayaAttention.forward†  s·  € ð $Ô)¨#¨2¨#Ô.ˆà%Ð+¨ˆØ"×&Ò& xÑ0Ô0ˆØ ×$Ò$ VÑ,Ô,ˆ	ð 26·²¸}ÈoÐ_hÑ1iÔ1iÑ.ˆ�j ,Ø#'§<¢<°¸jÑ#IÔ#IÑ ˆ�jà#×-Ò-¨a°Ñ3Ô3ˆØ×)Ò)¨!¨QÑ/Ô/ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð
%
ð  $œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
%
ð 
%
ð ð
%
ð 
%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;ˆØ—k’k +Ñ.Ô.ˆà˜LÐ(Ð(rD   )NNN)rv   rw   rx   rÛ   r"   rS   r1   rT   ry   Údictrg   r   r
   r|   r   r   rs   r~   r   s   @rC   r  r  k  sî   ø€ € € € € ØGÐGð*˜zð *°cð *ð *ð *ð *ð *ð *ð6 15Ø(,ØHLð.)ð .)à”|ð.)ð ˜S #˜Xœ¨Ñ-ð.)ð  ™ð	.)ð
 # 5¤<°´Ð#=Ô>ÀÑEð.)ð Ð+Ô,ð.)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)rD   r  c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        dej        dz  dee	e
f         dz  dedz  d	eej        ej        f         dz  d
ee         deej        ej        dz  f         fd„Zˆ xZS )ÚZayaDecoderLayerr&   rž   c                 óš  •— t          ¦   «                              ¦   «          |j        | _        t          ||¬¦  «        | _        t          ||¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        ¦  «        | _        t          |j        ¦  «        | _        d S )Nr  ©r„   )r0   r1   rQ   r  Ú	self_attnÚZayaSparseMoeBlockÚmlpr‚   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚZayaResidualScalingÚpost_attention_residual_scaleÚpost_mlp_residual_scaler  s      €rC   r1   zZayaDecoderLayer.__init__¸  s«   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå&¨fÀ	ÐJÑJÔJˆŒÝ% f¨iÑ8Ô8ˆŒÝ*¨6Ô+=À6ÔCVÐWÑWÔWˆÔÝ(3°FÔ4FÈFÔL_Ð(`Ñ(`Ô(`ˆÔ%Ý-@ÀÔASÑ-TÔ-TˆÔ*Ý':¸6Ô;MÑ'NÔ'NˆÔ$Ð$Ð$rD   NrŠ   Úprev_router_hidden_statesrñ   rº   r  rô   rG   c                 ó¢  — |}|                       |                     | j         j        j        ¬¦  «        ¦  «        } | j        d||||dœ|¤Ž\  }}|                      ||¦  «        }|                      |                     | j        j        j        ¬¦  «        ¦  «        }|                      ||¦  «        \  }}|                      ||¦  «        }||fS )NrM   )rŠ   rñ   rº   r  © )	r*  rW   rˆ   rN   r&  r-  r+  r(  r.  )	r=   rŠ   r/  rñ   rº   r  rô   ÚresidualÚ_s	            rC   rs   zZayaDecoderLayer.forwardÃ  sð   € ð !ˆð ×,Ò,¨X¯[ª[¸tÔ?SÔ?ZÔ?`¨[Ñ-aÔ-aÑbÔbˆà)˜4œ>ð 
Ø'Ø)Ø+Ø 3ð	
ð 
ð
 ð
ð 
Ñˆ�qð ×5Ò5°mÀXÑNÔNˆØ×5Ò5°h·k²kÈÔHeÔHlÔHr°kÑ6sÔ6sÑtÔtˆà37·8²8ØØ%ñ4
ô 4
Ñ0ˆÐ0ð
 ×4Ò4°]ÀHÑMÔMˆàÐ7Ð7Ð7rD   rt   )rv   rw   rx   r"   rS   r1   rT   ry   r!  rg   r   r
   r|   r   r   rs   r~   r   s   @rC   r#  r#  ·  s  ø€ € € € € ð	O˜zð 	O°cð 	Oð 	Oð 	Oð 	Oð 	Oð 	Oð :>Ø04Ø(,ØHLð 8ð  8à”|ð 8ð $)¤<°$Ñ#6ð 8ð ˜S #˜Xœ¨Ñ-ð	 8ð
  ™ð 8ð # 5¤<°´Ð#=Ô>ÀÑEð 8ð Ð+Ô,ð 8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð 8ð  8ð  8ð  8ð  8ð  8ð  8ð  8rD   r#  c                   óH   ‡ — e Zd Zdefˆ fd„Zdej        dej        fd„Zˆ xZS )r,  rQ   c                 ó   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _        t          j        t	          j        |¦  «        ¦  «        | _	        t          j        t	          j        |¦  «        ¦  «        | _
        d S ru   )r0   r1   r   r†   rT   r‡   Úhidden_states_scalerà   Úhidden_states_biasÚresidual_scaleÚresidual_bias)r=   rQ   rB   s     €rC   r1   zZayaResidualScaling.__init__ç  s�   ø€ Ý‰Œ×ÒÑÔÐÝ#%¤<µ´
¸;Ñ0GÔ0GÑ#HÔ#HˆÔ Ý"$¤,­u¬{¸;Ñ/GÔ/GÑ"HÔ"HˆÔÝ œl­5¬:°kÑ+BÔ+BÑCÔCˆÔÝœ\­%¬+°kÑ*BÔ*BÑCÔCˆÔÐÐrD   rŠ   r2  c                 óT   — || j         z   | j        z  }|| j        z   | j        z  }||z   S ru   )r7  r6  r9  r8  )r=   rŠ   r2  s      rC   rs   zZayaResidualScaling.forwardî  s7   € à&¨Ô)@Ñ@ÀDÔD\Ñ\ˆØ˜tÔ1Ñ1°TÔ5HÑHˆØ˜xÑ'Ð'rD   )	rv   rw   rx   rS   r1   rT   ry   rs   r~   r   s   @rC   r,  r,  æ  sq   ø€ € € € € ðD Cð Dð Dð Dð Dð Dð Dð( U¤\ð (¸U¼\ð (ð (ð (ð (ð (ð (ð (ð (rD   r,  c                   óP   ‡ — e Zd Zdededefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚZayaRouterMLPrQ   Únum_expertsr)  c                 óL  •— t          ¦   «                              ¦   «          t          ||¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j        ||d¬¦  «        | _        t	          j	        ¦   «         | _
        d S )Nr%  Tr    F)r0   r1   r‚   rè   r   r®   Úfc1Úfc2Úout_projÚGELUÚact_fn)r=   rQ   r=  r)  rB   s       €rC   r1   zZayaRouterMLP.__init__ö  s†   ø€ Ý‰Œ×ÒÑÔÐÝ °Ð>Ñ>Ô>ˆŒ	Ý”9˜[¨+¸DÐAÑAÔAˆŒÝ”9˜[¨+¸DÐAÑAÔAˆŒÝœ	 +¨{ÀÐGÑGÔGˆŒÝ”g‘i”iˆŒˆˆrD   rŠ   rG   c                 óö   — |                       |¦  «        }|                      |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|                      |¦  «        S ru   )rè   rC  r?  r@  rA  )r=   rŠ   s     rC   rs   zZayaRouterMLP.forwardþ  s_   € ØŸ	š	 -Ñ0Ô0ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØŸš D§H¢H¨]Ñ$;Ô$;Ñ<Ô<ˆØ�}Š}˜]Ñ+Ô+Ð+rD   )
rv   rw   rx   rS   rX   r1   rT   ry   rs   r~   r   s   @rC   r<  r<  õ  sx   ø€ € € € € ð  Cð  °cð  Èð  ð  ð  ð  ð  ð  ð, U¤\ð ,°e´lð ,ð ,ð ,ð ,ð ,ð ,ð ,ð ,rD   r<  c                   ó–   ‡ — e Zd Zdeddfˆ fd„Z	 ddej        dej        dz  deej        ej        ej        ej        f         fd„Zˆ xZ	S )	Ú
ZayaRouterrž   rG   Nc                 óŒ  •— t          ¦   «                              ¦   «          || _        |j        | _        || _        |j        | _        |j        dz   | _        |j        | _        |j	        | _	        t          j        | j        | j	        d¬¦  «        | _        | j        dk    | _        | j        r0t          j        t          j        | j	        ¦  «        ¦  «        | _        t%          | j	        | j        |j        ¦  «        | _        |                      dt          j        | j        t          j        ¬¦  «        ¦  «         d| j        d<   d S )	Nr!   Tr    r   Úbalancing_biasesrM   ç      ð¿r]   )r0   r1   r&   rQ   rž   r=  Únum_router_classesÚnum_experts_per_tokÚtop_kÚrouter_hidden_sizer   r®   Ú	down_projÚuse_edar†   rT   r‡   Úrouter_states_scaler<  r)  Ú
router_mlpr:   rà   r�   rH  r  s      €rC   r1   zZayaRouter.__init__  s  ø€ õ
 	‰Œ×ÒÑÔÐàˆŒØ!Ô-ˆÔØ"ˆŒà!Ô-ˆÔà"(Ô"4°qÑ"8ˆÔØÔ/ˆŒ
Ø"(Ô";ˆÔåœ 4Ô#3°TÔ5LÐSWÐXÑXÔXˆŒà”~¨Ò*ˆŒØŒ<ð 	YÝ')¤|µE´J¸tÔ?VÑ4WÔ4WÑ'XÔ'XˆDÔ$å'¨Ô(?ÀÔAXÐZ`ÔZmÑnÔnˆŒà×ÒÐ/µ´¸TÔ=TÕ\aÔ\iÐ1jÑ1jÔ1jÑkÔkÐkØ$(ˆÔ˜bÑ!Ð!Ð!rD   rŠ   Úrouter_statesc                 óü  — d| j         f}|j        d         }|                      |¦  «        }| j        r|�||| j        z  z   }|d d …| d …f                              ¦   «         }|                      |¦  «        }t          j        |d¬¦  «        }| 	                    ¦   «          
                    t          j        ¦  «        | j        z   }	t          j        |	| j         d¬¦  «        \  }
}t          j        |d|¬¦  «        }|| j        j        k    }|                     |d¦  «        }|                     |d¦  «        }|                     d| j        ¦  «        |                     |¦  «        |                     |¦  «        |fS )Nr]   r!   rb   rL   )rZ   Úindexr   )rL  rd   rN  rO  rP  r;   rQ  rT   rù   ÚdetachrW   r�   rH  ÚtopkÚgatherr&   r=  Úmasked_fillr—   rJ  )r=   rŠ   rR  Úfinal_shapeÚ
seq_lengthÚrouter_hidden_statesÚrouter_hidden_states_nextÚrouter_logitsÚrouter_probsÚbiased_router_probsr3  Úrouter_indicesÚskip_experts                rC   rs   zZayaRouter.forward"  s}  € ð
 ˜4œ:Ð&ˆØ"Ô(¨Ô+ˆ
à#Ÿ~š~¨mÑ<Ô<ÐàŒ<ð 	c˜MÐ5Ø#7¸-È$ÔJbÑ:bÑ#bÐ à$8¸¸¸¸Z¸K¸L¸L¸Ô$I×$OÒ$OÑ$QÔ$QÐ!ØŸšÐ(<Ñ=Ô=ˆÝ”} ]¸Ð;Ñ;Ô;ˆà*×1Ò1Ñ3Ô3×6Ò6µu´}ÑEÔEÈÔH]Ñ]ÐÝ!œJÐ':¸D¼JÈBÐOÑOÔOÑˆˆ>Ý”| L°a¸~ÐNÑNÔNˆð %¨¬Ô(?Ò?ˆØ#×/Ò/°¸QÑ?Ô?ˆØ'×3Ò3°KÀÑCÔCˆð ×!Ò! " dÔ&=Ñ>Ô>Ø× Ò  Ñ-Ô-Ø×"Ò" ;Ñ/Ô/Ø%ð	
ð 	
rD   ru   ©
rv   rw   rx   rS   r1   rT   ry   r|   rs   r~   r   s   @rC   rF  rF    sª   ø€ € € € € ð)ð ð)ð 
ð	)ð )ð )ð )ð )ð )ð> .2ð
ð 
à”|ð
ð ”| dÑ*ð
ð 
ˆuŒ|˜Uœ\¨5¬<¸¼ÐEÔ	Fð	
ð 
ð 
ð 
ð 
ð 
ð 
ð 
rD   rF  c                   ób   ‡ — e Zd ZdZˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚZayaExpertsz2Collection of expert weights stored as 3D tensors.c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )NrL   )r0   r1   r=  rQ   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r†   rT   ÚemptyÚgate_up_projrN  r	   Ú
hidden_actrC  râ   s     €rC   r1   zZayaExperts.__init__H  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆrD   rŠ   Útop_k_indexÚtop_k_weightsrG   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesrL   r!   r   )r]   r¾   rb   r]   )rT   Ú
zeros_liker}   r   rø   Úone_hotr=  ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearrj  ÚchunkrC  rN  Ú
index_add_rW   rN   )r=   rŠ   rl  rm  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rC   rs   zZayaExperts.forwardQ  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)	rv   rw   rx   rÛ   r1   rT   ry   rs   r~   r   s   @rC   rd  rd  D  s€   ø€ € € € € à<Ð<ð0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #rD   rd  c            
       ó€   ‡ — e Zd Zdefˆ fd„Z	 ddej        dej        dz  deej        ej        dz  f         fd„Zˆ xZ	S )	r'  rž   c                 óš   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        d S ru   )r0   r1   rF  r�  rd  Úexpertsr  s      €rC   r1   zZayaSparseMoeBlock.__init__m  s=   ø€ Ý‰Œ×ÒÑÔÐÝ˜v yÑ1Ô1ˆŒ	Ý" 6Ñ*Ô*ˆŒˆˆrD   NrŠ   r/  rG   c                 óæ   — |                       ||¬¦  «        \  }}}}|j        \  }}}|                     ||z  |¦  «        }	|                      |	||¦  «        }
|
                     |||¦  «        }
|
|fS )N)rR  )r�  rd   r¿   r†  )r=   rŠ   r/  r3  r^  r`  Ú
batch_sizerZ  Úemb_dimÚhidden_states_flatÚexpert_outputs              rC   rs   zZayaSparseMoeBlock.forwardr  s—   € ð FJÇYÂYØÐ)Bð FOñ F
ô F
ÑBˆˆ<˜Ð)Bð +8Ô*=Ñ'ˆ
�J Ø*×/Ò/°
¸ZÑ0GÈÑQÔQÐØŸšÐ%7¸ÈÑVÔVˆØ%×*Ò*¨:°zÀ7ÑKÔKˆàÐ7Ð7Ð7rD   ru   rb  r   s   @rC   r'  r'  l  sš   ø€ € € € € ð+¨#ð +ð +ð +ð +ð +ð +ð :>ð8ð 8à”|ð8ð $)¤<°$Ñ#6ð8ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð	8ð 8ð 8ð 8ð 8ð 8ð 8ð 8rD   r'  c                   óž   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬¦  «        eed	œZ ej        ¦   «         ˆ fd
„¦   «         Zˆ xZS )ÚZayaPreTrainedModelr&   ÚmodelTr#  rº   Fr   )rT  )r]  rŠ   Ú
attentionsc                 ó   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rft	          j        |j        ¦  «         t	          j        |j        ¦  «         t	          j        |j	        ¦  «         t	          j        |j
        ¦  «         d S t          |t          ¦  «        r4t	          j        |j        ¦  «         t	          j        |j        ¦  «         d S t          |t          ¦  «        rt	          j        |j        ¦  «         d S t          |t           ¦  «        rE|j        rt	          j        |j        ¦  «         t	          j        |j        ¦  «         d|j        d<   d S t          |t(          ¦  «        rF| j        j        }t	          j        |j        d|¬¦  «         t	          j        |j        d|¬¦  «         d S t          |t4          ¦  «        r›|j        D ]•}|j        }|j        |         dk    rt<          |j        |                  } ||j        |¬¦  «        \  }}t?          ||› d�¦  «                              |¦  «         t?          ||› d�¦  «                              |¦  «         Œ”d S d S )	NrI  r]   rï   )r�   Ústdr)   r*   r,   r.   )!r0   Ú_init_weightsre   r,  r   Úones_r6  Úzeros_r7  r8  r9  Ú	ZayaModelÚinput_hidden_states_scaleÚinput_hidden_states_biasrÝ   rá   rF  rO  rP  rH  rd  r&   Úinitializer_rangeÚnormal_rj  rN  r$   r7   r9   r(   r   rP   Úcopy_)r=   rð   r‘  r+   r?   r@   r3  rB   s          €rC   r’  z!ZayaPreTrainedModel._init_weights—  sq  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ1Ñ2Ô2ð 	XÝŒJ�vÔ1Ñ2Ô2Ð2ÝŒK˜Ô1Ñ2Ô2Ð2ÝŒJ�vÔ,Ñ-Ô-Ð-ÝŒK˜Ô,Ñ-Ô-Ð-Ð-Ð-Ý˜¥	Ñ*Ô*ð 	XÝŒJ�vÔ7Ñ8Ô8Ð8ÝŒK˜Ô7Ñ8Ô8Ð8Ð8Ð8Ý˜¥
Ñ+Ô+ð 	XÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜¥
Ñ+Ô+ð 	XØŒ~ð 7Ý”
˜6Ô5Ñ6Ô6Ð6ÝŒK˜Ô/Ñ0Ô0Ð0Ø*.ˆFÔ# BÑ'Ð'Ð'Ý˜¥Ñ,Ô,ð 	XØ”+Ô/ˆCÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 3Ñ4Ô4ð 	XØ$Ô0ð Xð X�
Ø%ÔE�ØÔ# JÔ/°9Ò<Ð<Ý#6°vÔ7GÈ
Ô7SÔ#T�LØ#/ <°´È*Ð#UÑ#UÔ#UÑ �˜qÝ˜ :Ð 8Ð 8Ð 8Ñ9Ô9×?Ò?ÀÑNÔNÐNÝ˜ :Ð AÐ AÐ AÑBÔB×HÒHÈÑWÔWÐWÐWð	Xð 	XðXð XrD   )rv   rw   rx   r"   rz   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   rF  r#  r  Ú_can_record_outputsrT   r}   r’  r~   r   s   @rC   r�  r�  „  sÃ   ø€ € € € € € àÐÐÑØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà"ÐØ"&Ðà'˜¨
¸!Ð<Ñ<Ô<Ø)Ø#ðð Ðð €U„]�_„_ðXð Xð Xð Xñ „_ðXð Xð Xð Xð XrD   r�  c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )r•  r&   c                 óž  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        t          j        t)          j        ‰j        ¦  «        ¦  «        | _        t          j        t)          j        ‰j        ¦  «        ¦  «        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r1  )r#  )Ú.0rž   r&   s     €rC   ú
<listcomp>z&ZayaModel.__init__.<locals>.<listcomp>À  s$   ø€ ÐbÐbÐb°YÕ˜f iÑ0Ô0ÐbÐbÐbrD   r%  ©r&   F)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrQ   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrÁ   r‚   r)  rè   r$   Ú
rotary_embÚgradient_checkpointingr†   rT   r‡   r–  rà   r—  Ú	post_initrâ   s    `€rC   r1   zZayaModel.__init__¹  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØbÐbÐbÐbÅ%ÈÔH`ÑBaÔBaÐbÑbÔbñ
ô 
ˆŒõ   Ô 2¸Ô8KÐLÑLÔLˆŒ	Ý-°VÐ<Ñ<Ô<ˆŒØ&+ˆÔ#Ý)+¬µe´jÀÔASÑ6TÔ6TÑ)UÔ)UˆÔ&Ý(*¬µU´[ÀÔASÑ5TÔ5TÑ(UÔ(UˆÔ%ð 	�ŠÑÔÐÐÐrD   NÚ	input_idsrñ   rm   rº   Úinputs_embedsÚ	use_cacherô   rG   c                 ó  ‡ ‡‡‡‡— |d u |d uz  rt          d¦  «        ‚|€‰                      |¦  «        }|r|€t          ‰ j        ¬¦  «        }‰€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   Š‰ 	                    d¦  «        Št          |x}	t          ¦  «        sL‰ j        |||‰dœŠˆfd„ˆfd„d	œŠˆfd
„t          ‰ j        j        ¦  «        D ¦   «         }	t          di ‰¤Ž|	d<   |Šˆˆˆ fd„t          ‰ j        j        ¦  «        D ¦   «         }
‰‰ j        z   ‰ j        z                       t
          j        ¦  «        Šd }t'          ‰ j        ¦  «        D ]J\  }}‰ j        j        |         } |‰|f|	|         |	                     d¦  «        dœ||
|         dœ|¤Ž\  Š}ŒK‰                      ‰                     ‰ j        j        j        ¬¦  «        ¦  «        Št3          ‰|r|nd ¬¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrª  r   r!   )rE   ©r&   r·  rñ   rº   rm   c                  ó   •— t          di ‰ ¤ŽS ©Nr1  ©r   ©Úmask_kwargss   €rC   ú<lambda>z#ZayaModel.forward.<locals>.<lambda>ï  s   ø€ Õ"4Ð"CÐ"C°{Ð"CÐ"C€ rD   c                  ó   •— t          di ‰ ¤ŽS r¼  ©r   r¾  s   €rC   rÀ  z#ZayaModel.forward.<locals>.<lambda>ð  s   ø€ Õ*KÐ*ZÐ*ZÈkÐ*ZÐ*Z€ rD   ©Úhybridr  c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS r1  r1  ©r¨  r+   Úmask_creation_functionss     €rC   ú
<dictcomp>z%ZayaModel.forward.<locals>.<dictcomp>ò  s7   ø€ ð #ð #ð #ØFP�
Ð?Ð3°JÔ?ÑAÔAð#ð #ð #rD   r  c                 ó@   •— i | ]}|‰                      ‰‰|¦  «        “ŒS r1  )r³  )r¨  r+   rŠ   rm   r=   s     €€€rC   rÈ  z%ZayaModel.forward.<locals>.<dictcomp>ù  s;   ø€ ð 
ð 
ð 
àð ˜Ÿš¨°|ÀZÑPÔPð
ð 
ð 
rD   )r  r  )rñ   rº   r  rM   )Úlast_hidden_staterº   r1  )Ú
ValueErrorr¯  r   r&   Úget_seq_lengthrT   rU   rd   rE   rÇ   re   r!  r6   r7   r   r—  r–  rW   r�   Ú	enumeraterÁ   rO   rè   rˆ   rN   r   )r=   r¶  rñ   rm   rº   r·  r¸  rô   Úpast_seen_tokensÚcausal_mask_mappingr  r/  ÚidxÚdecoder_layerr+   rŠ   rÇ  r¿  s   `  `           @@@rC   rs   zZayaModel.forwardË  sÖ  øøøøø€ ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	Yàœ+Ø!.Ø"0Ø#2Ø ,ðð ˆKð DÐCÐCÐCØ"ZÐ"ZÐ"ZÐ"Zð'ð 'Ð#ð#ð #ð #ð #ÝTWÐX\ÔXcÔXoÑTpÔTpð#ñ #ô #Ðõ +JÐ*XÐ*XÈKÐ*XÐ*XÐ Ñ'à%ˆð
ð 
ð 
ð 
ð 
ð 
å! $¤+Ô"9Ñ:Ô:ð
ñ 
ô 
Ðð (¨$Ô*GÑGÈ4ÔKiÑi×mÒmÝŒMñ
ô 
ˆð %)Ð!å"+¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�ØœÔ0°Ô5ˆJð 8E°}ØØ)ð
8ð 2°*Ô=Ø/×3Ò3°FÑ;Ô;ð ð  ð !0Ø$7¸
Ô$Cð
8ð 
8ð ð
8ð 
8Ñ4ˆMÐ4Ð4ð Ÿ	š	 -×"2Ò"2¸¼Ô9IÔ9OÐ"2Ñ"PÔ"PÑQÔQˆå%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
rD   )NNNNNN)rv   rw   rx   r"   r1   r   r    r   rT   Ú
LongTensorry   r
   ÚFloatTensorÚboolr   r   r   rs   r~   r   s   @rC   r•  r•  ·  s  ø€ € € € € ð˜zð ð ð ð ð ð ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ðL
ð L
àÔ# dÑ*ðL
ð œ tÑ+ðL
ð Ô&¨Ñ-ð	L
ð
  ™ðL
ð Ô(¨4Ñ/ðL
ð ˜$‘;ðL
ð Ð+Ô,ðL
ð 
 ðL
ð L
ð L
ñ „^ñ „_ñ  ÔðL
ð L
ð L
ð L
ð L
rD   r•  zZyphra/ZAYA1-8B)Ú
checkpointc                   ó(  ‡ — e Zd ZddiZdZˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej	        dz  dej
        dz  d	ej	        dz  d
edz  dej        dz  dej	        dz  dedz  dedz  deej
        z  dee         defd„¦   «         ¦   «         Zedd„¦   «         Zˆ xZS )ÚZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        | j        j	        ¬¦  «        | _
        |                      ¦   «          d S )Nr    )r0   r1   r•  rŽ  r­  r   r®   rQ   r&   Úlm_head_biasÚlm_headrµ  )r=   r&   rô   rB   s      €rC   r1   zZayaForCausalLM.__init__"  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý˜vÑ&Ô&ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈTÌ[ÔMeÐfÑfÔfˆŒØ�ŠÑÔÐÐÐrD   Nr   r¶  rñ   rm   rº   r·  Úlabelsr¸  Úoutput_router_logitsÚlogits_to_keeprô   rG   c
                 ór  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aÉ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ZayaForCausalLM

        >>> model = ZayaForCausalLM.from_pretrained("meta-zaya/Zaya-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-zaya/Zaya-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r¶  rñ   rm   rº   r·  r¸  rÜ  )ÚlossÚlogitsrº   rŠ   r�  r]  r1  )r&   rÜ  rŽ  rÊ  re   rS   ÚslicerÚ  Úloss_functionr­  r   rº   rŠ   r�  r]  )r=   r¶  rñ   rm   rº   r·  rÛ  r¸  rÜ  rÝ  rô   ÚoutputsrŠ   Úslice_indicesrà  rß  s                   rC   rs   zZayaForCausalLM.forward)  s  € ðB %9Ð$DÐ Ð È$Ì+ÔJjð 	ð +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå(ØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
rD   c                 ó¶   ‡‡	— |                       ¦   «         }|||||dœŠ	ˆ	fd„ˆ	fd„dœŠˆfd„t          |j        ¦  «        D ¦   «         }t          di ‰	¤Ž|d<   |S )Nrº  c                  ó   •— t          di ‰ ¤ŽS r¼  r½  r¾  s   €rC   rÀ  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>v  s   ø€ Õ0Ð?Ð?°;Ð?Ð?€ rD   c                  ó   •— t          di ‰ ¤ŽS r¼  rÂ  r¾  s   €rC   rÀ  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>w  s   ø€ Õ&GÐ&VÐ&VÈ+Ð&VÐ&V€ rD   rÃ  c                 ó2   •— i | ]}| ‰|         ¦   «         “ŒS r1  r1  rÆ  s     €rC   rÈ  z=ZayaForCausalLM.create_masks_for_generate.<locals>.<dictcomp>y  s7   ø€ ð 
ð 
ð 
ØBLˆJÐ;Ð/°
Ô;Ñ=Ô=ð
ð 
ð 
rD   r  r1  )Úget_text_configr6   r7   r   )
r&   r·  rñ   rº   rm   r3  Útext_configr  rÇ  r¿  s
           @@rC   Úcreate_masks_for_generatez)ZayaForCausalLM.create_masks_for_generatei  s®   øø€ ð ×,Ò,Ñ.Ô.ˆà!Ø*Ø,Ø.Ø(ð
ð 
ˆð @Ð?Ð?Ð?ØVÐVÐVÐVð#
ð #
Ðð
ð 
ð 
ð 
ÝPSÐT_ÔTkÑPlÔPlð
ñ 
ô 
ˆõ  ?ÐMÐMÀÐMÐMˆ�VÑØÐrD   )	NNNNNNNNr   ru   )rv   rw   rx   Ú_tied_weights_keysÚ_is_statefulr1   r   r   rT   rÒ  ry   r
   rÓ  rÔ  rS   r   r   r   rs   r{   rë  r~   r   s   @rC   r×  r×    sc  ø€ € € € € à*Ð,GÐHÐØ€Lðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eœlÑ*ð<
ð Ð+Ô,ð<
ð 
#ð<
ð <
ð <
ñ „^ñ Ôð<
ð| ðð ð ñ „\ðð ð ð ð rD   r×  )r�  r•  r×  )rï   )r!   )JÚcollections.abcr   Útypingr   r   rT   Útorch.nn.functionalr   rø   rÃ   Útorch.nnr   Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_zayar"   ÚModuler$   r‚   ry   rS   r›   r�   rÝ   rî   rX   rþ   r
  r  r#  r,  r<  rF  rd  r'  r�  r•  r×  Ú__all__r1  rD   rC   ú<module>r     sž  ðð, %Ð $Ð $Ð $Ð $Ð $Ø  Ð  Ð  Ð  Ð  Ð  Ð  Ð  à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sÐ sØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ðM<ð M<ð M<ð M<ð M<˜"œ)ñ M<ô M<ð M<ð` Ð˜YÑ'Ô'ðJð Jð Jð Jð J�"”)ñ Jô Jñ (Ô'ðJð(	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uðl!ð l!ð l!ð l!ð l!˜œ	ñ l!ô l!ð l!ð^(ð (ð (ð (ð (�”ñ (ô (ð (ð,(ð (ð (ð ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð4#ð #ð #ð #ðLI)ð I)ð I)ð I)ð I)�B”Iñ I)ô I)ð I)ðX,8ð ,8ð ,8ð ,8ð ,8Ð1ñ ,8ô ,8ð ,8ð^(ð (ð (ð (ð (˜"œ)ñ (ô (ð (ð,ð ,ð ,ð ,ð ,�B”Iñ ,ô ,ð ,ð <
ð <
ð <
ð <
ð <
�”ñ <
ô <
ð <
ð~ ð$#ð $#ð $#ð $#ð $#�"”)ñ $#ô $#ñ Ôð$#ðN8ð 8ð 8ð 8ð 8˜œñ 8ô 8ð 8ð0 ð/Xð /Xð /Xð /Xð /X˜/ñ /Xô /Xñ „ð/Xðd ðb
ð b
ð b
ð b
ð b
Ð#ñ b
ô b
ñ „ðb
ðJ €Ð,Ð-Ñ-Ô-ð_ð _ð _ð _ð _Ð)¨?ñ _ô _ñ .Ô-ð_ðD BÐ
AÐ
A€€€rD   