§
    ‚Štj†r  ã                   óÖ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  G d„ dej1        ¦  «        Z2 ed¦  «         G d„ dej1        ¦  «        ¦   «         Z3 G d„ dej1        ¦  «        Z4 G d„ dej1        ¦  «        Z5e G d„ d ej1        ¦  «        ¦   «         Z6 G d!„ d"ej1        ¦  «        Z7d#„ Z8 ed$¦  «        d?d%„¦   «         Z9d&ej:        d'e;d(ej:        fd)„Z<	 d@d+ej1        d,ej:        d-ej:        d.ej:        d/ej:        dz  d0e=d1e=d2e$e&         fd3„Z> ee9¦  «         G d4„ d5ej1        ¦  «        ¦   «         Z? G d6„ d7e¦  «        Z@ G d8„ d9e"¦  «        ZAe' G d:„ d;eA¦  «        ¦   «         ZBe' G d<„ d=eAe¦  «        ¦   «         ZCg d>¢ZDdS )Aé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚAfmoeConfigc                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚAfmoeRotaryEmbeddingÚinv_freqNÚconfigc                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultr%   F)Ú
persistentÚoriginal_inv_freq)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   Úrope_parametersr(   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)Úselfr&   ÚdeviceÚrope_init_fnr%   Ú	__class__s        €úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/afmoe/modeling_afmoe.pyr-   zAfmoeRotaryEmbedding.__init__4   sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUó    r7   ztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   é   ©Údtype)r7   rC   )	r1   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚint64ÚtoÚfloat)r&   r7   r<   ÚbaseÚdimÚattention_factorr%   s          r:   r2   z4AfmoeRotaryEmbedding.compute_default_rope_parametersD   sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r;   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   éÿÿÿÿr!   ÚmpsÚcpuF)Údevice_typeÚenabledrA   ©rM   rB   )r%   rK   ÚexpandÚshaperJ   r7   Ú
isinstanceÚtypeÚstrr   Ú	transposerG   ÚcatÚcosr3   ÚsinrC   )
r6   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrS   ÚfreqsÚembr]   r^   s
             r:   ÚforwardzAfmoeRotaryEmbedding.forwardb   s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*©N)NNN)Ú__name__Ú
__module__Ú__qualname__rG   ÚTensorÚ__annotations__r"   r-   Ústaticmethodr   ÚintÚtuplerK   r2   Úno_gradr   re   Ú__classcell__©r9   s   @r:   r$   r$   1   sù   ø€ € € € € € ØŒlÐÐÑðVð V˜{ð Vð Vð Vð Vð Vð Vð  à%)Ø+/Ø"ð*ð *Ø˜dÑ"ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r;   r$   ÚRMSNormc                   óF   ‡ — e Zd Zddeddfˆ fd„Zdej        fd„Zd„ Zˆ xZ	S )	ÚAfmoeRMSNormç�íµ ÷Æ°>Úepsr=   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )z;
        AfmoeRMSNorm is equivalent to T5LayerNorm
        N)r,   r-   r   Ú	ParameterrG   ÚonesÚweightÚvariance_epsilon)r6   rE   rv   r9   s      €r:   r-   zAfmoeRMSNorm.__init__t   sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr;   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |z                       |¦  «        S )NrA   rP   T)Úkeepdim)	rC   rJ   rG   Úfloat32ÚpowÚmeanÚrsqrtr{   rz   )r6   Úhidden_statesÚinput_dtypeÚvariances       r:   re   zAfmoeRMSNorm.forward|   s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØ”˜mÑ+×/Ò/°Ñ<Ô<Ð<r;   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)rn   rz   rW   r{   )r6   s    r:   Ú
extra_reprzAfmoeRMSNorm.extra_reprƒ   s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr;   )ru   )
rg   rh   ri   rK   r-   rG   rj   re   r†   rp   rq   s   @r:   rt   rt   r   sƒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð=¨¬ð =ð =ð =ð =ðJð Jð Jð Jð Jð Jð Jr;   rt   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚAfmoeMLPNc                 ó   •— t          ¦   «                              ¦   «          || _        |j        | _        |€|j        n|| _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _	        t          |j                 | _        d S ©NF©Úbias)r,   r-   r&   rE   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r6   r&   r�   r9   s      €r:   r-   zAfmoeMLP.__init__ˆ   s±   ø€ Ý‰Œ×ÒÑÔÐØˆŒØ!Ô-ˆÔØ=NÐ=V Ô!9Ð!9Ð\mˆÔÝœ 4Ô#3°TÔ5KÐRWÐXÑXÔXˆŒÝ”y Ô!1°4Ô3IÐPUÐVÑVÔVˆŒÝœ 4Ô#9¸4Ô;KÐRWÐXÑXÔXˆŒÝ˜VÔ.Ô/ˆŒˆˆr;   c                 ó¨   — |                       |                      |                      |¦  «        ¦  «        |                      |¦  «        z  ¦  «        }|S rf   )r‘   r“   r�   r�   )r6   r_   r‘   s      r:   re   zAfmoeMLP.forward’   sA   € Ø—N’N 4§;¢;¨t¯~ª~¸aÑ/@Ô/@Ñ#AÔ#AÀDÇLÂLÐQRÁOÄOÑ#SÑTÔTˆ	ØÐr;   rf   )rg   rh   ri   r-   re   rp   rq   s   @r:   rˆ   rˆ   ‡   sL   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r;   rˆ   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAfmoeTokenChoiceRouterz©
    Token-choice top-K router for MoE routing.

    This router assigns each token to the top-K experts based on sigmoid scores, matching the released checkpoints.
    c                 óê   •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        |j	        |j        d¬¦  «        | _
        d S rŠ   )r,   r-   r&   Únum_experts_per_tokÚtop_kÚnum_expertsÚroute_scaler   rŽ   rE   Úgate©r6   r&   r9   s     €r:   r-   zAfmoeTokenChoiceRouter.__init__ž   sb   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ/ˆŒ
Ø!Ô-ˆÔØ!Ô-ˆÔÝ”I˜fÔ0°&Ô2DÈ5ÐQÑQÔQˆŒ	ˆ	ˆ	r;   r‚   Úexpert_biasc                 óž  — |j         \  }}}|                     d|¦  «        }|                      |¦  «                             t          j        ¦  «        }t	          j        |¦  «        }t	          j        ||z   | j        d¬¦  «        \  }}| 	                    d|¬¦  «        }| 
                    dd¬¦  «        dz   }	||	z  }|| j        z  }|||fS )NrP   r!   )ÚkrM   )rM   ÚindexT)rM   r}   g#B’¡œÇ;)rW   Úviewrœ   rJ   rG   r~   ÚsigmoidÚtopkr™   ÚgatherÚsumr›   )
r6   r‚   rž   Ú_Ú
hidden_dimÚrouter_logitsÚscoresÚselected_expertsÚ
top_scoresÚdenominators
             r:   re   zAfmoeTokenChoiceRouter.forward¦   sÏ   € Ø(Ô.Ñˆˆ1ˆjØ%×*Ò*¨2¨zÑ:Ô:ˆàŸ	š	 -Ñ0Ô0×3Ò3µE´MÑBÔBˆÝ”˜}Ñ-Ô-ˆå#œj¨°+Ñ)=ÀÄÐQRÐSÑSÔSÑˆÐØ—]’] qÐ0@�]ÑAÔAˆ
Ø —n’n¨°T�nÑ:Ô:¸UÑBˆØ +Ñ-ˆ
Ø $Ô"2Ñ2ˆ
Ø˜jÐ*:Ð:Ð:r;   ©	rg   rh   ri   Ú__doc__r-   rG   rj   re   rp   rq   s   @r:   r–   r–   —   sm   ø€ € € € € ðð ðRð Rð Rð Rð Rð; U¤\ð ;ÀÄð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r;   r–   c                   ób   ‡ — e Zd ZdZˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚAfmoeExpertsz2Collection of expert weights stored as 3D tensors.c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j        t          j
        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )NrA   )r,   r-   rš   rE   r¨   Úmoe_intermediate_sizeÚintermediate_dimr   rx   rG   ÚemptyÚgate_up_projr‘   r   r’   r“   r�   s     €r:   r-   zAfmoeExperts.__init__¹   s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ Ô,ˆŒØ &Ô <ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr;   r‚   Útop_k_indexÚtop_k_weightsr=   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesrA   r!   r   )rP   éþÿÿÿrU   rP   )rG   Ú
zeros_likero   r   Ú
functionalÚone_hotrš   ÚpermuteÚgreaterr¦   ÚnonzeroÚwhereÚlinearr¶   Úchunkr“   r‘   Ú
index_add_rJ   rC   )r6   r‚   r·   r¸   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_staterœ   ÚupÚcurrent_hidden_statess                 r:   re   zAfmoeExperts.forwardÂ   sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6r®   rq   s   @r:   r±   r±   µ   s€   ø€ € € € € à<Ð<ð0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r;   r±   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚAfmoeSparseMoeBlockzÎ
    Mixture of Experts (MoE) module for AFMoE.

    This module implements a sparse MoE layer with both shared experts (always active) and
    routed experts (activated based on token-choice routing).
    c                 óN  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          ||j        |j        z  ¦  «        | _        t          |¦  «        | _
        t          j        t          j        |j        ¦  «        d¬¦  «        | _        d S )NF)Úrequires_grad)r,   r-   r&   r–   Úrouterrˆ   r³   Únum_shared_expertsÚshared_expertsr±   Úexpertsr   rx   rG   Úzerosrš   rž   r�   s     €r:   r-   zAfmoeSparseMoeBlock.__init__å   s†   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ,¨VÑ4Ô4ˆŒÝ& v¨vÔ/KÈfÔNgÑ/gÑhÔhˆÔÝ# FÑ+Ô+ˆŒÝœ<­¬°FÔ4FÑ(GÔ(GÐW\Ð]Ñ]Ô]ˆÔÐÐr;   c                 ó8  — |j         \  }}}|                     d|¦  «        }|                      || j        ¦  «        \  }}}|                      |¦  «                             |||¦  «        }	|                      |||¦  «                             |||¦  «        }
|	|
z   S )NrP   )rW   r¢   rÓ   rž   rÕ   rÖ   )r6   r‚   Ú
batch_sizer<   r¨   Úhidden_states_flatr§   r¬   r«   Úshared_outputÚrouted_outputs              r:   re   zAfmoeSparseMoeBlock.forwardí   s¬   € Ø*7Ô*=Ñ'ˆ
�G˜ZØ*×/Ò/°°JÑ?Ô?Ðð +/¯+ª+°mÀTÔEUÑ*VÔ*VÑ'ˆˆ:Ð'ð ×+Ò+Ð,>Ñ?Ô?×DÒDÀZÐQXÐZdÑeÔeˆØŸšÐ%7Ð9IÈ:ÑVÔV×[Ò[Ø˜ ñ
ô 
ˆð ˜}Ñ,Ð,r;   )rg   rh   ri   r¯   r-   re   rp   rq   s   @r:   rÐ   rÐ   Ý   sV   ø€ € € € € ðð ð^ð ^ð ^ð ^ð ^ð-ð -ð -ð -ð -ð -ð -r;   rÐ   c                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..NrP   rA   rU   )rW   rG   r\   )r_   Úx1Úx2s      r:   Úrotate_halfrà   ü   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'r;   Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerà   )Úqr    r]   r^   Úunsqueeze_dimÚq_embedÚk_embeds          r:   Úapply_rotary_pos_embrè     sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr;   r‚   Ún_repr=   c                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rW   rV   Úreshape)r‚   ré   ÚbatchÚnum_key_value_headsÚslenr@   s         r:   Ú	repeat_kvrï     s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr;   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )NrA   r   rP   )rM   rC   )ÚpÚtrainingr!   )rï   Únum_key_value_groupsrG   Úmatmulr[   r   r½   Úsoftmaxr~   rJ   rC   r÷   rû   Ú
contiguous)rñ   rò   ró   rô   rõ   rö   r÷   rø   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r:   Úeager_attention_forwardr  )  sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r;   c                   óÄ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej        de	ej        ej        f         dej        dz  d	e
dz  d
ee         de	ej        ej        f         fd„Zˆ xZS )ÚAfmoeAttentionaJ  
    Multi-headed attention module with optional sliding window and gating.

    This attention mechanism supports both full attention and sliding window attention,
    and includes Q/K normalization and gating of the output. It inherits from [`LlamaAttention`] to minimize the amount
    of custom logic we need to maintain.
    r&   Ú	layer_idxc                 óà  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        | j        dz  | _
        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        |j        |         dk    | _        | j        r|j        nd | _        t/          | j        |j        ¬¦  «        | _        t/          | j        |j        ¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _        d S )Nr@   g      à¿Tr‹   Úsliding_attention©rv   F)r,   r-   r&   r  rD   rE   rF   r@   rí   rü   rö   Úattention_dropoutÚ	is_causalr   rŽ   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚlayer_typesÚis_local_attentionÚsliding_windowrt   Úrms_norm_epsÚq_normÚk_normr�   ©r6   r&   r  r9   s      €r:   r-   zAfmoeAttention.__init__L  sÍ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!Ø”} dÑ*ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒð
 #)Ô"4°YÔ"?ÐCVÒ"VˆÔØ7;Ô7NÐX˜fÔ3Ð3ÐTXˆÔå" 4¤=°fÔ6IÐJÑJÔJˆŒÝ" 4¤=°fÔ6IÐJÑJÔJˆŒÝœ 6Ô#5°vÔ7QÐTXÔTaÑ7aÐhmÐnÑnÔnˆŒˆˆr;   Nr‚   Úposition_embeddingsrõ   Úpast_key_valuerø   r=   c                 óì  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «        }|                      |¦  «                             |¦  «        }	|                      |¦  «                             |¦  «        }
|                      |¦  «        }|                      |¦  «                             dd¦  «        }|  	                    |	¦  «                             dd¦  «        }	|
                     dd¦  «        }
| j
        r|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j        ¦  «        \  }	}
t          j        | j        j        t$          ¦  «        } || ||	|
f|| j        sdn| j        | j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|t1          j        |¦  «        z  }|                      |¦  «        }||fS )NrP   r!   rA   rð   )rõ   r÷   rö   r  )rW   r@   r  r¢   r  r  r�   r  r[   r  r  rè   Úupdater  r   Úget_interfacer&   Ú_attn_implementationr  rû   r  rö   r  rÿ   rG   r£   r  )r6   r‚   r  rõ   r  rø   Úinput_shapeÚhidden_shapeÚquery_statesr   r  Úgate_statesr]   r^   Úattention_interfaceÚoutputr  r  s                     r:   re   zAfmoeAttention.forwardk  s  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔBˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔDˆØ—n’n ]Ñ3Ô3ˆà—{’{ <Ñ0Ô0×:Ò:¸1¸aÑ@Ô@ˆØ—[’[ Ñ,Ô,×6Ò6°q¸!Ñ<Ô<ˆ
Ø#×-Ò-¨a°Ñ3Ô3ˆàÔ"ð 	`Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ%Ø'5×'<Ò'<¸ZÈÐW[ÔWeÑ'fÔ'fÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð  3Ð2ØØØØð	
 
ð
 *Ø#œ}ÐH�C�C°$Ô2HØ”LØÔ.ð
 
ð 
 
ð ð
 
ð 
 
Ñˆ�ð �”Ð.˜kÐ.¨2Ð.Ð.Ð.×9Ò9Ñ;Ô;ˆØ�%œ-¨Ñ4Ô4Ñ4ˆØ—k’k &Ñ)Ô)ˆØ˜LÐ(Ð(r;   rf   )rg   rh   ri   r¯   r"   rm   r-   rG   rj   rn   r	   r   r   re   rp   rq   s   @r:   r  r  B  så   ø€ € € € € ðð ðo˜{ð o°sð oð oð oð oð oð oðH (,ð.)ð .)à”|ð.)ð # 5¤<°´Ð#=Ô>ð.)ð œ tÑ+ð	.)ð
  ™ð.)ð Ð+Ô,ð.)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð.)ð .)ð .)ð .)ð .)ð .)ð .)ð .)r;   r  c                   óÖ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej	        dz  d	e
dz  d
edz  deej        ej        f         dz  dee         dej        fd„Zˆ xZS )ÚAfmoeDecoderLayerzÌ
    AFMoE decoder layer with dual normalization.

    This layer applies self-attention followed by either a dense MLP or MoE block,
    with dual normalization (pre and post) around each component.
    r&   r  c                 ó  •— t          ¦   «                              ¦   «          |j        | _        || _        t	          ||¬¦  «        | _        t          |j        |j        ¬¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        t          |j        |j        ¬¦  «        | _        ||j        k    | _        | j        rt          |¦  «        | _        d S t!          |¦  «        | _        d S )N)r&   r  r
  )r,   r-   rE   r  r  Ú	self_attnrt   r  Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernormÚnum_dense_layersÚmoe_enabledrÐ   Úmlprˆ   r  s      €r:   r-   zAfmoeDecoderLayer.__init__¤  sò   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ"ˆŒå'¨vÀÐKÑKÔKˆŒõ  ,¨FÔ,>ÀFÔDWÐXÑXÔXˆÔÝ(4°VÔ5GÈVÔM`Ð(aÑ(aÔ(aˆÔ%õ ".¨fÔ.@ÀfÔFYÐ!ZÑ!ZÔ!ZˆÔÝ".¨vÔ/AÀvÔGZÐ"[Ñ"[Ô"[ˆÔð %¨Ô(?Ò?ˆÔØÔð 	(Ý*¨6Ñ2Ô2ˆDŒHˆHˆHå Ñ'Ô'ˆDŒHˆHˆHr;   Nr‚   rõ   r`   r  Ú	use_cacher  rø   r=   c           
      ó"  — |}|                       |¦  «        } | j        d||||||dœ|¤Ž\  }}	|                      |¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   }|S )N)r‚   rõ   r`   r  r0  r  © )r)  r(  r*  r+  r/  r,  )
r6   r‚   rõ   r`   r  r0  r  rø   Úresidualr§   s
             r:   re   zAfmoeDecoderLayer.forwardº  sÉ   € ð !ˆð ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø)Ø%Ø)ØØ 3ð
ð 
ð ð
ð 
Ñˆ�qð ×5Ò5°mÑDÔDˆØ  =Ñ0ˆð !ˆØ×.Ò.¨}Ñ=Ô=ˆØŸš Ñ/Ô/ˆØ×/Ò/°Ñ>Ô>ˆà  =Ñ0ˆØÐr;   )NNNNN)rg   rh   ri   r¯   r"   rm   r-   rG   rj   Ú
LongTensorr	   Úboolrn   r   r   ÚFloatTensorre   rp   rq   s   @r:   r&  r&  œ  s  ø€ € € € € ðð ð(˜{ð (°sð (ð (ð (ð (ð (ð (ð2 /3Ø04Ø'+Ø!%ØHLð!ð !à”|ð!ð œ tÑ+ð!ð Ô&¨Ñ-ð	!ð
  ™ð!ð ˜$‘;ð!ð # 5¤<°´Ð#=Ô>ÀÑEð!ð Ð+Ô,ð!ð 
Ô	ð!ð !ð !ð !ð !ð !ð !ð !r;   r&  c                   ó€   ‡ — e Zd ZU dZeed<   dZdgZdgZ e	e
d¬¦  «        eedœZg d	¢Zd
Zd
Zd
Zd
Zd
Zd
Zˆ fd„Zˆ xZS )ÚAfmoePreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r&   Úmodelr&  Úpast_key_valuesr   )r¡   )r©   r‚   Ú
attentions)r)  r*  r+  r,  r  r  Únormrž   Tc                 óÊ  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         dS t	          |t          ¦  «        r t          j        |j        j        ¦  «         dS t	          |t          ¦  «        rt          j        |j        ¦  «         dS dS )zInitialize the weightsrð   )r€   ÚstdN)r,   Ú_init_weightsr&   Úinitializer_rangerX   r±   ÚinitÚnormal_r¶   r‘   r–   Úzeros_rœ   rz   rÐ   rž   )r6   rñ   r>  r9   s      €r:   r?  z"AfmoePreTrainedModel._init_weightsþ  sÕ   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�f�lÑ+Ô+ð 	,ÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 6Ñ7Ô7ð 	,ÝŒK˜œÔ*Ñ+Ô+Ð+Ð+Ð+Ý˜Õ 3Ñ4Ô4ð 	,ÝŒK˜Ô*Ñ+Ô+Ð+Ð+Ð+ð	,ð 	,r;   )rg   rh   ri   r¯   r"   rk   Úbase_model_prefixÚ_no_split_modulesÚ_skip_keys_device_placementr   r–   r&  r  Ú_can_record_outputsÚ_keep_in_fp32_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚsupports_gradient_checkpointingr?  rp   rq   s   @r:   r8  r8  Þ  sÇ   ø€ € € € € € ðð ð
 ÐÐÑØÐØ,Ð-ÐØ#4Ð"5Ðà'˜Ð(>ÀaÐHÑHÔHØ*Ø$ðð Ðð
	ð 	ð 	Ðð €NØÐØÐØ!ÐØ"&ÐØ&*Ð#ð
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,ð 
,r;   r8  c                   óì   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j
        dz  de	j        dz  de	j        dz  de	j
        dz  d	edz  d
edz  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú
AfmoeModelz›
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AfmoeDecoderLayer`]

    Args:
        config: AfmoeConfig
    r&   c                 óÞ  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        t!          ‰¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r2  )r&  )Ú.0r  r&   s     €r:   ú
<listcomp>z'AfmoeModel.__init__.<locals>.<listcomp>  s$   ø€ ÐcÐcÐc°iÕ˜v yÑ1Ô1ÐcÐcÐcr;   r
  ©r&   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrE   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersÚlayersrt   r  r<  r$   Ú
rotary_embÚgradient_checkpointingÚ	post_initr�   s    `€r:   r-   zAfmoeModel.__init__  sÒ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØcÐcÐcÐcÅ5ÈÔIaÑCbÔCbÐcÑcÔcñ
ô 
ˆŒõ ! Ô!3¸Ô9LÐMÑMÔMˆŒ	Ý.°fÐ=Ñ=Ô=ˆŒØ&+ˆÔ#à�ŠÑÔÐÐÐr;   NÚ	input_idsrõ   Úinputs_embedsr`   r:  r0  rø   r=   c           
      óø  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s%| j        |||dœ}
t          di |
¤Žt          di |
¤Ždœ}	|}| j        j        r|| j        j        dz  z  }|                      ||¦  «        }t#          | j        ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||d	œ|¤Ž}Œ+|                      |¦  «        }t+          ||r|nd ¬
¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embedsrU  r   r!   )r7   )r&   rc  rõ   r:  )Úfull_attentionr	  g      à?)rõ   r`   r  r0  r  )Úlast_hidden_stater:  r2  )Ú
ValueErrorr
   r&   rZ  Úget_seq_lengthrG   rH   rW   r7   rã   rX   Údictr   r   Úmup_enabledrE   r_  Ú	enumerater^  r  r<  r   )r6   rb  rõ   rc  r`   r:  r0  rø   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr‚   r  ÚiÚdecoder_layers                  r:   re   zAfmoeModel.forward#  s  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 
	àœ+Ø!.Ø"0Ø#2ð	ð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð
 &ˆð Œ;Ô"ð 	KØ)¨T¬[Ô-DÀcÑ-IÑJˆMà"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ñ 6Ô 6ð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø.Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ%Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r;   )NNNNNN)rg   rh   ri   r¯   r"   r-   r   r   r    rG   r4  rj   r6  r	   r5  r   r   rn   r   re   rp   rq   s   @r:   rP  rP    s  ø€ € € € € ðð ð˜{ð ð ð ð ð ð ð ØØð .2Ø.2Ø26Ø04Ø(,Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô(¨4Ñ/ð	<
ð
 Ô&¨Ñ-ð<
ð  ™ð<
ð ˜$‘;ð<
ð Ð+Ô,ð<
ð 
Ð'Ñ	'ð<
ð <
ð <
ñ „_ñ  Ôñ „^ð<
ð <
ð <
ð <
ð <
r;   rP  c                   ó$  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd
e	j
        dz  de	j        dz  de	j
        dz  dedz  de	j        dz  de	j
        dz  dedz  dedz  dee	j        z  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚAfmoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr‚   Úlogitsc                 óþ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S rŠ   )
r,   r-   rP  r9  rX  r   rŽ   rE   rs  ra  r�   s     €r:   r-   zAfmoeForCausalLM.__init__k  sh   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý Ñ'Ô'ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ�ŠÑÔÐÐÐr;   Nr   rb  rõ   r`   r:  rc  Úlabelsr0  Úoutput_router_logitsÚlogits_to_keeprø   r=   c
                 ór  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}t          |||j        |j        |j        |j        ¬¦  «        S )aÏ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AfmoeForCausalLM

        >>> model = AfmoeForCausalLM.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rb  rõ   r`   r:  rc  r0  rx  )Úlossru  r:  r‚   r;  r©   r2  )r&   rx  r9  rf  rX   rm   Úslicers  Úloss_functionrX  r   r:  r‚   r;  r©   )r6   rb  rõ   r`   r:  rc  rw  r0  rx  ry  rø   Úoutputsr‚   Úslice_indicesru  r{  s                   r:   re   zAfmoeForCausalLM.forwardr  s  € ðB %9Ð$DÐ Ð È$Ì+ÔJjð 	ð +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDå(ØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r;   )	NNNNNNNNr   )rg   rh   ri   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr-   r   r   rG   r4  rj   r	   r6  r5  rm   r   r   r   re   rp   rq   s   @r:   rr  rr  e  s\  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hðð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eœlÑ*ð<
ð Ð+Ô,ð<
ð 
#ð<
ð <
ð <
ñ „^ñ Ôð<
ð <
ð <
ð <
ð <
r;   rr  )rr  rP  r8  )r!   )rð   )EÚcollections.abcr   Útypingr   rG   r   Ú r   rA  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_afmoer"   ÚModuler$   rt   rˆ   r–   r±   rÐ   rà   rè   rj   rm   rï   rK   r  r  r&  r8  rP  rr  Ú__all__r2  r;   r:   ú<module>r–     s  ðð* %Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð SÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ ,Ð ,Ð ,Ð ,Ð ,Ð ,ð><ð ><ð ><ð ><ð ><˜2œ9ñ ><ô ><ð ><ðB Ð˜YÑ'Ô'ðJð Jð Jð Jð J�2”9ñ Jô Jñ (Ô'ðJð(ð ð ð ð ˆrŒyñ ô ð ð ;ð ;ð ;ð ;ð ;˜RœYñ ;ô ;ð ;ð< ð$#ð $#ð $#ð $#ð $#�2”9ñ $#ô $#ñ Ôð$#ðN-ð -ð -ð -ð -˜"œ)ñ -ô -ð -ð>(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ðV)ð V)ð V)ð V)ð V)�R”Yñ V)ô V)ñ +Ô*ðV)ðr?ð ?ð ?ð ?ð ?Ð2ñ ?ô ?ð ?ðD*,ð *,ð *,ð *,ð *,˜?ñ *,ô *,ð *,ðZ ðV
ð V
ð V
ð V
ð V
Ð%ñ V
ô V
ñ „ðV
ðr ðJ
ð J
ð J
ð J
ð J
Ð+¨_ñ J
ô J
ñ „ðJ
ðZ EÐ
DÐ
D€€€r;   