§
    ‚Štj¦ô  ã                   ó\  — d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<  e3j=        e>¦  «        Z?d„ Z@ ed¦  «        dTd„¦   «         ZAdejB        deCdejB        fd„ZD	 dUd!ejE        d"ejB        d#ejB        d$ejB        d%ejB        dz  d&eFd'eFd(e-e/         fd)„ZG eeA¦  «         G d*„ d+ejE        ¦  «        ¦   «         ZHd,ejB        d-eCfd.„ZId/„ ZJd0„ ZKd1„ ZL G d2„ d3ejE        ¦  «        ZM G d4„ d5ej        jE        ¦  «        ZN G d6„ d7ejE        ¦  «        ZO G d8„ d9ejE        ¦  «        ZP G d:„ d;ejE        ¦  «        ZQe G d<„ d=ejE        ¦  «        ¦   «         ZR G d>„ d?ejE        ¦  «        ZS G d@„ dAedB¬C¦  «        ZT edD¦  «         G dE„ dFejE        ¦  «        ¦   «         ZU G dG„ dHe!¦  «        ZVe0 G dI„ dJe+¦  «        ¦   «         ZWe0 G dK„ dLeW¦  «        ¦   «         ZX	 	 	 dVdNejB        eYejB                 z  dz  dOeCdz  d%ejB        dz  dejB        eCz  fdP„ZZe0 G dQ„ dReWe¦  «        ¦   «         Z[g dS¢Z\dS )Wé    )ÚCallable)ÚOptionalÚ	TypedDictN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úlazy_load_kernel)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)Úresolve_internal_import)Úcapture_outputsé   )ÚGraniteMoeHybridConfigc                 óœ   — | dd| j         d         dz  …f         }| d| j         d         dz  d…f         }t          j        | |fd¬¦  «        S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      ú|/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/granitemoehybrid/modeling_granitemoehybrid.pyÚrotate_halfr5   8   s]   € à	
ˆ3Ð"�!”'˜"”+ Ñ"Ð"Ð"Ô	#€BØ	
ˆ3�”˜”˜qÑ Ð"Ð"Ð"Ô	#€BÝŒ9�r�c˜2�Y BÐ'Ñ'Ô'Ð'ó    Úrotary_pos_embc                 ó¾   — |                      |¦  «        }|                      |¦  «        }| |z  t          | ¦  «        |z  z   }||z  t          |¦  «        |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer5   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r4   Úapply_rotary_pos_embrA   ?   sc   € ð& �-Š-˜Ñ
&Ô
&€CØ
�-Š-˜Ñ
&Ô
&€CØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�3‰w�; q™>œ>¨CÑ/Ñ0€GØ�GÐÐr6   Úhidden_statesÚn_repÚreturnc                 ó¸   — | j         \  }}}}|dk    r| S | dd…dd…ddd…dd…f                              |||||¦  «        } |                      |||z  ||¦  «        S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r'   N)r.   ÚexpandÚreshape)rB   rC   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r4   Ú	repeat_kvrL   Y   s„   € ð
 2?Ô1DÑ.€EÐ  hØ�‚z€zØÐØ! ! ! ! Q Q Q¨¨a¨a¨a°°°Ð"2Ô3×:Ò:¸5ÐBUÐW\Ð^bÐdlÑmÔm€MØ× Ò  Ð(;¸eÑ(CÀTÈ8ÑTÔTÐTr6   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t          || j        ¦  «        }t          || j        ¦  «        }	t          j        ||                     dd¦  «        ¦  «        |z  }
|�|
|z   }
t
          j                             |
dt          j        ¬¦  «         	                    |j
        ¦  «        }
t
          j                             |
|| j        ¬¦  «        }
t          j        |
|	¦  «        }|                     dd¦  «                             ¦   «         }||
fS )Nr+   r   r*   )r-   Údtype)ÚpÚtrainingr'   )rL   Únum_key_value_groupsr/   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32ÚtorW   rT   rY   Ú
contiguous)rN   rO   rP   rQ   rR   rS   rT   rU   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r4   Úeager_attention_forwardrf   e   sé   € õ ˜3 Ô ;Ñ<Ô<€JÝ˜U FÔ$?Ñ@Ô@€Lå”<  z×';Ò';¸A¸qÑ'AÔ'AÑBÔBÀWÑL€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€LÝ”,˜|¨\Ñ:Ô:€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r6   c                   óÌ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        dej        dz  de	dz  d	e
ej        ej        f         dz  d
ee         de
ej        ej        f         fd„Zˆ xZS )ÚGraniteMoeHybridAttentionuæ   Hybrid variant that handles ``position_embeddings is None`` â€” granitemoe-hybrid configs can
    opt out of RoPE via ``position_embedding_type=None``, in which case the model passes ``None``
    instead of a ``(cos, sin)`` tuple.ÚconfigÚ	layer_idxc                 ó¨  •— t          ¦   «                              ¦   «          || _        || _        t	          |d|j        |j        z  ¦  «        | _        |j        |j        z  | _	        |j
        | _        |j        | _        d| _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        |j        | j        z  |j        ¬¦  «        | _        t          j        |j        | j        z  |j        |j        ¬¦  «        | _        d S )NrK   T©Úbias)ÚsuperÚ__init__ri   rj   ÚgetattrÚhidden_sizeÚnum_attention_headsrK   rI   rZ   Úattention_multiplierrS   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfri   rj   Ú	__class__s      €r4   ro   z"GraniteMoeHybridAttention.__init__„   s>  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ"ˆŒÝ ¨
°FÔ4FÈ&ÔJdÑ4dÑeÔeˆŒØ$*Ô$>À&ÔB\Ñ$\ˆÔ!ØÔ2ˆŒØ!'Ô!9ˆÔØˆŒå”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ Ô :¸T¼]Ñ JÐQWÔQfð
ñ 
ô 
ˆŒõ ”iØÔ&¨¬Ñ6¸Ô8JÐQWÔQfð
ñ 
ô 
ˆŒˆˆr6   NrB   rR   Úpast_key_valuesÚposition_embeddingsrU   rD   c                 ó&  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
|�|\  }}t          ||	||¦  «        \  }}	|�|                     |	|
| j	        ¦  «        \  }	}
t          j        | j        j        t          ¦  «        } || ||	|
|f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Nr*   r'   r+   rM   )rT   rS   )r.   rK   rx   Úviewr\   ry   rz   rA   Úupdaterj   r   Úget_interfaceri   Ú_attn_implementationrf   rY   rt   rS   rG   ra   r{   )r}   rB   rR   r   r€   rU   Úinput_shapeÚhidden_shapeÚquery_statesrb   rc   r<   r=   Úattention_interfacere   rd   s                   r4   Úforwardz!GraniteMoeHybridAttention.forward›   sÆ  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆØ—[’[ Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆ
Ø—{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàÐ*Ø*‰HˆC�Ý';¸LÈ*ÐVYÐ[^Ñ'_Ô'_Ñ$ˆL˜*àÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—k’k +Ñ.Ô.ˆØ˜LÐ(Ð(r6   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r(   Úintro   r/   ÚTensorr
   Útupler   r   rŠ   Ú__classcell__©r~   s   @r4   rh   rh   ~   så   ø€ € € € € ð*ð *ð
Ð5ð 
À#ð 
ð 
ð 
ð 
ð 
ð 
ð6 )-ØHLð%)ð %)à”|ð%)ð œ tÑ+ð%)ð  ™ð	%)ð
 # 5¤<°´Ð#=Ô>ÀÑEð%)ð Ð+Ô,ð%)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð%)ð %)ð %)ð %)ð %)ð %)ð %)ð %)r6   rh   Úinput_tensorÚpad_sizec                 ó¦   — t          | j        ¦  «        dk    r
ddddd|ddfnddd|ddf}t          j        j                             | |dd¬¦  «        S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)ÚmoderQ   )Úlenr.   r/   r   r]   Úpad)r•   r–   Ú	pad_shapes      r4   Úpad_tensor_by_sizerž   Æ   sj   € õ 47°|Ô7IÑ3JÔ3JÈaÒ3OÐ3O��A�q˜!˜Q ¨!¨QÐ/Ð/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€IåŒ8Ô×"Ò" <°ÀÐSTÐ"ÑUÔUÐUr6   c                 ó"  — t          | |¦  «        } t          | j        ¦  «        dk    r.|                      | j        d         d|| j        d         ¦  «        S |                      | j        d         d|| j        d         | j        d         ¦  «        S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r*   r+   )rž   r›   r.   rG   )r•   r–   Ú
chunk_sizes      r4   Úreshape_into_chunksr¡   Ñ   s’   € õ & l°HÑ=Ô=€Lå
ˆ<ÔÑÔ !Ò#Ð#à×#Ò# LÔ$6°qÔ$9¸2¸zÈ<ÔK]Ð^_ÔK`ÑaÔaÐað ×#Ò#ØÔ˜qÔ! 2 z°<Ô3EÀaÔ3HÈ,ÔJ\Ð]^ÔJ_ñ
ô 
ð 	
r6   c                 ó  — |                       d¦  «        } | d         j        g |                       ¦   «         ¢|‘R Ž } t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                      | d¦  «        } t          j        | d¬¦  «        }t          j        t          j        ||| j        t          j        ¬¦  «        d¬¦  «        }|                     | t          j	         ¦  «        }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r*   ©.N©ÚdevicerW   )Údiagonalr   éþÿÿÿr,   )
ÚsizerF   r/   ÚtrilÚonesr¥   ÚboolÚmasked_fillÚcumsumÚinf)r•   r    ÚmaskÚtensor_segsums       r4   Úsegment_sumr±   å   só   € ð ×"Ò" 2Ñ&Ô&€Jð 2�< 	Ô*Ô1ÐS°<×3DÒ3DÑ3FÔ3FÐSÈ
ÐSÐSÐS€LåŒ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqsÐtÑtÔt€DØ×+Ò+¨T¨E°1Ñ5Ô5€Lå”L °2Ð6Ñ6Ô6€Mõ Œ:•e”j ¨ZÀÔ@SÕ[`Ô[eÐfÑfÔfÐqrÐsÑsÔs€DØ!×-Ò-¨t¨eµe´i°ZÑ@Ô@€MØÐr6   c                 ó¦   — |�N|j         d         dk    r=|j         d         dk    r,| j        }| |dd…dd…df         z                       |¦  «        } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr'   r   )r.   rW   r`   )rB   rR   rW   s      r4   Úapply_mask_to_padding_statesr³   ù   si   € ð
 Ð! nÔ&:¸1Ô&=ÀÒ&AÐ&AÀnÔFZÐ[\ÔF]Ð`aÒFaÐFaØÔ#ˆØ&¨¸¸¸¸1¸1¸1¸d¸
Ô)CÑC×GÒGÈÑNÔNˆàÐr6   c            
       ó  ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej        de	dz  dej        dz  d	ej
        dz  fd
„Z	 	 dde	dz  dej        dz  fd„Z ed¦  «        	 	 	 dde	dz  dej        dz  d	ej
        dz  fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridMambaLayeruP  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    ri   rj   c           	      ó  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j        | _        t          |j
        | j        z  ¦  «        | _        || _        |j        | _        |j        | _        t"          |j                 | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        | j        d| j        z  | j        z  z   | _         tC          j"        | j         | j         |j        | j        | j         | j        dz
  ¬¦  «        | _#        | j        | j         z   | j        z   }tC          j$        | j        || j        ¬¦  «        | _%        tC          j&        tO          j(        | j        ¦  «        ¦  «        | _)        tO          j*        d| j        dz   ¦  «        }tC          j&        tO          j+        |¦  «        ¦  «        | _,        t[          | j        | j        ¬¦  «        | _.        tC          j&        tO          j(        | j        ¦  «        ¦  «        | _/        tC          j$        | j        | j        | j        ¬¦  «        | _0        tc          d¦  «        }te          |dd ¦  «        a3te          |dd ¦  «        a4tc          d	¦  «        }tk          |d
¬¦  «        a6tk          |d¬¦  «        a7tk          |d¬¦  «        a8ts          tl          tn          tp          th          tf          f¦  «        a:tt          stv           <                    d¦  «         ntv           <                    d¦  «         |j=        |         | _>        d S )Nr+   r'   )Úin_channelsÚout_channelsrm   Úkernel_sizeÚgroupsÚpaddingrl   ©Úepszcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzOThe fast path for GraniteMoeHybrid will be used when running the model on a GPU)?rn   ro   Úmamba_n_headsÚ	num_headsrq   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer�   Úmamba_expandÚintermediate_sizerj   Úmamba_conv_biasÚuse_conv_biasÚ
hidden_actÚ
activationr	   ÚactÚmamba_proj_biasÚuse_biasÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headrK   Úmamba_chunk_sizer    Útime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimr   ÚConv1dÚconv1drv   Úin_projÚ	Parameterr/   rª   Údt_biasÚarangeÚlogÚA_logÚGraniteMoeHybridRMSNormGatedÚnormÚDÚout_projr   rp   r¾   r¿   r%   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r}   ri   rj   Úprojection_sizeÚAÚcausal_conv1dÚ	mamba_ssmr~   s          €r4   ro   z#GraniteMoeHybridMambaLayer.__init__  s:  ø€ Ý‰Œ×ÒÑÔÐØÔ-ˆŒØ!Ô-ˆÔØ$Ô2ˆÔØ &Ô 3ˆÔÝ!$ VÔ%8¸4Ô;KÑ%KÑ!LÔ!LˆÔØ"ˆŒØ#Ô3ˆÔØ Ô+ˆŒÝ˜&Ô+Ô,ˆŒØÔ.ˆŒà"(Ô"5ˆÔàÔ-ˆŒØÔ+ˆŒØ Ô1ˆŒà%Ô5ˆÔØ#Ô1ˆÔØ#Ô1ˆÔàÔ.°°T´]Ñ1BÀTÔEXÑ1XÑXˆŒÝ”iØœØœØÔ'ØÔ-Ø”=ØÔ)¨AÑ-ð
ñ 
ô 
ˆŒð Ô0°4´=Ñ@À4Ä>ÑQˆÝ”yØÔØØ”ð
ñ 
ô 
ˆŒõ ”|¥E¤J¨t¬~Ñ$>Ô$>Ñ?Ô?ˆŒõ ŒL˜˜DœN¨QÑ.Ñ/Ô/ˆÝ”\¥%¤)¨A¡,¤,Ñ/Ô/ˆŒ
Ý0°Ô1GÈTÔMdÐeÑeÔeˆŒ	Ý”�eœj¨¬Ñ8Ô8Ñ9Ô9ˆŒåœ	 $Ô"8¸$Ô:JÐQUÔQ^Ð_Ñ_Ô_ˆŒõ )¨Ñ9Ô9ˆÝ& }Ð6LÈdÑSÔSÐÝ" =Ð2DÀdÑKÔKÐõ % [Ñ1Ô1ˆ	Ý!8ØÐ$^ð"
ñ "
ô "
Ðõ %<ØÐ$Wð%
ñ %
ô %
Ð!õ ,CØÐ$^ð,
ñ ,
ô ,
Ð(õ
 "%å&Ý)Ý0Ý Ý$ðñ"
ô "
Ðõ &ð 	sÝ×Òð>ñô ð ð õ ×ÒÐ qÑrÔrÐrà Ô,¨YÔ7ˆŒˆˆr6   NrB   Úcache_paramsrR   Úseq_idxc                 ó2  — t          ||¦  «        }|                      |¦  «        }|j        \  }}}| j        | j        z  }	|d uo|                     | j        ¦  «        }
|
r:|j        | j                 j        d         }|j        | j                 j	        d         }|
�rž|dk    �r—| 
                    d¦  «                             | j        | j        | j        gd¬¦  «        \  }}}t          ||| j        j         
                    d¦  «        | j        j        | j        ¦  «        }t)          j        || j        |	|	gd¬¦  «        \  }}}t)          j        | j                             ¦   «         ¦  «         }|d d …d df         d d …d d …d f                              d| j        | j        ¦  «                             t(          j        ¬¦  «        }|d d …d d …d f                              dd| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }| j        d d …d df                              d| j        ¦  «        }|                     || j        |j        d         | j        z  ¦  «        }|                     || j        |j        d         | j        z  ¦  «        }|                     || j        | j        ¦  «        }t?          |||||||d |d¬¦
  «
        }|                     || j        | j        z  ¦  «        }|                       ||¦  «        }|  !                    |¦  «        d d …d df         }�n\t)          j        | j                             ¦   «         ¦  «         }| j"        d	t/          d
¦  «        fk    ri nd| j"        i}| j#        r�|€�tI          || j        j         
                    d¦  «        | j        j        | j        |f| j        | j%        || j        | j         j        | j         j&        | j!        j        | j!        j        | j        | j        dddœ|¤Ž}�nu|                     | j        | j        | j        gd¬¦  «        \  }}}| '                    dd¦  «        }|
rt)          j(        ||gd¬¦  «        }|�PtR          j*         +                    || j,        |j        d         z
  df¦  «        }| -                    || j        ¦  «         | j        dvr>|  .                    |                      |¦  «        dd |j        d         …f         ¦  «        }n@t_          || j        j         
                    d¦  «        | j        j        | j        |¬¦  «        }|
r|d d …d d …| d …f         }| '                    dd¦  «        }t          ||¦  «        }t)          j        || j        |	|	gd¬¦  «        \  }}}ta          |                     ||d| j        ¦  «        |||                     ||| j        d¦  «        |                     ||| j        d¦  «        f| j%        | j        d |d| j        d|
r|nd dœ|¤Ž\  }}|�|�| 1                    || j        ¦  «        }|                     ||d¦  «        }|                       ||¦  «        }|  !                    |¦  «        }|S )Nr   r'   r*   r,   .©rW   T)ÚzrÞ   Údt_softplusrM   r®   Údt_limitF)rä   r    rô   rÌ   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_statesr+   )ÚsiluÚswish)r1   Úweightrm   rÌ   rô   )r    rä   r÷   rô   r  rÞ   rø   Úinitial_states)2r³   rÜ   r.   rÓ   rÄ   Úhas_previous_staterj   ÚlayersÚconv_statesÚrecurrent_statesÚsqueezeÚsplitrÈ   rÙ   rÂ   r¾   rÛ   r  rm   rÌ   r/   Úexprá   ÚfloatrF   rK   r`   r_   rÞ   rä   r‚   ræ   rã   rå   rÖ   rY   rè   r    Úvariance_epsilonr\   r0   r   r]   rœ   rÆ   Úupdate_conv_staterÍ   r¿   rç   Úupdate_recurrent_state)r}   rB   ró   rR   rô   Úprojected_statesÚ
batch_sizeÚseq_lenÚ_Úgroups_time_state_sizeÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stateÚgateÚhidden_states_B_CÚdtÚBÚCrð   rÞ   rä   Úhidden_states_reshapedÚoutÚdt_limit_kwargsr  Úscan_outputÚ	ssm_states                              r4   Úcuda_kernels_forwardz/GraniteMoeHybridMambaLayer.cuda_kernels_forwardr  s  € õ 5°]ÀNÑSÔSˆØŸ<š<¨Ñ6Ô6Ðð "/Ô!4Ñˆ
�G˜QØ!%¤°Ô1DÑ!DÐà!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐØ!ð 	VØ%Ô,¨T¬^Ô<ÔHÈÔKˆJØ*Ô1°$´.ÔAÔRÐSTÔUˆOð "ñ L	1 g°¢l¡lØ*:×*BÒ*BÀ1Ñ*EÔ*E×*KÒ*KØÔ'¨¬¸¼ÐGÈRð +Lñ +ô +Ñ'ˆDÐ# Rõ
 !5Ø!ØØ”Ô"×*Ò*¨1Ñ-Ô-Ø”Ô Ø”ñ!ô !Ðõ #(¤+Ø!ØÔ'Ð)?ÐAWÐXØð#ñ #ô #ÑˆM˜1˜aõ ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ�!�!�!�T˜3�,”    1 1 1 d 
Ô+×2Ò2°2°t´}ÀdÔFYÑZÔZ×]Ò]ÕdiÔdqÐ]ÑrÔrˆAØ�A�A�A�q�q�q˜$�J”×&Ò& r¨2¨t¬}Ñ=Ô=ˆBØ”l 1 1 1 d¨C <Ô0×7Ò7¸¸D¼MÑJÔJˆGØ”�q�q�q˜$ �|Ô$×+Ò+¨B°´Ñ>Ô>ˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ—’�z 4¤=°!´'¸!´*ÀÄÑ2MÑNÔNˆAØ%2×%7Ò%7¸
ÀDÄNÐTXÔTaÑ%bÔ%bÐ"Ý2ØØ&ØØØØØØØØ ðñ ô ˆMð *×.Ò.¨z¸4¼>ÈDÌMÑ;YÑZÔZˆMØ ŸIšI m°TÑ:Ô:ˆMð —-’- Ñ.Ô.¨q¨q¨q°$¸¨|Ô<ˆC‰Cõ ”˜4œ:×+Ò+Ñ-Ô-Ñ.Ô.Ð.ˆAØ$(Ô$8¸SÅ%ÈÁ,Ä,Ð<OÒ$OÐ$O˜b˜bÐV`ÐbfÔbvÐUwˆOð Œ}ð X1 Ð!5Ý6Ø$Ø”KÔ&×.Ò.¨qÑ1Ô1Ø”KÔ$Ø”LØðð ”fØ#œØ#Ø#œØ#'¤9Ô#3Ø $¤	Ô :Ø#'¤=Ô#7Ø!%¤Ô!3Ø œMØ œMØ%*Ø(-ð#ð ð$ &ð%ð �‘ð, /?×.DÒ.DØÔ+¨T¬]¸D¼NÐKÐQSð /Eñ /ô /Ñ+�Ð'¨ð
 %6×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!Ø)ð [õ ).¬	°:Ð?PÐ2QÐWYÐ(ZÑ(ZÔ(ZÐ%ØÐ+Ý"$¤-×"3Ò"3Ø)ØÔ.Ð1BÔ1HÈÔ1LÑLÈaÐPñ#ô #�Kð !×2Ò2°;ÀÄÑOÔOÐOà”?Ð*;Ð;Ð;Ø(,¯ª°·²Ð=NÑ1OÔ1OÐPSÐUrÐWhÔWnÐoqÔWrÐUrÐPrÔ1sÑ(tÔ(tÐ%Ð%å(8Ø+Ø#œ{Ô1×9Ò9¸!Ñ<Ô<Ø!œ[Ô-Ø#'¤?Ø 'ð)ñ )ô )Ð%ð *ð KØ(9¸!¸!¸!¸Q¸Q¸QÀÀÀ	À	¸/Ô(JÐ%Ø$5×$?Ò$?ÀÀ1Ñ$EÔ$EÐ!å$@ÐARÐTbÑ$cÔ$cÐ!Ý&+¤kØ%ØÔ+Ð-CÐE[Ð\Øð'ñ 'ô 'Ñ#�˜q !õ *CØ!×&Ò& z°7¸BÀÄÑNÔNØØØ—F’F˜: w°´¸rÑBÔBØ—F’F˜: w°´¸rÑBÔBð*ð  $œØ”fØØ#Ø(,Ø œLØ $Ø6LÐ#V ? ?ÐRVð*ð *ð &ð*ð *Ñ&�˜Yð$ Ð(¨\Ð-EØ ,× CÒ CÀIÈtÌ~Ñ ^Ô ^�Ià)×.Ò.¨z¸7ÀBÑGÔG�à"Ÿiši¨°TÑ:Ô:�ð —m’m KÑ0Ô0�Øˆ
r6   c                 óÞ  ‡ ‡2— |j         \  }}}|j        }t          ||¦  «        }‰                      |¦  «        }|                     ‰ j        ‰ j        ‰ j        gd¬¦  «        \  }	}
}|
                     dd¦  «        }
|d uo| 	                    ‰ j
        ¦  «        }|r|j        ‰ j
                 j        d         }|r“|dk    r�|                     |
‰ j
        ¦  «        d‰ j         d …f         }t          j        |‰ j        j                             d¦  «        z  d¬¦  «        }
‰ j        r|
‰ j        j        z   }
‰                      |
¦  «        }
nÎ|rt          j        ||
gd¬¦  «        }
|�Pt0          j                             |
‰ j        |
j         d         z
  df¦  «        }|                     |‰ j
        ¦  «         ‰                      ‰                      |
¦  «        dd |
j         d         …f         ¦  «        }
|r|
d| d …f         }
|
                     dd¦  «        }
t          |
|¦  «        }
t          j        |
‰ j        ‰ j        ‰ j        z  ‰ j        ‰ j        z  gd¬¦  «        \  }}}t          j        ‰ j                             ¦   «         ¦  «         }|�rf|dk    �r_|j        ‰ j
                 j         d         j!        }|d d …dd d …f         d d …d df         }|                     dd¦  «         "                    ||j         d         ‰ j#        ¦  «        }‰ j$        d          "                    ‰ j$        j         d         ‰ j#        ¦  «        }t          j        j         %                    || &                    |j        ¦  «        z   ¦  «        }t          j'        |‰ j(        d         ‰ j(        d         ¦  «        }|d          "                    ‰ j        ‰ j#        ‰ j        ¦  «         &                    t          j)        ¬	¦  «        }t          j        |d         |z  ¦  «         &                    |¬
¦  «        }| *                    |‰ j        d¦  «        dd d d …f         }| "                    |‰ j        ‰ j        ‰ j        z  |j         d         ¦  «         +                    ¦   «         }| *                    |d|j         d         ¦  «        }|d         |dd d d …f         z  }| *                    |d‰ j#        ¦  «        }||d         z   &                    |¬
¦  «        }|j        ‰ j
                 j         d         |z  |z   }| ,                    |‰ j
        ¦  «        }| *                    |‰ j        d¦  «        dd d d …f         }| "                    |‰ j        ‰ j        ‰ j        z  |j         d         ¦  «         +                    ¦   «         }| *                    |d|j         d         ¦  «        }| &                    |j!        |j        ¬¦  «        }| -                    |‰ j        z  ‰ j#        ‰ j        ¦  «        }| -                    |‰ j        z  ‰ j        d¦  «        }t          j.        ||¦  «        }| -                    |‰ j        ‰ j#        ¦  «        }‰ j/        d          "                    ‰ j/        j         d         ‰ j#        ¦  «        }|||z  z    &                    |j        ¦  «        }| *                    |d¦  «        d d …d df         }�n0t0          j         %                    |‰ j$        z   ¦  «        }t          j'        |‰ j(        d         ‰ j(        d         ¦  «        }| *                    ||d‰ j#        ¦  «                             ¦   «         }| *                    ||d‰ j        ¦  «                             ¦   «         }| *                    ||d‰ j        ¦  «                             ¦   «         }| 0                    ‰ j        ‰ j        z  d‰ j        ¬¦  «        }| 0                    ‰ j        ‰ j        z  d‰ j        ¬¦  «        }‰ j1        |‰ j1        z  z
  ‰ j1        z  Š2‰ j/        d         te          |‰2¦  «        z  }||d         z  }| &                    |j        ¦  «        |z  }ˆ2ˆ fd„||||fD ¦   «         \  }}}}| 3                    dddd¦  «        }t          j4        |d¬¦  «        }t          j        tk          |¦  «        ¦  «        }|d d …d d …d d …d d d …d d …f         |d d …d d …d d d …d d …d d …f         z  } |                      d¬¦  «        }!|!d         | 3                    ddddd¦  «        d         z  }"|"                     d¬¦  «        }#|#d         |d d …d d …d f         z                       d¬¦  «        }$t          j        |d d …d d …d d …dd …f         |z
  ¦  «        }%||% 3                    dddd¦  «        d         z  }&|&dd d d …f         |d         z                       d¬¦  «        }'|rF|j        ‰ j
                 j         d         d d …d f          &                    |'j        |'j!        ¬¦  «        nt          j6        |'d d …d d…f         ¦  «        }(t          j        |(|'gd¬¦  «        }'t          j        tk          t0          j                             |d d …d d …d d …df         d¦  «        ¦  «        ¦  «        })|)                     dd¦  «        })|)d         |'d d …d d …d df         z                       d¬¦  «        }*|*d d …d d…f         |*d d …df         }+}'t          j        |¦  «        },|dd d d …f         |'d d …d d …d df         z  }-|, 3                    dddd¦  «        }.|-                     d¦  «        |.d         z  }/|$|/z   }| *                    |d‰ j        ‰ j#        ¦  «        }||z   }‰2dk    r|d d …d |…d d …d d …f         }| *                    ||d¦  «        }|+�|�| ,                    |+‰ j
        ¦  «        }+‰  7                    ||	¦  «        }0‰  8                    |0 &                    |¦  «        ¦  «        }1|1S )Nr*   r,   r'   r+   r   .r£   ).NNrö   ©r¥   r¤   )r-   Úoutput_sizec                 ó<   •— g | ]}t          |‰‰j        ¦  «        ‘ŒS © )r¡   r    )Ú.0Útr–   r}   s     €€r4   ú
<listcomp>z<GraniteMoeHybridMambaLayer.torch_forward.<locals>.<listcomp>—  s)   ø€ Ð%zÐ%zÐ%zÐ\]Õ&9¸!¸XÀtÄÑ&WÔ&WÐ%zÐ%zÐ%zr6   r   r˜   r§   )rW   r¥   )r'   r   )9r.   rW   r³   rÜ   r  rÈ   rÙ   rÂ   r\   r  rj   r  r  r  rÆ   r/   ÚsumrÛ   r  r
  rÊ   rm   rÍ   r0   r   r]   rœ   rÓ   rÄ   r  rá   r  r	  r¥   rF   rK   rÞ   Úsoftplusr`   ÚclamprÖ   r_   rG   ra   r  r‚   Úbmmrä   Úrepeat_interleaver    rž   Úpermuter­   r±   Ú
zeros_likerã   rå   )3r}   Úinput_statesró   rR   r  r  r  rW   r  r  r  r  r  r  r  rB   r  r  rð   Úcache_devicerÞ   ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyrä   Ú
D_residualÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesr"  Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr!  Úcontextualized_statesr–   s3   `                                                 @r4   Útorch_forwardz(GraniteMoeHybridMambaLayer.torch_forward  st  øø€ ð ".Ô!3Ñˆ
�G˜QØÔ"ˆõ 4°LÀ.ÑQÔQˆØŸ<š<¨Ñ5Ô5ÐØ&6×&<Ò&<ØÔ'¨¬¸¼ÐGÈRð '=ñ '
ô '
Ñ#ˆÐ ð .×7Ò7¸¸!Ñ<Ô<Ðà!-°TÐ!9Ð!m¸l×>]Ò>]Ð^bÔ^lÑ>mÔ>mÐØ!ð 	LØ%Ô,¨T¬^Ô<ÔHÈÔKˆJð "ð 	B g°¢l lØ&×8Ò8Ð9JÈDÌNÑ[Ô[Ð\_ÐbfÔbwÐawÐaxÐaxÐ\xÔyˆKå %¤	Ø˜dœkÔ0×8Ò8¸Ñ;Ô;Ñ;Àð!ñ !ô !Ðð Ô!ð IØ$5¸¼Ô8HÑ$HÐ!Ø $§¢Ð):Ñ ;Ô ;ÐÐà%ð WÝ$)¤I¨zÐ;LÐ.MÐSUÐ$VÑ$VÔ$VÐ!ØÐ'Ý œm×/Ò/Ø%¨Ô(=Ð@QÔ@WÐXZÔ@[Ñ([Ð]^Ð'_ñô �ð ×.Ò.¨{¸D¼NÑKÔKÐKà $§¢¨¯ªÐ5FÑ)GÔ)GÈÐMjÐO`ÔOfÐgiÔOjÐMjÐHjÔ)kÑ lÔ lÐØ%ð FØ$5°c¸G¸8¸9¸9°nÔ$EÐ!Ø 1× ;Ò ;¸A¸qÑ AÔ AÐå8Ð9JÈNÑ[Ô[ÐÝ#œkØØÔ# T¤]°TÔ5HÑ%HÈ$Ì-ÐZ^ÔZmÑJmÐnØð
ñ 
ô 
Ñˆ�q˜!õ ŒY�t”z×'Ò'Ñ)Ô)Ñ*Ô*Ð*ˆØ!ñ F	[ g°¢l¡là'Ô.¨t¬~Ô>ÔOÐPQÔRÔYˆLð �A�A�A�q˜!˜!˜!�G”˜Q˜Q˜Q  c˜\Ô*ˆBØ—’˜a Ñ#Ô#×*Ò*¨:°r´xÀ´|ÀTÄ]ÑSÔSˆBà”l 9Ô-×4Ò4°T´\Ô5GÈÔ5JÈDÌMÑZÔZˆGå”Ô$×-Ò-¨b°7·:²:¸b¼hÑ3GÔ3GÑ.GÑHÔHˆBÝ”˜R Ô!5°aÔ!8¸$Ô:NÈqÔ:QÑRÔRˆBØ�/Ô"×)Ò)¨$¬.¸$¼-ÈÔI\Ñ]Ô]×`Ò`ÕglÔgtÐ`ÑuÔuˆAå”)˜B˜yœM¨AÑ-Ñ.Ô.×2Ò2¸,Ð2ÑGÔGˆBð
 —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAà�I”  3¨¨a¨a¨a <¤Ñ0ˆBð *×1Ò1°*¸bÀ$Ä-ÑPÔPˆMØ˜ iÔ0Ñ0×4Ò4¸LÐ4ÑIÔIˆCð &Ô,¨T¬^Ô<ÔMÈaÔPÐSUÑUÐX[Ñ[ˆJØ%×<Ò<¸ZÈÌÑXÔXˆJð —	’	˜* d¤m°RÑ8Ô8¸¸dÀAÀAÀA¸ÔFˆAØ—’˜ T¤]°D´NÀdÄmÑ4SÐUVÔU\Ð]_ÔU`ÑaÔa×lÒlÑnÔnˆAØ—	’	˜* b¨!¬'°"¬+Ñ6Ô6ˆAð $Ÿš¨a¬h¸a¼g˜ÑFÔFˆJà",§/¢/°*¸t¼~Ñ2MÈtÌ}Ð^bÔ^qÑ"rÔ"rÐØŸš 
¨T¬^Ñ ;¸TÔ=PÐRSÑTÔTˆJÝ”	Ð-¨zÑ:Ô:ˆAØ—’�z 4¤>°4´=ÑAÔAˆAð ”�yÔ!×(Ò(¨¬¬°a¬¸$¼-ÑHÔHˆAØ�] QÑ&Ñ&×*Ò*¨1¬7Ñ3Ô3ˆAð —	’	˜* bÑ)Ô)¨!¨!¨!¨T°3¨,Ô7ˆA‰Aõ ”×'Ò'¨¨T¬\Ñ(9Ñ:Ô:ˆBÝ”˜R Ô!5°aÔ!8¸$Ô:NÈqÔ:QÑRÔRˆBØ)×1Ò1°*¸gÀrÈ4Ì=ÑYÔY×_Ò_ÑaÔaˆMØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ—	’	˜* g¨r°4Ô3FÑGÔG×MÒMÑOÔOˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØ×#Ò# D¤N°d´mÑ$CÈÐX\ÔXfÐ#ÑgÔgˆAØœ¨'°D´OÑ*CÑCÀtÄÑVˆHàœ 	Ô*Õ-?ÀÈxÑ-XÔ-XÑXˆJð *¨B¨y¬MÑ9ˆMØ—’�]Ô(Ñ)Ô)¨BÑ.ˆAð &{Ð%zÐ%zÐ%zÐ%zÐboÐqrÐtuÐwxÐayÐ%zÑ%zÔ%zÑ"ˆM˜1˜a ð —	’	˜!˜Q  1Ñ%Ô%ˆAÝ”| A¨2Ð.Ñ.Ô.ˆHõ ”	�+ a™.œ.Ñ)Ô)ˆAð ˜q˜q˜q ! ! ! Q Q Q¨¨a¨a¨a°°°Ð2Ô3°a¸¸¸¸1¸1¸1¸dÀAÀAÀAÀqÀqÀqÈ!È!È!Ð8KÔ6LÑLˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜yœ\¨A¯IªI°a¸¸A¸qÀ!Ñ,DÔ,DÀYÔ,OÑOˆNØ×"Ò" rÐ"Ñ*Ô*ˆAð ˜	”l ]°1°1°1°a°a°a¸°:Ô%>Ñ>×CÒCÈÐCÑJÔJˆFõ !œ9 X¨a¨a¨a°°°°A°A°A°r°s°s¨lÔ%;¸hÑ%FÑGÔGˆLØ˜,×.Ò.¨q°"°b¸!Ñ<Ô<¸YÔGÑGˆGØ˜c 4¨¨¨˜lÔ+¨m¸IÔ.FÑF×KÒKÐPQÐKÑRÔRˆFð *ð5�Ô# D¤NÔ3ÔDÀQÔGÈÈÈÈ4ÈÔP×SÒSÐZ`ÔZfÐouÔo|ÐSÑ}Ô}Ð}åÔ% f¨Q¨Q¨Q°°°¨U¤mÑ4Ô4ð õ
 ”Y °Ð8¸aÐ@Ñ@Ô@ˆFÝœ)¥Kµ´×0AÒ0AÀ(È1È1È1ÈaÈaÈaÐQRÐQRÐQRÐTVÈ;ÔBWÐY_Ñ0`Ô0`Ñ$aÔ$aÑbÔbˆKØ%×/Ò/°°1Ñ5Ô5ˆKØ% oÔ6¸ÀÀÀÀ1À1À1ÀdÈCÀÔ9PÑP×UÒUÐZ[ÐUÑ\Ô\ˆJØ *¨1¨1¨1¨c¨r¨c¨6Ô 2°J¸q¸q¸qÀ"¸uÔ4E�IˆFõ $œi¨Ñ1Ô1ˆOØ  T¨1¨1¨1 œo°°q°q°q¸!¸!¸!¸TÀ3°Ô0GÑGˆNØ'6×'>Ò'>¸qÀ!ÀQÈÑ'JÔ'JÐ$Ø#×'Ò'¨Ñ+Ô+Ð.FÀyÔ.QÑQˆEð ˜‘ˆAà—	’	˜* b¨$¬.¸$¼-ÑHÔHˆAà�J‘ˆAà˜!Š|ˆ|Ø�a�a�a˜˜'˜ 1 1 1 a a aÐ'Ô(�Ø—	’	˜* g¨rÑ2Ô2ˆAð Ð$¨Ð)AØ(×?Ò?À	È4Ì>ÑZÔZ�	à—i’i  4Ñ(Ô(ˆð
 !%§¢¨k¯nªn¸UÑ.CÔ.CÑ DÔ DÐØ$Ð$r6   rÛ   c                 ó|  — t           r>d| j        j        j        j        v r&t          ¦   «         s|                      ||||¦  «        S |�t          d¦  «        ‚|j        }|�G|j	        d         dk    r6|j	        d         dk    r%||d d …d d …d f         z   
                    |¦  «        }|                      |||¦  «        S )NÚcudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r'   r   )rê   rÜ   r  r¥   Útyper!   r#  ÚNotImplementedErrorrW   r.   r`   rO  )r}   rB   ró   rR   rô   rU   rW   s          r4   rŠ   z"GraniteMoeHybridMambaLayer.forwardÞ  sÝ   € õ "ð 	c f°´Ô0CÔ0JÔ0OÐ&OÐ&OÕXpÑXrÔXrÐ&OØ×,Ò,¨]¸LÈ.ÐZaÑbÔbÐbØÐÝ%Ønñô ð ð Ô#ˆØÐ%¨.Ô*>¸qÔ*AÀAÒ*EÐ*EÈ.ÔJ^Ð_`ÔJaÐdeÒJeÐJeà*¨^¸A¸A¸A¸q¸q¸qÀ$¸JÔ-GÑG×KÒKÈEÑRÔRˆMà×!Ò! -°¸~ÑNÔNÐNr6   ©NNNr‹   )rŒ   r�   rŽ   r�   r(   r�   ro   r/   r‘   r
   Ú	IntTensorr#  rO  r   rŠ   r“   r”   s   @r4   rµ   rµ     s‡  ø€ € € € € ðð ð\8Ð5ð \8À#ð \8ð \8ð \8ð \8ð \8ð \8ðB &*Ø.2Ø*.ðbð bà”|ðbð ˜d‘lðbð œ tÑ+ð	bð
 ” 4Ñ'ðbð bð bð bðP &*Ø.2ð	D%ð D%ð ˜d‘lðD%ð œ tÑ+ð	D%ð D%ð D%ð D%ðN Ð˜HÑ%Ô%ð &*Ø.2Ø*.ðOð Oð ˜d‘lðOð œ tÑ+ð	Oð
 ” 4Ñ'ðOð Oð Oñ &Ô%ðOð Oð Oð Oð Or6   rµ   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )râ   ç�íµ ÷Æ°>c                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        d S ©N©rn   ro   r   rÝ   r/   rª   r  r  ©r}   rq   r½   r~   s      €r4   ro   z%GraniteMoeHybridRMSNormGated.__init__ö  sB   ø€ Ý‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr6   Nc                 óœ  — |j         }|                     t          j        ¦  «        }|�?|t          j                             |                     t          j        ¦  «        ¦  «        z  }|                     d¦  «                             dd¬¦  «        }|t          j	        || j
        z   ¦  «        z  }| j        |                     |¦  «        z  S ©Nr+   r*   T)Úkeepdim)rW   r`   r/   r_   r   r]   r  ÚpowÚmeanÚrsqrtr  r  )r}   rB   r  Úinput_dtypeÚvariances        r4   rŠ   z$GraniteMoeHybridRMSNormGated.forwardû  sª   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆàÐØ)­B¬M×,>Ò,>¸t¿wºwÅuÄ}Ñ?UÔ?UÑ,VÔ,VÑVˆMØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆàŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r6   ©rW  rY  )rŒ   r�   rŽ   ro   rŠ   r“   r”   s   @r4   râ   râ   õ  sQ   ø€ € € € € ð$ð $ð $ð $ð $ð $ð
	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;r6   râ   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGraniteMoeHybridMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    ri   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        t          j	        | j        | j        dz  d¬¦  «        | _
        t          j	        | j        | j        d¬¦  «        | _        d S )Nr+   Frl   )rn   ro   rq   Ú
input_sizeÚshared_intermediate_sizer	   rË   rÌ   r   rv   Úinput_linearÚoutput_linear©r}   ri   r~   s     €r4   ro   zGraniteMoeHybridMLP.__init__  s…   ø€ Ý‰Œ×ÒÑÔÐà Ô,ˆŒØ!Ô:ˆÔÝ  Ô!2Ô3ˆŒÝœI d¤o°tÔ7GÈ!Ñ7KÐRWÐXÑXÔXˆÔÝœY tÔ'7¸¼ÈuÐUÑUÔUˆÔÐÐr6   rB   rD   c                 óÐ   — |                       |¦  «        }|                     dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr+   r*   r,   r   r'   )rj  ÚchunkrÌ   rk  )r}   rB   Úchunked_hidden_statess      r4   rŠ   zGraniteMoeHybridMLP.forward  sj   € Ø×)Ò)¨-Ñ8Ô8ˆØ -× 3Ò 3°A¸2Ð 3Ñ >Ô >ÐØŸšÐ(=¸aÔ(@ÑAÔAÐDYÐZ[ÔD\Ñ\ˆØ×*Ò*¨=Ñ9Ô9ˆØÐr6   ©
rŒ   r�   rŽ   r�   r(   ro   r/   r‘   rŠ   r“   r”   s   @r4   rf  rf    s|   ø€ € € € € ðð ðVÐ5ð Vð Vð Vð Vð Vð Vð U¤\ð °e´lð ð ð ð ð ð ð ð r6   rf  c                   óÔ   ‡ — e Zd ZU ej        ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d         de
dz  ded	ef         fd
„¦   «         Z ej        ¦   «         ed„ ¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridRotaryEmbeddingÚinv_freqNri   c                 ó²  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        | j        j        d         | _        | j        }| j        dk    rt          | j                 } || j        |¦  «        \  }| _
        |                      d|d¬¦  «         |                      d|                     ¦   «         d¬¦  «         d S )NÚ	rope_typeÚdefaultrs  F)Ú
persistentÚoriginal_inv_freq)rn   ro   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenri   Úrope_parametersru  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r}   ri   r¥   Úrope_init_fnrs  r~   s        €r4   ro   z(GraniteMoeHybridRotaryEmbedding.__init__$  sÊ   ø€ Ý‰Œ×ÒÑÔÐØ"(Ô"@ˆÔØ$*Ô$BˆÔ!àˆŒàœÔ4°[ÔAˆŒØ!%Ô!EˆØŒ>˜YÒ&Ð&Ý.¨t¬~Ô>ˆLØ+7¨<¸¼ÀVÑ+LÔ+LÑ(ˆ�$Ô(à×Ò˜Z¨¸eÐÑDÔDÐDØ×ÒÐ0°(·.².Ñ2BÔ2BÈuÐÑUÔUÐUÐUÐUr6   r¥   ztorch.devicer  rD   ztorch.Tensorc                 óü   — | j         d         }t          | dd¦  «        p| j        | j        z  }d}d|t	          j        d|dt          j        ¬¦  «                             |t          j        ¬¦  «        |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarK   Ng      ð?r   r+   rö   r¤   )	r|  rp   rq   rr   r/   rß   Úint64r`   r  )ri   r¥   r  Úbaser-   Úattention_factorrs  s          r4   r}  z?GraniteMoeHybridRotaryEmbedding.compute_default_rope_parameters4  sŒ   € ð& Ô% lÔ3ˆÝ�f˜j¨$Ñ/Ô/Ðc°6Ô3EÈÔIcÑ3cˆàÐð Ø•U”\ ! S¨!µ5´;Ð?Ñ?Ô?×BÒBÈ&ÕX]ÔXcÐBÑdÔdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óN  — | j         d d d …d f                              ¦   «                              |j        d         dd¦  «                             |j        ¦  «        }|d d …d d d …f                              ¦   «         }t          |j        j        t          ¦  «        r|j        j        dk    r|j        j        nd}t          |d¬¦  «        5  |                     ¦   «         |                     ¦   «         z   
                    dd¦  «        }t          j        ||fd¬	¦  «        }|                     ¦   «         | j        z  }|                     ¦   «         | j        z  }	d d d ¦  «         n# 1 swxY w Y   |                     |j        ¬
¦  «        |	                     |j        ¬
¦  «        fS )Nr   r*   r'   ÚmpsÚcpuF)Údevice_typeÚenabledr+   r,   rö   )rs  r  rF   r.   r`   r¥   Ú
isinstancerR  Ústrr#   r\   r/   r0   r<   r~  r=   rW   )
r}   r1   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrŠ  ÚfreqsÚembr<   r=   s
             r4   rŠ   z'GraniteMoeHybridRotaryEmbedding.forwardR  s·  € ð !œM¨$°°°°4¨-Ô8×>Ò>Ñ@Ô@×GÒGÈÔHZÐ[\ÔH]Ð_aÐcdÑeÔe×hÒhÐijÔiqÑrÔrÐØ ,¨Q¨Q¨Q°°a°a°a¨ZÔ 8× >Ò >Ñ @Ô @Ðå'1°!´(´-ÅÑ'EÔ'EÐkÈ!Ì(Ì-Ð[`ÒJ`ÐJ`�a”h”m�mÐfkˆÝ¨¸UÐCÑCÔCð 	5ð 	5Ø&×,Ò,Ñ.Ô.Ð1F×1LÒ1LÑ1NÔ1NÑN×YÒYÐZ[Ð]^Ñ_Ô_ˆEÝ”)˜U E˜N°Ð3Ñ3Ô3ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCØ—'’'‘)”)˜dÔ4Ñ4ˆCð		5ð 	5ð 	5ñ 	5ô 	5ð 	5ð 	5ð 	5ð 	5ð 	5ð 	5øøøð 	5ð 	5ð 	5ð 	5ð �vŠv˜AœGˆvÑ$Ô$ c§f¢f°1´7 fÑ&;Ô&;Ð;Ð;s   ÃBE&Å&E*Å-E*rY  rT  )rŒ   r�   rŽ   r/   r‘   Ú__annotations__r(   ro   Ústaticmethodr   r�   r’   r  r}  Úno_gradr   rŠ   r“   r”   s   @r4   rr  rr  !  sú   ø€ € € € € € ØŒlÐÐÑðVð VÐ5ð Vð Vð Vð Vð Vð Vð  à04Ø+/Ø"ð*ð *Ø&¨Ñ-ð*à˜Ô(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ô	%ð	*ð *ð *ñ „\ð*ð: €U„]�_„_Øð<ð <ñ Ôñ „_ð<ð <ð <ð <ð <r6   rr  c                   ór   ‡ — e Zd ZdZdefˆ fd„Zdej        deej        ej        ej        f         fd„Z	ˆ xZ
S )ÚGraniteMoeHybridTopKRoutera¡  Top-k gating that returns the routing decisions without grouping tokens by expert.

    Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live
    here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the
    experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm``
    paths can compile cleanly.
    ri   c                 óä   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        t          j	        | j        |j
        ¦  «        ¦  «        | _        d S rY  )rn   ro   Únum_local_expertsÚnum_expertsÚnum_experts_per_tokÚtop_kr   rÝ   r/   Úemptyrq   r  rl  s     €r4   ro   z#GraniteMoeHybridTopKRouter.__init__k  sU   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØÔ/ˆŒ
Ý”l¥5¤;¨tÔ/?ÀÔASÑ#TÔ#TÑUÔUˆŒˆˆr6   rB   rD   c                 óô   — t          j        || j        ¦  «                             ¦   «         }|                     | j        d¬¦  «        \  }}t          j        |d¬¦  «                             |¦  «        }|||fS )Nr*   r,   )	ÚFÚlinearr  r  Útopkrœ  r/   r^   Útype_as)r}   rB   Úrouter_logitsÚtop_k_logitsÚtop_k_indexÚtop_k_weightss         r4   rŠ   z"GraniteMoeHybridTopKRouter.forwardq  so   € Ýœ °´Ñ<Ô<×BÒBÑDÔDˆØ$1×$6Ò$6°t´zÀrÐ$6Ñ$JÔ$JÑ!ˆ�kÝœ l¸Ð;Ñ;Ô;×CÒCÀMÑRÔRˆØ˜M¨=Ð8Ð8r6   )rŒ   r�   rŽ   r�   r(   ro   r/   r‘   r’   rŠ   r“   r”   s   @r4   r—  r—  b  sŽ   ø€ € € € € ðð ðVÐ5ð Vð Vð Vð Vð Vð Vð9 U¤\ð 9°e¸E¼LÈ%Ì,ÐX]ÔXdÐ<dÔ6eð 9ð 9ð 9ð 9ð 9ð 9ð 9ð 9r6   r—  c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚGraniteMoeHybridExpertsz2Collection of expert weights stored as 3D tensors.ri   c                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr+   )rn   ro   r™  rš  rq   Ú
hidden_dimrÈ   Úintermediate_dimr   rÝ   r/   r�  Úgate_up_projÚ	down_projr	   rË   Úact_fnrl  s     €r4   ro   z GraniteMoeHybridExperts.__init__|  s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô 8ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr6   rB   r¥  r¦  rD   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr+   r'   r   )r*   r§   r,   r*   )r/   r2  r•  r   r]   Úone_hotrš  r1  Úgreaterr,  ÚnonzeroÚwherer   r¬  rn  r®  r­  Ú
index_add_r`   rW   )r}   rB   r¥  r¦  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater  ÚupÚcurrent_hidden_statess                 r4   rŠ   zGraniteMoeHybridExperts.forward…  sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6rp  r”   s   @r4   r¨  r¨  x  s�   ø€ € € € € à<Ð<ð0Ð5ð 0ð 0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r6   r¨  c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGraniteMoeHybridMoEzISparsely-gated mixture-of-experts block: router decides, experts compute.ri   c                 ó°   •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          |¦  «        | _        d S rY  )rn   ro   rq   rh  r—  Úrouterr¨  Úexpertsrl  s     €r4   ro   zGraniteMoeHybridMoE.__init__£  sE   ø€ Ý‰Œ×ÒÑÔÐØ Ô,ˆŒÝ0°Ñ8Ô8ˆŒÝ.¨vÑ6Ô6ˆŒˆˆr6   Úlayer_inputrD   c                 óö   — |                      ¦   «         \  }}}|                     d|¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }	|	                     ||| j        ¦  «        S )Nr*   )r¨   rG   rÂ  rÃ  r‚   rh  )
r}   rÄ  ÚbszÚlengthÚemb_sizerB   r¥  r¦  r  Úlayer_outputs
             r4   rŠ   zGraniteMoeHybridMoE.forward©  sv   € Ø +× 0Ò 0Ñ 2Ô 2ÑˆˆV�XØ#×+Ò+¨B°Ñ9Ô9ˆØ(,¯ª°MÑ(BÔ(BÑ%ˆ�] AØ—|’| M°;ÀÑNÔNˆØ× Ò   f¨d¬oÑ>Ô>Ð>r6   rp  r”   s   @r4   rÀ  rÀ     sq   ø€ € € € € ØSÐSð7Ð5ð 7ð 7ð 7ð 7ð 7ð 7ð? 5¤<ð ?°E´Lð ?ð ?ð ?ð ?ð ?ð ?ð ?ð ?r6   rÀ  c                   ód   — e Zd ZU dZej        ed<   ej        ed<   eed<   eed<   ej        ed<   dS )ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_krô   N)	rŒ   r�   rŽ   r�   r/   Ú
LongTensorr“  r�   rU  r(  r6   r4   rË  rË  ±  sb   € € € € € € ðð ð  Ô#Ð#Ð#Ñ#ØÔ#Ð#Ð#Ñ#ØÐÐÑØÐÐÑØŒ_ÐÐÑÐÐr6   rË  F)ÚtotalÚRMSNormc                   óT   ‡ — e Zd Zd	deddfˆ fd„Zdej        dej        fd„Zd„ Zˆ xZ	S )
ÚGraniteMoeHybridRMSNormrW  r½   rD   Nc                 ó¬   •— t          ¦   «                              ¦   «          t          j        t	          j        |¦  «        ¦  «        | _        || _        dS )zF
        GraniteMoeHybridRMSNorm is equivalent to T5LayerNorm
        NrZ  r[  s      €r4   ro   z GraniteMoeHybridRMSNorm.__init__Ë  sD   ø€ õ 	‰Œ×ÒÑÔÐÝ”l¥5¤:¨kÑ#:Ô#:Ñ;Ô;ˆŒØ #ˆÔÐÐr6   rB   c                 ó  — |j         }|                     t          j        ¦  «        }|                     d¦  «                             dd¬¦  «        }|t          j        || j        z   ¦  «        z  }| j        |                     |¦  «        z  S r]  )	rW   r`   r/   r_   r_  r`  ra  r  r  )r}   rB   rb  rc  s       r4   rŠ   zGraniteMoeHybridRMSNorm.forwardÓ  s|   € Ø#Ô)ˆØ%×(Ò(­¬Ñ7Ô7ˆØ ×$Ò$ QÑ'Ô'×,Ò,¨R¸Ð,Ñ>Ô>ˆØ%­¬°H¸tÔ?TÑ4TÑ(UÔ(UÑUˆØŒ{˜]×-Ò-¨kÑ:Ô:Ñ:Ð:r6   c                 óH   — t          | j        j        ¦  «        › d| j        › �S )Nz, eps=)r’   r  r.   r  )r}   s    r4   Ú
extra_reprz"GraniteMoeHybridRMSNorm.extra_reprÚ  s&   € Ý˜œÔ)Ñ*Ô*ÐIÐI°$Ô2GÐIÐIÐIr6   rd  )
rŒ   r�   rŽ   r  ro   r/   r‘   rŠ   rØ  r“   r”   s   @r4   rÔ  rÔ  É  sŒ   ø€ € € € € ð$ð $¨ð $¸$ð $ð $ð $ð $ð $ð $ð; U¤\ð ;°e´lð ;ð ;ð ;ð ;ðJð Jð Jð Jð Jð Jð Jr6   rÔ  c                   ó  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 ddej        dej        dz  de	dz  d	e
dz  d
eej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridDecoderLayerri   rj   c                 óN  •— t          ¦   «                              ¦   «          |j        | _        d | _        t	          |j        |j        ¬¦  «        | _        t	          |j        |j        ¬¦  «        | _        |j        dk    rt          |¦  «        nd | _
        |j        | _        t          |¦  «        | _        d | _        |j        |         dk    rt!          ||¦  «        | _        nt#          ||¦  «        | _        |j        |         | _        t'          |dd¦  «        dk    | _        d S )Nr¼   r   Úlinear_attentionr™  )rn   ro   rq   Ú	self_attnrÔ  rÐ   Úinput_layernormÚpost_attention_layernormr™  rÀ  Úblock_sparse_moeÚresidual_multiplierrf  Ú
shared_mlpÚmambaÚlayers_block_typerµ   rh   Ú
block_typerp   Úhas_expertsr|   s      €r4   ro   z%GraniteMoeHybridDecoderLayer.__init__ß  s  ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔàˆŒÝ6°vÔ7IÈvÔObÐcÑcÔcˆÔÝ(?ÀÔ@RÐX^ÔXkÐ(lÑ(lÔ(lˆÔ%ð @FÔ?WÐZ[Ò?[Ð?[Õ 3°FÑ ;Ô ;Ð ;ÐaeˆÔØ#)Ô#=ˆÔ Ý-¨fÑ5Ô5ˆŒØˆŒ
àÔ# IÔ.Ð2DÒDÐDÝ3°F¸IÑFÔFˆDŒJˆJå6°v¸yÑIÔIˆDŒNØ Ô2°9Ô=ˆŒõ # 6Ð+>ÀÑBÔBÀQÒFˆÔÐÐr6   NFrB   rR   r   Ú	use_cacher€   rU   rD   c           	      óˆ  — |}|                       |¦  «        }| j        � | j        d|||dœ|¤Ž}n | j        d|||||dœ|¤Ž\  }}||| j        z  z   }|}|                      |¦  «        }| j        r.|                      |¦  «        }	|	|                      |¦  «        z   }n|                      |¦  «        }||| j        z  z   }|S )N)rB   ró   rR   )rB   rR   r   rç  r€   r(  )rÞ  rã  rÝ  rá  rß  ræ  rà  râ  )
r}   rB   rR   r   rç  r€   rU   Úresidualr  Úmoe_hidden_statess
             r4   rŠ   z$GraniteMoeHybridDecoderLayer.forwardö  s  € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆàŒ:Ð!Ø&˜DœJð Ø+Ø,Ø-ðð ð ð	ð ˆMˆMð  .˜tœ~ð  Ø+Ø-Ø /Ø#Ø$7ð ð  ð ð ð  ÑˆM˜1ð ! =°4Ô3KÑ#KÑKˆØ ˆØ×5Ò5°mÑDÔDˆàÔð 	;Ø $× 5Ò 5°mÑ DÔ DÐØ-°·²ÀÑ0NÔ0NÑNˆMˆMà ŸOšO¨MÑ:Ô:ˆMà  =°4Ô3KÑ#KÑKˆØÐr6   )NNFN)rŒ   r�   rŽ   r(   r�   ro   r   r/   r‘   r
   r«   r’   r   rË  ÚFloatTensorrŠ   r“   r”   s   @r4   rÚ  rÚ  Þ  s  ø€ € € € € ðGÐ5ð GÀ#ð Gð Gð Gð Gð Gð Gð. ð /3Ø(,Ø!&ØHLð(ð (à”|ð(ð œ tÑ+ð(ð  ™ð	(ð
 ˜$‘;ð(ð # 5¤<°´Ð#=Ô>ÀÑEð(ð Ð4Ô5ð(ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð(ð (ð (ñ „^ð(ð (ð (ð (ð (r6   rÚ  c                   óŠ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚGraniteMoeHybridPreTrainedModelri   ÚmodelTrÚ  r   )rB   Ú
attentionsc           
      ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rMt	          j        |j        d| j        j        ¬¦  «         t	          j        |j	        d| j        j        ¬¦  «         n;t          |t          ¦  «        r&t	          j        |j        d| j        j        ¬¦  «         t          |t          ¦  «        r{t	          j        |j        ¦  «         t	          j        |j        t#          j        t#          j        d|j        dz   ¦  «        ¦  «        ¦  «         t	          j        |j        ¦  «         d S t          |t,          ¦  «        rt	          j        |j        ¦  «         d S d S )NrM   )r`  Ústdr'   )rn   Ú_init_weightsrŒ  r¨  ÚinitÚnormal_r¬  ri   Úinitializer_ranger­  r—  r  rµ   Úones_rÞ   Úcopy_rá   r/   rà   rß   rÂ   rä   râ   )r}   rN   r~   s     €r4   rò  z-GraniteMoeHybridPreTrainedModel._init_weights4  sG  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ5Ñ6Ô6ð 	UÝŒL˜Ô,°3¸D¼KÔ<YÐZÑZÔZÐZÝŒL˜Ô)°¸¼Ô9VÐWÑWÔWÐWÐWÝ˜Õ :Ñ;Ô;ð 	UÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTÝ�fÕ8Ñ9Ô9ð 	&ÝŒJ�v”~Ñ&Ô&Ð&ÝŒJ�v”|¥U¤Y­u¬|¸A¸vÔ?OÐRSÑ?SÑ/TÔ/TÑ%UÔ%UÑVÔVÐVÝŒJ�v”xÑ Ô Ð Ð Ð Ý˜Õ <Ñ=Ô=ð 	&ÝŒJ�v”}Ñ%Ô%Ð%Ð%Ð%ð	&ð 	&r6   )rŒ   r�   rŽ   r(   r“  Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrÚ  rh   Ú_can_record_outputsÚ_is_statefulr/   r•  rò  r“   r”   s   @r4   rí  rí  "  s­   ø€ € € € € € à"Ð"Ð"Ñ"ØÐØ&*Ð#Ø7Ð8ÐØ#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&Ðà5Ø/ðð Ðð €Là€U„]�_„_ð&ð &ð &ð &ñ „_ð&ð &ð &ð &ð &r6   rí  c                   óè   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridModelri   c                 ó  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        ‰j        | j        ¦  «        | _        t          j	        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          ‰j        ‰j        ¬¦  «        | _        ‰j        dk    rt#          ‰¦  «        nd | _        d| _        ‰j        | _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r(  )rÚ  )r)  rj   ri   s     €r4   r+  z2GraniteMoeHybridModel.__init__.<locals>.<listcomp>M  s$   ø€ ÐnÐnÐnÀÕ)¨&°)Ñ<Ô<ÐnÐnÐnr6   r¼   ÚropeF)rn   ro   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrq   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr  rÔ  rÐ   rã   Úposition_embedding_typerr  Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initrl  s    `€r4   ro   zGraniteMoeHybridModel.__init__F  sò   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒåœL¨Ô):¸FÔ<NÐPTÔP`ÑaÔaˆÔÝ”mØnÐnÐnÐnÍeÐTZÔTlÑNmÔNmÐnÑnÔnñ
ô 
ˆŒõ ,¨FÔ,>ÀFÔDWÐXÑXÔXˆŒ	ØEKÔEcÐgmÒEmÐEmÕ9¸&ÑAÔAÐAÐswˆŒØ&+ˆÔ#Ø$*Ô$?ˆÔ!ð 	�ŠÑÔÐÐÐr6   NÚ	input_idsrR   rŽ  r   Úinputs_embedsrç  rU   rD   c           	      óØ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|| j        z  }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j	        ¬¦  «        |z   }| 
                    d¦  «        }t          |x}	t          ¦  «        s%| j        |||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}d }| j        �|                      ||¦  «        }t!          | j        ¦  «        D ])\  }} ||f|	| j        j        |                  |||dœ|¤Ž}Œ*|                      |¦  «        }t)          ||¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)ri   r   r'   r%  )ri   r  rR   r   )Úfull_attentionrÜ  )rR   r   rç  r€   )Úlast_hidden_stater   r(  )Ú
ValueErrorr  r  r   ri   Úget_seq_lengthr/   rß   r.   r¥   r9   rŒ  Údictr   r   r  Ú	enumerater  rä  rã   r   )r}   r  rR   rŽ  r   r  rç  rU   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrB   r€   ÚiÚdecoder_layers                  r4   rŠ   zGraniteMoeHybridModel.forwardW  sí  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMà%¨Ô(AÑAˆàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå°Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2ð	ð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ$CÐ$RÐ$RÀkÐ$RÐ$Rð#ð #Ðð &ˆØ"ÐØŒ?Ð&Ø"&§/¢/°-ÀÑ"NÔ"NÐå )¨$¬+Ñ 6Ô 6ð 	ð 	ÑˆAˆ}Ø)˜MØðà2°4´;Ô3PÐQRÔ3SÔTØ /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r6   )NNNNNN)rŒ   r�   rŽ   r(   ro   r   r$   r&   r/   rÐ  r‘   r
   rë  r«   r   rË  r’   r   rŠ   r“   r”   s   @r4   r  r  D  s  ø€ € € € € ðÐ5ð ð ð ð ð ð ð" ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð<
ð <
àÔ# dÑ*ð<
ð œ tÑ+ð<
ð Ô&¨Ñ-ð	<
ð
  ™ð<
ð Ô(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð4Ô5ð<
ð 
Ð(Ñ	(ð<
ð <
ð <
ñ „_ñ  Ôñ „^ð<
ð <
ð <
ð <
ð <
r6   r  r+   Úgate_logitsrš  c                 óÆ  ‡— | �t          | t          ¦  «        sdS t          | t          ¦  «        r/| d         j        Št          j        ˆfd„| D ¦   «         d¬¦  «        }t          j        j                             |d¬¦  «        }t          j        ||d¬¦  «        \  }}t          j        j         	                    ||¦  «        }|€@t          j
        |                     ¦   «         d¬¦  «        }	t          j
        |d¬¦  «        }
�n.|j        \  }}|j        d         ||z  z  }|ddd…dd…ddf                              |||||f¦  «                             d||¦  «                             ‰¦  «        }t          j        |                     ¦   «         |z  d¬¦  «        t          j        |d¬¦  «        z  }	|ddd…dd…df                              ||||f¦  «                             d|¦  «                             ‰¦  «        }t          j        ||z  d¬¦  «        t          j        |d¬¦  «        z  }
t          j        |	|
                     d¦  «        z  ¦  «        }||z  S )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                 ó:   •— g | ]}|                      ‰¦  «        ‘ŒS r(  )r`   )r)  Ú
layer_gateÚcompute_devices     €r4   r+  z,load_balancing_loss_func.<locals>.<listcomp>»  s&   ø€ Ð-jÐ-jÐ-jÐPZ¨j¯mªm¸NÑ.KÔ.KÐ-jÐ-jÐ-jr6   r,   r*   )rŒ  r’   r¥   r/   r0   r   r]   r^   r¡  r±  r`  r  r.   rF   rG   r`   r,  r9   )r#  rš  rœ  rR   Úconcatenated_gate_logitsÚrouting_weightsr  Úselected_expertsr·  Útokens_per_expertÚrouter_prob_per_expertr  Úsequence_lengthr  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_lossr'  s                    @r4   Úload_balancing_loss_funcr1  ™  s�  ø€ ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqå�+�uÑ%Ô%ð sØ$ QœÔ.ˆÝ#(¤9Ð-jÐ-jÐ-jÐ-jÐ^iÐ-jÑ-jÔ-jÐpqÐ#rÑ#rÔ#rÐ å”hÔ)×1Ò1Ð2JÐPRÐ1ÑSÔS€Oåœ* _°eÀÐDÑDÔDÑ€AÐå”(Ô%×-Ò-Ð.>ÀÑLÔL€KàÐå!œJ {×'8Ò'8Ñ':Ô':ÀÐBÑBÔBÐõ "'¤¨OÀÐ!CÑ!CÔ!CÐÑà&4Ô&:Ñ#ˆ
�OØ4Ô:¸1Ô=À*ÈÑB^Ñ_Ðð ˜4    A A A t¨TÐ1Ô2ßŠVÐ&¨
°OÀUÈKÐXÑYÔYßŠW�R˜ Ñ,Ô,ßŠR�ÑÔð	 	õ "œI k×&7Ò&7Ñ&9Ô&9Ð<QÑ&QÐWXÐYÑYÔYÕ\aÔ\eØ! qð]
ñ ]
ô ]
ñ 
Ðð ˜4    A A A tÐ+Ô,ßŠVÐ&¨
°OÀ[ÐQÑRÔRßŠW�R˜Ñ%Ô%ßŠR�ÑÔð	 	)õ "'¤¨?Ð=]Ñ+]ÐcdÐ!eÑ!eÔ!eÕhmÔhqØ,°!ði
ñ i
ô i
ñ "
Ðõ ”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r6   c                   ó  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 dde
j        d	z  de
j        d	z  de
j        d	z  ded	z  de
j        d	z  de
j        d	z  ded	z  dee
j        z  deez  fd„¦   «         ¦   «         Zˆ xZS )ÚGraniteMoeHybridForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   Úlogitsri   c                 ó^  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |j	        | _	        |j
        | _        |j        | _        |j        | _        |                      ¦   «          d S )NFrl   )rn   ro   r  rî  r
  r   rv   rq   r4  Úrouter_aux_loss_coefr™  rš  r›  Úlogits_scalingr  rl  s     €r4   ro   z$GraniteMoeHybridForCausalLM.__init__ñ  s–   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý*¨6Ñ2Ô2ˆŒ
Ø Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ$*Ô$?ˆÔ!Ø!Ô3ˆÔØ#)Ô#=ˆÔ Ø$Ô3ˆÔð 	�ŠÑÔÐÐÐr6   Nr   r  rR   rŽ  r   r  ÚlabelsÚoutput_router_logitsÚlogits_to_keeprD   c	           	      ó2  — |�|n| j         j        } | j        d|||||dœ|	¤Ž}
|
j        }t	          |t
          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }|| j         j        z  }d}|� | j	        ||fd| j         j
        i|	¤Ž}d}|rHt          |
j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t%          ||||
j        |
j        |
j        |
j        ¬¦  «        S )a–  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm-granite/granite-4.0-h-tiny")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-4.0-h-tiny")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r  rR   rŽ  r   r  r
  )ÚlossÚaux_lossr6  r   rB   rï  r£  r(  )ri   r;  rî  r  rŒ  r�   Úslicer4  r9  Úloss_functionr
  r1  r£  rš  r›  r8  r`   r¥   r   r   rB   rï  )r}   r  rR   rŽ  r   r  r:  r;  r<  rU   ÚoutputsrB   Úslice_indicesr6  r>  r?  s                   r4   rŠ   z#GraniteMoeHybridForCausalLM.forwardþ  s‘  € ðJ %9Ð$DÐ Ð È$Ì+ÔJjð 	ð �$”*ð 
ØØ)Ø%Ø+Ø'ð
ð 
ð ð
ð 
ˆð  Ô1ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆØ˜$œ+Ô4Ñ4ˆàˆØÐà%�4Ô%ØØðð ð  œ;Ô1ðð ð	ð ˆDð ˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�Ý(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r6   )NNNNNNNr   )rŒ   r�   rŽ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr(   ro   r   r    r/   rÐ  r‘   r
   rë  r«   r�   r’   r   rŠ   r“   r”   s   @r4   r3  r3  ë  s`  ø€ € € € € à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HðÐ5ð ð ð ð ð ð ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø-.ðQ
ð Q
àÔ# dÑ*ðQ
ð œ tÑ+ðQ
ð Ô&¨Ñ-ð	Q
ð
  ™ðQ
ð Ô(¨4Ñ/ðQ
ð Ô  4Ñ'ðQ
ð # T™kðQ
ð ˜eœlÑ*ðQ
ð 
Ð*Ñ	*ðQ
ð Q
ð Q
ñ Ôñ „^ðQ
ð Q
ð Q
ð Q
ð Q
r6   r3  )r3  r  rí  )r'   )rM   )Nr+   N)]Úcollections.abcr   Útypingr   r   r/   Útorch.nn.functionalr   r]   rŸ  Ú r   ró  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   r   Úintegrations.accelerater   Úintegrations.hub_kernelsr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    r!   r"   Úutils.genericr#   r$   Úutils.import_utilsr%   Úutils.output_capturingr&   Úconfiguration_granitemoehybridr(   Ú
get_loggerrŒ   rë   r5   rA   r‘   r�   rL   ÚModuler  rf   rh   rž   r¡   r±   r³   rµ   râ   rf  rr  r—  r¨  rÀ  rË  rÔ  rÚ  rí  r  r’   r1  r3  Ú__all__r(  r6   r4   ú<module>r_     s
  ðð* %Ð $Ð $Ð $Ð $Ð $Ø &Ð &Ð &Ð &Ð &Ð &Ð &Ð &à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )ðð ð ð ð ð ð ð ð ð ð ð ð >Ð =Ð =Ð =Ð =Ð =Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lÐ lØ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø BÐ BÐ BÐ BÐ BÐ Bð 
ˆÔ	˜HÑ	%Ô	%€ð(ð (ð (ð ÐÐ*Ñ+Ô+ðð ð ñ ,Ô+ðð2	U˜Uœ\ð 	U°#ð 	U¸%¼,ð 	Uð 	Uð 	Uð 	Uð& ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð2 ÐÐ)Ñ*Ô*ðA)ð A)ð A)ð A)ð A) ¤	ñ A)ô A)ñ +Ô*ðA)ðNV U¤\ð V¸Sð Vð Vð Vð Vð
ð 
ð 
ð(ð ð ð(	ð 	ð 	ðlOð lOð lOð lOð lO ¤ñ lOô lOð lOð^;ð ;ð ;ð ;ð ; 5¤8¤?ñ ;ô ;ð ;ð$ð ð ð ð ˜"œ)ñ ô ð ð4><ð ><ð ><ð ><ð >< b¤iñ ><ô ><ð ><ðB9ð 9ð 9ð 9ð 9 ¤ñ 9ô 9ð 9ð, ð$#ð $#ð $#ð $#ð $#˜bœiñ $#ô $#ñ Ôð$#ðN?ð ?ð ?ð ?ð ?˜"œ)ñ ?ô ?ð ?ð"ð ð ð ð  )°5ð ñ ô ð ð0 Ð˜YÑ'Ô'ðJð Jð Jð Jð J˜bœiñ Jô Jñ (Ô'ðJð(Að Að Að Að AÐ#=ñ Aô Að AðH ð&ð &ð &ð &ð & oñ &ô &ñ „ð&ðB ðQ
ð Q
ð Q
ð Q
ð Q
Ð;ñ Q
ô Q
ñ „ðQ
ðl #Ø
Ø*.ð	O&ð O&Ø”  e¤lÔ 3Ñ3°dÑ:ðO&à�t‘ðO&ð ”L 4Ñ'ð	O&ð
 „\�CÑðO&ð O&ð O&ð O&ðd ðe
ð e
ð e
ð e
ð e
Ð"AÀ?ñ e
ô e
ñ „ðe
ðP fÐ
eÐ
e€€€r6   