§
    ‚Štjè  ã                   óX  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ d	d
lmZmZmZmZ ddlmZ  ej        e¦  «        Z G d„ ded¬¦  «        Z G d„ dej        ¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Z G d„ de¦  «        Zg d¢ZdS )é    )Ú	TypedDictN)Únné   )ÚACT2FN)ÚCache)ÚUnpack)Úloggingé   )ÚGraniteMoeDecoderLayerÚGraniteMoeForCausalLMÚGraniteMoeModelÚGraniteMoePreTrainedModelé   )ÚGraniteMoeSharedConfigc                   ód   — e Zd ZU dZej        ed<   ej        ed<   eed<   eed<   ej        ed<   dS )ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_kÚseq_idxN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚ
LongTensorÚ__annotations__ÚintÚ	IntTensor© ó    ú{/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/granitemoeshared/modular_granitemoeshared.pyr   r   $   sb   € € € € € € ðð ð  Ô#Ð#Ð#Ñ#ØÔ#Ð#Ð#Ñ#ØÐÐÑØÐÐÑØŒ_ÐÐÑÐÐr"   r   F)Útotalc                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚGraniteMoeSharedMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    Úconfigc                 óD  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t
          |j                 | _        t          j	        | j        | j        dz  d¬¦  «        | _
        t          j	        | j        | j        d¬¦  «        | _        d S )Nr
   F)Úbias)ÚsuperÚ__init__Úhidden_sizeÚ
input_sizeÚshared_intermediate_sizer   Ú
hidden_actÚ
activationr   ÚLinearÚinput_linearÚoutput_linear©Úselfr'   Ú	__class__s     €r#   r+   zGraniteMoeSharedMLP.__init__E   s…   ø€ Ý‰Œ×ÒÑÔÐà Ô,ˆŒØ!Ô:ˆÔÝ  Ô!2Ô3ˆŒÝœI d¤o°tÔ7GÈ!Ñ7KÐRWÐXÑXÔXˆÔÝœY tÔ'7¸¼ÈuÐUÑUÔUˆÔÐÐr"   Úhidden_statesÚreturnc                 óÐ   — |                       |¦  «        }|                     dd¬¦  «        }|                      |d         ¦  «        |d         z  }|                      |¦  «        }|S )Nr
   éÿÿÿÿ)Údimr   r   )r2   Úchunkr0   r3   )r5   r7   Úchunked_hidden_statess      r#   ÚforwardzGraniteMoeSharedMLP.forwardN   sj   € Ø×)Ò)¨-Ñ8Ô8ˆØ -× 3Ò 3°A¸2Ð 3Ñ >Ô >ÐØŸšÐ(=¸aÔ(@ÑAÔAÐDYÐZ[ÔD\Ñ\ˆØ×*Ò*¨=Ñ9Ô9ˆØÐr"   )
r   r   r   r   r   r+   r   ÚTensorr>   Ú__classcell__©r6   s   @r#   r&   r&   <   s|   ø€ € € € € ðð ðVÐ5ð Vð Vð Vð Vð Vð Vð U¤\ð °e´lð ð ð ð ð ð ð ð r"   r&   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej        dej        dz  dej        dz  d	e	dz  d
e
dz  de
dz  deej        ej        f         dz  dee         deej        eej        ej        f         dz  f         fd„Zˆ xZS )ÚGraniteMoeSharedDecoderLayerr'   Ú	layer_idxc                 óŽ   •— t          ¦   «                              ||¦  «         |j        dk    rd nt          |¦  «        | _        d S )Nr   )r*   r+   r.   r&   Ú
shared_mlp)r5   r'   rD   r6   s      €r#   r+   z%GraniteMoeSharedDecoderLayer.__init__W   sC   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+Ø"(Ô"AÀQÒ"FÐ"F˜$˜$ÕL_Ð`fÑLgÔLgˆŒˆˆr"   NFr7   Úattention_maskÚposition_idsÚpast_key_valuesÚoutput_attentionsÚ	use_cacheÚposition_embeddingsÚkwargsr8   c                 ó4  — |}	|                       |¦  «        } | j        d|||||||dœ|¤Ž\  }}
|	|| j        z  z   }|}	|                      |¦  «        }|                      |¦  «        }| j        €|}n||                      |¦  «        z   }|	|| j        z  z   }|S )N)r7   rG   rH   rI   rJ   rK   rL   r!   )Úinput_layernormÚ	self_attnÚresidual_multiplierÚpost_attention_layernormÚblock_sparse_moerF   )r5   r7   rG   rH   rI   rJ   rK   rL   rM   ÚresidualÚ_Úmoe_hidden_statess               r#   r>   z$GraniteMoeSharedDecoderLayer.forward[   sÜ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆð *˜4œ>ð 	
Ø'Ø)Ø%Ø+Ø/ØØ 3ð	
ð 	
ð ð	
ð 	
Ñˆ�qð ! =°4Ô3KÑ#KÑKˆà ˆØ×5Ò5°mÑDÔDˆØ ×1Ò1°-Ñ@Ô@ÐàŒ?Ð"Ø-ˆMˆMà-°·²ÀÑ0NÔ0NÑNˆMØ  =°4Ô3KÑ#KÑKˆØÐr"   )NNNFFN)r   r   r   r   r   r+   r   r?   r   r   ÚboolÚtupler   r   ÚFloatTensorr>   r@   rA   s   @r#   rC   rC   V   s2  ø€ € € € € ðhÐ5ð hÀ#ð hð hð hð hð hð hð /3Ø04Ø(,Ø).Ø!&ØHLð%ð %à”|ð%ð œ tÑ+ð%ð Ô&¨Ñ-ð	%ð
  ™ð%ð   $™;ð%ð ˜$‘;ð%ð # 5¤<°´Ð#=Ô>ÀÑEð%ð Ð4Ô5ð%ð 
ˆuÔ  %¨Ô(9¸5Ô;LÐ(LÔ"MÐPTÑ"TÐTÔ	Uð%ð %ð %ð %ð %ð %ð %ð %r"   rC   c                   ó    — e Zd ZU eed<   dgZdS )ÚGraniteMoeSharedPreTrainedModelr'   rC   N)r   r   r   r   r   Ú_no_split_modulesr!   r"   r#   r[   r[   ƒ   s'   € € € € € € Ø"Ð"Ð"Ñ"Ø7Ð8ÐÐÐr"   r[   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGraniteMoeSharedModelr'   c                 óº   •‡— t          ¦   «                              ‰¦  «         t          j        ˆfd„t	          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r!   )rC   )Ú.0rD   r'   s     €r#   ú
<listcomp>z2GraniteMoeSharedModel.__init__.<locals>.<listcomp>Œ   s$   ø€ ÐnÐnÐnÀÕ)¨&°)Ñ<Ô<ÐnÐnÐnr"   )r*   r+   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersr4   s    `€r#   r+   zGraniteMoeSharedModel.__init__‰   sV   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý”mØnÐnÐnÐnÍeÐTZÔTlÑNmÔNmÐnÑnÔnñ
ô 
ˆŒˆˆr"   )r   r   r   r   r+   r@   rA   s   @r#   r^   r^   ˆ   sE   ø€ € € € € ð
Ð5ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r"   r^   c                   ó,   ‡ — e Zd ZddiZdefˆ fd„Zˆ xZS )ÚGraniteMoeSharedForCausalLMzlm_head.weightzmodel.embed_tokens.weightr'   c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S )N)r*   r+   r^   ÚmodelÚ	post_initr4   s     €r#   r+   z$GraniteMoeSharedForCausalLM.__init__“   s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý*¨6Ñ2Ô2ˆŒ
à�ŠÑÔÐÐÐr"   )r   r   r   Ú_tied_weights_keysr   r+   r@   rA   s   @r#   rh   rh   �   sQ   ø€ € € € € Ø*Ð,GÐHÐðÐ5ð ð ð ð ð ð ð ð ð ð r"   rh   )rh   r^   r[   )Útypingr   r   r   Úactivationsr   Úcache_utilsr   Úprocessing_utilsr   Úutilsr	   Úgranitemoe.modeling_granitemoer   r   r   r   Úconfiguration_granitemoesharedr   Ú
get_loggerr   Úloggerr   ÚModuler&   rC   r[   r^   rh   Ú__all__r!   r"   r#   ú<module>rx      sü  ðð Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à !Ð !Ð !Ð !Ð !Ð !Ø  Ð  Ð  Ð  Ð  Ð  Ø &Ð &Ð &Ð &Ð &Ð &Ø Ð Ð Ð Ð Ð ðð ð ð ð ð ð ð ð ð ð ð ð CÐ BÐ BÐ BÐ BÐ Bð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð  )°5ð ñ ô ð ð0ð ð ð ð ˜"œ)ñ ô ð ð4*ð *ð *ð *ð *Ð#9ñ *ô *ð *ðZ9ð 9ð 9ð 9ð 9Ð&?ñ 9ô 9ð 9ð

ð 
ð 
ð 
ð 
˜Oñ 
ô 
ð 
ðð ð ð ð Ð"7ñ ô ð ð fÐ
eÐ
e€€€r"   