§
    ‚ŠtjüT  ã                   ó�  — d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ	 ddl
mZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z,  G d„ de&¦  «        Z- G d„ dej.        ¦  «        Z/ G d„ dej.        ¦  «        Z0 G d„ dej.        ¦  «        Z1 G d „ d!ej.        ¦  «        Z2 G d"„ d#ej.        ¦  «        Z3 G d$„ d%ej.        ¦  «        Z4 G d&„ d'e¦  «        Z5 G d(„ d)e¦  «        Z6e G d*„ d+e6¦  «        ¦   «         Z7 G d,„ d-e6e¦  «        Z8g d.¢Z9dS )/z"Modular components for DBRX model.é    )ÚCallable)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Úload_balancing_loss_funcé   )Ú
DbrxConfigc                   ó   — e Zd ZdS )ÚDbrxRotaryEmbeddingN)Ú__name__Ú
__module__Ú__qualname__© ó    úc/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/dbrx/modular_dbrx.pyr    r    -   s   € € € € € Ø€Dr%   r    c                   óª   ‡ — e Zd ZdZ	 ddedz  fˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
ej        ej        f         f
d
„Zˆ xZS )ÚDbrxAttentionzYModular DBRX attention component that can be reused across different model architectures.NÚ	layer_idxc                 óT  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        |j        | _	        || _
        |j        }|j        | _        |j        | _        |j        | _        | j        | j        z  | _        | j        dz  | _        |j        | _        d| _        t+          j        | j        | j        d| j        z  | j        z  z   d¬¦  «        | _        t+          j        | j        | j        d¬¦  «        | _        d S )Ng      à¿Tr   F©Úbias)ÚsuperÚ__init__ÚconfigÚd_modelÚhidden_sizeÚn_headsÚ	num_headsÚhead_dimÚmax_seq_lenÚmax_position_embeddingsr)   Úattn_configÚ
attn_pdropÚattention_dropoutÚclip_qkvÚ
kv_n_headsÚnum_key_value_headsÚnum_key_value_groupsÚscalingÚ
rope_thetaÚ	is_causalr   ÚLinearÚWqkvÚout_proj)Úselfr/   r)   Úkwargsr7   Ú	__class__s        €r&   r.   zDbrxAttention.__init__4   s  ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ!œ>ˆÔØœˆŒØÔ(¨D¬NÑ:ˆŒØ'-Ô'9ˆÔ$Ø"ˆŒàÔ(ˆØ!,Ô!7ˆÔØ#Ô,ˆŒØ#.Ô#9ˆÔ Ø$(¤N°dÔ6NÑ$NˆÔ!Ø”} dÑ*ˆŒØ%Ô0ˆŒØˆŒå”IØÔ˜dÔ.°°TÔ5MÑ1MÐPTÔP]Ñ1]Ñ]Ðdið
ñ 
ô 
ˆŒ	õ œ	 $Ô"2°DÔ4DÈ5ÐQÑQÔQˆŒˆˆr%   Úhidden_statesÚattention_maskÚposition_embeddingsÚpast_key_valuesÚreturnc                 ó®  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «        }| j        �| j         nd }	|                     |	| j        ¬¦  «        }|                     | j        | j        | j        z  | j        | j        z  gd¬¦  «        \  }
}}|
                     |¦  «         	                    dd¦  «        }
|                     |¦  «         	                    dd¦  «        }|                     |¦  «         	                    dd¦  «        }|\  }}t          |
|||¦  «        \  }
}|�|                     ||| j        ¦  «        \  }}t          j        | j        j        t"          ¦  «        } || |
|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )Néÿÿÿÿ)ÚminÚmaxr   ©Údimr   ç        )Údropoutr>   )Úshaper4   rB   r:   ÚclampÚsplitr1   r<   ÚviewÚ	transposer   Úupdater)   r   Úget_interfacer/   Ú_attn_implementationr   Útrainingr9   r>   ÚreshapeÚ
contiguousrC   )rD   rG   rH   rI   rJ   rE   Úinput_shapeÚhidden_shapeÚ
qkv_statesÚmin_valÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r&   ÚforwardzDbrxAttention.forwardP   s$  € ð $Ô)¨#¨2¨#Ô.ˆØ8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆà—Y’Y˜}Ñ-Ô-ˆ
Ø$(¤MÐ$=�4”=�.�.À4ˆØ×%Ò%¨'°t´}Ð%ÑEÔEˆ
à1;×1AÒ1AàÔ ØÔ(¨4¬=Ñ8ØÔ(¨4¬=Ñ8ðð
 ð 2Bñ 2
ô 2
Ñ.ˆ�j ,ð $×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆØ—_’_ \Ñ2Ô2×<Ò<¸QÀÑBÔBˆ
Ø#×(Ò(¨Ñ6Ô6×@Ò@ÀÀAÑFÔFˆà&‰ˆˆSÝ#7¸ÀjÐRUÐWZÑ#[Ô#[Ñ ˆ�jàÐ&Ø'6×'=Ò'=¸jÈ,ÐX\ÔXfÑ'gÔ'gÑ$ˆJ˜å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆØ˜LÐ(Ð(r%   ©N©NNN)r!   r"   r#   Ú__doc__Úintr.   ÚtorchÚTensorÚ
LongTensorr	   Útuplerk   Ú__classcell__©rF   s   @r&   r(   r(   1   sÚ   ø€ € € € € ØcÐcð
 !%ðRð Rð ˜‘:ðRð Rð Rð Rð Rð Rð> /3Ø7;Ø(,ð3)ð 3)à”|ð3)ð œ tÑ+ð3)ð #Ô-°Ñ4ð	3)ð
  ™ð3)ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)ð 3)r%   r(   c            
       ól   ‡ — e Zd Zˆ fd„Zdej        dej        dej        dej        dej        f
d„Zˆ xZS )ÚDbrxExpertGLUc                 ó^  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _	        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _
        t          j        t          j        | j        | j        z  | j        ¦  «        ¦  «        | _        |j                             dd¦  «        }t          |         | _        d S )NÚnameÚsilu)r-   r.   r1   Úffn_hidden_sizeÚmoe_num_expertsr   Ú	Parameterrp   ÚemptyÚw1Úv1Úw2Ú
ffn_act_fnÚgetr   Úactivation_fn)rD   r/   Úact_fn_namerF   s      €r&   r.   zDbrxExpertGLU.__init__‡   sé   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ%Ô5ˆÔØ%Ô5ˆÔå”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒÝ”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒÝ”,�uœ{¨4Ô+?À$ÔBVÑ+VÐX\ÔXhÑiÔiÑjÔjˆŒàÔ'×+Ò+¨F°FÑ;Ô;ˆÝ# KÔ0ˆÔÐÐr%   ÚxÚ	expert_w1Ú	expert_v1Ú	expert_w2rK   c                 óÜ   — |                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z  }|                      |                     ¦   «         ¦  «        }|S rl   )Úmatmulr„   Út)	rD   r†   r‡   rˆ   r‰   Ú	gate_projÚup_projÚintermediate_statesÚ	down_projs	            r&   rk   zDbrxExpertGLU.forward”   se   € ð —H’H˜YÑ'Ô'ˆ	Ø—(’(˜9Ñ%Ô%ˆØ×&Ò& yÑ1Ô1ˆ	Ø'¨'Ñ1ÐØ'×.Ò.¨y¯{ª{©}¬}Ñ=Ô=ˆ	ØÐr%   ©r!   r"   r#   r.   rp   rq   rk   rt   ru   s   @r&   rw   rw   †   s   ø€ € € € € ð1ð 1ð 1ð 1ð 1ðØ”ðØ*/¬,ðØCHÄ<ðØ\aÔ\hðà	Œðð ð ð ð ð ð ð r%   rw   c                   ó^   ‡ — e Zd Zˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZS )ÚDbrxExpertsc                 ó¸   •— t          ¦   «                              ¦   «          t          |¦  «        | _        |j        | _        |j        | _        |j        | _        d S rl   )r-   r.   rw   Úmlpr1   r{   r|   Únum_experts©rD   r/   rF   s     €r&   r.   zDbrxExperts.__init__    sO   ø€ Ý‰Œ×ÒÑÔÐÝ  Ñ(Ô(ˆŒØ!Ô-ˆÔØ%Ô5ˆÔØ!Ô1ˆÔÐÐr%   rG   Útop_k_indexÚtop_k_weightsrK   c                 ó‚  — |j         d         }|                     d| j        ¦  «        }t          j        ||j        |j        ¬¦  «        }t          j        ¦   «         5  t          j        j	         
                    || j        ¬¦  «        }|                     ddd¦  «        }t          j        |                     d¬¦  «        d¦  «                             ¦   «         }d d d ¦  «         n# 1 swxY w Y   d| j        | j        f}|D �]}	|	d         }	t          j        ¦   «         5  t          j        ||	         ¦  «        \  }
}d d d ¦  «         n# 1 swxY w Y   | j        j                             |¦  «        |	         }| j        j                             |¦  «        |	         }| j        j                             |¦  «        |	         }|                      ||         |||¦  «        }|                     d| j        ¦  «        |||
d f         z  }|                     d||¦  «         �Œ |                     |d| j        ¦  «        }|S )	Nr   rM   )ÚdtypeÚdevice)Únum_classesr   r   )rM   éþÿÿÿrP   )rT   r]   r{   rp   Ú
zeros_liker›   rœ   Úno_gradr   Ú
functionalÚone_hotr–   ÚpermuteÚgreaterÚsumÚnonzeror1   Úwherer•   r€   rW   r   r�   Ú
index_add_)rD   rG   r˜   r™   Ú
batch_sizeÚnext_statesÚexpert_maskÚ
expert_hitÚsplit_expert_shapeÚ
expert_idxÚidxÚ	token_idxr€   r   r�   Ústatess                   r&   rk   zDbrxExperts.forward§   s©  € ð #Ô(¨Ô+ˆ
Ø%×-Ò-¨b°$Ô2FÑGÔGˆåÔ& }¸MÔ<OÐXeÔXlÐmÑmÔmˆÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 ! $Ô"6¸Ô8HÐIÐØ$ð 		9ñ 		9ˆJØ# AœˆJÝ”‘”ð Fð FÝ!&¤¨[¸Ô-DÑ!EÔ!E‘��YðFð Fð Fñ Fô Fð Fð Fð Fð Fð Fð Føøøð Fð Fð Fð Fà””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ””×!Ò!Ð"4Ñ5Ô5°jÔAˆBØ—X’X˜m¨IÔ6¸¸BÀÑCÔCˆFØ—[’[  TÔ%9Ñ:Ô:¸]È9ÐVYÐ[_ÐK_Ô=`Ñ`ˆFØ×"Ò" 1 i°Ñ8Ô8Ð8Ñ8à!×&Ò& z°2°tÔ7KÑLÔLˆØÐs%   ÁA>C'Ã'C+Ã.C+Ä!EÅE	ÅE	r‘   ru   s   @r&   r“   r“   Ÿ   sz   ø€ € € € € ð2ð 2ð 2ð 2ð 2ðà”|ðð ”\ðð ”|ð	ð
 
Œðð ð ð ð ð ð ð r%   r“   c                   óh   ‡ — e Zd Zˆ fd„Zdej        deej        ej        ej        f         fd„Zˆ xZ	S )Ú
DbrxRouterc                 óÄ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          j        | j        |j        d¬¦  «        | _        d S ©NFr+   )	r-   r.   r{   r1   Úmoe_jitter_epsr   rA   r|   Úlayerr—   s     €r&   r.   zDbrxRouter.__init__Ç   sQ   ø€ Ý‰Œ×ÒÑÔÐØ!Ô1ˆÔØ$Ô3ˆÔÝ”Y˜tÔ/°Ô1GÈeÐTÑTÔTˆŒ
ˆ
ˆ
r%   rG   rK   c                 ó  — | j         rB| j        �;|t          j        |¦  «                             d| j        z
  d| j        z   ¦  «        z  }|                     d|j        d         ¦  «        }|                      |¦  «        }|S )Ng      ð?rM   )r\   r¶   rp   Ú
empty_likeÚuniform_rW   rT   r·   )rD   rG   Úrouter_logitss      r&   rk   zDbrxRouter.forwardÍ   s†   € ØŒ=ð 	˜TÔ0Ð<Ø�UÔ-¨mÑ<Ô<×EÒEØ�dÔ)Ñ)¨3°Ô1DÑ+Dñô ñ ˆMð &×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØŸ
š
 =Ñ1Ô1ˆØÐr%   )
r!   r"   r#   r.   rp   rq   rs   rr   rk   rt   ru   s   @r&   r³   r³   Æ   su   ø€ € € € € ðUð Uð Uð Uð Uð U¤\ð °e¸E¼LÈ%Ì,ÐX]ÔXhÐ<hÔ6ið ð ð ð ð ð ð ð r%   r³   c                   óf   ‡ — e Zd ZdZˆ fd„Zd„ Zdej        deej        ej        f         fd„Z	ˆ xZ
S )ÚDbrxFFNz0Modular DBRX MLP/FFN component with MoE support.c                 óð   •— t          ¦   «                              ¦   «          t          |j        ¦  «        | _        t          |j        ¦  «        | _        |j        j        | _        |j        j        | _	        d S rl   )
r-   r.   r³   Ú
ffn_configÚrouterr“   ÚexpertsÚmoe_normalize_expert_weightsÚ	moe_top_kÚtop_k)rD   r/   rE   rF   s      €r&   r.   zDbrxFFN.__init__Ú   s\   ø€ Ý‰Œ×ÒÑÔÐÝ  Ô!2Ñ3Ô3ˆŒÝ" 6Ô#4Ñ5Ô5ˆŒà,2Ô,=Ô,ZˆÔ)ØÔ&Ô0ˆŒ
ˆ
ˆ
r%   c                 óî   — t           j        j                             |d|j        ¬¦  «        }t          j        || j        d¬¦  «        \  }}| j        � |t          j        || j        dd¬¦  «        z  }||fS )Nr   )rQ   r›   rM   rP   T)ÚprQ   Úkeepdim)	rp   r   r¡   Úsoftmaxr›   ÚtopkrÄ   rÂ   Únorm)rD   r»   Úrouter_top_valueÚrouter_indicess       r&   Úroute_tokens_to_expertszDbrxFFN.route_tokens_to_expertsâ   s…   € ÝœÔ+×3Ò3°MÀqÐP]ÔPcÐ3ÑdÔdˆÝ+0¬:°mÀTÄZÐUWÐ+XÑ+XÔ+XÑ(Ð˜.ØÔ,Ð8Ø/µ%´*Ø  DÔ$EÈ2ÐW[ð3ñ 3ô 3ñ  Ðð   Ð/Ð/r%   rG   rK   c                 óŽ   — |                       |¦  «        }|                      |¦  «        \  }}|                      |||¦  «        }|S rl   )rÀ   rÍ   rÁ   )rD   rG   r»   r™   r˜   Úoutputs         r&   rk   zDbrxFFN.forwardë   sE   € ØŸš MÑ2Ô2ˆØ%)×%AÒ%AÀ-Ñ%PÔ%PÑ"ˆ�{Ø—’˜m¨[¸-ÑHÔHˆØˆr%   )r!   r"   r#   rn   r.   rÍ   rp   rq   rs   rk   rt   ru   s   @r&   r½   r½   ×   s   ø€ € € € € Ø:Ð:ð1ð 1ð 1ð 1ð 1ð0ð 0ð 0ð U¤\ð °e¸E¼LÈ%Ì,Ð<VÔ6Wð ð ð ð ð ð ð ð r%   r½   c                   ó¤   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej        dej        dej        dz  de	dz  d	e
d
eej        ej        f         fd„Zˆ xZS )ÚDbrxNormAttentionNormNr/   r)   c                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        t	          j        |j        d¬¦  «        | _        t          ||¬¦  «        | _	        t	          j        |j        d¬¦  «        | _
        d S )NFr+   ©r/   r)   )r-   r.   r)   Úresid_pdropr   Ú	LayerNormr0   Únorm_1r(   ÚattnÚnorm_2©rD   r/   r)   rF   s      €r&   r.   zDbrxNormAttentionNorm.__init__ó   s}   ø€ Ý‰Œ×ÒÑÔÐØ"ˆŒØ!Ô-ˆÔÝ”l 6¤>¸Ð>Ñ>Ô>ˆŒÝ!ØØð
ñ 
ô 
ˆŒ	õ ”l 6¤>¸Ð>Ñ>Ô>ˆŒˆˆr%   rG   rI   rH   rJ   rE   rK   c                 óR  — |}|                       |¦  «                             |j        ¦  «        } | j        d||||dœ|¤Ž\  }}t          j                             || j        | j        ¬¦  «        }||z   }|}|  	                    |¦  «                             |j        ¦  «        }||fS ©N)rG   rH   rI   rJ   )rÆ   r\   r$   )
rÖ   Útor›   r×   r   r¡   rS   rÔ   r\   rØ   )rD   rG   rI   rH   rJ   rE   Úresidual_statesÚ_s           r&   rk   zDbrxNormAttentionNorm.forwardþ   sÄ   € ð (ˆØŸš MÑ2Ô2×5Ò5°mÔ6IÑJÔJˆà$˜4œ9ð 
Ø'Ø)Ø 3Ø+ð	
ð 
ð
 ð
ð 
Ñˆ�qõ œ×-Ò-¨m¸tÔ?OÐZ^ÔZgÐ-ÑhÔhˆØ%¨Ñ7ˆà'ˆØŸš MÑ2Ô2×5Ò5°mÔ6IÑJÔJˆà Ð-Ð-r%   rl   )NN)r!   r"   r#   r   ro   r.   rp   rq   rr   r	   r   rs   rk   rt   ru   s   @r&   rÑ   rÑ   ò   sÍ   ø€ € € € € ð	?ð 	?˜zð 	?°c¸D±jð 	?ð 	?ð 	?ð 	?ð 	?ð 	?ð /3Ø(,ð.ð .à”|ð.ð #Ô-ð.ð œ tÑ+ð	.ð
  ™ð.ð ð.ð 
ˆuŒ|˜Uœ\Ð)Ô	*ð.ð .ð .ð .ð .ð .ð .ð .r%   rÑ   c                   ó|   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej        dej        dz  dej        dz  de	dz  d	e
f
d
„Zˆ xZS )Ú	DbrxBlockr/   r)   c                 óÜ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        || _        t          ||¬¦  «        | _        t          |¬¦  «        | _	        d S )NrÓ   ©r/   )
r-   r.   r0   r1   rÔ   r)   rÑ   Únorm_attn_normr½   ÚffnrÙ   s      €r&   r.   zDbrxBlock.__init__  sj   ø€ Ý‰Œ×ÒÑÔÐØ!œ>ˆÔØ!Ô-ˆÔØ"ˆŒÝ3ØØð
ñ 
ô 
ˆÔõ  &Ð)Ñ)Ô)ˆŒˆˆr%   NrG   rH   rI   rJ   rE   c                 ó¼   —  | j         d||||dœ|¤Ž\  }}|                      |¦  «        }t          j                             || j        | j        ¬¦  «        }||z   }|S rÛ   )rã   rä   r   r¡   rS   rÔ   r\   )rD   rG   rH   rI   rJ   rE   Úresid_statess          r&   rk   zDbrxBlock.forward&  sƒ   € ð ': dÔ&9ð '
Ø'Ø)Ø 3Ø+ð	'
ð '
ð
 ð'
ð '
Ñ#ˆ�mð Ÿš Ñ/Ô/ˆÝœ×-Ò-¨m¸tÔ?OÐZ^ÔZgÐ-ÑhÔhˆØ$ }Ñ4ˆØÐr%   rm   )r!   r"   r#   r   ro   r.   rp   rq   rr   r	   r   rk   rt   ru   s   @r&   rà   rà     s´   ø€ € € € € ð	*˜zð 	*°cð 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð /3Ø7;Ø(,ðð à”|ðð œ tÑ+ðð #Ô-°Ñ4ð	ð
  ™ðð ðð ð ð ð ð ð ð r%   rà   c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej        ¦   «         dej        fˆ fd	„¦   «         Zˆ xZS )
ÚDbrxPreTrainedModelr/   ÚtransformerTrà   rJ   F)rG   Ú
attentionsÚmodulec                 ó8  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        rVt          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         t          j        |j
        d|¬¦  «         d S d S )NrR   )ÚmeanÚstd)r-   Ú_init_weightsr/   Úinitializer_rangeÚ
isinstancerw   ÚinitÚnormal_r   r€   r�   )rD   rë   rî   rF   s      €r&   rï   z!DbrxPreTrainedModel._init_weightsL  s’   ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ+ˆÝ�f�mÑ,Ô,ð 	7ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6ÝŒL˜œ¨°#Ð6Ñ6Ô6Ð6Ð6Ð6ð	7ð 	7r%   )r!   r"   r#   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flex_attnÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphrà   r(   Ú_can_record_outputsrp   r    r   ÚModulerï   rt   ru   s   @r&   rè   rè   <  sµ   ø€ € € € € € ØÐÐÑØ%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ"&ÐØÐØ€NØ"Ðà"Ø#ðð Ðð
 €U„]�_„_ð7 B¤Ið 7ð 7ð 7ð 7ð 7ñ „_ð7ð 7ð 7ð 7ð 7r%   rè   c                   ó  ‡ — e Zd ZdZdefˆ fd„Zdej        fd„Zdej        fd„Z	e
ee	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dedz  dej        dz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	DbrxModela©  Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer.

    Args:
        config ([`DbrxConfig`]): Model configuration class with all parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
    r/   c                 óô  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        ‰j        | _        t          ‰¦  «        | _        t          j	        ‰j        ‰j
        | j        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ‰j
        d¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r$   )rà   )Ú.0r)   r/   s     €r&   ú
<listcomp>z&DbrxModel.__init__.<locals>.<listcomp>g  s#   ø€ Ð$jÐ$jÐ$jÀi¥Y¨v°yÑ%AÔ%AÐ$jÐ$jÐ$jr%   Fr+   )r-   r.   Úpad_token_idÚpadding_idxÚ
vocab_sizeÚ	emb_pdropr    Ú
rotary_embr   Ú	Embeddingr0   ÚwteÚ
ModuleListÚrangeÚn_layersÚblocksrÕ   Únorm_fÚgradient_checkpointingÚ	post_initr—   s    `€r&   r.   zDbrxModel.__init__`  sÒ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø!Ô.ˆÔØ Ô+ˆŒØÔ)ˆŒÝ-¨fÑ5Ô5ˆŒÝ”< Ô 1°6´>À4ÔCSÑTÔTˆŒÝ”mÐ$jÐ$jÐ$jÐ$jÕSXÐY_ÔYhÑSiÔSiÐ$jÑ$jÔ$jÑkÔkˆŒÝ”l 6¤>¸Ð>Ñ>Ô>ˆŒØ&+ˆÔ#ð 	�ŠÑÔÐÐÐr%   rK   c                 ó   — | j         S rl   ©r  ©rD   s    r&   Úget_input_embeddingszDbrxModel.get_input_embeddingsn  s	   € ØŒxˆr%   Úvaluec                 ó   — || _         d S rl   r  ©rD   r  s     r&   Úset_input_embeddingszDbrxModel.set_input_embeddingsq  s   € ØˆŒˆˆr%   NÚ	input_idsrH   Úposition_idsrJ   Úinputs_embedsÚ	use_cacherE   c           
      óF  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          | j        ||||¬¦  «        }	|}
|                      |
|¦  «        }| j        d | j        j        …         D ]} ||
f||	|||dœ|¤Ž}
Œ|                      |
¦  «        }
t          |
|¬¦  «        S )	Nz:You must specify exactly one of input_ids or inputs_embedsrâ   r   r   )rœ   )r/   r  rH   rJ   r  )rI   rH   r  rJ   r  )Úlast_hidden_staterJ   )Ú
ValueErrorr
   r/   r  Úget_seq_lengthrp   ÚarangerT   rœ   Ú	unsqueezer   r
  r  Únum_hidden_layersr  r   )rD   r  rH   r  rJ   r  r  rE   Úpast_seen_tokensÚcausal_maskrG   rI   Údecoder_layers                r&   rk   zDbrxModel.forwardt  sˆ  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ŸHšH YÑ/Ô/ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆð #Ÿošo¨m¸\ÑJÔJÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà$7Ø*Ø)Ø /Ø#ðð ð ðð ˆMˆMð Ÿš MÑ2Ô2ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r%   )NNNNNN)r!   r"   r#   rn   r   r.   r   r  r  r  r   r   r   rp   rr   rq   r	   ÚFloatTensorÚboolr   r   r   rk   rt   ru   s   @r&   r  r  V  sQ  ø€ € € € € ðð ð˜zð ð ð ð ð ð ð b¤lð ð ð ð ð¨"¬,ð ð ð ð ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð5
ð 5
àÔ# dÑ*ð5
ð œ tÑ+ð5
ð Ô&¨Ñ-ð	5
ð
  ™ð5
ð Ô(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ô,ð5
ð 
 ð5
ð 5
ð 5
ñ „^ñ „_ñ  Ôð5
ð 5
ð 5
ð 5
ð 5
r%   r  c                   óš  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	ej	        fd
„Z
dej	        fd„Zd	ej        fd„Zdej        fd„Zdefd„Zd	efd„Zee	 	 	 	 	 	 	 	 	 d dej        dz  dej        dz  dej        dz  dedz  dej        dz  dej        dz  dedz  dedz  deej        z  dee         d	efd„¦   «         ¦   «         Zˆ xZS )!ÚDbrxForCausalLMzlm_head.weightztransformer.wte.weightÚlm_headÚcolwise_gather_outputrG   Úlogitsr/   c                 ód  •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        t          j        |j        |j        d¬¦  «        | _        |j	        j
        | _        |j	        j        | _        |j	        j        | _        |                      ¦   «          d S rµ   )r-   r.   r  ré   r  r   rA   r1   r.  r¿   Úmoe_loss_weightÚrouter_aux_loss_coefr|   r–   rÃ   Únum_experts_per_tokr  r—   s     €r&   r.   zDbrxForCausalLM.__init__´  s“   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆÔØ Ô+ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒØ$*Ô$5Ô$EˆÔ!Ø!Ô,Ô<ˆÔØ#)Ô#4Ô#>ˆÔ Ø�ŠÑÔÐÐÐr%   rK   c                 ó4   — | j                              ¦   «         S rl   )ré   r  r  s    r&   r  z$DbrxForCausalLM.get_input_embeddings¾  s   € ØÔ×4Ò4Ñ6Ô6Ð6r%   r  c                 ó:   — | j                              |¦  «         d S rl   )ré   r  r  s     r&   r  z$DbrxForCausalLM.set_input_embeddingsÁ  s   € ØÔ×-Ò-¨eÑ4Ô4Ð4Ð4Ð4r%   c                 ó   — | j         S rl   ©r.  r  s    r&   Úget_output_embeddingsz%DbrxForCausalLM.get_output_embeddingsÄ  s
   € ØŒ|Ðr%   Únew_embeddingsc                 ó   — || _         d S rl   r8  )rD   r:  s     r&   Úset_output_embeddingsz%DbrxForCausalLM.set_output_embeddingsÇ  s   € Ø%ˆŒˆˆr%   Údecoderc                 ó   — || _         d S rl   ©ré   )rD   r=  s     r&   Úset_decoderzDbrxForCausalLM.set_decoderÊ  s   € Ø"ˆÔÐÐr%   c                 ó   — | j         S rl   r?  r  s    r&   Úget_decoderzDbrxForCausalLM.get_decoderÍ  s   € ØÔÐr%   Nr   r  rH   r  rJ   r  Úlabelsr  Úoutput_router_logitsÚlogits_to_keeprE   c
                 ó  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}d}|rHt          |j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t#          ||||j        |j        |j        |j        ¬¦  «        S )a‹  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >> from transformers import AutoTokenizer, DbrxForCausalLM

        >> model = DbrxForCausalLM.from_pretrained("transformers-community/dbrx-instruct")
        >> tokenizer = AutoTokenizer.from_pretrained("transformers-community/dbrx-instruct")

        >> prompt = "Hey, are you conscious? Can you talk to me?"
        >> inputs = tokenizer(prompt, return_tensors="pt")

        >> # Generate
        >> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```
        N)r  rH   r  rJ   r  r  rD  )ÚlossÚaux_lossr0  rJ   rG   rê   r»   r$   )r/   rD  ré   r!  rñ   ro   Úslicer.  Úloss_functionr  r   r»   r–   r4  r3  rÜ   rœ   r   rJ   rG   rê   )rD   r  rH   r  rJ   r  rC  r  rD  rE  rE   ÚoutputsrG   Úslice_indicesr0  rG  rH  s                    r&   rk   zDbrxForCausalLM.forwardÐ  so  € ðN %9Ð$DÐ Ð È$Ì+ÔJjð 	ð
 +;¨$Ô*:ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDàˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r%   )	NNNNNNNNr   ) r!   r"   r#   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr   r.   r   r  r  r  rA   r9  r<  r  r@  rB  r   r   rp   rr   rq   r	   r*  r+  ro   r   r   r   rk   rt   ru   s   @r&   r-  r-  ¯  s&  ø€ € € € € Ø*Ð,DÐEÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð˜zð ð ð ð ð ð ð7 b¤lð 7ð 7ð 7ð 7ð5¨"¬,ð 5ð 5ð 5ð 5ð r¤yð ð ð ð ð&°B´Ið &ð &ð &ð &ð# 9ð #ð #ð #ð #ð ˜Yð  ð  ð  ð  ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
  ™ðP
ð Ô(¨4Ñ/ðP
ð Ô  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eœlÑ*ðP
ð Ð+Ô,ðP
ð 
#ðP
ð P
ð P
ñ „^ñ ÔðP
ð P
ð P
ð P
ð P
r%   r-  )r-  r  rè   ):rn   Úcollections.abcr   Útypingr   rp   r   Ú r   rò   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   Úmixtral.modeling_mixtralr   Úconfiguration_dbrxr   r    rÿ   r(   rw   r“   r³   r½   rÑ   rà   rè   r  r-  Ú__all__r$   r%   r&   ú<module>rb     sÆ  ðð )Ð (à $Ð $Ð $Ð $Ð $Ð $Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /ðð ð ð ð ð ð RÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5ðð ð ð ð ð ð ð ð ð ð
 @Ð ?Ð ?Ð ?Ð ?Ð ?Ø *Ð *Ð *Ð *Ð *Ð *ð	ð 	ð 	ð 	ð 	Ð.ñ 	ô 	ð 	ðR)ð R)ð R)ð R)ð R)�B”Iñ R)ô R)ð R)ðjð ð ð ð �B”Iñ ô ð ð2$ð $ð $ð $ð $�"”)ñ $ô $ð $ðNð ð ð ð �”ñ ô ð ð"ð ð ð ð ˆbŒiñ ô ð ð6%.ð %.ð %.ð %.ð %.˜BœIñ %.ô %.ð %.ðPð ð ð ð Ð*ñ ô ð ðD7ð 7ð 7ð 7ð 7˜/ñ 7ô 7ð 7ð4 ðU
ð U
ð U
ð U
ð U
Ð#ñ U
ô U
ñ „ðU
ðps
ð s
ð s
ð s
ð s
Ð)¨?ñ s
ô s
ð s
ðl BÐ
AÐ
A€€€r%   