§
    ‚Štj_G  ã            
       ó0  — d Z ddlZddlmc mZ ddlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,  ej-        e.¦  «        Z/	 	 	 d4dej0        e1ej0                 z  dz  de2dz  dej0        dz  dej0        e2z  fd„Z3e G d„ dej4        ¦  «        ¦   «         Z5 G d„ dej4        ¦  «        Z6 G d„ dej4        ¦  «        Z7 G d„ d e)¦  «        Z8 G d!„ d"e*¦  «        Z9 G d#„ d$e"¦  «        Z: G d%„ d&e¦  «        Z; G d'„ d(e(¦  «        Z< G d)„ d*e'¦  «        Z= G d+„ d,e#¦  «        Z> G d-„ d.e%¦  «        Z? G d/„ d0e&¦  «        Z@ G d1„ d2e$¦  «        ZAg d3¢ZBdS )5zPyTorch Mixtral model.é    N)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úuse_experts_implementation)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚlogging)ÚOutputRecorderé   )	ÚMistralAttentionÚMistralForCausalLMÚMistralForQuestionAnsweringÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚMistralPreTrainedModelÚMistralRMSNormÚMistralRotaryEmbeddingé   )ÚMixtralConfigÚgate_logitsÚnum_expertsÚattention_maskÚreturnc                 óÆ  ‡— | �t          | t          ¦  «        sdS t          | t          ¦  «        r/| d         j        Št          j        ˆfd„| D ¦   «         d¬¦  «        }t          j        j                             |d¬¦  «        }t          j        ||d¬¦  «        \  }}t          j        j         	                    ||¦  «        }|€@t          j
        |                     ¦   «         d¬¦  «        }	t          j
        |d¬¦  «        }
�n.|j        \  }}|j        d         ||z  z  }|ddd…dd…ddf                              |||||f¦  «                             d||¦  «                             ‰¦  «        }t          j        |                     ¦   «         |z  d¬¦  «        t          j        |d¬¦  «        z  }	|ddd…dd…df                              ||||f¦  «                             d|¦  «                             ‰¦  «        }t          j        ||z  d¬¦  «        t          j        |d¬¦  «        z  }
t          j        |	|
                     d¦  «        z  ¦  «        }||z  S )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   c                 ó:   •— g | ]}|                      ‰¦  «        ‘ŒS © )Úto)Ú.0Ú
layer_gateÚcompute_devices     €úi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mixtral/modular_mixtral.pyú
<listcomp>z,load_balancing_loss_func.<locals>.<listcomp>W   s&   ø€ Ð-jÐ-jÐ-jÐPZ¨j¯mªm¸NÑ.KÔ.KÐ-jÐ-jÐ-jó    ©Údiméÿÿÿÿ)Ú
isinstanceÚtupleÚdeviceÚtorchÚcatr   Ú
functionalÚsoftmaxÚtopkÚone_hotÚmeanÚfloatÚshapeÚexpandÚreshaper'   ÚsumÚ	unsqueeze)r    r!   Útop_kr"   Úconcatenated_gate_logitsÚrouting_weightsÚ_Úselected_expertsÚexpert_maskÚtokens_per_expertÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthÚnum_hidden_layersÚexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_lossr*   s                    @r+   Úload_balancing_loss_funcrO   5   s�  ø€ ð: Ð¥*¨[½%Ñ"@Ô"@ÐØˆqå�+�uÑ%Ô%ð sØ$ QœÔ.ˆÝ#(¤9Ð-jÐ-jÐ-jÐ-jÐ^iÐ-jÑ-jÔ-jÐpqÐ#rÑ#rÔ#rÐ å”hÔ)×1Ò1Ð2JÐPRÐ1ÑSÔS€Oåœ* _°eÀÐDÑDÔDÑ€AÐå”(Ô%×-Ò-Ð.>ÀÑLÔL€KàÐå!œJ {×'8Ò'8Ñ':Ô':ÀÐBÑBÔBÐõ "'¤¨OÀÐ!CÑ!CÔ!CÐÑà&4Ô&:Ñ#ˆ
�OØ4Ô:¸1Ô=À*ÈÑB^Ñ_Ðð ˜4    A A A t¨TÐ1Ô2ßŠVÐ&¨
°OÀUÈKÐXÑYÔYßŠW�R˜ Ñ,Ô,ßŠR�ÑÔð	 	õ "œI k×&7Ò&7Ñ&9Ô&9Ð<QÑ&QÐWXÐYÑYÔYÕ\aÔ\eØ! qð]
ñ ]
ô ]
ñ 
Ðð ˜4    A A A tÐ+Ô,ßŠVÐ&¨
°OÀ[ÐQÑRÔRßŠW�R˜Ñ%Ô%ßŠR�ÑÔð	 	)õ "'¤¨?Ð=]Ñ+]ÐcdÐ!eÑ!eÔ!eÕhmÔhqØ,°!ði
ñ i
ô i
ñ "
Ðõ ”9Ð.Ð1G×1QÒ1QÐRSÑ1TÔ1TÑTÑUÔU€LØ˜+Ñ%Ð%r-   c                   óh   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        dej        dej        fd„Zˆ xZ	S )	ÚMixtralExpertsz2Collection of expert weights stored as 3D tensors.Úconfigc                 ó´  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        d| j        z  | j        ¦  «        ¦  «        | _        t          j	        t          j        | j        | j        | j        ¦  «        ¦  «        | _        t          |j                 | _        d S )Nr   )ÚsuperÚ__init__Únum_local_expertsr!   Úhidden_sizeÚ
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	Parameterr4   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©ÚselfrR   Ú	__class__s     €r+   rU   zMixtralExperts.__init__‹   s£   ø€ Ý‰Œ×ÒÑÔÐØ!Ô3ˆÔØ Ô,ˆŒØ &Ô 8ˆÔÝœL­¬°TÔ5EÀqÈ4ÔK`ÑG`ÐbfÔbqÑ)rÔ)rÑsÔsˆÔÝœ¥e¤k°$Ô2BÀDÄOÐUYÔUjÑ&kÔ&kÑlÔlˆŒÝ˜VÔ.Ô/ˆŒˆˆr-   Úhidden_statesÚtop_k_indexÚtop_k_weightsr#   c                 ó€  — t          j        |¦  «        }t          j        ¦   «         5  t           j        j                             || j        ¬¦  «        }|                     ddd¦  «        }t          j        | 	                    d¬¦  «        d¦  «         
                    ¦   «         }d d d ¦  «         n# 1 swxY w Y   |D ]þ}|d         }|| j        k    rŒt          j        ||         ¦  «        \  }}	||	         }
t          j                             |
| j        |         ¦  «                             dd¬¦  «        \  }}|                      |¦  «        |z  }t          j                             || j        |         ¦  «        }|||	|d f         z  }|                     d|	|                     |j        ¦  «        ¦  «         Œÿ|S )N)Únum_classesr   r   r   )r0   éþÿÿÿr.   r0   )r4   Ú
zeros_likeÚno_gradr   r6   r9   r!   ÚpermuteÚgreaterr?   ÚnonzeroÚwhereÚlinearr]   Úchunkr`   r^   Ú
index_add_r'   Údtype)rb   rd   re   rf   Úfinal_hidden_statesrF   Ú
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r+   ÚforwardzMixtralExperts.forward”   sø  € õ $Ô.¨}Ñ=Ô=ÐÝŒ]‰_Œ_ð 	Sð 	SÝœ(Ô-×5Ò5°kÈtÔO_Ð5Ñ`Ô`ˆKØ%×-Ò-¨a°°AÑ6Ô6ˆKÝœ {§¢¸8 Ñ'DÔ'DÀaÑHÔH×PÒPÑRÔRˆJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sð
 %ð 
	nð 
	nˆJØ# AœˆJØ˜TÔ-Ò-Ð-ØÝ#(¤;¨{¸:Ô/FÑ#GÔ#GÑ ˆI�yØ)¨)Ô4ˆMÝ”}×+Ò+¨M¸4Ô;LÈZÔ;XÑYÔY×_Ò_Ð`aÐgiÐ_ÑjÔj‰HˆD�"Ø$(§K¢K°Ñ$5Ô$5¸Ñ$:Ð!Ý$&¤M×$8Ò$8Ð9NÐPTÔP^Ð_iÔPjÑ$kÔ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÔ<eÑ$eÐ!Ø×*Ò*¨1¨iÐ9N×9QÒ9QÐReÔRkÑ9lÔ9lÑmÔmÐmÐmà"Ð"s   ¨A>B2Â2B6Â9B6)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   rU   r4   ÚTensorr}   Ú__classcell__©rc   s   @r+   rQ   rQ   ‡   sŒ   ø€ € € € € à<Ð<ð0˜}ð 0ð 0ð 0ð 0ð 0ð 0ð#à”|ð#ð ”\ð#ð ”|ð	#ð
 
Œð#ð #ð #ð #ð #ð #ð #ð #r-   rQ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMixtralTopKRouterc                 óü   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        t          j	        t          j        | j        | j        ¦  «        ¦  «        | _        d S ©N)rT   rU   Únum_experts_per_tokrA   rV   r!   rW   rX   r   r[   r4   r\   Úweightra   s     €r+   rU   zMixtralTopKRouter.__init__°   s^   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø!Ô3ˆÔØ Ô,ˆŒÝ”l¥5¤;¨tÔ/?ÀÄÑ#QÔ#QÑRÔRˆŒˆˆr-   c                 ó\  — |                      d| j        ¦  «        }t          j        || j        ¦  «        }t
          j        j                             | 	                    ¦   «         d¬¦  «        }t          j
        || j        d¬¦  «        \  }}||                     dd¬¦  «        z  }|}|||fS )Nr0   r.   T)r/   Úkeepdim)r>   rX   ÚFrp   rŠ   r4   r   r6   r7   r;   r8   rA   r?   )rb   rd   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress          r+   r}   zMixtralTopKRouter.forward·   s¥   € Ø%×-Ò-¨b°$´/ÑBÔBˆÝœ °´Ñ<Ô<ˆÝ”xÔ*×2Ò2°=×3FÒ3FÑ3HÔ3HÈbÐ2ÑQÔQˆÝ+0¬:°lÀDÄJÐTVÐ+WÑ+WÔ+WÑ(Ð˜.ØÐ,×0Ò0°RÀÐ0ÑFÔFÑFÐØ(ˆØ˜m¨^Ð;Ð;r-   )r~   r   r€   rU   r}   rƒ   r„   s   @r+   r†   r†   ¯   sL   ø€ € € € € ðSð Sð Sð Sð Sð<ð <ð <ð <ð <ð <ð <r-   r†   c                   ó\   ‡ — e Zd Zˆ fd„Zdej        deej        ej        f         fd„Zˆ xZS )ÚMixtralSparseMoeBlockc                 óÈ   •— t          ¦   «                              ¦   «          |j        | _        |j        | _        t          |¦  «        | _        t          |¦  «        | _	        d S rˆ   )
rT   rU   r‰   rA   Úrouter_jitter_noiseÚjitter_noiser†   rz   rQ   Úexpertsra   s     €r+   rU   zMixtralSparseMoeBlock.__init__Â   sP   ø€ Ý‰Œ×ÒÑÔÐØÔ/ˆŒ
Ø"Ô6ˆÔÝ% fÑ-Ô-ˆŒ	Ý% fÑ-Ô-ˆŒˆˆr-   rd   r#   c                 ó†  — |j         \  }}}| j        rF| j        dk    r;|t          j        |¦  «                             d| j        z
  d| j        z   ¦  «        z  }|                     d|j         d         ¦  «        }|                      |¦  «        \  }}}|                      |||¦  «        }| 	                    |||¦  «        }|S )Nr   g      ð?r0   )
r<   Útrainingr—   r4   Ú
empty_likeÚuniform_Úviewrz   r˜   r>   )rb   rd   rI   rJ   rX   rD   rf   re   s           r+   r}   zMixtralSparseMoeBlock.forwardÉ   sÇ   € Ø2?Ô2EÑ/ˆ
�O ZØŒ=ð 	x˜TÔ.°Ò2Ð2Ø�UÔ-¨mÑ<Ô<×EÒEÀcÈDÔL]ÑF]Ð_bÐeiÔevÑ_vÑwÔwÑwˆMØ%×*Ò*¨2¨}Ô/BÀ2Ô/FÑGÔGˆØ(,¯	ª	°-Ñ(@Ô(@Ñ%ˆˆ=˜+ØŸš ]°KÀÑOÔOˆØ%×-Ò-¨j¸/È:ÑVÔVˆØÐr-   )	r~   r   r€   rU   r4   r‚   r2   r}   rƒ   r„   s   @r+   r”   r”   Á   sj   ø€ € € € € ð.ð .ð .ð .ð .ð U¤\ð °e¸E¼LÈ%Ì,Ð<VÔ6Wð ð ð ð ð ð ð ð r-   r”   c                   ó   — e Zd ZdS )ÚMixtralRMSNormN©r~   r   r€   r&   r-   r+   rŸ   rŸ   Ô   ó   € € € € € Ø€Dr-   rŸ   c                   ó   — e Zd ZdS )ÚMixtralRotaryEmbeddingNr    r&   r-   r+   r£   r£   Ø   r¡   r-   r£   c                   ó   — e Zd ZdS )ÚMixtralAttentionNr    r&   r-   r+   r¥   r¥   Ü   r¡   r-   r¥   c                   óÆ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej        deej        ej        f         dz  dej        dz  dej	        dz  d	e
dz  d
ee         dej        fd„Zˆ xZS )ÚMixtralDecoderLayerrR   Ú	layer_idxc                 ó2  •— t          ¦   «                              ¦   «          |j        | _        t          ||¦  «        | _        t          |¦  «        | _        t          |j        |j        ¬¦  «        | _	        t          |j        |j        ¬¦  «        | _
        d S )N)Úeps)rT   rU   rW   r¥   Ú	self_attnr”   ÚmlprŸ   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)rb   rR   r¨   rc   s      €r+   rU   zMixtralDecoderLayer.__init__á   s€   ø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔå)¨&°)Ñ<Ô<ˆŒå(¨Ñ0Ô0ˆŒÝ-¨fÔ.@ÀfÔFYÐZÑZÔZˆÔÝ(6°vÔ7IÈvÔObÐ(cÑ(cÔ(cˆÔ%Ð%Ð%r-   Nrd   Úposition_embeddingsr"   Úposition_idsÚpast_key_valuesÚkwargsr#   c           	      óÌ   — |}|                       |¦  «        } | j        d|||||dœ|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rd   r°   r"   r±   r²   r&   )r®   r«   r¯   r¬   )	rb   rd   r°   r"   r±   r²   r³   ÚresidualrD   s	            r+   r}   zMixtralDecoderLayer.forwardë   sœ   € ð !ˆØ×,Ò,¨]Ñ;Ô;ˆØ)˜4œ>ð 
Ø'Ø 3Ø)Ø%Ø+ð
ð 
ð ð
ð 
Ñˆ�qð ! =Ñ0ˆØ ˆØ×5Ò5°mÑDÔDˆØŸš Ñ/Ô/ˆØ  =Ñ0ˆØÐr-   )NNNN)r~   r   r€   r   ÚintrU   r4   r‚   r2   Ú
LongTensorr   r   r   r}   rƒ   r„   s   @r+   r§   r§   à   sð   ø€ € € € € ðd˜}ð d¸ð dð dð dð dð dð dð IMØ.2Ø04Ø(,ðð à”|ðð # 5¤<°´Ð#=Ô>ÀÑEðð œ tÑ+ð	ð
 Ô&¨Ñ-ðð  ™ðð Ð+Ô,ðð 
Œðð ð ð ð ð ð ð r-   r§   c                   ó`   — e Zd Z eed¬¦  «        eedœZ ej	        ¦   «         d„ ¦   «         Z
dS )ÚMixtralPreTrainedModelr   )Úindex)rŽ   rd   Ú
attentionsc                 óL  — t          j        | |¦  «         | j        j        }t	          |t
          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         d S t	          |t          ¦  «        rt          j        |j        d|¬¦  «         d S d S )Ng        )r:   Ústd)r   Ú_init_weightsrR   Úinitializer_ranger1   rQ   ÚinitÚnormal_r]   r^   r†   rŠ   )rb   Úmoduler½   s      r+   r¾   z$MixtralPreTrainedModel._init_weights  s§   € åÔ% d¨FÑ3Ô3Ð3ØŒkÔ+ˆÝ�f�nÑ-Ô-ð 	;ÝŒL˜Ô,°3¸CÐ@Ñ@Ô@Ð@ÝŒL˜Ô)°¸Ð=Ñ=Ô=Ð=Ð=Ð=Ý˜Õ 1Ñ2Ô2ð 	;ÝŒL˜œ¨S°cÐ:Ñ:Ô:Ð:Ð:Ð:ð	;ð 	;r-   N)r~   r   r€   r   r†   r§   r¥   Ú_can_record_outputsr4   rk   r¾   r&   r-   r+   r¹   r¹     s]   € € € € € à'˜Ð(9ÀÐCÑCÔCØ,Ø&ðð Ðð €U„]�_„_ð;ð ;ñ „_ð;ð ;ð ;r-   r¹   c                   óœ   — e Zd Z	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dedz  dej        dz  dedz  de	e
         d	efd
„ZdS )ÚMixtralModelNÚ	input_idsr"   r±   r²   Úinputs_embedsÚ	use_cacher³   r#   c           
      ót  — |d u |d uz  rt          d¦  «        ‚|r|€t          | j        ¬¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }| j        j
        €t          nt          }	 |	| j        ||||¬¦  «        }
|}|                      ||¬¦  «        }| j        d | j        j        …         D ]} ||f|
||||dœ|¤Ž}Œ|                      |¦  «        }t#          ||¬	¦  «        S )
Nz:You must specify exactly one of input_ids or inputs_embeds)rR   r   r   )r3   )rR   rÇ   r"   r²   r±   )r±   )r"   r±   r²   rÈ   r°   )Úlast_hidden_stater²   )Ú
ValueErrorr   rR   Úembed_tokensÚget_seq_lengthr4   Úaranger<   r3   r@   Úsliding_windowr
   r   Ú
rotary_embÚlayersrK   Únormr   )rb   rÆ   r"   r±   r²   rÇ   rÈ   r³   Úpast_seen_tokensÚmask_functionÚcausal_maskrd   r°   Údecoder_layers                 r+   r}   zMixtralModel.forward  s¢  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLà.2¬kÔ.HÐ.PÕ*Ð*ÕVwˆØ#�mØ”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆØ"Ÿošo¨mÈ,˜oÑWÔWÐà!œ[Ð)H¨4¬;Ô+HÐ)HÔIð 		ð 		ˆMØ)˜MØðà*Ø)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆå%Ø+Ø+ð
ñ 
ô 
ð 	
r-   )NNNNNN)r~   r   r€   r4   r·   r‚   r   ÚFloatTensorÚboolr   r   r   r}   r&   r-   r+   rÅ   rÅ     s¼   € € € € € ð .2Ø.2Ø04Ø(,Ø26Ø!%ð4
ð 4
àÔ# dÑ*ð4
ð œ tÑ+ð4
ð Ô&¨Ñ-ð	4
ð
  ™ð4
ð Ô(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ô,ð4
ð 
 ð4
ð 4
ð 4
ð 4
ð 4
ð 4
r-   rÅ   c                   óì   ‡ — e Zd ZddiZˆ fd„Z	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	edz  d
ej	        dz  dej        dz  de
dz  de
dz  deej        z  dee         defd„Zˆ xZS )ÚMixtralForCausalLMzlm_head.weightzmodel.embed_tokens.weightc                 óº   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |j        | _        |j        | _        |j        | _        d S rˆ   )rT   rU   rÅ   ÚmodelÚrouter_aux_loss_coefrV   r!   r‰   ra   s     €r+   rU   zMixtralForCausalLM.__init__S  sQ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý! &Ñ)Ô)ˆŒ
Ø$*Ô$?ˆÔ!Ø!Ô3ˆÔØ#)Ô#=ˆÔ Ð Ð r-   Nr   rÆ   r"   r±   r²   rÇ   ÚlabelsrÈ   Úoutput_router_logitsÚlogits_to_keepr³   r#   c
                 ó  — |�|n| j         j        } | j        d|||||||dœ|
¤Ž}|j        }t	          |	t
          ¦  «        rt          |	 d¦  «        n|	}|                      |dd…|dd…f         ¦  «        }d}|� | j        ||| j	        fi |
¤Ž}d}|rHt          |j        | j        | j        |¦  «        }|�%|| j        |                     |j        ¦  «        z  z  }t#          ||||j        |j        |j        |j        ¬¦  «        S )a~  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MixtralForCausalLM

        >>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rÆ   r"   r±   r²   rÇ   rÈ   rß   )ÚlossÚaux_lossÚlogitsr²   rd   r»   rŽ   r&   )rR   rß   rÜ   rÊ   r1   r¶   ÚsliceÚlm_headÚloss_functionÚ
vocab_sizerO   rŽ   r!   r‰   rÝ   r'   r3   r   r²   rd   r»   )rb   rÆ   r"   r±   r²   rÇ   rÞ   rÈ   rß   rà   r³   Úoutputsrd   Úslice_indicesrä   râ   rã   s                    r+   r}   zMixtralForCausalLM.forwardZ  sn  € ðJ %9Ð$DÐ Ð È$Ì+ÔJjð 	ð
 +5¨$¬*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ð	+
ð 	+
ð ð	+
ð 	+
ˆð  Ô1ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô% f¨f°d´oÐPÐPÈÐPÐPˆDàˆØð 	MÝ/ØÔ%ØÔ ØÔ(Øñ	ô ˆHð Ð!Ø˜Ô1°H·K²KÀÄÑ4LÔ4LÑLÑL�å(ØØØØ#Ô3Ø!Ô/ØÔ)Ø!Ô/ð
ñ 
ô 
ð 	
r-   )	NNNNNNNNr   )r~   r   r€   Ú_tied_weights_keysrU   r4   r·   r‚   r   r×   rØ   r¶   r   r   r   r}   rƒ   r„   s   @r+   rÚ   rÚ   P  s:  ø€ € € € € Ø*Ð,GÐHÐð>ð >ð >ð >ð >ð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ðP
ð P
àÔ# dÑ*ðP
ð œ tÑ+ðP
ð Ô&¨Ñ-ð	P
ð
  ™ðP
ð Ô(¨4Ñ/ðP
ð Ô  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eœlÑ*ðP
ð Ð+Ô,ðP
ð 
#ðP
ð P
ð P
ð P
ð P
ð P
ð P
ð P
r-   rÚ   c                   ó   — e Zd ZdS )Ú MixtralForSequenceClassificationNr    r&   r-   r+   rí   rí   ­  r¡   r-   rí   c                   ó   — e Zd ZdS )ÚMixtralForTokenClassificationNr    r&   r-   r+   rï   rï   ±  r¡   r-   rï   c                   ó   — e Zd ZdS )ÚMixtralForQuestionAnsweringNr    r&   r-   r+   rñ   rñ   µ  r¡   r-   rñ   )rÚ   rñ   rÅ   r¹   rí   rï   )Nr   N)Cr�   r4   Útorch.nn.functionalr   r6   r�   Ú r   rÀ   Úactivationsr   Úcache_utilsr   r   Úintegrationsr	   Úmasking_utilsr
   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.output_capturingr   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   r   Úconfiguration_mixtralr   Ú
get_loggerr~   Úloggerr‚   r2   r¶   rO   ÚModulerQ   r†   r”   rŸ   r£   r¥   r§   r¹   rÅ   rÚ   rí   rï   rñ   Ú__all__r&   r-   r+   ú<module>r     sŒ  ðð& Ð à €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø QÐ QÐ QÐ QÐ QÐ QÐ QÐ QØ -Ð -Ð -Ð -Ð -Ð -Ø &Ð &Ð &Ð &Ð &Ð &Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 1Ð 0Ð 0Ð 0Ð 0Ð 0ð 
ˆÔ	˜HÑ	%Ô	%€ð
 #Ø
Ø*.ð	O&ð O&Ø”  e¤lÔ 3Ñ3°dÑ:ðO&à�t‘ðO&ð ”L 4Ñ'ð	O&ð
 „\�CÑðO&ð O&ð O&ð O&ðd ð$#ð $#ð $#ð $#ð $#�R”Yñ $#ô $#ñ Ôð$#ðN<ð <ð <ð <ð <˜œ	ñ <ô <ð <ð$ð ð ð ð ˜BœIñ ô ð ð&	ð 	ð 	ð 	ð 	�^ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð'ñ 	ô 	ð 	ð#ð #ð #ð #ð #Ð4ñ #ô #ð #ðL;ð ;ð ;ð ;ð ;Ð3ñ ;ô ;ð ;ð$5
ð 5
ð 5
ð 5
ð 5
�<ñ 5
ô 5
ð 5
ðpZ
ð Z
ð Z
ð Z
ð Z
Ð+ñ Z
ô Z
ð Z
ðz	ð 	ð 	ð 	ð 	Ð'Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$Añ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð"=ñ 	ô 	ð 	ðð ð €€€r-   