§
    ‚Štjå  ã                   ó^  — d dl Z d dlmZ d dl mZ ddlmZmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$  ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z% G d„ de ¦  «        Z& G d„ de¦  «        Z' G d„ de#¦  «        Z( G d„ de¦  «        Z) G d„ de"¦  «        Z* G d„ de$¦  «        Z+ G d „ d!e!¦  «        Z, G d"„ d#e¦  «        Z- G d$„ d%e¦  «        Z. G d&„ d'e¦  «        Z/ G d(„ d)e¦  «        Z0g d*¢Z1dS )+é    N)Ústrict)Únné   )ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚMistralConfig)ÚQwen2AttentionÚQwen2DecoderLayerÚQwen2ForCausalLMÚQwen2ForQuestionAnsweringÚQwen2ForSequenceClassificationÚQwen2ForTokenClassificationÚQwen2MLPÚ
Qwen2ModelÚQwen2PreTrainedModelÚQwen2RMSNormÚQwen2RotaryEmbeddingz$mistralai/Ministral-8B-Instruct-2410)Ú
checkpointc                   ó>   — e Zd ZU dZdZdZee         dz  ed<   d„ Z	dS )ÚMinistralConfiga”  
    Example:

    ```python
    >>> from transformers import MinistralModel, MinistralConfig

    >>> # Initializing a Ministral 8B style configuration
    >>> configuration = MinistralConfig()

    >>> # Initializing a model from the Ministral 8B style configuration
    >>> model = MinistralModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú	ministralNÚlayer_typesc                 ó�   — | j         €| j        | _         | j        €| j        �dndg| j        z  | _        t          j        | fi |¤Ž d S )NÚsliding_attentionÚfull_attention)Únum_key_value_headsÚnum_attention_headsr"   Úsliding_windowÚnum_hidden_layersr   Ú__post_init__)ÚselfÚkwargss     úm/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/ministral/modular_ministral.pyr*   zMinistralConfig.__post_init__C   sg   € ØÔ#Ð+Ø'+Ô'?ˆDÔ$àÔÐ#à'+Ô':Ð'FÐ#Ð#ÐL\ð àÔ&ñ 'ˆDÔõ 	Ô& tÐ6Ð6¨vÐ6Ð6Ð6Ð6Ð6ó    )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer"   ÚlistÚstrÚ__annotations__r*   © r.   r-   r    r    ,   sO   € € € € € € ðð ð  €Jà$(€K��c”˜TÑ!Ð(Ð(Ñ(ð	7ð 	7ð 	7ð 	7ð 	7r.   r    c                   ó   — e Zd ZdS )ÚMinistralMLPN©r/   r0   r1   r7   r.   r-   r9   r9   O   ó   € € € € € Ø€Dr.   r9   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚMinistralAttentionÚ	layer_idxc                 ó`  •— t          ¦   «                              ||¦  «         t          j        |j        |j        | j        z  d¬¦  «        | _        t          j        |j        |j        | j        z  d¬¦  «        | _	        t          j        |j        |j        | j        z  d¬¦  «        | _
        d S )NF)Úbias)ÚsuperÚ__init__r   ÚLinearÚhidden_sizer'   Úhead_dimÚq_projr&   Úk_projÚv_proj)r+   Úconfigr>   Ú	__class__s      €r-   rB   zMinistralAttention.__init__T   s™   ø€ Ý‰Œ×Ò˜ Ñ+Ô+Ð+å”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒÝ”i Ô 2°FÔ4NÐQUÔQ^Ñ4^ÐejÐkÑkÔkˆŒˆˆr.   )r/   r0   r1   ÚintrB   Ú__classcell__©rJ   s   @r-   r=   r=   S   sO   ø€ € € € € ðl¨#ð lð lð lð lð lð lð lð lð lð lr.   r=   c                   ó   — e Zd ZdS )ÚMinistralRMSNormNr:   r7   r.   r-   rO   rO   \   r;   r.   rO   c                   ó   — e Zd ZdS )ÚMinistralDecoderLayerNr:   r7   r.   r-   rQ   rQ   `   r;   r.   rQ   c                   ó   — e Zd ZdS )ÚMinistralPreTrainedModelNr:   r7   r.   r-   rS   rS   d   r;   r.   rS   c                   ó   — e Zd ZdS )ÚMinistralRotaryEmbeddingNr:   r7   r.   r-   rU   rU   h   r;   r.   rU   c                   óâ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej	        dz  dedz  dej        dz  d	edz  d
ee         defd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚMinistralModelrI   c                 óN   •— t          ¦   «                              |¦  «         | `d S )N)rA   rB   Úhas_sliding_layers)r+   rI   rJ   s     €r-   rB   zMinistralModel.__init__m   s'   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØÐ#Ð#Ð#r.   NÚ	input_idsÚattention_maskÚposition_idsÚpast_key_valuesÚinputs_embedsÚ	use_cacher,   Úreturnc           
      óâ  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }| 	                    d¦  «        }t          |x}	t          ¦  «        s&| j        ||||dœ}
t          d
i |
¤Žt          d
i |
¤Ždœ}	|}|                      ||¦  «        }t          | j        d | j        j        …         ¦  «        D ]*\  }} ||f|	| j        j        |                  ||||dœ|¤Ž}Œ+|                      |¦  «        }t)          ||r|nd ¬	¦  «        S )Nz:You must specify exactly one of input_ids or inputs_embeds)rI   r   é   )Údevice)rI   r^   r[   r]   r\   )r%   r$   )r[   r\   r]   r_   Úposition_embeddings)Úlast_hidden_stater]   r7   )Ú
ValueErrorÚembed_tokensr   rI   Úget_seq_lengthÚtorchÚarangeÚshaperc   Ú	unsqueezeÚ
isinstanceÚdictr	   r
   Ú
rotary_embÚ	enumerateÚlayersr)   r"   Únormr   )r+   rZ   r[   r\   r]   r^   r_   r,   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_statesrd   ÚiÚdecoder_layers                  r-   ÚforwardzMinistralModel.forwardq   sò  € ð ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÐ Ø ×-Ò-¨iÑ8Ô8ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLõ °Ð?Ð-ÅÑFÔFð 	ð œ+Ø!.Ø"0Ø#2Ø ,ðð ˆKõ #5Ð"CÐ"C°{Ð"CÐ"CÝ%FÐ%UÐ%UÈÐ%UÐ%Uð#ð #Ðð
 &ˆØ"Ÿošo¨m¸\ÑJÔJÐå )¨$¬+Ð6U¸¼Ô8UÐ6UÔ*VÑ WÔ Wð 		ð 		ÑˆAˆ}Ø)˜MØðà2°4´;Ô3JÈ1Ô3MÔNØ)Ø /Ø#Ø$7ðð ð ðð ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆÝ&Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r.   )NNNNNN)r/   r0   r1   r    rB   r   r   r   ri   Ú
LongTensorÚTensorr   ÚFloatTensorÚboolr   r   r   ry   rL   rM   s   @r-   rW   rW   l   s  ø€ € € € € ð$˜ð $ð $ð $ð $ð $ð $ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ð:
ð :
àÔ# dÑ*ð:
ð œ tÑ+ð:
ð Ô&¨Ñ-ð	:
ð
  ™ð:
ð Ô(¨4Ñ/ð:
ð ˜$‘;ð:
ð Ð+Ô,ð:
ð 
!ð:
ð :
ð :
ñ „^ñ „_ñ  Ôð:
ð :
ð :
ð :
ð :
r.   rW   c                   ó   — e Zd ZdS )ÚMinistralForCausalLMNr:   r7   r.   r-   r   r   ±   r;   r.   r   c                   ó   — e Zd ZdS )Ú"MinistralForSequenceClassificationNr:   r7   r.   r-   r�   r�   µ   r;   r.   r�   c                   ó   — e Zd ZdS )ÚMinistralForTokenClassificationNr:   r7   r.   r-   rƒ   rƒ   ¹   r;   r.   rƒ   c                   ó   — e Zd ZdS )ÚMinistralForQuestionAnsweringNr:   r7   r.   r-   r…   r…   ½   r;   r.   r…   )r    rS   rW   r   r�   rƒ   r…   )2ri   Úhuggingface_hub.dataclassesr   r   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr	   r
   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   Úutils.output_capturingr   Úmistral.configuration_mistralr   Úqwen2.modeling_qwen2r   r   r   r   r   r   r   r   r   r   r   r    r9   r=   rO   rQ   rS   rU   rW   r   r�   rƒ   r…   Ú__all__r7   r.   r-   ú<module>r’      s¶  ðð  €€€Ø .Ð .Ð .Ð .Ð .Ð .Ø Ð Ð Ð Ð Ð à .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø &Ð &Ð &Ð &Ð &Ð &Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð €ÐAÐBÑBÔBØð7ð 7ð 7ð 7ð 7�mñ 7ô 7ñ „ñ CÔBð7ðB	ð 	ð 	ð 	ð 	�8ñ 	ô 	ð 	ðlð lð lð lð l˜ñ lô lð lð	ð 	ð 	ð 	ð 	�|ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð-ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð3ñ 	ô 	ð 	ðB
ð B
ð B
ð B
ð B
�Zñ B
ô B
ð B
ðJ	ð 	ð 	ð 	ð 	Ð+ñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð)Gñ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð&Añ 	ô 	ð 	ð	ð 	ð 	ð 	ð 	Ð$=ñ 	ô 	ð 	ðð ð €€€r.   