tjrNddlmZddlZddlmZddlmZddlmZddlm Z ddl m Z dd l m Z mZmZdd lmZdd lmZdd lmZmZmZdd lmZmZddlmZddlmZe dGddejZ GddejZ!GddejZ"edeGddeZ#edeGdde Z$eGd d!eZ%ed"Gd#d$e%Z&ed%Gd&d'e%e Z'gd(Z(dS))) dataclassN)nn)ACT2FN)Cache)GenerationMixin)use_kernel_forward_from_hub)BaseModelOutputWithPastBaseModelOutputWithPooling ModelOutput)PreTrainedModel)Unpack)TransformersKwargsauto_docstringtorch_compilable_check)can_return_tuplemerge_with_config_defaults) AutoModel)Mistral3ConfigRMSNormcTeZdZd deddffd ZdejdejfdZdZxZ S) Mistral3RMSNormư>epsreturnNcttjt j||_||_dS)z> Mistral3RMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizer __class__s l/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mistral3/modeling_mistral3.pyr zMistral3RMSNorm.__init__*sD l5:k#:#:;; # hidden_statesc|j}|tj}|ddd}|tj||jzz}|j||zS)NrT)keepdim) dtypetor"float32powmeanrsqrtr%r$)r&r+ input_dtypevariances r)forwardzMistral3RMSNorm.forward2s|#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU {]--k::::r*cHt|jjd|jS)Nz, eps=)tupler$shaper%r&s r) extra_reprzMistral3RMSNorm.extra_repr9s& )**II$2GIIIr*)r) __name__ __module__ __qualname__floatr r"Tensorr7r< __classcell__r(s@r)rr(s$$$$$$$$$$;U\;el;;;;JJJJJJJr*rcZeZdZdZdeffd ZdejdejdejfdZxZ S)Mistral3PatchMergerz< Learned merging of spatial_merge_size ** 2 patches configct||_|jj}|j|_|jjj|_tj||jdzz|d|_ dS)NrFbias) rr rF vision_configr'spatial_merge_size patch_sizerLinear merging_layer)r&rFr'r(s r)r zMistral3PatchMerger.__init__Bss  *6 "(";+3>Y{T5La5O'OQ\chiiir*image_features image_sizesrcfd|D}d|D}|jd}g}t||D]\}}||\}} ||| |dddd} t jj | j j } | |j dzzd } | | t j |d}|}|S) NcPg|]"}|djz|djzf#S)rr)rL).0 image_sizer&s r) z/Mistral3PatchMerger.forward..LsA   U_Z]do -z!}/O P   r*cg|] \}}||z SrW)rShws r)rUz/Mistral3PatchMerger.forward..Ps :::daAE:::r*r-rrr) kernel_sizestridedim)r: enumeratesplitviewpermute unsqueezer"r functionalunfoldrKtappendcatrN) r&rOrPtokens_per_imagedpermuted_tensor image_index image_tokensrXrY image_gridgrids ` r)r7zMistral3PatchMerger.forwardKsa    cn   ;:k:::   $)2>3G3GHX3Y3Y)Z)Z ) ) %K{+DAq%**1a33;;Aq!DDNNqQQJ8&--(?H_.D99Q!8!!;;R@@BBDDD  " "4 ( ( ( (?:::++N;;r*) r=r>r?__doc__rr r"rAr7rBrCs@r)rErE=sj~jjjjjjelRWR^r*rEcHeZdZdeffd ZdejdejfdZxZS)Mistral3MultiModalProjectorrFc@tt|jj|jj|_t||_ t|j trdnt|j |_tj|jj|jz|jj|j|_t&|j|_tj|jj|jj|j|_dS)N)rrrH)rr rrJr' text_config rms_norm_epsnormrE patch_merger isinstancevision_feature_layerintlennum_feature_layersrrMmultimodal_projector_biaslinear_1rprojector_hidden_actactlinear_2r&rFr(s r)r z$Mistral3MultiModalProjector.__init__ds #F$8$D&J\Jijjj /77F7== cAA3vGbCcCc   ,t/F F   *1    &56   *F,>,JQWQq    r*rOrPc||}|||}||}||}||}|SN)rurvr}rr)r&rOrPr+s r)r7z#Mistral3MultiModalProjector.forwardvsa>22**>;GG n55 //  m44 r*) r=r>r?rr r"rAr7rBrCs@r)rqrqcsj ~      $elr*rqzT Base class for Mistral3 causal language model (or autoregressive) outputs.  custom_introceZdZUdZdZejdzed<dZejdzed<dZ e dzed<dZ e ejdzed<dZ e ejdzed<dZejdzed<dS) Mistral3CausalLMOutputWithPasta4 loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nlosslogitspast_key_valuesr+ attentionsimage_hidden_states)r=r>r?rorr" FloatTensor__annotations__rrrr+r9rrrWr*r)rrs  &*D% d ")))'+FE  $+++$(OUT\(((59M5*+d299926Je'(4/66648*T188888r*rzM Base class for Mistral3 outputs, with hidden states and attentions. c2eZdZUdZdZejdzed<dS)Mistral3ModelOutputWithPasta past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache). Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nr)r=r>r?rorr"rrrWr*r)rrs7  59*T188888r*rc@eZdZUeed<dZdZdZdgZdZ dZ dZ dZ dZ dS)Mistral3PreTrainedModelrFmodel)imagetextTrN)r=r>r?rrbase_model_prefixinput_modalitiessupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph_supports_flex_attn_supports_attention_backendrWr*r)rrsV (&*##4"5N!"&r*rzx The Mistral3 model which consists of a vision backbone and a language model, without a language modeling head. c6eZdZdeffd Zeeed ddej dej de e e ze e zdzd e dzd eed eezf d Zd ejdej dej fdZeee dd ejdzdej dzdej dzdejdzdedzdej dzde e e ze e zdzde dzdej dzd eed eezfdZxZS) Mistral3ModelrFct|tj|j|_t ||_tj|j|_ | dSr) rr r from_configrJ vision_towerrqmulti_modal_projectorrslanguage_model post_initrs r)r zMistral3Model.__init__sm    %1&2FGG%@%H%H"'3F4FGG r*zWObtains image last hidden states from the vision tower and apply multimodal projection.rN pixel_valuesrPrxoutput_hidden_stateskwargsrc ` d|D}|j|f|ddd| t|tr j|}n$ fd|D}t j|d}||d|}|jj |j j z} t j ||j | zd} t j|d| }| _ S) Nci|] \}}||| SrrW)rSkvs r) z4Mistral3Model.get_image_features..sCCC41aQ]!Q]]]r*T)rPr return_dictc*g|]}j|SrW)r+)rS layer_idx image_outputss r)rUz4Mistral3Model.get_image_features..s!ddd)}29=dddr*r-r\r)device)itemsrrwryr+r"rgrsqueezerLrFrK as_tensorrprodtolistr_ pooler_output) r&rrPrxrrselected_image_featurehs_poolrOdownsample_ratio split_sizesrs @r)get_image_featuresz Mistral3Model.get_image_featuressUDC6<<>>CCC))  #!%       *C 0 0 @%2%@AU%V " "ddddOcdddG%*YwB%?%?%? "334J4R4RST4U4UWbcc,7$+:XX _[1F G G GK[ [ a afh a i i p p r r ^%;%;A%>%> LL&4 #r* input_ids inputs_embedsrOc |e||tj|jjtj|jk}|d}n||jjk}|}|j d|j dz}| d |j}t||j dz| kd|d||S)z Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is equal to the length of multimodal features. If the lengths are different, an error is raised. N)r/rr-rrz6Image features and image tokens do not match, tokens: z , features: )get_input_embeddingsr"tensorrFimage_token_idlongrallsumr:rbr0rnumel)r&rrrOspecial_image_maskn_image_tokensn_image_featuress r)get_placeholder_maskz"Mistral3Model.get_placeholder_masks  !.2M$2K2K2M2M T[7uzR_Rfggg33" "4!7!7!;!;  !*dk.H!H +//11)/2^5I!5LL/99"==@@AUVV ]04 48L8L8N8N N s^ s saq s s   "!r*attention_mask position_idsr use_cachec |du|duz rtd|||}||||| dj} t j| d|j|j} | ||| } | | | }|j d |||||d| } t| j | j| j| j|| ndS) Nz:You must specify exactly one of input_ids or inputs_embedsT)rrxrPrrr\)rrO)rrrrr)last_hidden_staterr+rrrW) ValueErrorrrrr"rgr0rr/rmasked_scatterrrrrr+r)r&rrrrrrrxrrPrrOroutputss r)r7zMistral3Model.forwards\  -t"; < [YZZ Z  7D5577 BBM  #!44)%9' 5   #Y~1===@@AUWdWjkkN!%!:!:~";"" *889K^\\M%$% )%+'      +%7#3!/)2>2JPT     r*)NN) NNNNNNNNN)r=r>r?rr rrrr"rrArylistboolrrr9r r LongTensorrrrr7rBrCs@r)rrsW ~ ^nDH,0 !!'!\!"DIoS 9D@ ! #Tk ! +, ! + +!!!  !F")":?:K"]b]n""""0 .215.204(,26CG!%+// / #d*/ '$./  t+ / &- /  / (4// "DIoS 9D@/ $;/ \D(/ +,/  , ,/ / / ^ / / / / / r*rzV The MISTRAL3 model which consists of a vision backbone and a language model. ceZdZddiZdeffd ZdejfdZe e e dde j d e jd eeezeezdzd eedeezf d Ze e e dde jdzde j dzde jdzde jdzdedzde j dzde jdzdedzdee jzd e jdzd eedeezfdZ dfd ZxZS) Mistral3ForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightrFct|t||_t j|jj|jjd|_ | dS)NFrH) rr rrrrMrsr' vocab_sizelm_headrrs r)r z)Mistral3ForConditionalGeneration.__init__Mse    "6** y!3!?ASA^ejkkk  r*rc|jSr)rr;s r)get_output_embeddingsz6Mistral3ForConditionalGeneration.get_output_embeddingsSs |r*NrrPrxrc .|jjd|||d|S)N)rrPrxrW)rr)r&rrPrxrs r)rz3Mistral3ForConditionalGeneration.get_image_featuresVs:-tz, %#!5      r*rrrrrrlabelsrlogits_to_keepc l|jd|||||||| d| } | d} t| trt| dn| }|| dd|ddf}d}| |jd|||jjjd| }t||| j | j | j | j S)a1 Example: ```python >>> from PIL import Image >>> import httpx >>> from io import BytesIO >>> from transformers import AutoProcessor, Mistral3ForConditionalGeneration >>> model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503") >>> processor = AutoProcessor.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503") >>> prompt = "[INST][IMG]What is the image?[/INST]" >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, text=prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(**inputs, max_new_tokens=15) >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "What is the image?The image depicts two cats lying on a pink blanket." ```)rrrrrrrrPrN)rrr)rrrr+rrrW)rrwryslicer loss_functionrFrsrrrr+rr)r&rrrrrrrrrrPrrr+ slice_indicesrrs r)r7z(Mistral3ForConditionalGeneration.forwardgsR$*  %)%+'#        8B>SV8W8Wk~ot444]k mAAA}aaa,?@AA  %4%f9P9[_eD.#3!/) ' ;     r*Fc tj|f|||||d|} |s|dds|| d<| S)N)rrrris_first_iterationrTr)rprepare_inputs_for_generationget) r&rrrrrrrr model_inputsr(s r)rz>Mistral3ForConditionalGeneration.prepare_inputs_for_generationst=uww<  +'))1       8VZZ T%B%B 8 ,8L (r*r) NNNNNNNNrN)NNNNNF)r=r>r?_tied_weights_keysrr rModulerrrrr"rrAryrrrr9r rrrrrr7rrBrCs@r)rrEsn +,VW~ ry  DH   '  \  "DIoS 9D@  +,  + +    ^    .215.204(,26*.!%-.+/C C #d*C '$.C  t+ C &- C  C (4/C  4'C $;C el*C \D(C +,C  / /C C C ^ C P r*r)rrr)) dataclassesrr"r activationsr cache_utilsr generationr integrationsr modeling_outputsr r r modeling_utilsr processing_utilsrutilsrrr utils.genericrrautorconfiguration_mistral3rrrrErqrrrrr__all__rWr*r)rsp,"!!!!! !!!!!! ))))))777777``````````------&&&&&&OOOOOOOOOOIIIIIIII222222Y''JJJJJbiJJ('J(#####")###L")8  99999[99   90   9 9 9 9 9"9 9 9   9 ' ' ' ' 'o ' ' ' { { { { { +{ {  { | CCCCC'>CC  CL [ Z Zr*