Ph dZddlmZmZddlZddlmZddlmZddlm Z ddl m Z dd l m Z mZmZmZdd lmZdd lmZmZd d lmZej0eZdZ Gddej>Z!Gddej>Z"Gddej>Z#Gd d!ej>Z$Gd"d#ej>Z%Gd$d%ej>Z&Gd&d'e Z'Gd(d)ej>Z(eGd*d+eZ)eGd,d-e)Z*ed./Gd0d1e)Z+Gd2d3ej>Z,Gd4d5ej>Z-Gd6d7ej>Z.ed8/Gd9d:e)Z/gd;Z0y)=zPyTorch MobileViTV2 model.)OptionalUnionN)nn)CrossEntropyLoss)ACT2FN)GradientCheckpointingLayer)BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionSemanticSegmenterOutput)PreTrainedModel)auto_docstringlogging)MobileViTV2Configvaluedivisor min_valuereturnc|||}t|t||dz z|z|z}|d|zkr||z }t|S)a Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the original TensorFlow repo. It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py g?)maxint)rrr new_values n/var/www/html/saasai/venv/lib/python3.12/site-packages/transformers/models/mobilevitv2/modeling_mobilevitv2.pymake_divisibler*sS  Is57Q;#677BWLMI3;W y>z-infinfmin_valmax_valc.t|t||SN)rminrr r!s rclipr&9s wGU+ ,,rceZdZ ddedededededededed ed eeefd d ffd Zde jd e jfdZ xZ S)MobileViTV2ConvLayerconfig in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationrNc $t |t|dz dz |z} ||zdk7rtd|d|d||zdk7rtd|d|dt j ||||| |||d |_| r t j|d d d d |_nd|_| rdt| trt| |_ yt|jtrt|j|_ y|j|_ yd|_ y)NrrrzInput channels (z) are not divisible by z groups.zOutput channels (zeros) r*r+r,r-paddingr0r.r/ padding_modegh㈵>g?T) num_featuresepsmomentumaffinetrack_running_stats)super__init__r ValueErrorrConv2d convolution BatchNorm2d normalization isinstancestrr activation hidden_act) selfr)r*r+r,r-r.r/r0r1r2r5 __class__s rr=zMobileViTV2ConvLayer.__init__?s* {Q!+,x7  1 $/ }UV\U]]efg g99#%#    !#)$( "D "&D  .#."("8F--s3"():):";"("3"3"DOrfeaturesc|j|}|j|j|}|j|j|}|Sr#)r@rBrE)rGrIs rforwardzMobileViTV2ConvLayer.forwardusK##H-    )))(3H ?? &x0Hr)rrFrTT) __name__ __module__ __qualname__rrboolrrDr=torchTensorrK __classcell__rHs@rr(r(>s"&+/4#!4#4# 4#  4#  4#4#4#4# 4#dCi(4# 4#l rr(c xeZdZdZ d dedededededdf fd Zd ejdejfd Z xZ S) MobileViTV2InvertedResidualzY Inverted residual block (MobileNetv2): https://huggingface.co/papers/1801.04381 r)r*r+r-r0rNc @t|ttt ||j zd}|dvrt d|d|dk(xr||k(|_t|||d|_ t|||d||||_ t|||dd |_ y) N)rrzInvalid stride .r)r*r+r,r)r*r+r,r-r.r0Fr*r+r,r2) r<r=rrround expand_ratior> use_residualr( expand_1x1conv_3x3 reduce_1x1)rGr)r*r+r-r0expanded_channelsrHs rr=z$MobileViTV2InvertedResidual.__init__s *3u[6CVCV5V/W+XZ[\  vha89 9#q[K{l/J.  :KYZ - )*$  / )%  rrIc|}|j|}|j|}|j|}|jr||zS|Sr#)r]r^r_r\)rGrIresiduals rrKz#MobileViTV2InvertedResidual.forwardsI??8,==*??8,&*&7&7x("EXEr)r rLrMrN__doc__rrr=rPrQrKrRrSs@rrUrUsc lm ' 69 IL VY eh  BF FFrrUc teZdZ d dedededededdf fd Zd ejdejfd ZxZ S) MobileViTV2MobileNetLayerr)r*r+r- num_stagesrNct|tj|_t |D]5}t ||||dk(r|nd}|jj||}7y)Nrr)r*r+r-)r<r=r ModuleListlayerrangerUappend) rGr)r*r+r-rgirjrHs rr=z"MobileViTV2MobileNetLayer.__init__sc ]]_ z"A/')!"avQ E JJ  e $&K#rrIc8|jD] }||} |Sr#rj)rGrI layer_modules rrKz!MobileViTV2MobileNetLayer.forwards JJL#H-H'r)rr rLrMrNrrr=rPrQrKrRrSs@rrfrfsVqr'''69'IL'VY'kn' '  rrfcheZdZdZdededdffd ZdejdejfdZ xZ S) MobileViTV2LinearSelfAttentionay This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper: https://huggingface.co/papers/2206.02680 Args: config (`MobileVitv2Config`): Model configuration object embed_dim (`int`): `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)` r) embed_dimrNc t|t||dd|zzdddd|_t j |j |_t|||dddd|_||_y)NrrTF)r)r*r+r/r,r1r2p) r<r=r(qkv_projrDropout attn_dropoutout_projrt)rGr)rtrHs rr=z'MobileViTV2LinearSelfAttention.__init__s{ ,!a)m,#   JJ)<)<=,!"#   #r hidden_statesc|j|}tj|d|j|jgd\}}}tjj j |d}|j|}||z}tj|dd}tjj j||j|z}|j|}|S)Nr)split_size_or_sectionsdimrTrkeepdim) rxrPsplitrtr functionalsoftmaxrzsumrelu expand_asr{) rGr|qkvquerykeyrcontext_scorescontext_vectorouts rrKz&MobileViTV2LinearSelfAttention.forwardsmmM* "KKQX\XfXfDgmnosE,,44U4C**>:~->r4Hhh!!&&u-0H0H0OOmmC  rrcrSs@rrsrss> #0#S#T#2U\\ellrrsc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2FFNr)rtffn_latent_dim ffn_dropoutrNc t|t|||ddddd|_t j ||_t|||ddddd|_t j ||_y)NrTF)r)r*r+r,r-r/r1r2) r<r=r(conv1rrydropout1conv2dropout2)rGr)rtrrrHs rr=zMobileViTV2FFN.__init__s| )!'#    ;/ )&"#     ;/ rr|c|j|}|j|}|j|}|j|}|Sr#)rrrr)rGr|s rrKzMobileViTV2FFN.forward's@ =1  m4  =1  m4 r rLrMrNrrfloatr=rPrQrKrRrSs@rrrsY ! 0!00 0  0  0@U\\ellrrc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2TransformerLayerr)rtrdropoutrNcPt|tjd||j|_t |||_tj||_ tjd||j|_ t||||j|_ y)Nr num_groups num_channelsr8rv)r<r=r GroupNormlayer_norm_epslayernorm_beforers attentionryrlayernorm_afterrrffn)rGr)rtrrrHs rr=z$MobileViTV2TransformerLayer.__init__0s~  "  W]WlWl m7 J W- !||qyV\VkVkl!&)^VEWEWXrr|c|j|}|j|}||z}|j|}|j|}||z}|Sr#)rrrr)rGr|layernorm_1_outattention_output layer_outputs rrKz#MobileViTV2TransformerLayer.forward>sY// >>>/:(=8 ++M: xx - #m3 rrrrSs@rrr/s^  Y! Y Y Y  Y  Y U\\ ell rrcheZdZdedededdffd ZdejdejfdZxZ S) MobileViTV2Transformerr)n_layersd_modelrNc<t ||j}||zg|z}|Dcgc]}t|dzdz}}t j |_t|D].}t||||}|j j|0ycc}w)N)rtr) r<r=ffn_multiplierrrrirjrkrrl) rGr)rrrffn_dimsd block_idxtransformer_layerrHs rr=zMobileViTV2Transformer.__init__Ks .."W,-82::ACbB':]]_ xI ;'(9:M!  JJ  / 0 );sBr|c8|jD] }||} |Sr#ro)rGr|rps rrKzMobileViTV2Transformer.forward\s  JJL(7M'rrqrSs@rrrJsA101C1#1RV1"U\\ellrrceZdZdZ ddededededededed d ffd Zd ejd e eje eefffd Z dejde eefd ejfdZ dejd ejfdZ xZ S)MobileViTV2LayerzE MobileViTV2 layer: https://huggingface.co/papers/2206.02680 r)r*r+ attn_unit_dim n_attn_blocksr0r-rNct ||j|_|j|_|}|dk(r)t ||||dk(r|nd|dkDr|dznd|_|}nd|_t||||j||_ t|||ddd|_ t||||_ tjd||j|_t|||dd d|_y) Nrr)r*r+r-r0)r*r+r,r.F)r*r+r,r1r2)rrrT)r<r= patch_size patch_width patch_heightrUdownsampling_layerr(conv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projection) rGr)r*r+rrr0r- cnn_out_dimrHs rr=zMobileViTV2Layer.__init__gs  !,,"--# Q;&A')!)QvA*2Q,QA 'D #'K&*D #- #$//   - #$#   2&-ZghTZTiTij 4 #$"  r feature_mapc"|j\}}}}tjj||j|j f|j|j f}|j |||j|j zd}|||ffS)N)r,r-r)shaperrunfoldrrreshape)rGr batch_sizer* img_height img_widthpatchess r unfoldingzMobileViTV2Layer.unfoldings9D9J9J6 KY--&& **D,<,<=%%t'7'78'  //*k4;L;LtO_O_;_acdY///rr output_sizec|j\}}}}|j|||z|}tjj |||j |j f|j |j f}|S)N)rr,r-)rrrrfoldrr)rGrrrin_dimr n_patchesrs rfoldingzMobileViTV2Layer.foldingsz4;MM1 FJ //*fz.A9Mmm(( #**D,<,<=%%t'7'78 ) rrIc6|jr|j|}|j|}|j|}|j|\}}|j |}|j |}|j ||}|j|}|Sr#)rrrrrrrr)rGrIrrs rrKzMobileViTV2Layer.forwards  " "..x8H==*==* $~~h7""7+..)<<5''1r)rrr)rLrMrNrdrrr=rPrQtuplerrrKrRrSs@rrrbs; !; ;  ;  ;  ; ; ;  ; z 0U\\ 0eELL%PSUXPX/z-MobileViTV2Encoder.forward..5sX$Fq!-$Fs)last_hidden_stater|) enumeraterjrr )rGr|rrall_hidden_statesrmrps rrKzMobileViTV2Encoder.forward&sj #7BD(4OA|(7M#$58H$H! 5 X]4E$FXX X- ]noor)FT) rLrMrNrr=rPrQrOrrr rKrRrSs@rrrsbO#0O#TO#h&+ p||p#p p u44 5 prrcNeZdZUeed<dZdZdZdgZde jddfd Z y) MobileViTV2PreTrainedModelr) mobilevitv2 pixel_valuesTrmodulerNct|tjtjtjfrm|j j jd|jj|j%|jj jyyt|tjrJ|jj j|j j jdyy)zInitialize the weightsr)meanstdNg?)rCrLinearr?rAweightdatanormal_r)initializer_ranger/zero_rfill_)rGrs r _init_weightsz(MobileViTV2PreTrainedModel._init_weightsBs fryy"))R^^D E MM   & &CT[[5R5R & S{{&   &&('  - KK   " " $ MM   $ $S ).r) rLrMrNr__annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesrModulerrrrrr:s9 %$O&*#+, *BII *$ *rrc eZdZd dedeffd ZdZe d dee jdeedeede e e ffd ZxZS) MobileViTV2Modelr) expand_outputc t||||_||_t t d|j zdddd}t||j|ddd d |_ t||_ |jy ) a expand_output (`bool`, *optional*, defaults to `True`): Whether to expand the output of the model. If `True`, the model will output pooled features in addition to hidden states. If `False`, only the hidden states will be returned. rrrr%rWrrrTr*r+r,r-r1r2N) r<r=r)rrr&rr(r conv_stemrencoder post_init)rGr)rrrHs rr=zMobileViTV2Model.__init__Qs   *$ rF333R LVWce . ++$" *&1  rc|jD]e\}}|jj|}t|ts0|j jD]}|j j|gy)zPrunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrjrCrrr prune_heads)rGheads_to_prune layer_indexheadsmobilevitv2_layerrs r _prune_headszMobileViTV2Model._prune_headsmsi#1"6"6"8 K $ 2 2; ? +-=>):)F)F)L)L%%//;;EB*M#9rrrrrc||n|jj}||n|jj}| td|j |}|j |||}|j r |d}tj|ddgd}n|d}d}|s|||fn|f}||ddzSt|||j S) Nz You have to specify pixel_valuesrrrrFrr)r pooler_outputr|) r)ruse_return_dictr>rrrrPr r r|) rGrrrembedding_outputencoder_outputsr pooled_outputoutputs rrKzMobileViTV2Model.forwardws%9$D $++JjJj &1%>,7,, !5#'     / 2 "JJ'8r2hPUVM / 2  M;H;T'7[lZnFOAB// /7/')77  r)T)NNN)rLrMrNrrOr=r(rrrPrQrrr rKrRrSs@rrrOs~08C04/3&* ' u||,' 'tn' d^ ' u>> ? ' ' rrz MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. ) custom_introceZdZdeddffd Ze d deejdee deejdee de e e ff d Z xZS) !MobileViTV2ForImageClassificationr)rNcLt|||j|_t||_t d|j zd}|jdkDr!tj||jntj|_ |jy)NrrWrr) in_features out_features) r<r= num_labelsrrrrrr Identity classifierr )rGr)r+rHs rr=z*MobileViTV2ForImageClassification.__init__s   +++F3%cF,C,C&CQO   1$ II,V=N=N O  rrrlabelsrcB||n|jj}|j|||}|r |jn|d}|j |}d}||j |||j}|s|f|ddz} ||f| zS| St |||jS)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr*rr)losslogitsr|)r)r-rr,r: loss_functionr r|) rGrrr;routputsr0r>r=r1s rrKz)MobileViTV2ForImageClassification.forwards&1%, #%"!  rrIc|jdd}|j|}|j|}tjj ||dd}|S)Nr+bilinearFsizemode align_corners)rrFrrr interpolate)rGrI spatial_sizes rrKzMobileViTV2ASPPPooling.forwardsS~~bc* ##H-==*==,,XLzin,orrqrSs@rrCrCsB  0  s  RU  Z^   rrCcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2ASPPz ASPP module defined in DeepLab papers: https://huggingface.co/papers/1606.00915, https://huggingface.co/papers/1706.05587 r)rNc t|td|jzd}|}|j}t |j dk7r tdtj|_ t|||dd}|jj||jj|j Dcgc]}t|||d|d c}t|||}|jj|t|d |z|dd|_tj |j" |_ycc}w) NrrWrrz"Expected 3 values for atrous_ratesrrrY)r*r+r,r0r2rv)r<r=rraspp_out_channelslen atrous_ratesr>rriconvsr(rlextendrCprojectryaspp_dropout_probr) rGr)encoder_out_channelsr*r+ in_projectionrate pool_layerrHs rr=zMobileViTV2ASPP.__init__s= -cF4K4K.KUVW* // v"" #q (AB B]]_ , #%!   -( #// 0D% +!- !!#) 0  ,FKN  *%+ L 0|YZkq  zzF$<$<= ) sErIcg}|jD]}|j||tj|d}|j |}|j |}|S)Nrr)rVrlrPcatrXr)rGrIpyramidconvpooled_featuress rrKzMobileViTV2ASPP.forward)sWJJD NN4> *))G+,,w/,,7r rLrMrNrdrr=rPrQrKrRrSs@rrPrPs8*>0*>T*>X rrPcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2DeepLabV3zJ DeepLabv3 architecture: https://huggingface.co/papers/1706.05587 r)rNc t|t||_t j |j |_t||j|jdddd|_ y)NrFT)r*r+r,r1r2r/) r<r=rPasppr Dropout2dclassifier_dropout_probrr(rSr8r:rGr)rHs rr=zMobileViTV2DeepLabV3.__init__:s] #F+ ||F$B$BC . 00**#  rr|cr|j|d}|j|}|j|}|S)Nr)rgrr:)rGr|rIs rrKzMobileViTV2DeepLabV3.forwardJs699]2./<<)??8,rrcrSs@rrere5s7 0 T U\\ellrrezZ MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. ceZdZdeddffd Ze d deejdeejdee dee de e e ff d Z xZS) "MobileViTV2ForSemanticSegmentationr)rNct|||j|_t|d|_t ||_|jy)NF)r)r<r=r8rrresegmentation_headr rjs rr=z+MobileViTV2ForSemanticSegmentation.__init__WsE   +++F%H!5f!= rrr;rrch||n|jj}||n|jj}|$|jjdk(r t d|j |d|}|r |j n|d}|j|}d}|Ytjj||jdddd } t|jj } | | |}|s|r |f|ddz} n |f|d dz} ||f| zS| St|||r|j d Sdd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Examples: ```python >>> import requests >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```Nrz/The number of labels should be greater than oneTr*r+rHFrI) ignore_indexr)r=r>r| attentions)r)rr-r8r>rr|rorrrMrrsemantic_loss_ignore_indexr ) rGrr;rrr@encoder_hidden_statesr>r=upsampled_logitsloss_fctr1s rrKz*MobileViTV2ForSemanticSegmentation.forwardassH%9$D $++JjJj &1%  !}}88V\\"#.Zu 9  (T[[5[5[\H,f5D# WQR[0 WQR[0)-)9TGf$ Ev E&3G'//   NR   rrA)rLrMrNrr=rrrPrQrOrrr rKrRrSs@rrmrmQs 0T04)-/3&* I u||,I &I 'tn I d^ I u-- . I I rrm)r4rmrr)rWN)1rdtypingrrrPrtorch.nnr activationsrmodeling_layersr modeling_outputsr r r r modeling_utilsrutilsrrconfiguration_mobilevitv2r get_loggerrLloggerrrrr&rr(rUrfrsrrrrrrrr4rCrPrerm__all__rrrrs"!" %!9 .,8   H % #  HSM UX ).f e ----Y^- =299=B-F"))-Fb .<RYY<~&RYY&R"))6RYY0o1odcpcpL***(O 1O O d 4 (B4  4 pRYY09bii9z2998 U )CU  U p r