tjg&dZddlZddlZddlmZddlmZddlZddlm Z ddlm Z ddl m Z dd lmZdd lmZdd lmZdd lmZmZmZdd lmZmZddlmZddlmZmZm Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,ede GddeZ-ede Gdde Z.ede GddeZ/Gdde'Z0eeGddeZ1eeGddeZ2Gd d!e+Z3Gd"d#e)Z4Gd$d%e#Z5Gd&d'e"Z6Gd(d)e%Z7Gd*d+e%Z8eGd,d-e(Z9Gd.d/e$Z:Gd0d1e9Z;Gd2d3e*Z<Gd4d5e,Z=Gd6d7e&Z>ed89Gd:d;e9Z?gd<Z@dS)=zPyTorch CLIPSeg model.N) dataclass)Any)strict)nn)initialization)BaseModelOutputWithPooling)PreTrainedModel)Unpack) ModelOutputTransformersKwargsauto_docstring)can_return_tuplemerge_with_config_defaults)capture_outputs) CLIPConfigCLIPTextConfigCLIPVisionConfig) CLIPMLP CLIPAttention CLIPEncoderCLIPEncoderLayer CLIPModel CLIPOutputCLIPPreTrainedModelCLIPTextEmbeddings CLIPTextModelCLIPVisionEmbeddingsCLIPVisionModelzCIDAS/clipseg-rd64) checkpointc&eZdZdZeZdS)CLIPSegTextConfiga Example: ```python >>> from transformers import CLIPSegTextConfig, CLIPSegTextModel >>> # Initializing a CLIPSegTextConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegTextConfig() >>> # Initializing a CLIPSegTextModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegTextModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```N__name__ __module__ __qualname____doc__AttributeErrorprojection_dimi/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/clipseg/modular_clipseg.pyr#r#0'  $^%%NNNr,r#c&eZdZdZeZdS)CLIPSegVisionConfiga Example: ```python >>> from transformers import CLIPSegVisionConfig, CLIPSegVisionModel >>> # Initializing a CLIPSegVisionConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegVisionConfig() >>> # Initializing a CLIPSegVisionModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```Nr$r+r,r-r0r0Fr.r,r0ceZdZUdZdZeeeedfzed<dZ eed<dZ eed<d Z e ezed <d Z eed <d Zeed<dZeed<dZeed<dS) CLIPSegConfiga extract_layers (`list[int]`, *optional*, defaults to `[3, 6, 9]`): Layers to extract when forwarding the query image through the frozen visual backbone of CLIP. reduce_dim (`int`, *optional*, defaults to 64): Dimensionality to reduce the CLIP vision embedding. conditional_layer (`int`, *optional*, defaults to 0): The layer to use of the Transformer encoder whose activations will be combined with the condition embeddings using FiLM (Feature-wise Linear Modulation). If 0, the last layer is used. use_complex_transposed_convolution (`bool`, *optional*, defaults to `False`): Whether to use a more complex transposed convolution in the decoder, enabling more fine-grained segmentation.. Example: ```python >>> from transformers import CLIPSegConfig, CLIPSegModel >>> # Initializing a CLIPSegConfig with CIDAS/clipseg-rd64 style configuration >>> configuration = CLIPSegConfig() >>> # Initializing a CLIPSegModel (with random weights) from the CIDAS/clipseg-rd64 style configuration >>> model = CLIPSegModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a CLIPSegConfig from a CLIPSegTextConfig and a CLIPSegVisionConfig >>> # Initializing a CLIPSegText and CLIPSegVision configuration >>> config_text = CLIPSegTextConfig() >>> config_vision = CLIPSegVisionConfig() >>> config = CLIPSegConfig(text_config=config_text, vision_config=config_vision) ```)r .extract_layers@ reduce_dimdecoder_num_attention_headsdecoder_attention_dropout quick_geludecoder_hidden_actidecoder_intermediate_sizerconditional_layerF"use_complex_transposed_convolutionN)r%r&r'r(r5listinttuple__annotations__r7r9r;floatr=strr>r?r@boolr+r,r-r2r2\s! ! F3M5*C/047>>>7;Je',-4;;;;;r,rMceZdZUdZdZejdzed<dZejdzed<dZ ejdzed<dZ ejdzed<dZ e ed<dZ eed<d eefd ZdS) CLIPSegImageSegmentationOutputa loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Binary cross entropy loss for segmentation. logits (`torch.FloatTensor` of shape `(batch_size, height, width)`): Classification scores for each pixel. conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, projection_dim)`): Conditional embeddings used for segmentation. pooled_output (`torch.FloatTensor` of shape `(batch_size, embed_dim)`): Pooled output of the [`CLIPSegVisionModel`]. vision_model_output (`BaseModelOutputWithPooling`): The output of the [`CLIPSegVisionModel`]. decoder_output (`CLIPSegDecoderOutput`): The output of the [`CLIPSegDecoder`]. NlossrNconditional_embeddings pooled_outputvision_model_outputdecoder_outputreturncXtd|DS)Nc3lK|]/}t|tr|n|V0dSN) isinstancer to_tuple).0vs r- z:CLIPSegImageSegmentationOutput.to_tuple..s=^^1Z;%?%?FQZZ\\\Q^^^^^^r,)rCvalues)selfs r-r_z'CLIPSegImageSegmentationOutput.to_tuples,^^PTP[P[P]P]^^^^^^r,)r%r&r'r(rUrQrRrDrNrVrWrXr rYrMrCrr_r+r,r-rTrTs  &*D% d ")))'+FE  $+++7;E-4;;;.2M5$t+2226:3:::+/N(///_%*______r,rTc>eZdZddejdejffd ZxZS)CLIPSegVisionEmbeddingsT pixel_valuesrZcLt||dSr]superforward)rdrginterpolate_pos_encoding __class__s r-rkzCLIPSegVisionEmbeddings.forwards"  &>?????r,T)r%r&r'rQrRTensorrk __classcell__rms@r-rfrfsd@@E$5@Y^Ye@@@@@@@@@@r,rfceZdZdS)CLIPSegTextEmbeddingsNrJr+r,r-rsrsrKr,rsceZdZdS)CLIPSegAttentionNrJr+r,r-rururKr,ruceZdZdS) CLIPSegMLPNrJr+r,r-rwrwrKr,rwceZdZdS)CLIPSegEncoderLayerNrJr+r,r-ryryrKr,rycPeZdZdZdejdejdeejfdZdS)CLIPSegDecoderLayerz CLIPSeg decoder layer, which is identical to `CLIPSegEncoderLayer`, except that normalization is applied after self-attention/MLP, rather than before. rOattention_maskrZc |}|jd||d|\}}||z}||}|}||}||z}||}|S)N)rOr|r+) self_attn layer_norm1mlp layer_norm2)rdrOr|kwargsresidual_s r-rkzCLIPSegDecoderLayer.forwards !)4> ')     q !=0 ((77  //  =0 ((77 r,N) r%r&r'r(rQrorCrRrkr+r,r-r{r{sZ |  u ! r,r{cLeZdZeegedZejdZ dS)CLIPSegPreTrainedModel)rOrPctj|||jj}t |t rt j|jj d|dzt j|j j d|dzt j |j tj|j jddd St |t"rt j|jd|jdz|zt j|jj |jj|zt j|j j |jj|zt j |j tj|jdd St |t.r|jdzd|jjzdzz|z}|jdz|z}t j|jj |t j|jj |t j|jj |t j|jj |d St |t:r||jjdzd|jjzdzz|z}d|jjzdz|z}t j|jj |t j|j j |d St |tBrXt j|j"j |j#dz|zt j|j$j |j%dz|zd Sd S) zInitialize the weightsr:g{Gz?)meanstd)rg)rrN)&r _init_weightsconfiginitializer_factorr^rsinitnormal_token_embeddingweightposition_embeddingcopy_ position_idsrQarangeshapeexpandrfclass_embedding embed_dimpatch_embeddinginitializer_range num_positionsrunum_hidden_layersq_projk_projv_projout_projrw hidden_sizefc1fc2 CLIPSegModeltext_projectiontext_embed_dimvisual_projectionvision_embed_dim)rdmodulefactor in_proj_std out_proj_stdfc_stds r-rz$CLIPSegPreTrainedModel._init_weightssL %dF333/ f3 4 4  L/6Sftm T T T T L29RV W W W W Jv*EL9L9RSU9V,W,W,^,^_f,g,g h h h h h  7 8 8  L/cv?OQU?UX^?^ _ _ _ _ L/6FM<[^dz+CLIPSegDecoder.__init__..Fs/ b b bPQRYv+79J K K b b br,reluc.g|]}tSr+)r{)r`rdecoder_configs r-rz+CLIPSegDecoder.__init__..Ns"$t$t$tQ%8%H%H$t$t$tr,) rj__init__r?rrr*r7film_mulfilm_addr@r patch_size SequentialConv2dReLUConvTranspose2dtransposed_convolutionlenr5 ModuleListrangereducescopydeepcopyrr9num_attention_headsr>intermediate_size hidden_actlayers post_init)rdrtransposed_kernelsdepthrrms ` @r-rzCLIPSegDecoder.__init__&sA    !'!9 &"79JKK  &"79JKK  4 "("6"AQ"FH\HgklHl!m *,- &+V->AWXYYY "%%* 21 5-a0   "%*A;Ma;PYklmYn + +D ' '+-*<!1f&:&EfNbNm+++D 'F)**} b b b bUZ[`UaUa b b b   v';<<%+%6"-3-O*+1+K($*!m$t$t$t$tRWX[\b\qXrXrRsRs$t$t$tuu  r,rOrVrrZc |ddd}d}tt||j|jD]\}\}}} || ||z}n | |}||jkrZ|||dddz||z}|ddd}||fddi|}|ddddddfdd}ttj |j d} |j d} | | |j d| | }||d} t!| S)a/ conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*): The conditional embeddings for the query images. If provided, the model will use this instead of computing the embeddings from the conditional_pixel_values. Nrrrrr|)rN) enumerateziprrr?rpermuter transposerBmathsqrtrviewrsqueezerM) rdrOrVr activationsoutputi activationlayerreducesize batch_sizerNs r-rkzCLIPSegDecoder.forwardRs$DDbD) .7KVZVb8c8c.d.d B B *A* E6! ++f4 ++D***'=>>PQSTVWAXAXX[_[h[h*\\ 1a00U6AA$A&AAFF122qqq!++Aq1149V\!_--..+1!4 Za$EE,,V44<>> from transformers import AutoTokenizer, CLIPSegTextModel >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegTextModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled (EOS token) states ```r+rird super_kwargsrms r-rkzCLIPSegTextModel.forward~s! uww.....r,)r%r&r'rCr rkrprqs@r-rr}sJ/1K)K//////////r,rc \eZdZ ddejdzdedzdeedee zffd Z xZ S) CLIPSegVisionModelTrgNrlrrZc :tj||fi|S)a+ Examples: ```python >>> import httpx >>> from io import BytesIO >>> from PIL import Image >>> from transformers import AutoProcessor, CLIPSegVisionModel >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegVisionModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> with httpx.stream("GET", url) as response: ... image = Image.open(BytesIO(response.read())) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```rirdrgrlrrms r-rkzCLIPSegVisionModel.forwards&8uww|-EPPPPPr,rn) r%r&r'rQrRrGr r rCr rkrprqs@r-rrs15QQ'$.Q#'+Q+, Q + + QQQQQQQQQQr,rc leZdZfdZ d dejdedeede e zffd Z d deffd Z xZ S) rc 6tjdi|S)a Examples: ```python >>> import torch >>> from transformers import AutoTokenizer, CLIPSegModel >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> with torch.inference_mode(): ... text_features = model.get_text_features(**inputs) ```r+)rjget_text_featuresrs r-rzCLIPSegModel.get_text_featuress")uww(88<888r,TrgrlrrZc :tj||fi|S)a Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegModel >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.inference_mode(): ... image_features = model.get_image_features(**inputs) ```)rjget_image_featuresrs r-rzCLIPSegModel.get_image_featuress(2*uww),8P[[TZ[[[r,c >tjdd|i|dS)a return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegModel >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> inputs = processor( ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True ... ) >>> with torch.inference_mode(): ... outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities ```rlNr+ri)rdrlrrms r-rkzCLIPSegModel.forwards-6 ZZ1IZ\ZZZZZr,rn)r%r&r'rrQrRrGr r rCr rrkrprqs@r-rrs99999(*.\\'\#'\+, \ + + \\\\\\6[[[[[[[[[[[[r,rzn CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation. ) custom_introceZdZUeed<deffd Z ddedzdejdzdejdzdejdzdejdzd ej f d Z e e ddej dzd ej dzdej dzd ej dzdejdzdej dzdej dzdedeed eezfdZxZS)CLIPSegForImageSegmentationrct|t||_|j|_t ||_|dSr])rjrrclipr5rdecoderr)rdrrms r-rz$CLIPSegForImageSegmentation.__init__sZ     (( $3%f--  r,Nr input_idsr|rconditional_pixel_valuesrZc|pt||krtdtj5|j|||j}dddn #1swxYwYn~|mt||krtdtj5|j|j}dddn #1swxYwYntd|S)Nz@Make sure to pass as many prompt texts as there are query images)r|rzAMake sure to pass as many prompt images as there are query imagesz[Invalid conditional, should be either provided as `input_ids` or `conditional_pixel_values`)r ValueErrorrQrrr pooler_outputr)rdrrr|rrrVs r-get_conditional_embeddingsz6CLIPSegForImageSegmentation.get_conditional_embeddings s  9~~++ !cddd  )-)D)Dn<*E**'               & 1+,, :: !deee n n)-)E)EF^)_)_)m& n n n n n n n n n n n n n n nm &%s##A''A+.A+* CCCTrgrVlabelsrlrc  tj5d| d<|jjd ||d| } | j} | jfd|jD} t| j| j| j| j } dddn #1swxYwY|&| |j d||||}nU|j d|j dkrtd |j d |j jkrtd |j| |fi| } | j}d}|9||j}t'j}|||}t+|||| | | S)a~ conditional_pixel_values (`torch.FloatTensor`, *optional*): The pixel values of the conditional images. conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*): The conditional embeddings for the query images. If provided, the model will use this instead of computing the embeddings from the conditional_pixel_values. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> import torch >>> from transformers import AutoProcessor, CLIPSegForImageSegmentation >>> from transformers.image_utils import load_image >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined") >>> model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = load_image(url) >>> texts = ["a cat", "a remote", "a blanket"] >>> inputs = processor(text=texts, images=[image] * len(texts), padding=True, return_tensors="pt") >>> with torch.inference_mode(): ... outputs = model(**inputs) >>> logits = outputs.logits >>> print(logits.shape) torch.Size([3, 352, 352]) ```Toutput_hidden_states)rgrlc&g|] }|dzS)rr+)r`rrOs r-rz7CLIPSegForImageSegmentation.forward..es"MMMA=Q/MMMr,)last_hidden_staterrOrPNr)rrr|rrzWMake sure to pass as many conditional embeddings as there are query images in the batchrzcMake sure that the feature dimension of the conditional embeddings matches `config.projection_dim`.)rUrNrVrWrXrYr+)rQrrrrrOr5r rrPrrrrr*rrNtodevicerBCEWithLogitsLossrT)rdrrgrrVr|rrrlrvision_outputsrWrdecoder_outputsrNrUloss_fnrOs @r-rkz#CLIPSegForImageSegmentation.forward)s9b]__  -1F) *9TY9))AN +8M*8MMMMM9LMMMK8"0"B,:,:)4 N               , " )%)%D%D'-a0#-))A &E&& " "&+A.,2DQ2GGG m&+A.$+2LLL 0 '$,  "     !'  YYv}--F*,,G766**D-#9' .*     sA#BB B)NNNNN)NNNNNNNT)r%r&r'r2rDrrBrQrorRrrr LongTensorrGr r rCrIrkrprqs@r-rrs }"&)-.2,08< &&$J&<$&& t+ & lT) & #(,"5 &  &&&&:/315=A;?.204*.)-n n $t+n '$.n #("3d": n !& 1D 8 n  t+ n &-n  4'n #'n +,n   n n n ^n n n n n r,r)r2r#r0rrrrr)Ar(rr dataclassesrtypingrrQhuggingface_hub.dataclassesrrrrmodeling_outputsr modeling_utilsr processing_utilsr utilsr r r utils.genericrrutils.output_capturingrclip.configuration_cliprrrclip.modeling_cliprrrrrrrrrrr r#r0r2rIrMrTrfrsrurwryr{rrrrrrr__all__r+r,r-rs !!!!!! ......&&&&&&::::::------&&&&&&DDDDDDDDDDIIIIIIII555555RRRRRRRRRR                          /000&&&&&&&10&(/000&&&&&*&&10&(/000+5+5+5+5+5J+5+510+5\     J    <<<<<;<< <" _____[__ _6@@@@@2@@@      .        }                *   *>(((((0(((V     [   U3U3U3U3U3+U3U3U3p/////}///(QQQQQQQQ@H[H[H[H[H[9H[H[H[V X X X X X "8X X  X v   r,