tj_ddlZddlmZddlmZmZmZddlmZm Z m Z m Z m Z m Z mZmZddlmZddlmZddlmZmZed eGd d eZed eGd deZed eGddeZGdde ZGddeZGdde ZGdde ZGdde ZGdde ZgdZ dS)N)strict)InstructBlipConfigInstructBlipQFormerConfigInstructBlipVisionConfig)'BaseModelOutputWithVisionQformerOutputs$InstructBlipForConditionalGeneration/InstructBlipForConditionalGenerationModelOutputInstructBlipModelInstructBlipPreTrainedModelInstructBlipQFormerModelInstructBlipVisionModelTransformersKwargs)BaseModelOutputWithPooling)Unpack)auto_docstringcan_return_tuplez"Salesforce/instructblip-flan-t5-xl) checkpointceZdZdZdS)InstructBlipVideoVisionConfigaH Example: ```python >>> from transformers import InstructBlipVideoVisionConfig, InstructBlipVideoVisionModel >>> # Initializing a InstructBlipVideoVisionConfig with Salesforce/instructblip-flan-t5-xl style configuration >>> configuration = InstructBlipVideoVisionConfig() >>> # Initializing a InstructBlipVideoVisionModel (with random weights) from the Salesforce/instructblip-flan-t5-xl style configuration >>> model = InstructBlipVideoVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```N__name__ __module__ __qualname____doc__}/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/instructblipvideo/modular_instructblipvideo.pyrr(s    rrceZdZdZdS)InstructBlipVideoQFormerConfiga3 cross_attention_frequency (`int`, *optional*, defaults to 2): The frequency of adding cross-attention to the Transformer layers. encoder_hidden_size (`int`, *optional*, defaults to 1408): The hidden size of the hidden states for cross-attention. Examples: ```python >>> from transformers import InstructBlipVideoQFormerConfig, InstructBlipVideoQFormerModel >>> # Initializing a InstructBlipVideo Salesforce/instructblip-flan-t5-xl style configuration >>> configuration = InstructBlipVideoQFormerConfig() >>> # Initializing a model (with random weights) from the Salesforce/instructblip-flan-t5-xl style configuration >>> model = InstructBlipVideoQFormerModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```Nrrrrr r <s    rr cDeZdZUdZddiZdZedzed<eZ dS)InstructBlipVideoConfiga qformer_config (`dict`, *optional*): Dictionary of configuration options used to initialize [`InstructBlipVideoQFormerConfig`]. num_query_tokens (`int`, *optional*, defaults to 32): The number of query tokens passed through the Transformer. Example: ```python >>> from transformers import ( ... InstructBlipVideoVisionConfig, ... InstructBlipVideoQFormerConfig, ... OPTConfig, ... InstructBlipVideoConfig, ... InstructBlipVideoForConditionalGeneration, ... ) >>> # Initializing a InstructBlipVideoConfig with Salesforce/instructblip-flan-t5-xl style configuration >>> configuration = InstructBlipVideoConfig() >>> # Initializing a InstructBlipVideoForConditionalGeneration (with random weights) from the Salesforce/instructblip-flan-t5-xl style configuration >>> model = InstructBlipVideoForConditionalGeneration(configuration) >>> # Accessing the model configuration >>> configuration = model.config >>> # We can also initialize a InstructBlipVideoConfig from a InstructBlipVideoVisionConfig, InstructBlipVideoQFormerConfig and any PreTrainedConfig >>> # Initializing Instructblipvideo vision, Instructblipvideo Q-Former and language model configurations >>> vision_config = InstructBlipVideoVisionConfig() >>> qformer_config = InstructBlipVideoQFormerConfig() >>> text_config = OPTConfig() >>> config = InstructBlipVideoConfig(vision_config=vision_config, qformer_config=qformer_config, text_config=text_config) ```video_token_idvideo_token_indexN) rrrr attribute_mapr$int__annotations__AttributeErrorimage_token_indexrrrr"r"TsM" " H&':;M$(sTz(((&((rr"ceZdZdZdS) InstructBlipVideoPreTrainedModel)videotextNrrrinput_modalitiesrrrr+r+s(rr+ceZdZdZdS)InstructBlipVideoVisionModelr,Nr.rrrr1r1srr1ceZdZdS)InstructBlipVideoQFormerModelNrrrrrrr3r3Drr3ceZdZdS)4InstructBlipVideoForConditionalGenerationModelOutputNr4rrrr7r7r5rr7ceZdZee ddejdejdejdzdejdzdejdzdejdzd ejdzd ejdzd e d e dzd e e de e zfdZdS)InstructBlipVideoModelNF pixel_valuesqformer_input_idsqformer_attention_mask input_idsattention_maskdecoder_input_idsdecoder_attention_mask inputs_embedsinterpolate_pos_encoding use_cachekwargsreturnc R|j\} } }}}|| | z|||}|jd || d| }|d}tj|ddtj|j}|j |jddd}tj|ddtj|j}|tj |}| | d}| | d}| |j}tj ||gd}|jd |||||d| }|dddd|dddf}||}|| |jj| zd}|I|j|}||jjk}|tj |}nd||tj|jjtj|jk}|d}|d |j}| |j|j}|||}|jjr|jd ||| d| }n|jd ||||| d | }t7||| S) Nr:rBrdtypedevicedimr=r> query_embedsencoder_hidden_statesencoder_attention_maskrAr>rC)rAr>r?r@rC)vision_outputsqformer_outputslanguage_model_outputsr)shapereshape vision_modeltorchonessizelongrK query_tokensexpand ones_likerepeat_interleavetocatqformerlanguage_projectionconfignum_query_tokenslanguage_modelget_input_embeddingsr#tensorall unsqueezerJmasked_scatteruse_decoder_only_language_modelr7)selfr:r;r<r=r>r?r@rArBrCrD batch_sizeframeschannelheightwidthrT image_embedsimage_attention_maskr^query_attention_mask query_outputs query_outputlanguage_model_inputsspecial_image_maskoutputss rforwardzInstructBlipVideoModel.forwards$6B5G2 FGVU#++J,?&RWXX ** %%=     &a(  %z,*;*;*=*=crc*B%*]i]pqqq(// 0B10Er2NN $z,*;*;*=*=crc*B%*]i]pqqq ! )%*_5F%G%G "-??A?NN!7!I!I&VW!I!X!X!7!:!:;O;V!W!W!&,@BX+Y_`!a!a!a$  '1%".#7      %Q'+A\->->q-A-A+A111(DE !% 8 8 F F!6 = =j$+JfioJoqs t t  FD/DDFFyQQM!*dk.H!H %!&!;!;!.2M$2K2K2M2M T[7uzR_Rfggg33" "4!7!7!;!; /99"==@@AUVV 5 8 89M}Ob c c%445GI^__ ; 6 )d)+-# GG*d)+-"3'=#   GD))#*    r)NNNNNNFN)rrrrrrZ FloatTensor LongTensorTensorboolrrtupler7r}rrrr9r9s* ;?.22659:>-1).!%[ [ '[ !,[ !& 04 7 [ $t+ [ (4/ [ !+d2[ !& 04 7[ |d*[ #'[ $;[ +,[  E E[ [ [ ^[ [ [ rr9ceZdZee ddejdejdejdzdedzde e de e zf d Z d Zd ejd ejfd Zee ddejdejdejdzd ejdzdejdzdejdzdejdzd ejdzdejdzdededzde e de ezfdZej ddejdejdzdejdzd ejdzdejdzd ejdzdedejfdZdS))InstructBlipVideoForConditionalGenerationNFr:r;r<rBrDrEc |j\}}}} } |||z|| | }|jd ||d|} t| j| j| j| j| d} | d} tj | ddtj | j } |j | jddd}tj | ddtj | j }|tj|}||d}||d}||j }tj||gd}|jd |||| | d |}|| _|dddd| dddf}||}|||jj|zd}|| _| S) a pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`): The tensors corresponding to the input images. qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. rGN)last_hidden_state pooler_output hidden_states attentionsrTrUrrHrIrLrNrOr)rWrXrYrrrrrrZr[r\r]rKr^r_r`rarbrcrdrUrerfrg)ror:r;r<rBrDrprqrrrsrtrTrurvr^rwrUryvideo_featuress rget_video_featuresz(6(6%0)    &a(  %z,*;*;*=*=crc*B%*]i]pqqq(// 0B10Er2NN $z,*;*;*=*=crc*B%*]i]pqqq ! )%*_5F%G%G "-??A?NN!7!I!I&VW!I!X!X!7!:!:;O;V!W!W!&,@BX+Y_`!a!a!a&$, '1%".#7      *9&&q)!!!-C|/@/@/C/C-CQQQ*FG 11,??(// DKInstructBlipVideoForConditionalGeneration.get_placeholder_mask=s  !.2M$2K2K2M2M T[7uzR_Rfggg33" "4!7!7!;!;  !*dk.H!H /99"==@@AUVV!!rr>r?r@labelsrCc T|j|f||| d| } | j}| j}| j}|||}|t j|}||j|j }| ||}| ||}|j j r>|jd ||| d| }|d}d}| |jd || |j jjd| }n"|jd ||||| | d| }|j}|j}t)|||||S) a qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)): The sequence used as a prompt to be fed to the Q-Former module. qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*): Mask to avoid performing attention on padding token indices. Examples: ```python >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration >>> import torch >>> from huggingface_hub import hf_hub_download >>> import av >>> import numpy as np >>> def read_video_pyav(container, indices): ... ''' ... Decode the video with PyAV decoder. ... Args: ... container (`av.container.input.InputContainer`): PyAV container. ... indices (`list[int]`): List of frame indices to decode. ... Returns: ... result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3). ... ''' ... frames = [] ... container.seek(0) ... start_index = indices[0] ... end_index = indices[-1] ... for i, frame in enumerate(container.decode(video=0)): ... if i > end_index: ... break ... if i >= start_index and i in indices: ... frames.append(frame) ... return np.stack([x.to_ndarray(format="rgb24") for x in frames]) >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto") >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b") >>> file_path = hf_hub_download( ... repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset" ... ) >>> container = av.open(file_path) >>> # sample uniformly 4 frames from the videWhy is this video funny?o >>> total_frames = container.streams.video[0].frames >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int) >>> clip = read_video_pyav(container, indices) >>> prompt = "What is happening in the video?" >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device) >>> outputs = model.generate( ... **inputs, ... do_sample=False, ... num_beams=5, ... max_length=256, ... repetition_penalty=1.5, ... length_penalty=1.0, ... ) >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip() >>> print(generated_text) "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front" ```r;r<rBNrArSr)logitsr vocab_size)rAr>r?r@rrC)lossrrTrUrVr)rrrUrTrirZr`rbrKrJrrmrfrnrh loss_function text_configrrrr7)ror:r;r<r=r>r?r@rArrBrCrDrrzrUrTr{r|rrs rr}z1InstructBlipVideoForConditionalGeneration.forwardLs`CZ$BY C /#9%= C C  C C !/ <(8'6  7D5577 BBM  !"_Y77N 5 8 89M}Ob c c!66yP]6^^%445GI^__ ; 6 $)d)+-# G QZFD!)t)!&T[=T=_ci *d)+-"3'=# Gr=r)hasattr_preprocess_acceleraterWrrrfr$rgr bos_token_idrZrjr]rKrepeatrir`rbrJrrmrhis_encoder_decodergenerate)ror:r;r<r=r>rArBgenerate_kwargsrprrz video_tokens start_tokensr{inputsr|s rrz2InstructBlipVideoForConditionalGeneration.generatesD 4 ) ) *  ' ' ) ) )!'* BFBYBY /#9%= CZC C  !/ <   $ =>A]]`aa +t{/F/S.TT !L,uzR^Refff %,,Z;; 7D5577 BBM  !"_Y77N 5 8 89M}Ob c c!66yP]6^^%445GI^__ #0NSS")< ,"+F; .$%.KKK?KKr)NF) NNNNNNNFN)NNNNNF)rrrrrrZr~rrrrrrrrrr7r}no_gradrrrrrrs ;?05 DD'D!+D!& 04 7 D #'+ D +, D 8 8DDD^DL^^^ "e.> "uO` " " " " ;?.22659:>26*.).!%D D 'D !,D !& 04 7 D $t+ D (4/ D !+d2D !& 04 7D (4/D  4'D #'D $;D +,D  E ED D D ^D LU]__6::>-12626).CC'C!+d2C!& 04 7 C #d* C (4/ C(4/C#'C  CCC_CCCrr)r"r rr1r+r3r9r)!rZhuggingface_hub.dataclassesr;transformers.models.instructblip.configuration_instructbliprrr6transformers.models.instructblip.modeling_instructbliprrr r r r r rmodeling_outputsrprocessing_utilsrutilsrrrr r"r+r1r3r7r9r__all__rrrrs  ......                    ;:::::&&&&&&55555555?@@@     $<  A@ $?@@@     %>  A@ ,?@@@')')')')')0')')A@')T)))))'B)))#:     $<        ;j   ^ ^ ^ ^ ^ .^ ^ ^ Bggggg0TgggT   r