
    <hj                        S SK JrJr  S SKrS SKrS SKJrJr  S SKJ	r	J
r
JrJrJrJrJr  SSKJr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJrJr  \R:                  " \5      r " S S\5      r  " S S\5      r! " S S\5      r" " S S\5      r# " S S\5      r$ " S S\5      r% " S S\
5      r& " S S\5      r' " S S\	5      r(/ SQr)g)     )OptionalUnionN)InstructBlipQFormerConfigInstructBlipVisionConfig)$InstructBlipForConditionalGeneration/InstructBlipForConditionalGenerationModelOutputInstructBlipModelInstructBlipPreTrainedModelInstructBlipQFormerModelInstructBlipVisionModelTransformersKwargs   )PretrainedConfig)FlashAttentionKwargs)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMES)Unpack)logging   )CONFIG_MAPPING
AutoConfigc                       \ rS rSrSrg)InstructBlipVideoVisionConfig.    N__name__
__module____qualname____firstlineno____static_attributes__r       w/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/instructblipvideo/modular_instructblipvideo.pyr   r   .       r!   r   c                       \ rS rSrSrg)InstructBlipVideoQFormerConfig2   r   Nr   r   r!   r"   r%   r%   2   r#   r!   r%   c                   r   ^  \ rS rSrSrSrSS0r\\\	S.r
     SU 4S jjr\S\	S	\S
\4S j5       rSrU =r$ )InstructBlipVideoConfig6   a
  
[`InstructBlipVideoConfig`] is the configuration class to store the configuration of a
[`InstructBlipVideoForConditionalGeneration`]. It is used to instantiate a Instructblipvideo model according to the specified
arguments, defining the vision model, Q-Former model and language model configs. Instantiating a configuration with
the defaults will yield a similar configuration to that of the Instructblipvideo
[Salesforce/instruct-blip-flan-t5](https://huggingface.co/Salesforce/instruct-blip-flan-t5) architecture.

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    vision_config (`dict`, *optional*):
        Dictionary of configuration options used to initialize [`InstructBlipVideoVisionConfig`].
    qformer_config (`dict`, *optional*):
        Dictionary of configuration options used to initialize [`InstructBlipVideoQFormerConfig`].
    text_config (`dict`, *optional*):
        Dictionary of configuration options used to initialize any [`PretrainedConfig`].
    num_query_tokens (`int`, *optional*, defaults to 32):
        The number of query tokens passed through the Transformer.

    video_token_index (`int`, *optional*):
        Token index of special video token.
    kwargs (*optional*):
        Dictionary of keyword arguments.

Example:

```python
>>> from transformers import (
...     InstructBlipVideoVisionConfig,
...     InstructBlipVideoQFormerConfig,
...     OPTConfig,
...     InstructBlipVideoConfig,
...     InstructBlipVideoForConditionalGeneration,
... )

>>> # Initializing a InstructBlipVideoConfig with Salesforce/instruct-blip-flan-t5 style configuration
>>> configuration = InstructBlipVideoConfig()

>>> # Initializing a InstructBlipVideoForConditionalGeneration (with random weights) from the Salesforce/instruct-blip-flan-t5 style configuration
>>> model = InstructBlipVideoForConditionalGeneration(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config

>>> # We can also initialize a InstructBlipVideoConfig from a InstructBlipVideoVisionConfig, InstructBlipVideoQFormerConfig and any PretrainedConfig

>>> # Initializing Instructblipvideo vision, Instructblipvideo Q-Former and language model configurations
>>> vision_config = InstructBlipVideoVisionConfig()
>>> qformer_config = InstructBlipVideoQFormerConfig()
>>> text_config = OPTConfig()

>>> config = InstructBlipVideoConfig.from_text_vision_configs(vision_config, qformer_config, text_config)
```instructblipvideovideo_token_idvideo_token_index)text_configqformer_configvision_configc                   > [         TU ]  " S0 UD6  Uc  0 n[        R                  S5        Uc  0 n[        R                  S5        Uc  0 n[        R                  S5        [	        S0 UD6U l        [        S0 UD6U l        UR                  SS5      n[        U   " S0 UD6U l
        X@l        XPl        U R
                  R                  U R                  l        U R                  R                  [         ;   U l        SU l        SU l        g )	NzZvision_config is None. initializing the InstructBlipVideoVisionConfig with default values.z\qformer_config is None. Initializing the InstructBlipVideoQFormerConfig with default values.zTtext_config is None. Initializing the text config with default values (`OPTConfig`).
model_typeoptg      ?g{Gz?r   )super__init__loggerinfor   r/   r%   r.   getr   r-   num_query_tokensr,   hidden_sizeencoder_hidden_sizer1   r   use_decoder_only_language_modelinitializer_factorinitializer_range)	selfr/   r.   r-   r8   r,   kwargstext_model_type	__class__s	           r"   r4    InstructBlipVideoConfig.__init__x   s     	"6" MKKtu!NKKvwKKKno:K]K<N~N%//,>)/:I[I 0!2262D2D2P2P//3/?/?/J/JNo/o,"%!%r!   r/   r.   r-   c                 n    U " SUR                  5       UR                  5       UR                  5       S.UD6$ )z
Instantiate a [`InstructBlipVideoConfig`] (or a derived class) from a InstructBlipVideo vision model, Q-Former and
language model configurations.

Returns:
    [`InstructBlipVideoConfig`]: An instance of a configuration object
)r/   r.   r-   r   )to_dict)clsr/   r.   r-   r?   s        r"    from_vision_qformer_text_configs8InstructBlipVideoConfig.from_vision_qformer_text_configs   sD       
'//1)113#++-
 	
 	
r!   )r<   r=   r8   r.   r-   r;   r,   r/   )NNN    N)r   r   r   r   __doc__r1   attribute_mapr   r%   r   sub_configsr4   classmethodr   rF   r    __classcell__)rA   s   @r"   r(   r(   6   sv    5n %J-M "86K !&F 
4
 7
 &	
 
r!   r(   c                       \ rS rSrSrg) InstructBlipVideoPreTrainedModel   r   Nr   r   r!   r"   rO   rO      r#   r!   rO   c                       \ rS rSrSrg)InstructBlipVideoVisionModel   r   Nr   r   r!   r"   rR   rR      r#   r!   rR   c                       \ rS rSrSrg)InstructBlipVideoQFormerModel   r   Nr   r   r!   r"   rU   rU      r#   r!   rU   c                       \ rS rSrSrg)4InstructBlipVideoForConditionalGenerationModelOutput   r   Nr   r   r!   r"   rX   rX      r#   r!   rX   c            !       `   \ rS rSr           SS\R
                  S\R
                  S\\R                     S\\R
                     S\\R                     S\\R                     S	\\R                     S
\\R                     S\\	   S\\	   S\\	   S\	S\\	   S\
\   S\\\4   4S jjrSrg)InstructBlipVideoModel   Npixel_valuesqformer_input_idsqformer_attention_mask	input_idsattention_maskdecoder_input_idsdecoder_attention_maskinputs_embedsoutput_attentionsoutput_hidden_statesreturn_dictinterpolate_pos_encoding	use_cacher?   returnc                    Ub  UOU R                   R                  nUR                  u  nnnnnUR                  UU-  UUU5      nU R	                  UU	U
UUS9nUS   n[
        R                  " UR                  5       S S [
        R                  UR                  S9nU R                  R                  UR                  S   SS5      n[
        R                  " UR                  5       S S [
        R                  UR                  S9nUc  [
        R                  " U5      nUR                  USS9nUR                  USS9n[
        R                  " UU/SS9nU R                  UUUUUU	U
US9nUS   S S 2S UR                  S5      2S S 24   nU R!                  U5      nUR                  XR                   R"                  U-  S5      nUcR  U R$                  R'                  5       " U5      nX@R                   R(                  :H  nUc  [
        R                  " U5      nOiXR'                  5       " [
        R*                  " U R                   R(                  [
        R                  UR                  S95      :H  nUR-                  S5      nUR/                  S5      R1                  U5      R3                  UR                  5      nUR3                  UR                  UR4                  5      nUR7                  UU5      nU R                   R8                  (       a  U R$                  " SUUU	U
UUS.UD6nOU R$                  " SUUUUU	U
UUS	.UD6n[;        UUUS
9$ )N)r]   re   rf   rg   rh   r   dtypedevicedim   )r`   ra   query_embedsencoder_hidden_statesencoder_attention_maskre   rf   rg   rd   ra   re   rf   rg   ri   )rd   ra   rb   rc   re   rf   rg   ri   )vision_outputsqformer_outputslanguage_model_outputsr   )configuse_return_dictshapereshapevision_modeltorchonessizelongro   query_tokensexpand	ones_likerepeat_interleavecatqformerlanguage_projectionr8   language_modelget_input_embeddingsr+   tensorall	unsqueeze	expand_astorn   masked_scatterr;   rX   )r>   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   r?   
batch_sizeframeschannelheightwidthrw   image_embedsimage_attention_maskr   query_attention_maskquery_outputsquery_outputlanguage_model_inputsspecial_image_maskoutputss                                 r"   forwardInstructBlipVideoModel.forward   sv   " &1%<k$++B]B] 6B5G5G2
FGVU#++J,?&RWX**%/!5#%= + 
 &a(  %zz,*;*;*=cr*B%**]i]p]pq ((//0B0B10Er2N$zz,*;*;*=cr*B%**]i]p]pq!)%*__5F%G"-??A?N!7!I!I&VW!I!X!&,@BX+Y_`!a'1%".#7/!5# % 	
 %Q'+A\->->q-A+A1(DE !% 8 8 F !6 = =j++JfJfioJoqs t  //DDFyQM!*kk.H.H!H%!&!;!.2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;/99"=GGVYYZgZnZno 5 8 89M9M}ObOb c%445GI^_;;66)) +-"3%9'# G )) 
+-"3'="3%9'#
 
G D))#*
 	
r!   r   )NNNNNNNNNFN)r   r   r   r   r   FloatTensorr   
LongTensorTensorboolr   r   r   tuplerX   r   r    r   r!   r"   r[   r[      sB   
 >B15598<=A04,0/3&*).$(i
''i
 !,,i
 !))9)9 :	i

 E--.i
 !!1!12i
 $E$4$45i
 !))9)9 :i
  -i
 $D>i
 'tni
 d^i
 #'i
 D>i
 -.i
  
uJJ	K!i
 i
r!   r[   c            #          \ rS rSr   SS\R
                  S\R                  S\\R                     S\\   S\\   4
S jjr	   SS\R
                  S\R                  S\\R                     S\\   S\\   4
S	 jjr
S
\R                  S\R
                  4S jr            SS\R
                  S\R
                  S\\R                     S
\\R
                     S\\R                     S\\R                     S\\R                     S\\R
                     S\\   S\\   S\\R                     S\\   S\S\\   S\\   S\\\4   4 S jjr\R$                  " 5             SS\R
                  S\\R                     S\\R                     S
\\R                     S\\R                     S\\R
                     S\S\R                  4S jj5       rSrg))InstructBlipVideoForConditionalGenerationi0  Nr]   r^   r_   rh   rg   c           	      ^   UR                   u  pgpn
UR                  Xg-  XU
5      nU R                  UUSS9nUS   n[        R                  " UR                  5       SS [        R                  UR                  S9nU R                  R                  UR                   S   SS5      n[        R                  " UR                  5       SS [        R                  UR                  S9nUc  [        R                  " U5      nUR                  USS9nUR                  USS9n[        R                  " X/SS9nU R                  UUUUUSS	9nUS   SS2SUR                  S5      2SS24   nU R                  U5      nUR                  X`R                  R                   U-  S5      nU(       a  UUU4$ U$ )
z
Encodes images into continuous embeddings that can be forwarded to the language model.

Args:
    pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        The tensors corresponding to the input images.
T)r]   rh   rg   r   Nrl   rm   rp   rr   )r`   ra   rs   rt   ru   rg   )r|   r}   r~   r   r   r   r   ro   r   r   r   r   r   r   r   rz   r8   )r>   r]   r^   r_   rh   rg   r   r   r   r   r   rw   r   r   r   r   r   r   r   s                      r"   get_video_features<InstructBlipVideoForConditionalGeneration.get_video_features1  s   " 6B5G5G2
GU#++J,?RWX**%%= + 

 &a(  %zz,*;*;*=cr*B%**]i]p]pq ((//0B0B10Er2N$zz,*;*;*=cr*B%**]i]p]pq!)%*__5F%G"-??A?N!7!I!I&VW!I!X!&,@+Y_`!a'1%".#7 % 
 %Q'+A\->->q-A+A1(DE !% 8 8 F !6 = =j++JfJfioJoqs t(.-GG$$r!   c                     g )Nr   )r>   r]   r^   r_   rh   rg   s         r"   get_image_features<InstructBlipVideoForConditionalGeneration.get_image_featuresm  s     	r!   r`   rd   c           	         Ucj  X R                  5       " [        R                  " U R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  S5      R                  U5      R                  UR                  5      nU$ )zI
Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`.
rm   rl   )r   r   r   rz   r+   r   ro   r   r   r   r   )r>   r`   rd   r   s       r"   get_placeholder_mask>InstructBlipVideoForConditionalGeneration.get_placeholder_maskw  s     !.2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*kk.H.H!H/99"=GGVYYZgZnZno!!r!   ra   rb   rc   re   rf   labelsri   r?   rj   c                 x   Ub  UOU R                   R                  nU R                  UUUUSS9u  nnnU(       d  UR                  5       OUnU(       d  UR                  5       OUnUc  U R	                  5       " U5      nUc  [
        R                  " U5      nUR                  UR                  UR                  5      nU R                  XHS9nUR                  UU5      nU R                   R                  (       aj  U R                  " SUUU	U
UUS.UD6nU(       a  UR                  OUS   nSnUb3  U R                  " SUXR                   R                   R"                  S.UD6nOLU R                  " SUUUUU	U
UUUS.	UD6nU(       a  UR$                  OUS   nU(       a  UR                  OUS	   n['        UUUUUS
9$ )a	  
qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
    The sequence used as a prompt to be fed to the Q-Former module.
qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
    Mask to avoid performing attention on padding token indices.

Examples:

```python
>>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration
>>> import torch
>>> from huggingface_hub import hf_hub_download
>>> import av
>>> import numpy as np

>>> def read_video_pyav(container, indices):
...     '''
...     Decode the video with PyAV decoder.
...     Args:
...         container (`av.container.input.InputContainer`): PyAV container.
...         indices (`list[int]`): List of frame indices to decode.
...     Returns:
...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
...     '''
...     frames = []
...     container.seek(0)
...     start_index = indices[0]
...     end_index = indices[-1]
...     for i, frame in enumerate(container.decode(video=0)):
...         if i > end_index:
...             break
...         if i >= start_index and i in indices:
...             frames.append(frame)
...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])

>>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto")
>>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

>>> file_path = hf_hub_download(
...       repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
... )
>>> container = av.open(file_path)

>>> # sample uniformly 4 frames from the videWhy is this video funny?o
>>> total_frames = container.streams.video[0].frames
>>> indices = np.arange(0, total_frames, total_frames / 4).astype(int)
>>> clip = read_video_pyav(container, indices)

>>> prompt = "What is happening in the video?"
>>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device)

>>> outputs = model.generate(
...     **inputs,
...     do_sample=False,
...     num_beams=5,
...     max_length=256,
...     repetition_penalty=1.5,
...     length_penalty=1.0,
... )
>>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
>>> print(generated_text)
"A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front"
```NTr^   r_   rh   rg   rd   rv   r   )logitsr   
vocab_size)	rd   ra   rb   rc   re   rf   rg   r   ri   rr   )lossr   rw   rx   ry   r   )rz   r{   r   to_tupler   r   r   r   ro   rn   r   r   r;   r   r   loss_functionr-   r   r   rX   )r>   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   r   rg   rh   ri   r?   r   rw   r   r   r   r   r   s                          r"   r   1InstructBlipVideoForConditionalGeneration.forward  s   b &1%<k$++B]B]?C?V?V/#9%= @W @
<~} ;F002>8C..0  557	BM!"__Y7N 5 8 89M9M}ObOb c!66y6^%445GI^_;;66)) +-"3%9'# G (3W^^
FD!)) !&[[=T=T=_=_ci
 )) +-"3'="3%9'# G $/7<<GAJD'2W^^
FC))#*
 	
r!   c                 J   [        U S5      (       a  U R                  5         UR                  S   n	U R                  UUUUSS9u  pnUc  Uc  U R                  R
                  /U R                  R                  -  S-  nXR                  R                  R                  /-   n[        R                  " U/[        R                  UR                  S9nUR                  U	S5      nU R                  5       " U5      nUc  [        R                  " U5      nU
R!                  UR                  UR"                  5      n
U R%                  XFS9nUR'                  X5      nXeS	.nU R(                  R                  R*                  (       d  UUS
'   U R(                  R,                  " S0 UDUD6nU$ )aA  
Overrides `generate` function to be able to use the model as a conditional generator.

Args:
    pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or
        (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed.
    qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt to be fed to the Q-Former module.
    qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt for the generation.
    attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Embedded representation of the inputs. Should be float, not int tokens.
    interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
        Whether to interpolate the positional encoding of the image embeddings.

Returns:
    captions (list): A list of strings of length batch_size * num_captions.
hf_device_mapr   Tr      rm   rr   r   )rd   ra   r`   r   )hasattr_preprocess_accelerater|   r   rz   r,   r8   r-   bos_token_idr   r   r   ro   repeatr   r   r   rn   r   r   r   is_encoder_decodergenerate)r>   r]   r^   r_   r`   ra   rd   rh   generate_kwargsr   r   rw   r   video_tokensstart_tokensr   inputsr   s                     r"   r   2InstructBlipVideoForConditionalGeneration.generate  s   D 4))'')!''*
?C?V?V/#9%= @W @
<}    $ = =>A]A]]`aa+{{/F/F/S/S.TT!LL,uzzR^ReRef	%,,Z;	 557	BM!"__Y7N 5 8 89M9M}ObOb c!66y6^%445G_#0S""))<<"+F;%%..KK?Kr!   r   )NFF)NNNNNNNNNNFN)NNNNNF)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rX   r   no_gradr   r    r   r!   r"   r   r   0  s   
 >B38&+9%''9% !++9% !))9)9 :	9%
 #+4.9% d^9%@ >B38&+'' !++ !))9)9 :	
 #+4. d^"e.>.> "uO`O` "& >B15598<=A59,0/3-1&*).$(N
''N
 !,,N
 !))9)9 :	N

 E--.N
 !!1!12N
 $E$4$45N
 !))9)9 :N
   1 12N
 $D>N
 'tnN
 ))*N
 d^N
 #'N
 D>N
  +,!N
" 
uJJ	K#N
` ]]_ 9==A045959).C''C $E$4$45C !))9)9 :	C
 E,,-C !!1!12C   1 12C #'C 
		C Cr!   r   )r(   r%   r   rR   rO   rU   r[   r   )*typingr   r   r   torch.utils.checkpoint;transformers.models.instructblip.configuration_instructblipr   r   6transformers.models.instructblip.modeling_instructblipr   r   r	   r
   r   r   r   configuration_utilsr   modeling_flash_attention_utilsr   models.auto.modeling_autor   processing_utilsr   utilsr   autor   r   
get_loggerr   r5   r   r%   r(   rO   rR   rU   rX   r[   r   __all__r   r!   r"   <module>r      s     #     4 B J &  - 
		H	%	$< 		%> 	z
. z
z	'B 		#: 		$< 		;j 	j
. j
Zj0T jZ		r!   