§
    ‚Štj	ë  ã                   óú  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4  e&j5        e6¦  «        Z7 G d„ dej8        ¦  «        Z9 G d„ dej8        ¦  «        Z:e$ G d„ de¦  «        ¦   «         Z;	 dMdej8        dej<        d ej<        d!ej<        d"ej<        dz  d#e=d$e=fd%„Z> G d&„ d'ej8        ¦  «        Z? G d(„ d)ej8        ¦  «        Z@ G d*„ d+e¦  «        ZA G d,„ d-ej8        ¦  «        ZB G d.„ d/e;¦  «        ZC G d0„ d1ej8        ¦  «        ZD G d2„ d3ej8        ¦  «        ZE G d4„ d5ej8        ¦  «        ZF G d6„ d7ej8        ¦  «        ZG G d8„ d9ej8        ¦  «        ZH G d:„ d;e¦  «        ZI G d<„ d=ej8        ¦  «        ZJ G d>„ d?e;¦  «        ZK e$d@¬A¦  «        e G dB„ dCe"¦  «        ¦   «         ¦   «         ZL e$dD¬A¦  «         G dE„ dFe;¦  «        ¦   «         ZMe$e G dG„ dHe¦  «        ¦   «         ¦   «         ZN e$dI¬A¦  «         G dJ„ dKe;e¦  «        ¦   «         ZOg dL¢ZPdS )Né    )ÚCallable)Ú	dataclass)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelÚAutoModelForCausalLMÚAutoModelForSeq2SeqLMé   )ÚInstructBlipVideoConfigÚInstructBlipVideoQFormerConfigÚInstructBlipVideoVisionConfigc                   óz   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej	        d
e
dej        fd„Zˆ xZS )Ú!InstructBlipVideoVisionEmbeddingsÚconfigc                 ó  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        dd| j        ¦  «        ¦  «        | _        t          j        d| j        | j        | j        ¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        t          j
        d| j        | j        ¦  «        ¦  «        | _        d S )Nr$   r   )Úin_channelsÚout_channelsÚkernel_sizeÚstrider    )ÚsuperÚ__init__r*   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	ParameterÚtorchÚrandnÚclass_embeddingÚConv2dÚpatch_embeddingÚnum_patchesÚnum_positionsÚposition_embedding©Úselfr*   Ú	__class__s     €ú~/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/instructblipvideo/modeling_instructblipvideo.pyr1   z*InstructBlipVideoVisionEmbeddings.__init__<   sß   ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¸1¸d¼nÑ,MÔ,MÑNÔNˆÔå!œyØ¨¬ÀDÄOÐ\`Ô\kð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔå"$¤,­u¬{¸1¸dÔ>PÐRVÔR`Ñ/aÔ/aÑ"bÔ"bˆÔÐÐó    Ú
embeddingsÚheightÚwidthÚreturnc                 ó”  — |j         d         dz
  }| j        j         d         dz
  }t          j                             ¦   «         s||k    r||k    r| j        S | j        dd…dd…f         }| j        dd…dd…f         }|j         d         }|| j        z  }	|| j        z  }
t          |dz  ¦  «        }|                     d|||¦  «        }|                     dddd¦  «        }t          j
                             ||	|
fdd	¬
¦  «        }|                     dddd¦  «                             dd|¦  «        }t          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r$   Néÿÿÿÿg      à?r   r   r    ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)Úshaper>   r7   ÚjitÚ
is_tracingr5   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r@   rD   rE   rF   r<   r=   Úclass_pos_embedÚpatch_pos_embedrO   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               rB   Úinterpolate_pos_encodingz:InstructBlipVideoVisionEmbeddings.interpolate_pos_encodingN   sr  € ð !Ô& qÔ)¨AÑ-ˆØÔ/Ô5°aÔ8¸1Ñ<ˆõ Œy×#Ò#Ñ%Ô%ð 	+¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?ØÔ*Ð*àÔ1°!°!°!°R°a°R°%Ô8ˆØÔ1°!°!°!°Q°R°R°%Ô8ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCrC   FÚpixel_valuesr^   c                 ó(  — |j         \  }}}}| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j                             |dd¦  «                             |¦  «        }	t          j
        |	|gd¬¦  «        }
|r|                      |
||¦  «        }n| j        }|
|d d …d |
                     d¦  «        …d d …f                              |¦  «        z   }
|
S )N)Údtyper    r$   rI   rN   )rP   r;   Úweightra   ÚtoÚflattenÚ	transposer9   Úexpandr7   rX   r^   r>   rK   )r@   r_   r^   Ú
batch_sizeÚ_rE   rF   Útarget_dtypeÚpatch_embedsÚclass_embedsrD   r>   s               rB   Úforwardz)InstructBlipVideoVisionEmbeddings.forwardv   s  € Ø'3Ô'9Ñ$ˆ
�A�v˜uØÔ+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆØÔ+×2Ò2°:¸qÀ"ÑEÔE×HÒHÈÑVÔVˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
Ø#ð 	9Ø!%×!>Ò!>¸zÈ6ÐSXÑ!YÔ!YÐÐà!%Ô!8ÐØÐ"4°Q°Q°QÐ8L¸*¿/º/È!Ñ:LÔ:LÐ8LÈaÈaÈaÐ5OÔ"P×"SÒ"SÐT`Ñ"aÔ"aÑaˆ
ØÐrC   ©F)Ú__name__Ú
__module__Ú__qualname__r'   r1   r7   ÚTensorÚintr^   ÚFloatTensorÚboolrl   Ú__classcell__©rA   s   @rB   r)   r)   ;   sÂ   ø€ € € € € ðcÐ<ð cð cð cð cð cð cð$&D°5´<ð &DÈð &DÐUXð &DÐ]bÔ]ið &Dð &Dð &Dð &DðPð  EÔ$5ð ÐQUð ÐbgÔbnð ð ð ð ð ð ð ð rC   r)   c                   ó2   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú"InstructBlipVideoQFormerEmbeddingsz;Construct the embeddings from word and position embeddings.c                 óþ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j
        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt#          j        |j        ¦  «                             d¦  «        d¬¦  «         || _        d S )N)Úpadding_idx©ÚepsÚposition_ids©r$   rI   F)Ú
persistent)r0   r1   r   Ú	EmbeddingÚ
vocab_sizer2   Úpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚ	layernormÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr7   Úarangerf   r*   r?   s     €rB   r1   z+InstructBlipVideoQFormerEmbeddings.__init__ˆ   s×   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#%¤<°Ô0NÐPVÔPbÑ#cÔ#cˆÔ åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒð 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð ˆŒˆˆrC   Nr   c                 ó   — |�|                      ¦   «         d         }nd}|€(| j        d d …|||z   …f                              ¦   «         }|�b|                      |¦  «        }|                      |                     |j        ¦  «        ¦  «        }||z   }|�t          j        ||fd¬¦  «        }n|}|                     | j	        j
        j        ¦  «        }|  	                    |¦  «        }|                      |¦  «        }|S )Nr$   r   rN   )rK   r}   Úclonerƒ   r…   rc   Údevicer7   rX   rˆ   rb   ra   r‹   )r@   Ú	input_idsr}   Úquery_embedsÚpast_key_values_lengthÚ
seq_lengthrD   r…   s           rB   rl   z*InstructBlipVideoQFormerEmbeddings.forward—   s  € ð Ð Ø"ŸšÑ)Ô)¨!Ô,ˆJˆJàˆJàÐØÔ,¨Q¨Q¨QÐ0FÈÐVlÑIlÐ0lÐ-lÔm×sÒsÑuÔuˆLàÐ Ø×-Ò-¨iÑ8Ô8ˆJà"&×":Ò":¸<¿?º?È:ÔK\Ñ;]Ô;]Ñ"^Ô"^ÐØ#Ð&9Ñ9ˆJàÐ'Ý"œY¨°jÐ'AÀqÐIÑIÔI�
øà%ˆJà—]’] 4¤>Ô#8Ô#>Ñ?Ô?ˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐrC   ©NNNr   )rn   ro   rp   Ú__doc__r1   rl   ru   rv   s   @rB   rx   rx   …   s`   ø€ € € € € ØEÐEðð ð ð ð ð" ØØØ ðð ð ð ð ð ð ð rC   rx   c                   ó|   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZg d¢Z ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )Ú InstructBlipVideoPreTrainedModelr*   Úblip)ÚvideoÚtextT)rx   ÚInstructBlipVideoAttentionÚInstructBlipVideoEncoderLayerÚInstructBlipVideoQFormerLayerÚ*InstructBlipVideoQFormerMultiHeadAttentionÚ"InstructBlipVideoQFormerSelfOutputc                 ó:  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        r:t          j        |j        d|¬¦  «         t          j        |j	        d|¬¦  «         dS t	          |t          t          f¦  «        rt          j        |j        ¦  «         dS t	          |t          ¦  «        rQt          j        |j        t#          j        |j        j        d         ¦  «                             d¦  «        ¦  «         dS dS )zInitialize the weightsç        )ÚmeanÚstdrI   r~   N)r0   Ú_init_weightsr*   Úinitializer_rangeÚ
isinstancer)   ÚinitÚtrunc_normal_r>   r9   Ú)InstructBlipVideoForConditionalGenerationÚInstructBlipVideoModelÚzeros_Úquery_tokensrx   Úcopy_r}   r7   r�   rP   rf   )r@   ÚmoduleÚfactorrA   s      €rB   r¥   z.InstructBlipVideoPreTrainedModel._init_weightsÍ   s
  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ø”Ô.ˆÝ�fÕ?Ñ@Ô@ð 	iÝÔ˜vÔ8¸sÈÐOÑOÔOÐOÝÔ˜vÔ5¸CÀVÐLÑLÔLÐLÐLÐLÝ˜Õ!JÕLbÐ cÑdÔdð 	iÝŒK˜Ô+Ñ,Ô,Ð,Ð,Ð,Ý˜Õ BÑCÔCð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	irC   )rn   ro   rp   r%   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_no_split_modulesr7   Úno_gradr¥   ru   rv   s   @rB   r˜   r˜   ·   s¤   ø€ € € € € € à#Ð#Ð#Ñ#ØÐØ(ÐØ&*Ð#Ø"&ÐØÐØ€NØÐà!Ððð ð Ðð €U„]�_„_ð
ið 
ið 
ið 
iñ „_ð
ið 
ið 
ið 
ið 
irC   r˜   r¢   r¯   ÚqueryÚkeyÚvalueÚattention_maskÚscalingr‹   c                 óz  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )NrI   éþÿÿÿrN   )ÚpÚtrainingr$   r    )	r7   Úmatmulre   r   rU   Úsoftmaxr‹   rÄ   Ú
contiguous)
r¯   r¼   r½   r¾   r¿   rÀ   r‹   ÚkwargsÚattn_weightsÚattn_outputs
             rB   Úeager_attention_forwardrË   Ü   s­   € õ ”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$rC   c                   ó¢   ‡ — e Zd ZdZˆ fd„Zdej        dedefd„Zdej        de	ej        ej        d	z  e	ej                 d	z  f         fd
„Z
ˆ xZS )rœ   z=Multi-headed attention from 'Attention Is All You Need' paperc                 óV  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚| j        dz  | _	        d| _
        |j        | _        t          j        | j        d| j        z  d¬¦  «        | _        |j        rWt          j        t#          j        | j        ¦  «        ¦  «        }t          j        t#          j        | j        ¦  «        ¦  «        }nd }d }|�It#          j        |t#          j        |d¬¦  «        |f¦  «        }t          j        |¦  «        | j        _        t          j        | j        | j        ¦  «        | _        d S )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Fr   )Úbias)Úrequires_grad)r0   r1   r*   r2   r3   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚ	is_causalÚattention_dropoutr   ÚLinearÚqkvÚqkv_biasr6   r7   ÚzerosrX   Ú
zeros_likerÏ   Ú
projection)r@   r*   Úq_biasÚv_biasrÚ   rA   s        €rB   r1   z#InstructBlipVideoAttention.__init__ö   s€  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÈdÌnð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð ”] DÑ(ˆŒ
ØˆŒØ!'Ô!9ˆÔõ ”9˜Tœ^¨Q°´Ñ-?ÀeÐLÑLÔLˆŒàŒ?ð 	Ý”\¥%¤+¨d¬nÑ"=Ô"=Ñ>Ô>ˆFÝ”\¥%¤+¨d¬nÑ"=Ô"=Ñ>Ô>ˆFˆFàˆFØˆFàÐÝ”y &­%Ô*:¸6ÐQVÐ*WÑ*WÔ*WÐY_Ð!`ÑaÔaˆHÝœL¨Ñ2Ô2ˆDŒHŒMåœ) D¤N°D´NÑCÔCˆŒˆˆrC   ÚtensorÚseq_lenÚbszc                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S )Nr$   r    )rW   rÒ   rÓ   re   rÇ   )r@   rà   rá   râ   s       rB   Ú_shapez!InstructBlipVideoAttention._shape  s<   € Ø�{Š{˜3 ¨¬¸¼ÑGÔG×QÒQÐRSÐUVÑWÔW×bÒbÑdÔdÐdrC   Úhidden_statesrG   Nc                 ó  — |                      ¦   «         \  }}}|                      |¦  «        }|                     ||d| j        || j        z  ¦  «                             ddddd¦  «        }|d         |d         |d         }	}}t          j        | j        j        t          ¦  «        }
 |
| |||	fd| j
        sdn| j        | j        dœ|¤Ž\  }}|                     ||d	¦  «                             ¦   «         }|                      |¦  «        }||fS )
z#Input shape: Batch x Time x Channelr   r    r   r$   é   Nr¢   )r¿   r‹   rÀ   rI   )rK   rÙ   rS   rÒ   rT   r   Úget_interfacer*   Ú_attn_implementationrË   rÄ   r×   rÕ   rÇ   rÝ   )r@   rå   rÈ   râ   Útgt_lenr3   Ú	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerÊ   rÉ   s                rB   rl   z"InstructBlipVideoAttention.forward  s=  € ð #0×"4Ò"4Ñ"6Ô"6ÑˆˆW�ià—H’H˜]Ñ+Ô+ˆ	à×%Ò% c¨7°A°t´~ÀyÐTXÔTbÑGbÑcÔc×kÒkØˆq�!�Q˜ñ
ô 
ˆ	ð 2;¸1´¸yÈ¼|ÈYÐWXÌ\ ,�jˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØð		%
ð
  Ø#œ}ÐH�C�C°$Ô2HØ”Jð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð "×)Ò)¨#¨w¸Ñ;Ô;×FÒFÑHÔHˆØ—o’o kÑ2Ô2ˆà˜LÐ(Ð(rC   )rn   ro   rp   r–   r1   r7   rq   rr   rä   Útuplerl   ru   rv   s   @rB   rœ   rœ   ó   s¾   ø€ € € € € ØGÐGðDð Dð Dð Dð Dð>e˜Uœ\ð e°Cð e¸cð eð eð eð eð")à”|ð")ð 
ˆuŒ|˜Uœ\¨DÑ0°%¸¼Ô2EÈÑ2LÐLÔ	Mð	")ð ")ð ")ð ")ð ")ð ")ð ")ð ")rC   rœ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚInstructBlipVideoMLPc                 ó  •— t          ¦   «                              ¦   «          || _        t          |j                 | _        t          j        |j        |j	        ¦  «        | _
        t          j        |j	        |j        ¦  «        | _        d S ©N)r0   r1   r*   r	   Ú
hidden_actÚactivation_fnr   rØ   r2   Úintermediate_sizeÚfc1Úfc2r?   s     €rB   r1   zInstructBlipVideoMLP.__init__>  sf   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ# FÔ$5Ô6ˆÔÝ”9˜VÔ/°Ô1IÑJÔJˆŒÝ”9˜VÔ5°vÔ7IÑJÔJˆŒˆˆrC   rå   rG   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rô   )rø   rö   rù   ©r@   rå   s     rB   rl   zInstructBlipVideoMLP.forwardE  s=   € ØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆØŸš Ñ/Ô/ˆØÐrC   ©rn   ro   rp   r1   r7   rq   rl   ru   rv   s   @rB   rò   rò   =  sc   ø€ € € € € ðKð Kð Kð Kð Kð U¤\ð °e´lð ð ð ð ð ð ð ð rC   rò   c                   óh   ‡ — e Zd Zdefˆ fd„Zedej        dee	         dej
        fd„¦   «         Zˆ xZS )r�   r*   c                 óD  •— t          ¦   «                              ¦   «          |j        | _        t	          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _	        t          |¦  «        | _        t          j        | j        |j        ¬¦  «        | _        d S ©Nr{   )r0   r1   r2   r3   rœ   Ú	self_attnr   r†   r‡   Úlayer_norm1rò   ÚmlpÚlayer_norm2r?   s     €rB   r1   z&InstructBlipVideoEncoderLayer.__init__M  s   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒÝ3°FÑ;Ô;ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÝ'¨Ñ/Ô/ˆŒÝœ<¨¬¸FÔ<QÐRÑRÔRˆÔÐÐrC   rå   rÈ   rG   c                 óÄ   — |}|                       |¦  «        } | j        dd|i|¤Ž\  }}||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )Nrå   © )r  r   r  r  )r@   rå   rÈ   Úresidualrh   s        rB   rl   z%InstructBlipVideoEncoderLayer.forwardU  sŽ   € ð !ˆà×(Ò(¨Ñ7Ô7ˆØ)˜4œ>ð 
ð 
Ø'ð
àð
ð 
Ñˆ�qð &¨Ñ0ˆØ ˆØ×(Ò(¨Ñ7Ô7ˆØŸš Ñ/Ô/ˆà%¨Ñ0ˆàÐrC   )rn   ro   rp   r%   r1   r   r7   rq   r   r   rs   rl   ru   rv   s   @rB   r�   r�   L  s“   ø€ € € € € ðSÐ6ð Sð Sð Sð Sð Sð Sð ðà”|ðð Ð+Ô,ðð 
Ô	ð	ð ð ñ „^ðð ð ð ð rC   r�   c                   óZ   ‡ — e Zd ZdZdefˆ fd„Zedee         de	e
z  fd„¦   «         Zˆ xZS )ÚInstructBlipVideoEncodera"  
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`InstructBlipVideoEncoderLayer`].

    Args:
        config (`InstructBlipVideoConfig`):
            The corresponding vision configuration for the `InstructBlipVideoEncoder`.
    r*   c                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r  )r�   )Ú.0rh   r*   s     €rB   ú
<listcomp>z5InstructBlipVideoEncoder.__init__.<locals>.<listcomp>y  s"   ø€ Ð$tÐ$tÐ$tÈqÕ%BÀ6Ñ%JÔ%JÐ$tÐ$tÐ$trC   F)	r0   r1   r*   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersÚgradient_checkpointingr?   s    `€rB   r1   z!InstructBlipVideoEncoder.__init__v  sb   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”mÐ$tÐ$tÐ$tÐ$tÕTYÐZ`ÔZrÑTsÔTsÐ$tÑ$tÔ$tÑuÔuˆŒØ&+ˆÔ#Ð#Ð#rC   rÈ   rG   c                 óL   — |}| j         D ]} ||fi |¤Ž}Œt          |¬¦  «        S )N©Úlast_hidden_state)r  r   )r@   Úinputs_embedsrÈ   rå   Úencoder_layers        rB   rl   z InstructBlipVideoEncoder.forward|  sP   € ð &ˆØ!œ[ð 	ð 	ˆMØ)˜MØðð àðð ˆMˆMõ
 °Ð?Ñ?Ô?Ð?rC   )rn   ro   rp   r–   r%   r1   r   r   r   rð   r   rl   ru   rv   s   @rB   r  r  l  s˜   ø€ € € € € ðð ð,Ð6ð ,ð ,ð ,ð ,ð ,ð ,ð ð@ð Ð+Ô,ð@ð 
�Ñ	 ð	@ð @ð @ñ „^ð@ð @ð @ð @ð @rC   r  c                   óÌ   ‡ — e Zd ZU dZdZeed<   eedœZ	defˆ fd„Z
e ed¬¦  «        e	 	 ddej        dz  d	ed
ee         deez  fd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚInstructBlipVideoVisionModelr_   rš   r*   )rå   Ú
attentionsc                 ó  •— t          ¦   «                              |¦  «         || _        |j        }t	          |¦  «        | _        t          |¦  «        | _        t          j	        ||j
        ¬¦  «        | _        |                      ¦   «          d S rÿ   )r0   r1   r*   r2   r)   rD   r  Úencoderr   r†   r‡   Úpost_layernormÚ	post_init)r@   r*   r3   rA   s      €rB   r1   z%InstructBlipVideoVisionModel.__init__•  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ&ˆ	å;¸FÑCÔCˆŒÝ/°Ñ7Ô7ˆŒÝ œl¨9¸&Ô:OÐPÑPÔPˆÔà�ŠÑÔÐÐÐrC   F)Útie_last_hidden_statesNr^   rÈ   rG   c                 ó  — |€t          d¦  «        ‚|                      ||¬¦  «        } | j        dd|i|¤Ž}|j        }|                      |¦  «        }|d d …dd d …f         }|                      |¦  «        }t          ||¬¦  «        S )Nz You have to specify pixel_values)r^   r  r   ©r  Úpooler_outputr  )rÔ   rD   r  r  r  r   )r@   r_   r^   rÈ   rå   Úencoder_outputsr  Úpooled_outputs           rB   rl   z$InstructBlipVideoVisionModel.forward   s¿   € ð ÐÝÐ?Ñ@Ô@Ð@àŸš¨ÐOg˜ÑhÔhˆà+7¨4¬<ð ,
ð ,
Ø'ð,
àð,
ð ,
ˆð
 ,Ô=ÐØ ×/Ò/Ð0AÑBÔBÐà)¨!¨!¨!¨Q°°°¨'Ô2ˆØ×+Ò+¨MÑ:Ô:ˆå)Ø/Ø'ð
ñ 
ô 
ð 	
rC   c                 ó   — | j         S rô   )rD   ©r@   s    rB   Úget_input_embeddingsz1InstructBlipVideoVisionModel.get_input_embeddings¾  s
   € ØŒÐrC   ©NF)rn   ro   rp   Úmain_input_namer³   r'   r±   r�   rœ   Ú_can_record_outputsr1   r   r   r   r7   rs   rt   r   r   rð   r   rl   r&  ru   rv   s   @rB   r  r  Œ  s
  ø€ € € € € € Ø$€OØÐØ)Ð)Ð)Ñ)à6Ø0ðð Ðð
	Ð<ð 	ð 	ð 	ð 	ð 	ð 	ð  Ø€_¨EÐ2Ñ2Ô2Øð 26Ø).ð
ð 
àÔ'¨$Ñ.ð
ð #'ð
ð Ð+Ô,ð	
ð
 
Ð+Ñ	+ð
ð 
ð 
ñ „^ñ 3Ô2ñ  Ôð
ð6ð ð ð ð ð ð rC   r  c                   ó@   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddee         fd„Zˆ xZS )rŸ   Fc                 ó  •— t          ¦   «                              ¦   «          || _        |j        |j        z  dk    r.t          |d¦  «        st          d|j        |j        fz  ¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        | j        dz  | _
        d| _        |j        | _        t          j        |j        | j	        ¦  «        | _        |rJt          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        d S t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        d S )Nr   Úembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)rÎ   F)r0   r1   r*   r2   rÑ   ÚhasattrrÔ   rr   Úattention_head_sizeÚall_head_sizerÀ   rÖ   Úattention_probs_dropout_probr×   r   rØ   r¼   Úencoder_hidden_sizer½   r¾   ©r@   r*   Úis_cross_attentionrA   s      €rB   r1   z3InstructBlipVideoQFormerMultiHeadAttention.__init__Ã  sZ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?ÝØ^ØÔ% vÔ'AÐBñCñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØÔ/°Ñ5ˆŒØˆŒØ!'Ô!DˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Øð 	KÝ”y Ô!;¸TÔ=OÑPÔPˆDŒHÝœ 6Ô#=¸tÔ?QÑRÔRˆDŒJˆJˆJå”y Ô!3°TÔ5GÑHÔHˆDŒHÝœ 6Ô#5°tÔ7IÑJÔJˆDŒJˆJˆJrC   NrÈ   c                 óì  — |d u}|j         d d…         }g |¢d‘| j        ‘R }|r|}	|}n|}	g |	j         d d…         ¢d‘| j        ‘R }
|                      |¦  «                             |¦  «                             dd¦  «        }|                      |	¦  «                             |
¦  «                             dd¦  «        }|                      |	¦  «                             |
¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        } || ||||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |j         d d…         ¢d‘R Ž                      ¦   «         }||fS )NrI   r$   r    r¢   )r‹   rÀ   )rP   r.  r¼   rW   re   r½   r¾   r   rè   r*   ré   rË   rÄ   r×   rÀ   rS   rÇ   )r@   rå   r¿   Úencoder_hidden_statesÚencoder_attention_maskrÈ   r3  Úinput_shapeÚhidden_shapeÚcurrent_statesÚkv_shapeÚquery_layerÚ	key_layerÚvalue_layerrï   rÊ   rÉ   s                    rB   rl   z2InstructBlipVideoQFormerMultiHeadAttention.forwardÛ  s»  € ð 3¸$Ð>Ðà#Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆàð 	+Ø2ˆNØ3ˆNˆNà*ˆNàM�^Ô)¨#¨2¨#Ô.ÐM°ÐM°DÔ4LÐMÐMˆØ—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆØ—H’H˜^Ñ,Ô,×1Ò1°(Ñ;Ô;×EÒEÀaÈÑKÔKˆ	Ø—j’j Ñ0Ô0×5Ò5°hÑ?Ô?×IÒIÈ!ÈQÑOÔOˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)ÐE¨;Ô+<¸R¸a¸RÔ+@ÐEÀ"ÐEÐEÐE×PÒPÑRÔRˆØ˜LÐ(Ð(rC   rm   ©NNN)rn   ro   rp   r1   r   r   rl   ru   rv   s   @rB   rŸ   rŸ   Â  sw   ø€ € € € € ðKð Kð Kð Kð Kð Kð6 Ø"Ø#ð,)ð ,)ð Ð+Ô,ð,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)ð ,)rC   rŸ   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )r    c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S rÿ   )r0   r1   r   rØ   r2   Údenser†   r‡   r‰   rŠ   r‹   r?   s     €rB   r1   z+InstructBlipVideoQFormerSelfOutput.__init__  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrC   rå   Úinput_tensorrG   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rô   ©rA  r‹   r†   ©r@   rå   rB  s      rB   rl   z*InstructBlipVideoQFormerSelfOutput.forward  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐrC   rü   rv   s   @rB   r    r    
  ói   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð rC   r    c                   ó˜   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddej        dej        dz  dej        dz  dej        dz  dee         d	ej        fd
„Z	ˆ xZ
S )Ú!InstructBlipVideoQFormerAttentionFc                 óš   •— t          ¦   «                              ¦   «          t          ||¦  «        | _        t	          |¦  «        | _        d S rô   )r0   r1   rŸ   Ú	attentionr    Úoutputr2  s      €rB   r1   z*InstructBlipVideoQFormerAttention.__init__  s>   ø€ Ý‰Œ×ÒÑÔÐÝCÀFÐL^Ñ_Ô_ˆŒÝ8¸Ñ@Ô@ˆŒˆˆrC   Nrå   r¿   r5  r6  rÈ   rG   c                 ó\   —  | j         d||||dœ|¤Ž\  }}|                      ||¦  «        }|S )N)rå   r¿   r5  r6  r  )rK  rL  )	r@   rå   r¿   r5  r6  rÈ   rÊ   rh   Úattention_outputs	            rB   rl   z)InstructBlipVideoQFormerAttention.forward  sW   € ð (˜œð 
Ø'Ø)Ø"7Ø#9ð	
ð 
ð
 ð
ð 
‰ˆ�Qð  Ÿ;š; {°MÑBÔBÐØÐrC   rm   r>  )rn   ro   rp   r1   r7   rq   rs   r   r   rl   ru   rv   s   @rB   rI  rI    sÃ   ø€ € € € € ðAð Að Að Að Að Að 48Ø:>Ø;?ð ð  à”|ð ð Ô)¨DÑ0ð ð  %Ô0°4Ñ7ð	 ð
 !&Ô 1°DÑ 8ð ð Ð+Ô,ð ð 
Œð ð  ð  ð  ð  ð  ð  ð  rC   rI  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú$InstructBlipVideoQFormerIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S rô   )r0   r1   r   rØ   r2   r÷   rA  r§   rõ   Ústrr	   Úintermediate_act_fnr?   s     €rB   r1   z-InstructBlipVideoQFormerIntermediate.__init__2  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$rC   rå   rG   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rô   )rA  rS  rû   s     rB   rl   z,InstructBlipVideoQFormerIntermediate.forward:  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐrC   rü   rv   s   @rB   rP  rP  1  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð rC   rP  c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚInstructBlipVideoQFormerOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S rÿ   )r0   r1   r   rØ   r÷   r2   rA  r†   r‡   r‰   rŠ   r‹   r?   s     €rB   r1   z'InstructBlipVideoQFormerOutput.__init__A  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆrC   rå   rB  rG   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rô   rD  rE  s      rB   rl   z&InstructBlipVideoQFormerOutput.forwardG  rF  rC   rü   rv   s   @rB   rV  rV  @  rG  rC   rV  c                   óL   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddee         fd„Zd„ Zd„ Zˆ xZ	S )	rž   c                 óª  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        || _        ||j        z  dk    rt	          |d¬¦  «        | _        d| _	        nd| _	        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        d S )Nr$   r   T)r3  F)r0   r1   Úchunk_size_feed_forwardÚseq_len_dimrI  rK  Ú	layer_idxÚcross_attention_frequencyÚcrossattentionÚhas_cross_attentionrP  ÚintermediaterV  rL  Úintermediate_queryÚoutput_query)r@   r*   r]  rA   s      €rB   r1   z&InstructBlipVideoQFormerLayer.__init__O  sÃ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ:¸6ÑBÔBˆŒà"ˆŒà�vÔ7Ñ7¸1Ò<Ð<Ý"CÀFÐ_cÐ"dÑ"dÔ"dˆDÔØ'+ˆDÔ$Ð$à',ˆDÔ$å@ÀÑHÔHˆÔÝ4°VÑ<Ô<ˆŒå"FÀvÑ"NÔ"NˆÔÝ:¸6ÑBÔBˆÔÐÐrC   Nr   rÈ   c           
      ó  —  | j         |fd|i|¤Ž}|dk    rÎ|d d …d |…d d …f         }| j        r#|€t          d¦  «        ‚ | j        |f|||dœ|¤Ž}t	          | j        | j        | j        |¦  «        }	|j        d         |k    r`t	          | j	        | j        | j        |d d …|d …d d …f         ¦  «         
                    |	j        ¦  «        }
t          j        |	|
gd¬¦  «        }	n!t	          | j	        | j        | j        |¦  «        }	|	S )Nr¿   r   z>encoder_hidden_states must be given for cross-attention layers)r¿   r5  r6  r$   rN   )rK  r`  rÔ   r_  r   Úfeed_forward_chunk_queryr[  r\  rP   Úfeed_forward_chunkrc   r�   r7   rX   )r@   rå   r¿   r5  r6  Úquery_lengthrÈ   rN  Úquery_attention_outputÚlayer_outputÚlayer_output_texts              rB   rl   z%InstructBlipVideoQFormerLayer.forwardc  sˆ  € ð *˜4œ>Øð
ð 
à)ð
ð ð
ð 
Ðð ˜!ÒÐØ%5°a°a°a¸¸,¸ÈÈÈÐ6IÔ%JÐ"àÔ'ð 	Ø(Ð0Ý$Ð%eÑfÔfÐfØ)<¨Ô)<Ø*ð*à#1Ø*?Ø+Að	*ð *ð
 ð*ð *Ð&õ 5ØÔ-ØÔ,ØÔ Ø&ñ	ô ˆLð  Ô% aÔ(¨<Ò7Ð7Ý$=ØÔ+ØÔ0ØÔ$Ø$ Q Q Q¨¨¨°q°q°qÐ%8Ô9ñ	%ô %÷
 ’"�\Ô(Ñ)Ô)ð "õ  %œy¨,Ð8IÐ)JÐPQÐRÑRÔR�øå4ØÔ'ØÔ,ØÔ Ø ñ	ô ˆLð ÐrC   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rô   )ra  rL  ©r@   rN  Úintermediate_outputri  s       rB   rf  z0InstructBlipVideoQFormerLayer.feed_forward_chunk˜  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐrC   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rô   )rb  rc  rl  s       rB   re  z6InstructBlipVideoQFormerLayer.feed_forward_chunk_query�  s4   € Ø"×5Ò5Ð6FÑGÔGÐØ×(Ò(Ð)<Ð>NÑOÔOˆØÐrC   r•   )
rn   ro   rp   r1   r   r   rl   rf  re  ru   rv   s   @rB   rž   rž   N  s“   ø€ € € € € ðCð Cð Cð Cð Cð. Ø"Ø#Øð3ð 3ð Ð+Ô,ð3ð 3ð 3ð 3ðjð ð ð
ð ð ð ð ð ð rC   rž   c                   óP   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddee         fd„¦   «         Zˆ xZS )ÚInstructBlipVideoQFormerEncoderc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r  )rž   )r  r]  r*   s     €rB   r  z<InstructBlipVideoQFormerEncoder.__init__.<locals>.<listcomp>¨  s$   ø€ ÐoÐoÐoÀ)Õ*¨6°9Ñ=Ô=ÐoÐoÐorC   F)	r0   r1   r*   r   r  r  r  Úlayerr  r?   s    `€rB   r1   z(InstructBlipVideoQFormerEncoder.__init__¤  sg   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]ØoÐoÐoÐoÍuÐU[ÔUmÑOnÔOnÐoÑoÔoñ
ô 
ˆŒ
ð ',ˆÔ#Ð#Ð#rC   Nr   rÈ   c                 ó�   — t          | j        j        ¦  «        D ]}| j        |         } ||||f||dœ|¤Ž}Œt	          |¬¦  «        S )N)r6  rg  r  )r  r*   r  rs  r   )	r@   rå   r¿   r5  r6  rg  rÈ   ÚiÚlayer_modules	            rB   rl   z'InstructBlipVideoQFormerEncoder.forward¬  s   € õ �t”{Ô4Ñ5Ô5ð 
	ð 
	ˆAØœ: aœ=ˆLà(˜LØØØ%ðð (>Ø)ðð ð ðð ˆMˆMõ 9Ø+ð
ñ 
ô 
ð 	
rC   r•   )	rn   ro   rp   r1   r   r   r   rl   ru   rv   s   @rB   rp  rp  £  s|   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð ð Ø"Ø#Øð
ð 
ð Ð+Ô,ð
ð 
ð 
ñ Ôð
ð 
ð 
ð 
ð 
rC   rp  c                   ój  ‡ — e Zd ZdZdZdZdZdZe e	e
dd¬¦  «        g e	e
dd¬¦  «        gdœZdefˆ fd	„Zd
„ Zd„ Zeee	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dee         deej                 ez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚInstructBlipVideoQFormerModelz’
    Querying Transformer (Q-Former), used in InstructBlipVideo. Slightly modified from BLIP-2 as it also takes the
    instruction as input.
    Tr$   z
.attention)ÚindexÚ
layer_namez.crossattention)rå   r  Úcross_attentionsr*   c                 óÐ   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rô   )r0   r1   r*   rx   rD   rp  r  r  r?   s     €rB   r1   z&InstructBlipVideoQFormerModel.__init__Ü  sV   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒå<¸VÑDÔDˆŒå6°vÑ>Ô>ˆŒà�ŠÑÔÐÐÐrC   c                 ó   — | j         j        S rô   ©rD   rƒ   r%  s    rB   r&  z2InstructBlipVideoQFormerModel.get_input_embeddingsæ  s   € ØŒÔ.Ð.rC   c                 ó   — || j         _        d S rô   r~  )r@   r¾   s     rB   Úset_input_embeddingsz2InstructBlipVideoQFormerModel.set_input_embeddingsé  s   € Ø*/ˆŒÔ'Ð'Ð'rC   Nr‘   r¿   r}   r’   r5  r6  rÈ   rG   c                 ó‚  — |€|€t          d¦  «        ‚|�|j        d         nd}|                      |||¬¦  «        }	t          | j        |	|¬¦  «        }|�1|dk    r|	dd…d|…dd…f         n|	}
t          | j        |
||¬¦  «        } | j        |	f||||dœ|¤Ž}|j        }|dd…ddd…f         }t          ||¬	¦  «        S )
a$  
        query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
            Hidden states to be used in the attention computation. If cross-attention,
            will be used for the query (i.e., key and value will use the encoder_hidden_states).
        Nz7You have to specify query_embeds when input_ids is Noner$   r   )r‘   r}   r’   )r*   r  r¿   )r*   r  r¿   r5  )r¿   r5  r6  rg  r   )rÔ   rP   rD   r   r*   r  r  r   )r@   r‘   r¿   r}   r’   r5  r6  rÈ   rg  Úembedding_outputÚquery_embedding_outputr"  Úsequence_outputr#  s                 rB   rl   z%InstructBlipVideoQFormerModel.forwardì  sN  € ð$ Ð Ð!5ÝÐVÑWÔWÐWà0<Ð0H�|Ô)¨!Ô,Ð,ÈaˆàŸ?š?ØØ%Ø%ð +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð "Ð-ð O[Ð]^ÒN^ÐN^Ð%5°a°a°a¸¸,¸ÈÈÈÐ6IÔ%JÐ%JÐdtÐ"Ý%>Ø”{Ø4Ø5Ø&;ð	&ñ &ô &Ð"ð ,8¨4¬<Øð,
à)Ø"7Ø#9Ø%ð,
ð ,
ð ð,
ð ,
ˆð *Ô;ˆØ'¨¨¨¨1¨a¨a¨a¨Ô0ˆå;Ø-Ø'ð
ñ 
ô 
ð 	
rC   )NNNNN)rn   ro   rp   r–   rµ   r·   r¶   r¸   rž   r   rŸ   r)  r&   r1   r&  r€  r   r   r   r7   Ú
LongTensorrs   rq   r   r   rð   r   rl   ru   rv   s   @rB   rx  rx  Ç  s§  ø€ € € € € ðð ð
 #'ÐØ€NØÐØÐð 7àˆNÐEÈQÐ[gÐhÑhÔhð
ð ˆNÐEÈQÐ[lÐmÑmÔmð
ðð ÐðÐ=ð ð ð ð ð ð ð/ð /ð /ð0ð 0ð 0ð  ØØð 48Ø04Ø,0Ø:>Ø;?ð9
ð 9
àÔ#ð9
ð Ô)¨DÑ0ð9
ð Ô&¨Ñ-ð	9
ð
 ”l TÑ)ð9
ð  %Ô0°4Ñ7ð9
ð !&Ô 1°DÑ 8ð9
ð Ð+Ô,ð9
ð 
ˆuÔ Ô	!Ð$PÑ	Pð9
ð 9
ð 9
ñ „^ñ „_ñ  Ôð9
ð 9
ð 9
ð 9
ð 9
rC   rx  zV
    Class defining the outputs of [`InstructBlipVideoForConditionalGeneration`].
    )Úcustom_introc                   óÂ   — e Zd ZU dZdZeej                 dz  ed<   dZ	eej                 dz  ed<   dZ
edz  ed<   dZedz  ed<   dZeez  dz  ed<   dee         fd	„ZdS )
Ú4InstructBlipVideoForConditionalGenerationModelOutputaª  
    loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Language modeling loss from the language model.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head of the language model.
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
        Outputs of the language model.
    NÚlossÚlogitsÚvision_outputsÚqformer_outputsÚlanguage_model_outputsrG   c                 ó^   ‡ — t          ˆ fd„‰                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ót   •K  — | ]2}|d vr‰|         n!t          ‰|¦  «                             ¦   «         V — Œ3dS )©r‹  rŒ  r�  N)ÚgetattrÚto_tuple)r  Úkr@   s     €rB   ú	<genexpr>zPInstructBlipVideoForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>F  sf   øè è € ð 
ð 
ð ð ÐWÐWÐWð �ŒGˆGå˜˜qÑ!Ô!×*Ò*Ñ,Ô,ð
ð 
ð 
ð 
ð 
ð 
rC   )rð   Úkeysr%  s   `rB   r’  z=InstructBlipVideoForConditionalGenerationModelOutput.to_tupleE  sE   ø€ Ýð 
ð 
ð 
ð 
ð —Y’Y‘[”[ð	
ñ 
ô 
ñ 
ô 
ð 	
rC   )rn   ro   rp   r–   r‰  rð   r7   rs   r±   rŠ  r‹  r   rŒ  r   r�  r   r   r   r’  r  rC   rB   rˆ  rˆ  +  sÃ   € € € € € € ðð ð -1€Dˆ%�Ô!Ô
" TÑ
)Ð0Ð0Ñ0Ø.2€FˆE�%Ô#Ô$ tÑ+Ð2Ð2Ñ2Ø8<€NÐ.°Ñ5Ð<Ð<Ñ<ØKO€OÐAÀDÑHÐOÐOÑOØNRÐÐ2°_ÑDÀtÑKÐRÐRÑRð
˜% œ*ð 
ð 
ð 
ð 
ð 
ð 
rC   rˆ  z`
    InstructBlipVideo base Model consisting of language model, qformer and vision encoder.
    c                   óN  ‡ — e Zd ZdZdgZdefˆ fd„Zd„ Zdej	        dej
        fd„Zee	 	 	 	 	 	 	 	 ddej
        dej
        dej	        d	z  dej
        d	z  dej	        d	z  dej	        d	z  dej	        d	z  dej        d	z  deded	z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )r«   r_   r­   r*   c                 óæ  •— t          ¦   «                              |¦  «         t          |j        ¦  «        | _        t          j        t          j        d|j	        |j
        j        ¦  «        ¦  «        | _        t          |j
        ¦  «        | _        t          j        |j
        j        |j        j        ¦  «        | _        t%          j        |j        ¦  «        | _        |                      ¦   «          d S ©Nr$   )r0   r1   r  Úvision_configÚvision_modelr   r6   r7   rÛ   Únum_query_tokensÚqformer_configr2   r­   rx  ÚqformerrØ   Útext_configÚlanguage_projectionr!   Úfrom_configÚlanguage_modelr  r?   s     €rB   r1   zInstructBlipVideoModel.__init__W  s¹   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å8¸Ô9MÑNÔNˆÔÝœL­¬°Q¸Ô8OÐQWÔQfÔQrÑ)sÔ)sÑtÔtˆÔÝ4°VÔ5JÑKÔKˆŒå#%¤9¨VÔ-BÔ-NÐPVÔPbÔPnÑ#oÔ#oˆÔ Ý'Ô3°FÔ4FÑGÔGˆÔð 	�ŠÑÔÐÐÐrC   c                 ó
  — | j         }t          |¦  «        dk    r@d|vr<t          j                             ¦   «         dk    rt
                               d¦  «         t          | j        d¦  «        rd| j        j	        _
        dS dS ©z­
        Some pre-processing hacks to make the model `accelerate` compatible. Check
        https://github.com/huggingface/transformers/pull/21707 for more details.
        r$   r¡  a   The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models.Ú_hf_hookTN©Úhf_device_mapÚlenr7   ÚcudaÚdevice_countÚloggerÚwarningr-  r¡  r¤  Úio_same_device©r@   r¦  s     rB   Ú_preprocess_acceleratez-InstructBlipVideoModel._preprocess_accelerated  ó–   € ð
 Ô*ˆåˆ}ÑÔ Ò!Ð!Ð&6¸mÐ&KÐ&KÕPUÔPZ×PgÒPgÑPiÔPiÐlmÒPmÐPmå�NŠNðMñô ð õ �4Ô&¨
Ñ3Ô3ð 	?Ø:>ˆDÔÔ(Ô7Ð7Ð7ð	?ð 	?rC   r‘   r  c                 óN  — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     d¦  «         	                    |j        ¦  «        }|S ©zZ
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
        N©ra   r�   rI   )
r&  r7   rà   r*   Úimage_token_idÚlongr�   ÚallÚ	unsqueezerc   ©r@   r‘   r  Úspecial_image_masks       rB   Úget_placeholder_maskz+InstructBlipVideoModel.get_placeholder_maskx  ó£   € ð ÐØ!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;ÐÐà!*¨d¬kÔ.HÒ!HÐà/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØ!Ð!rC   NFÚqformer_input_idsÚqformer_attention_maskr¿   Údecoder_input_idsÚdecoder_attention_maskr^   Ú	use_cacherÈ   rG   c           	      óR  — |j         \  }}}}}|                     ||z  |||¦  «        } | j        d||	dœ|¤Ž}|d         }t          j        |                     ¦   «         dd…         t          j        |j        ¬¦  «        }| j         	                    |j         d         dd¦  «        }t          j        |                     ¦   «         dd…         t          j        |j        ¬¦  «        }|€t          j
        |¦  «        }|                     |d¬¦  «        }|                     |d¬¦  «        }|                     |j        ¦  «        }t          j        ||gd¬¦  «        } | j        d|||||dœ|¤Ž}|d         dd…d|                     d¦  «        …dd…f         }|                      |¦  «        }|                     || j        j        |z  d¦  «        }|€I | j                             ¦   «         |¦  «        }|| j        j        k    }|€t          j
        |¦  «        }nd| |                      ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }|                     d¦  «                             |j        ¦  «        }|                     |j        |j        ¦  «        }|                     ||¦  «        }| j        j        r | j        d|||
d	œ|¤Ž}n | j        d|||||
d
œ|¤Ž}t7          |||¬¦  «        S )aÕ  
        qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
            to serve as text prompt, which the Q-Former model will encode.

            Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for
            details.

            [What are input IDs?](../glossary#input-ids)
        qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

            Only relevant in case an encoder-decoder language model (like T5) is used.
        ©r_   r^   r   NrI   r²  rN   r$   ©r‘   r¿   r’   r5  r6  ©r  r¿   r¿  )r  r¿   r½  r¾  r¿  r�  r  )rP   rS   rš  r7   ÚonesrK   r´  r�   r­   rf   Ú	ones_likeÚrepeat_interleaverc   rX   r�  rŸ  r*   r›  r¡  r&  Úvideo_token_idrà   rµ  r¶  ra   Úmasked_scatterÚuse_decoder_only_language_modelrˆ  )r@   r_   r»  r¼  r‘   r¿   r½  r¾  r  r^   r¿  rÈ   rg   ÚframesÚchannelrE   rF   r‹  Úimage_embedsÚimage_attention_maskr­   Úquery_attention_maskÚquery_outputsÚquery_outputÚlanguage_model_inputsr¸  Úoutputss                              rB   rl   zInstructBlipVideoModel.forward‡  sº  € ðP 6BÔ5GÑ2ˆ
�F˜G V¨UØ#×+Ò+¨J¸Ñ,?ÀÈ&ÐRWÑXÔXˆà*˜Ô*ð 
Ø%Ø%=ð
ð 
ð ð
ð 
ˆð
 & aÔ(ˆõ  %œz¨,×*;Ò*;Ñ*=Ô*=¸c¸r¸cÔ*BÍ%Ì*Ð]iÔ]pÐqÑqÔqÐð Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ2ÑNÔNˆÝ$œz¨,×*;Ò*;Ñ*=Ô*=¸c¸r¸cÔ*BÍ%Ì*Ð]iÔ]pÐqÑqÔqÐà!Ð)Ý%*¤_Ð5FÑ%GÔ%GÐ"à-×?Ò?ÀÈAÐ?ÑNÔNÐØ!7×!IÒ!IÈ&ÐVWÐ!IÑ!XÔ!XÐØ!7×!:Ò!:Ð;OÔ;VÑ!WÔ!WÐÝ!&¤Ð,@ÐBXÐ+YÐ_`Ð!aÑ!aÔ!aÐØ$˜œð 
Ø'Ø1Ø%Ø".Ø#7ð
ð 
ð ð
ð 
ˆð % QÔ'¨¨¨Ð+A¨\×->Ò->¸qÑ-AÔ-AÐ+AÀ1À1À1Ð(DÔEˆð !%× 8Ò 8¸Ñ FÔ FÐð !6× =Ò =¸jÈ$Ì+ÔJfÐioÑJoÐqsÑ tÔ tÐØÐ ØF˜DÔ/×DÒDÑFÔFÀyÑQÔQˆMØ!*¨d¬kÔ.HÒ!HÐØÐ%Ý!&¤°Ñ!;Ô!;�øà!.Ð2M°$×2KÒ2KÑ2MÔ2MÝ”˜Tœ[Ô7½u¼zÐR_ÔRfÐgÑgÔgñ3ô 3ò "Ðð "4×!7Ò!7¸Ñ!;Ô!;Ðà/×9Ò9¸"Ñ=Ô=×@Ò@ÀÔAUÑVÔVÐØ 5× 8Ò 8¸Ô9MÈ}ÔObÑ cÔ cÐØ%×4Ò4Ð5GÐI^Ñ_Ô_ˆàŒ;Ô6ð 	Ø)�dÔ)ð Ø+Ø-Ø#ðð ð ð	ð ˆGˆGð *�dÔ)ð Ø+Ø-Ø"3Ø'=Ø#ðð ð ðð ˆGõ DØ)Ø)Ø#*ð
ñ 
ô 
ð 	
rC   )NNNNNNFN)rn   ro   rp   r(  Ú_keep_in_fp32_modulesr%   r1   r®  r7   r…  rs   r¹  r   r   rq   rt   r   r   rð   rˆ  rl   ru   rv   s   @rB   r«   r«   N  s¦  ø€ € € € € ð %€OØ+Ð,ÐðÐ6ð ð ð ð ð ð ð?ð ?ð ?ð("¨eÔ.>ð "ÈuÔO`ð "ð "ð "ð "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø-1Ø).Ø!%ðq
ð q
àÔ'ðq
ð !Ô,ðq
ð !&Ô 0°4Ñ 7ð	q
ð
 Ô$ tÑ+ðq
ð Ô(¨4Ñ/ðq
ð !Ô+¨dÑ2ðq
ð !&Ô 0°4Ñ 7ðq
ð ”| dÑ*ðq
ð #'ðq
ð ˜$‘;ðq
ð Ð+Ô,ðq
ð 
ÐEÑ	Eðq
ð q
ð q
ñ „^ñ Ôðq
ð q
ð q
ð q
ð q
rC   r«   c                   ó<   — e Zd ZU dZdZedz  ed<   dZedz  ed<   dS )Ú'BaseModelOutputWithVisionQformerOutputszÝ
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    Nr‹  rŒ  )	rn   ro   rp   r–   r‹  r   r±   rŒ  r   r  rC   rB   rÕ  rÕ  ý  sJ   € € € € € € ðð ð 9=€NÐ.°Ñ5Ð<Ð<Ñ<ØKO€OÐAÀDÑHÐOÐOÑOÐOÐOrC   rÕ  a¯  
    InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                   óè  ‡ — e Zd ZU eed<   dZdZdgZdefˆ fd„Zd„ Z	de
j        fd„Zdˆ fd
„	Zd„ Zd„ Zdej        dej        fd„Zee	 	 	 	 	 	 	 	 	 ddej        dej        dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  deded	z  dee         deez  fd„¦   «         ¦   «         Z ej        ¦   «         	 	 	 	 	 	 ddej        dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  dedej        fd„¦   «         Zee	 	 d dej        dej        dej        d	z  ded	z  dee         deez  fd„¦   «         ¦   «         Zˆ xZ S )!rª   r*   r_   Tr­   c                 óX  •— t          ¦   «                              |¦  «         t                               |j        ¦  «        | _        t          j        t          j	        d|j
        |j        j        ¦  «        ¦  «        | _        t                               |j        ¦  «        | _        t          j        |j        j        |j        j        ¦  «        | _        |j        rt)          j        |j        ¦  «        }nt-          j        |j        ¦  «        }|| _        |                      ¦   «          d S r˜  )r0   r1   r  Ú_from_configr™  rš  r   r6   r7   rÛ   r›  rœ  r2   r­   rx  r�  rØ   rž  rŸ  rÉ  r"   r   r#   r¡  r  )r@   r*   r¡  rA   s      €rB   r1   z2InstructBlipVideoForConditionalGeneration.__init__  së   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å8×EÒEÀfÔFZÑ[Ô[ˆÔåœL­¬°Q¸Ô8OÐQWÔQfÔQrÑ)sÔ)sÑtÔtˆÔÝ4×AÒAÀ&ÔBWÑXÔXˆŒå#%¤9¨VÔ-BÔ-NÐPVÔPbÔPnÑ#oÔ#oˆÔ àÔ1ð 	SÝ1Ô=¸fÔ>PÑQÔQˆNˆNå2Ô>¸vÔ?QÑRÔRˆNà,ˆÔð 	�ŠÑÔÐÐÐrC   c                 ó:   — | j                              |¦  «         d S rô   )r¡  Úset_output_embeddings)r@   Únew_embeddingss     rB   rÚ  z?InstructBlipVideoForConditionalGeneration.set_output_embeddings/  s   € ØÔ×1Ò1°.ÑAÔAÐAÐAÐArC   rG   c                 ó4   — | j                              ¦   «         S rô   )r¡  Úget_output_embeddingsr%  s    rB   rÝ  z?InstructBlipVideoForConditionalGeneration.get_output_embeddings2  s   € ØÔ"×8Ò8Ñ:Ô:Ð:rC   Nc                 ó~   •— |€| j                              ¦   «         S t          ¦   «                              |¬¦  «        S )N)Úmodality)r¡  Úget_encoderr0   )r@   rß  rA   s     €rB   rà  z5InstructBlipVideoForConditionalGeneration.get_encoder5  s9   ø€ ØÐØÔ&×2Ò2Ñ4Ô4Ð4å‘7”7×&Ò&°Ð&Ñ9Ô9Ð9rC   c                 ó4   — | j                              ¦   «         S rô   )r¡  Úget_decoderr%  s    rB   râ  z5InstructBlipVideoForConditionalGeneration.get_decoder;  s   € ØÔ"×.Ò.Ñ0Ô0Ð0rC   c                 ó
  — | j         }t          |¦  «        dk    r@d|vr<t          j                             ¦   «         dk    rt
                               d¦  «         t          | j        d¦  «        rd| j        j	        _
        dS dS r£  r¥  r­  s     rB   r®  z@InstructBlipVideoForConditionalGeneration._preprocess_accelerate>  r¯  rC   r‘   r  c                 óN  — |€e| |                       ¦   «         t          j        | j        j        t          j        |j        ¬¦  «        ¦  «        k    }|                     d¦  «        }n|| j        j        k    }|                     d¦  «         	                    |j        ¦  «        }|S r±  )
r&  r7   rà   r*   rÇ  r´  r�   rµ  r¶  rc   r·  s       rB   r¹  z>InstructBlipVideoForConditionalGeneration.get_placeholder_maskR  rº  rC   Fr»  r¼  r¿   r½  r¾  Úlabelsr^   r¿  rÈ   c           
      óT  —  | j         |f|||
dœ|¤Ž}|j        }|j        }|j        }|€ |                      ¦   «         |¦  «        }|€t          j        |¦  «        }|                     |j        |j	        ¦  «        }|  
                    ||¬¦  «        }|                     ||¦  «        }| j        j        r> | j        d	|||dœ|¤Ž}|d         }d}|	�  | j        d	||	| j        j        j        dœ|¤Ž}n" | j        d	|||||	|dœ|¤Ž}|j        }|j        }t)          |||||¬¦  «        S )
a˜  
        qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
            The sequence used as a prompt to be fed to the Q-Former module.
        qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
            Mask to avoid performing attention on padding token indices.

        Examples:

        ```python
        >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration
        >>> import torch
        >>> from huggingface_hub import hf_hub_download
        >>> import av
        >>> import numpy as np

        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`list[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])

        >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto")
        >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

        >>> file_path = hf_hub_download(
        ...       repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample uniformly 4 frames from the videWhy is this video funny?o
        >>> total_frames = container.streams.video[0].frames
        >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int)
        >>> clip = read_video_pyav(container, indices)

        >>> prompt = "What is happening in the video?"
        >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device)

        >>> outputs = model.generate(
        ...     **inputs,
        ...     do_sample=False,
        ...     num_beams=5,
        ...     max_length=256,
        ...     repetition_penalty=1.5,
        ...     length_penalty=1.0,
        ... )
        >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
        >>> print(generated_text)
        "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front"
        ```©r»  r¼  r^   N©r  rÃ  r   )rŠ  rå  r�   )r  r¿   r½  r¾  rå  r¿  )r‰  rŠ  r‹  rŒ  r�  r  )Úget_video_featuresr!  rŒ  r‹  r&  r7   rÅ  rc   r�   ra   r¹  rÈ  r*   rÉ  r¡  Úloss_functionrž  r�   r‰  rŠ  rˆ  )r@   r_   r»  r¼  r‘   r¿   r½  r¾  r  rå  r^   r¿  rÈ   Úvideo_featuresrÑ  rŒ  r‹  r¸  rÒ  rŠ  r‰  s                        rB   rl   z1InstructBlipVideoForConditionalGeneration.forwarda  sÚ  € ð` CZÀ$ÔBYØðC
à/Ø#9Ø%=ð	C
ð C
ð
 ðC
ð C
ˆð !/Ô <ÐØ(Ô8ˆØ'Ô6ˆàÐ Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ!Ý"œ_¨YÑ7Ô7ˆNà 5× 8Ò 8¸Ô9MÈ}ÔObÑ cÔ cÐØ!×6Ò6°yÐP]Ð6Ñ^Ô^ÐØ%×4Ò4Ð5GÐI^Ñ_Ô_ˆàŒ;Ô6ð 	$Ø)�dÔ)ð Ø+Ø-Ø#ðð ð ð	ð ˆGð ˜Q”ZˆFØˆDØÐ!Ø)�tÔ)ð Ø!¨&¸T¼[Ô=TÔ=_ðð Øciðð �øð
 *�dÔ)ð Ø+Ø-Ø"3Ø'=ØØ#ðð ð ðð ˆGð ”<ˆDØ”^ˆFåCØØØ)Ø+Ø#*ð
ñ 
ô 
ð 	
rC   c                 óì  — t          | d¦  «        r|                      ¦   «          |j        d         }	|                      ||||¬¦  «        }
|
j        }|€Ž|€o| j        j        g| j        j        z  dz  }|| j        j        j	        gz   }t          j        |gt          j        |j        ¬¦  «        }|                     |	d¦  «        } |                      ¦   «         |¦  «        }|€t          j        |¦  «        }|                     |j        |j        ¦  «        }|                      ||¬¦  «        }|                     ||¦  «        }||d	œ}| j        j        j        s||d
<    | j        j        di |¤|¤Ž}|S )aÙ  
        Overrides `generate` function to be able to use the model as a conditional generator.

        Args:
            pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or
                (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed.
            qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt to be fed to the Q-Former module.
            qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt for the generation.
            attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Embedded representation of the inputs. Should be float, not int tokens.
            interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the positional encoding of the image embeddings.

        Returns:
            captions (list): A list of strings of length batch_size * num_captions.
        r¦  r   rç  Nrç   r²  r$   rè  )r  r¿   r‘   r  )r-  r®  rP   ré  r!  r*   Úvideo_token_indexr›  rž  Úbos_token_idr7   rà   r´  r�   Úrepeatr&  rÅ  rc   ra   r¹  rÈ  r¡  Úis_encoder_decoderÚgenerate)r@   r_   r»  r¼  r‘   r¿   r  r^   Úgenerate_kwargsrg   rë  rÑ  Úvideo_tokensÚstart_tokensr¸  ÚinputsrÒ  s                    rB   rñ  z2InstructBlipVideoForConditionalGeneration.generateé  s¦  € õD �4˜Ñ)Ô)ð 	*à×'Ò'Ñ)Ô)Ð)à!Ô'¨Ô*ˆ
ØBF×BYÒBYØØ/Ø#9Ø%=ð	 CZñ C
ô C
ˆð !/Ô <ÐàÐ ØÐ Ø $¤Ô =Ð>ÀÄÔA]Ñ]Ð`aÑa�Ø+¨t¬{Ô/FÔ/SÐ.TÑT�Ý!œL¨,¨½u¼zÐR^ÔReÐfÑfÔf�	Ø%×,Ò,¨Z¸Ñ;Ô;�	Ø7˜D×5Ò5Ñ7Ô7¸	ÑBÔBˆMàÐ!Ý"œ_¨YÑ7Ô7ˆNà 5× 8Ò 8¸Ô9MÈ}ÔObÑ cÔ cÐØ!×6Ò6°yÐP]Ð6Ñ^Ô^ÐØ%×4Ò4Ð5GÐI^Ñ_Ô_ˆà#0ÀNÐSÐSˆØÔ"Ô)Ô<ð 	,Ø"+ˆF�;Ñà.�$Ô%Ô.ÐKÐK°ÐK¸?ÐKÐKˆàˆrC   c           	      ó  — |j         \  }}}}	}
|                     ||z  ||	|
¦  «        } | j        d
||dœ|¤Ž}t          |j        |j        |j        |j        |d¬¦  «        }|d         }t          j	        | 
                    ¦   «         dd…         t          j        |j        ¬¦  «        }| j                             |j         d         dd¦  «        }t          j	        | 
                    ¦   «         dd…         t          j        |j        ¬¦  «        }|€t          j        |¦  «        }|                     |d¬¦  «        }|                     |d¬¦  «        }|                     |j        ¦  «        }t          j        ||gd¬¦  «        } | j        d
|||||d	œ|¤Ž}||_        |d         dd…d| 
                    d¦  «        …dd…f         }|                      |¦  «        }|                     || j        j        |z  d¦  «        }||_        |S )a  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
            The sequence used as a prompt to be fed to the Q-Former module.
        qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
            Mask to avoid performing attention on padding token indices.
        rÁ  N)r  r!  rå   r  r‹  rŒ  r   rI   r²  rN   r$   rÂ  r  )rP   rS   rš  rÕ  r  r!  rå   r  r7   rÄ  rK   r´  r�   r­   rf   rÅ  rÆ  rc   rX   r�  rŒ  rŸ  r*   r›  )r@   r_   r»  r¼  r^   rÈ   rg   rÊ  rË  rE   rF   r‹  rÌ  rÍ  r­   rÎ  rŒ  rÐ  rë  s                      rB   ré  z<InstructBlipVideoForConditionalGeneration.get_video_features/  si  € ð( 6BÔ5GÑ2ˆ
�F˜G V¨UØ#×+Ò+¨J¸Ñ,?ÀÈ&ÐRWÑXÔXˆà5F°TÔ5Fð 6
Ø%Ø%=ð6
ð 6
ð ð6
ð 6
ˆõ
 AØ,Ô>Ø(Ô6Ø(Ô6Ø%Ô0Ø)Ø ð
ñ 
ô 
ˆð & aÔ(ˆõ  %œz¨,×*;Ò*;Ñ*=Ô*=¸c¸r¸cÔ*BÍ%Ì*Ð]iÔ]pÐqÑqÔqÐð Ô(×/Ò/°Ô0BÀ1Ô0EÀrÈ2ÑNÔNˆÝ$œz¨,×*;Ò*;Ñ*=Ô*=¸c¸r¸cÔ*BÍ%Ì*Ð]iÔ]pÐqÑqÔqÐà!Ð)Ý%*¤_Ð5FÑ%GÔ%GÐ"à-×?Ò?ÀÈAÐ?ÑNÔNÐØ!7×!IÒ!IÈ&ÐVWÐ!IÑ!XÔ!XÐØ!7×!:Ò!:Ð;OÔ;VÑ!WÔ!WÐÝ!&¤Ð,@ÐBXÐ+YÐ_`Ð!aÑ!aÔ!aÐØ&˜$œ,ð 
Ø'Ø1Ø%Ø".Ø#7ð
ð 
ð ð
ð 
ˆð *9ˆÔ&Ø& qÔ)¨!¨!¨!Ð-C¨|×/@Ò/@ÀÑ/CÔ/CÐ-CÀQÀQÀQÐ*FÔGˆð ×1Ò1°,Ñ?Ô?ˆð (×/Ò/°
¸D¼KÔ<XÐ[aÑ<aÐceÑfÔfˆØ'5ˆÔ$àÐrC   rô   )	NNNNNNNFN)NNNNNFr'  )!rn   ro   rp   r%   r±   r(  r¹   rÓ  r1   rÚ  r   ÚModulerÝ  rà  râ  r®  r7   r…  rs   r¹  r   r   rt   r   r   rð   rˆ  rl   r»   rñ  rÕ  ré  ru   rv   s   @rB   rª   rª     s”  ø€ € € € € € ð $Ð#Ð#Ñ#Ø$€Oà!ÐØ+Ð,ÐðÐ6ð ð ð ð ð ð ð(Bð Bð Bð; r¤yð ;ð ;ð ;ð ;ð:ð :ð :ð :ð :ð :ð1ð 1ð 1ð?ð ?ð ?ð("¨eÔ.>ð "ÈuÔO`ð "ð "ð "ð "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø26Ø*.Ø).Ø!%ðD
ð D
àÔ'ðD
ð !Ô,ðD
ð !&Ô 0°4Ñ 7ð	D
ð
 Ô$ tÑ+ðD
ð Ô(¨4Ñ/ðD
ð !Ô+¨dÑ2ðD
ð !&Ô 0°4Ñ 7ðD
ð Ô(¨4Ñ/ðD
ð Ô  4Ñ'ðD
ð #'ðD
ð ˜$‘;ðD
ð Ð+Ô,ðD
ð 
ÐEÑ	EðD
ð D
ð D
ñ „^ñ ÔðD
ðL €U„]�_„_ð 6:Ø:>Ø-1Ø26Ø26Ø).ðCð CàÔ'ðCð !Ô+¨dÑ2ðCð !&Ô 0°4Ñ 7ð	Cð
 Ô# dÑ*ðCð Ô(¨4Ñ/ðCð Ô(¨4Ñ/ðCð #'ðCð 
Ô	ðCð Cð Cñ „_ðCðJ Øð
 ;?Ø05ðDð DàÔ'ðDð !Ô+ðDð !&Ô 0°4Ñ 7ð	Dð
 #'¨¡+ðDð Ð+Ô,ðDð 
Ð8Ñ	8ðDð Dð Dñ „^ñ ÔðDð Dð Dð Dð DrC   rª   )r  r˜   rx  r«   rª   )r¢   )QÚcollections.abcr   Údataclassesr   Útypingr   r7   r   Ú r   r¨   Úactivationsr	   Ú
generationr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úautor!   r"   r#   Úconfiguration_instructblipvideor%   r&   r'   Ú
get_loggerrn   rª  r÷  r)   rx   r˜   rq   ÚfloatrË   rœ   rò   r�   r  r  rŸ   r    rI  rP  rV  rž   rp  rx  rˆ  r«   rÕ  rª   Ú__all__r  rC   rB   ú<module>r     så  ðð, %Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø )Ð )Ð )Ð )Ð )Ð )Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jÐ jØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IÐ Iðð ð ð ð ð ð ð ð ð ð 
ˆÔ	˜HÑ	%Ô	%€ðGð Gð Gð Gð G¨¬	ñ Gô Gð GðT/ð /ð /ð /ð /¨¬ñ /ô /ð /ðd ð ið  ið  ið  ið  i ñ  iô  iñ „ð iðV ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð.G)ð G)ð G)ð G)ð G) ¤ñ G)ô G)ð G)ðTð ð ð ð ˜2œ9ñ ô ð ðð ð ð ð Ð$>ñ ô ð ð@@ð @ð @ð @ð @˜rœyñ @ô @ð @ð@3ð 3ð 3ð 3ð 3Ð#Cñ 3ô 3ð 3ðlE)ð E)ð E)ð E)ð E)°´ñ E)ô E)ð E)ðPð ð ð ð ¨¬ñ ô ð ð ð  ð  ð  ð  ¨¬	ñ  ô  ð  ð2ð ð ð ð ¨2¬9ñ ô ð ðð ð ð ð  R¤Yñ ô ð ðRð Rð Rð Rð RÐ$>ñ Rô Rð Rðj!
ð !
ð !
ð !
ð !
 b¤iñ !
ô !
ð !
ðHa
ð a
ð a
ð a
ð a
Ð$Dñ a
ô a
ð a
ðH €ððñ ô ð
 ð
ð 
ð 
ð 
ð 
¸;ñ 
ô 
ñ „ñô ð
ð: €ððñ ô ð
g
ð g
ð g
ð g
ð g
Ð=ñ g
ô g
ñô ð
g
ðT Ø
ð	Pð 	Pð 	Pð 	Pð 	PÐ.Hñ 	Pô 	Pñ „ñ „ð	Pð €ððñ ô ðað að að að aÐ0PÐRañ aô añô ðaðHð ð €€€rC   