§
    ‚Štj–’  ã                   ó¨  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlmZ  ej        e¦  «        Zee G d„ de¦  «        ¦   «         ¦   «         Z G d„ dej         ¦  «        Z! G d„ dej         ¦  «        Z" G d„ dej         ¦  «        Z# G d„ dej         ¦  «        Z$ G d„ dej         ¦  «        Z% G d„ dej         ¦  «        Z& G d„ dej         ¦  «        Z' G d „ d!e¦  «        Z( G d"„ d#ej         ¦  «        Z) G d$„ d%ej         ¦  «        Z*e G d&„ d'e¦  «        ¦   «         Z+ G d(„ d)ej         ¦  «        Z, G d*„ d+ej         ¦  «        Z-e,e-d,œZ. ed-¬.¦  «         G d/„ d0e+¦  «        ¦   «         Z/ G d1„ d2ej         ¦  «        Z0 ed3¬.¦  «         G d4„ d5e+¦  «        ¦   «         Z1g d6¢Z2dS )7zPyTorch TVP Modelé    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)Úload_backbone)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	TvpConfigc                   ó¬   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej        df         dz  ed<   dZe
ej        df         dz  ed<   dS )ÚTvpVideoGroundingOutputa€  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Temporal-Distance IoU loss for video grounding.
    logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
        Contains start_time/duration and end_time/duration. It is the time slot of the videos corresponding to the
        input texts.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
    NÚlossÚlogits.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   © ó    úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/tvp/modeling_tvp.pyr   r   $   s’   € € € € € € ð	ð 	ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø:>€M�5˜Ô*¨CÐ/Ô0°4Ñ7Ð>Ð>Ñ>Ø7;€J��eÔ'¨Ð,Ô-°Ñ4Ð;Ð;Ñ;Ð;Ð;r"   r   c                   ó:   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zd„ Zd„ Zˆ xZ	S )ÚTvpLossa~  
    This class computes the losses for `TvpForVideoGrounding`. The process happens in two steps: 1) we compute
    hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair of matched
    ground-truth / prediction (supervise class and box).

    Args:
        losses (`list[str]`):
            List of all the losses to be applied.
    c                 óÌ   •— t          ¦   «                              ¦   «          | j        | j        | j        dœ| _        |D ]}|| j        vrt          d|› d�¦  «        ‚Œ|| _        d S )N©ÚiouÚdistanceÚdurationzLoss z not supported)ÚsuperÚ__init__Úloss_iouÚloss_distanceÚloss_durationÚloss_mapÚ
ValueErrorÚlosses)Úselfr2   r   Ú	__class__s      €r#   r,   zTvpLoss.__init__C   s‚   ø€ Ý‰Œ×ÒÑÔÐà”=ØÔ*ØÔ*ð
ð 
ˆŒð
 ð 	?ð 	?ˆDØ˜4œ=Ð(Ð(Ý Ð!=¨Ð!=Ð!=Ð!=Ñ>Ô>Ð>ð )ð ˆŒˆˆr"   c                 óê   — t          j        ||¦  «        t          j        ||¦  «        z
  }t          j        ||¦  «        t          j        ||¦  «        z
  }d|                     d¬¦  «        |z  z
  }|S )z6
        Measure the intersection over union.
        r   r   ©Úmin)r   r7   ÚmaxÚclamp)	r3   Ú
start_timeÚend_timeÚcandidates_start_timeÚcandidates_end_timer*   ÚinterÚunionr(   s	            r#   r-   zTvpLoss.loss_iouP   sp   € õ ”	Ð-¨xÑ8Ô8½5¼9ÐEZÐ\fÑ;gÔ;gÑgˆÝ”	Ð-¨xÑ8Ô8½5¼9ÐEZÐ\fÑ;gÔ;gÑgˆØ�%—+’+ !�+Ñ$Ô$ uÑ,Ñ,ˆàˆ
r"   c                 óJ  — t          j        t          j        ||¦  «        d¦  «        }t          j        t          j        ||¦  «        d¦  «        }t          j        t          j        ||¦  «        t          j        ||¦  «        z
  |¦  «                             d¬¦  «        }|S )z5
        Measure the distance of mid points.
        g       @gš™™™™™É?r6   )r   ÚdivÚaddr8   r7   r9   )	r3   r:   r;   r<   r=   r*   Úmid_candidatesÚmid_groundtruthÚdistance_diffs	            r#   r.   zTvpLoss.loss_distanceZ   sŽ   € õ œ¥5¤9Ð-BÐDWÑ#XÔ#XÐZ]Ñ^Ô^ˆÝœ)¥E¤I¨j¸(Ñ$CÔ$CÀSÑIÔIˆÝœ	ÝŒI�n oÑ6Ô6½¼À>ÐSbÑ9cÔ9cÑcÐemñ
ô 
ç
Š%�Cˆ%‰.Œ.ð 	ð Ðr"   c                 óú   — t          j        ||¦  «        }t          j        ||¦  «        }t          j        t          j        t          j        ||¦  «        |¦  «        ¦  «        }|                     d¬¦  «        }|S )z5
        Measure the difference of duration.
        gš™™™™™Ù?r6   )r   ÚsubÚsquarerA   r9   )	r3   r:   r;   r<   r=   r*   Úduration_candidatesÚduration_groundtruthÚduration_diffs	            r#   r/   zTvpLoss.loss_durationf   sp   € õ $œiÐ(;Ð=RÑSÔSÐÝ$œy¨°:Ñ>Ô>ÐÝœ¥U¤Y­u¬yÐ9LÐNbÑ/cÔ/cÐemÑ%nÔ%nÑoÔoˆØ%×+Ò+°Ð+Ñ4Ô4ˆàÐr"   c                 ó*  — |\  }}}t          j        ||¦  «        }|dd…df                              ¦   «         |dd…df                              ¦   «         }}i }	| j        D ]1}
|	                     |
 | j        |
         |||||¦  «        i¦  «         Œ2|	S )am  
        This performs the loss computation.

        Args:
            logits (`torch.FloatTensor`):
                The output logits of head module.
            labels (`list[torch.FloatTensor]`):
                List of tensors ([start, end, duration]), which contains start time, end time of the video corresponding to the text, and also the duration.
        Nr   r   )r   ÚmulÚfloatr2   Úupdater0   )r3   r   Úlabelsr*   r:   r;   Ú
candidatesr<   r=   Úlosses_dictr   s              r#   ÚforwardzTvpLoss.forwardq   sÃ   € ð *0Ñ&ˆ�*˜hÝ”Y˜v xÑ0Ô0ˆ
Ø5?ÀÀÀÀ1ÀÔ5E×5KÒ5KÑ5MÔ5MÈzÐZ[ÐZ[ÐZ[Ð]^ÐZ^ÔO_×OeÒOeÑOgÔOgÐ2ÐàˆØ”Kð 	ð 	ˆDØ×ÒØÐ*�t”} TÔ*¨:°xÐAVÐXkÐmuÑvÔvÐwñô ð ð ð Ðr"   )
r   r   r   r   r,   r-   r.   r/   rS   Ú__classcell__©r4   s   @r#   r%   r%   8   s~   ø€ € € € € ðð ðð ð ð ð ðð ð ð
ð 
ð 
ð	ð 	ð 	ðð ð ð ð ð ð r"   r%   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVisionModelc           	      ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        |j        �|j        j        d         }n—t          | j        d¦  «        r2t          | j        j        d¦  «        r| j        j        j        d         }nPt          | j        d¦  «        r,t          | j        j        d¦  «        r| j        j        j        }nt          d¦  «        ‚t          j        ||j        ddddd¬	¦  «        | _        d S )
NéÿÿÿÿÚconfigÚhidden_sizesÚhidden_sizezBackbone config not foundr   r   F)Úkernel_sizeÚstrideÚpaddingÚgroupsÚbias)r+   r,   r   ÚbackboneÚbackbone_configr[   ÚhasattrrZ   r\   r1   r   ÚConv2dÚgrid_encoder_conv)r3   rZ   Úin_channelsr4   s      €r#   r,   zTvpVisionModel.__init__‰   s  ø€ Ý‰Œ×ÒÑÔÐÝ% fÑ-Ô-ˆŒàÔ!Ð-Ø Ô0Ô=¸bÔAˆKˆKÝ�T”] HÑ-Ô-ð 	:µ'¸$¼-Ô:NÐP^Ñ2_Ô2_ð 	:Øœ-Ô.Ô;¸BÔ?ˆKˆKÝ�T”] HÑ-Ô-ð 	:µ'¸$¼-Ô:NÐP]Ñ2^Ô2^ð 	:Øœ-Ô.Ô:ˆKˆKåÐ8Ñ9Ô9Ð9å!#¤ØØÔØØØØØð"
ñ "
ô "
ˆÔÐÐr"   c                 óÒ  — |j         \  }}}}}|                     ||z  |||¦  «        }|                      |¦  «        d         d         }|                      |¦  «        }t          j                             |dd¬¦  «        }t          j                             |d¬¦  «        }|j         dd …         \  }	}
}|                     |||	|
|¦  «        }|                     ddd	d
d¦  «        }|S )NÚfeature_mapsr   é   )r]   r^   T)Úinplaceéýÿÿÿr   r   é   )	ÚshapeÚviewrb   rf   r   Ú
functionalÚ
max_pool2dÚreluÚpermute)r3   Úpixel_valuesÚ
batch_sizeÚ
num_framesÚnum_channelsÚheightÚwidthÚgrid_feat_outputsÚgridÚnew_channelÚ
new_heightÚ	new_widths               r#   rS   zTvpVisionModel.forward    sè   € Ø>JÔ>PÑ;ˆ
�J ¨f°eà#×(Ò(¨°jÑ)@À,ÐPVÐX]Ñ^Ô^ˆØ ŸMšM¨,Ñ7Ô7¸ÔGÈÔJÐØ×%Ò%Ð&7Ñ8Ô8ˆÝŒ}×'Ò'¨¸!ÀAÐ'ÑFÔFˆÝŒ}×!Ò! $°Ð!Ñ5Ô5ˆØ-1¬Z¸¸¸¬_Ñ*ˆ�Z à�yŠy˜ Z°¸jÈ)ÑTÔTˆà�|Š|˜A˜q ! Q¨Ñ*Ô*ˆØˆr"   ©r   r   r   r,   rS   rT   rU   s   @r#   rW   rW   ˆ   sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð.ð ð ð ð ð ð r"   rW   c                   ój   ‡ — e Zd ZdZˆ fd„Zdej        dededej        fd„Zdd	e	fd
„Z
dd	e	fd„Zˆ xZS )ÚTvpVisualInputEmbeddingz;
    Takes input of both image and video (multi-frame)
    c                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¦  «        | _        t          j        |j	        |j        ¦  «        | _
        t          j        d|j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        |j	        | _	        d S )Nr   ©Úeps)r+   r,   r   Ú	EmbeddingÚmax_position_embeddingsr\   Úposition_embeddingsÚ max_grid_row_position_embeddingsÚrow_position_embeddingsÚ max_grid_col_position_embeddingsÚcol_position_embeddingsÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout©r3   rZ   r4   s     €r#   r,   z TvpVisualInputEmbedding.__init__µ   sÍ   ø€ Ý‰Œ×ÒÑÔÐå#%¤<°Ô0NÐPVÔPbÑ#cÔ#cˆÔ Ý')¤|°FÔ4[Ð]cÔ]oÑ'pÔ'pˆÔ$Ý')¤|°FÔ4[Ð]cÔ]oÑ'pÔ'pˆÔ$Ý%'¤\°!°VÔ5GÑ%HÔ%HˆÔ"Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"<Ñ=Ô=ˆŒØ06Ô0WˆÔ-Ø06Ô0WˆÔ-Ð-Ð-r"   Ú	embeddingrx   ry   Úreturnc                 ó  — dx}}|| j         k    r
|| j         z  }|| j        k    r
|| j        z  }|                     dddd¦  «        }t          j                             |||fdd¬¦  «        }|                     dddd¦  «        }|S )z¸
        This method allows to interpolate the pre-trained pad weights , to be able to use the model on collection of high
        resolution images (high resolution videos).

        r   r   r   rj   ÚbicubicF©Úscale_factorÚmodeÚalign_corners)rˆ   rŠ   rs   r   rp   Úinterpolate)r3   r”   rx   ry   Úh0Úw0s         r#   Úinterpolate_pos_encodingz0TvpVisualInputEmbedding.interpolate_pos_encodingÁ   s©   € ð ˆˆˆRà�DÔ9Ò9Ð9Ø˜$Ô?Ñ?ˆBà�4Ô8Ò8Ð8Ø˜Ô>Ñ>ˆBØ×%Ò% a¨¨A¨qÑ1Ô1ˆ	Ý”M×-Ò-ØØ˜b˜ØØð	 .ñ 
ô 
ˆ	ð ×%Ò% a¨¨A¨qÑ1Ô1ˆ	ØÐr"   FrŸ   c                 óL  — |j         \  }}}}t          | j        |¦  «        }t          j        |t          j        |j        ¬¦  «        }|                      |¦  «        }	dt          |j         ¦  «        dz
  z  |d|fz   }
 |	j	        |
Ž }	t          | j
        |¦  «        }t          j        |t          j        |j        ¬¦  «        }|                      |¦  «        }|d||f} |j	        |Ž }|	|z   }|r1|| j        k    s|| j
        k    r||                      |||¦  «        z   }n||z   }|S )af  
        Args:
            grid: (batch_size, height, width, hidden_dim)
            interpolate_pos_encoding: (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.
        Returns:
            grid + col_position_embeddings.view(*col_shape): (batch_size, *, height, width, hidden_dim)
        ©ÚdtypeÚdevice)r   r   r   )rn   r7   rˆ   r   ÚarangeÚlongr£   r‰   Úlenro   rŠ   r‹   rŸ   )r3   r{   rŸ   ru   rx   ry   Ú
hidden_dimÚ
row_heightÚrow_position_idsr‰   Ú	row_shapeÚ	row_widthÚcol_position_idsr‹   Ú	col_shapeÚpositional_embeddingss                   r#   Úadd_2d_positional_embeddingsz4TvpVisualInputEmbedding.add_2d_positional_embeddingsØ   sP  € ð 15´
Ñ-ˆ
�F˜E :õ ˜Ô>ÀÑGÔGˆ
Ý œ<¨
½%¼*ÈTÌ[ÐYÑYÔYÐà"&×">Ò">Ð?OÑ"PÔ"PÐØ�C ¤
™OœO¨aÑ/Ñ0°JÀÀ:Ð3NÑNˆ	à">Ð"9Ô">À	Ð"JÐõ ˜Ô=¸uÑEÔEˆ	Ý œ<¨	½¼ÈDÌKÐXÑXÔXÐà"&×">Ò">Ð?OÑ"PÔ"PÐØ  I¨zÐ:ˆ	à">Ð"9Ô">À	Ð"JÐà 7Ð:QÑ QÐð $ð 	0Ø�TÔ:Ò:Ð:¸eÀdÔFkÒ>kÐ>kà˜$×7Ò7Ð8MÈvÐW\Ñ]Ô]Ñ]ˆDˆDàÐ/Ñ/ˆDØˆr"   c                 óœ  — |j         \  }}}}}|                     d¦  «        }|                      ||¬¦  «        }|                     |d|¦  «        }|j         dd…         }	|j        }
t          j        |	t
          j        |
¬¦  «        }|                      |¦  «        }||z   }|  	                    |¦  «        }|  
                    |¦  «        }|S )a  
        Args:
            grid: Array of shape (batch_size, num_frames, height, width, num_channels).
                It contains processed frames extracted from videos, and is generated by Tvp image preprocessor. Note,
                num_frames can be 1
            interpolate_pos_encoding: (bool, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.

        Returns:
            embeddings: The embedding of grid with size (batch_size, height*width, num_channels)

        r   ©rŸ   rY   Nr¡   )rn   Úmeanr¯   ro   r£   r   Úzerosr¥   rŒ   r�   r’   )r3   r{   rŸ   ru   rv   rx   ry   rw   Úvisual_tokensÚvisual_tokens_shaper£   Útoken_type_idsrŒ   Ú
embeddingss                 r#   rS   zTvpVisualInputEmbedding.forward  sÑ   € ð ?C¼jÑ;ˆ
�J ¨¨|à�yŠy˜‰|Œ|ˆØ×0Ò0°ÐPhÐ0ÑiÔiˆàŸ	š	 *¨b°,Ñ?Ô?ˆØ+Ô1°#°2°#Ô6ÐØÔ%ˆõ œÐ%8ÅÄ
ÐSYÐZÑZÔZˆØ $× :Ò :¸>Ñ JÔ JÐà"Ð%:Ñ:ˆ
Ø—_’_ ZÑ0Ô0ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr"   ©F)r   r   r   r   r,   r   ÚTensorÚintrŸ   Úboolr¯   rS   rT   rU   s   @r#   r�   r�   °   s¿   ø€ € € € € ðð ð
Xð 
Xð 
Xð 
Xð 
Xð°%´,ð Èð ÐTWð Ð\aÔ\hð ð ð ð ð.'ð 'È4ð 'ð 'ð 'ð 'ðRð °dð ð ð ð ð ð ð ð r"   r�   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚTvpTextInputEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó´  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )N)Úpadding_idxrƒ   )r+   r,   r   r…   Ú
vocab_sizer\   Úpad_token_idÚword_embeddingsr†   r‡   Útype_vocab_sizerŒ   r�   rŽ   r�   r�   r‘   r’   r“   s     €r#   r,   zTvpTextInputEmbeddings.__init__$  s¥   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#%¤<°Ô0NÐPVÔPbÑ#cÔ#cˆÔ Ý%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr"   Nc                 ó^  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|�|j        n|j        }|€It          j        |t          j        |¬¦  «        }|                     d¦  «                             |¦  «        }|€!t          j        |t          j        |¬¦  «        }|€|                      |¦  «        }|  	                    |¦  «        }|  
                    |¦  «        }	||z   |	z   }
|                      |
¦  «        }
|                      |
¦  «        }
|
S )NrY   r   r¡   r   )Úsizer£   r   r¤   r¥   Ú	unsqueezeÚexpandr³   rÂ   r‡   rŒ   r�   r’   )r3   Ú	input_idsr¶   Úposition_idsÚinputs_embedsÚinput_shapeÚ
seq_lengthr£   r‡   rŒ   r·   s              r#   rS   zTvpTextInputEmbeddings.forward,  s(  € ØÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
Ø%.Ð%:�Ô!Ð!ÀÔ@TˆØÐÝ œ<¨
½%¼*ÈVÐTÑTÔTˆLØ'×1Ò1°!Ñ4Ô4×;Ò;¸KÑHÔHˆLØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNàÐ Ø ×0Ò0°Ñ;Ô;ˆMØ"×6Ò6°|ÑDÔDÐØ $× :Ò :¸>Ñ JÔ JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—_’_ ZÑ0Ô0ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr"   ©NNNN©r   r   r   r   r,   rS   rT   rU   s   @r#   r½   r½   !  sR   ø€ € € € € ØQÐQð>ð >ð >ð >ð >ðð ð ð ð ð ð ð r"   r½   c                   óT   ‡ — e Zd Zˆ fd„Zdej        dedefd„Z	 	 d	dedz  fd„Z	ˆ xZ
S )
ÚTvpAttentionc                 ó0  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r/t	          |d¦  «        st          d|j        › d|j        › �¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j        |j        ¦  «        | _        t          j
        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads rƒ   )r+   r,   r\   Únum_attention_headsrd   r1   rº   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer�   Úattention_probs_dropout_probÚattn_dropoutÚdenser�   rŽ   r�   r‘   r’   r“   s     €r#   r,   zTvpAttention.__init__F  sg  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð H 6Ô#5ð  Hð  HÐkqô  lFð  Hð  Hñô ð ð $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
ÝœJ vÔ'JÑKÔKˆÔå”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr"   ÚtensorÚsequence_lengthru   c                 ó’   — |                      ||| j        | j        ¦  «                             dd¦  «                             ¦   «         S )Nr   rj   )ro   rÓ   rÔ   Ú	transposeÚ
contiguous)r3   rÝ   rÞ   ru   s       r#   Ú_reshapezTvpAttention._reshapeZ  s8   € à�KŠK˜
 O°TÔ5MÈtÔOgÑhÔhßŠY�q˜!‰_Œ_ßŠZ‰\Œ\ð	
r"   NÚoutput_attentionsc                 ó~  — |j         d d…         \  }}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |||¦  «        }	|                      |||¦  «        }
|                      |||¦  «        }t          j        |	|
                     dd¦  «        ¦  «        }|t          j	        | j
        ¦  «        z  }|�||z   }t          j                             |d¬¦  «        }|                      |¦  «        }t          j        ||¦  «        }|                     dd¦  «                             ¦   «         }|                     ||| j        ¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|r||fn|f}|S )Nrj   rY   éþÿÿÿ©Údimr   )rn   r×   rØ   rÙ   râ   r   Úmatmulrà   ÚmathÚsqrtrÔ   r   rp   ÚsoftmaxrÛ   rá   ÚreshaperÕ   rÜ   r’   r�   )r3   r   Úattention_maskrã   ru   rÞ   Úmixed_query_layerÚmixed_key_layerÚmixed_value_layerÚquery_layerÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚattn_outputÚoutputss                   r#   rS   zTvpAttention.forwarda  s®  € ð '4Ô&9¸"¸1¸"Ô&=Ñ#ˆ
�OØ ŸJšJ }Ñ5Ô5ÐàŸ(š( =Ñ1Ô1ˆØ ŸJšJ }Ñ5Ô5Ðà—m’mÐ$5°È
ÑSÔSˆØ—M’M /°?ÀJÑOÔOˆ	Ø—m’mÐ$5°È
ÑSÔSˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐØ+­d¬i¸Ô8PÑ.QÔ.QÑQÐØÐ%Ø/°.Ñ@Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð ×+Ò+¨OÑ<Ô<ˆå”l ?°KÑ@Ô@ˆØ!×+Ò+¨A¨qÑ1Ô1×<Ò<Ñ>Ô>ˆØ!×)Ò)¨*°oÀtÔGYÑZÔZˆà—j’j Ñ-Ô-ˆØ—l’l ;Ñ/Ô/ˆØ—o’o k°MÑ&AÑBÔBˆà4EÐY�; Ð0Ð0ÈKÈ>ˆØˆr"   ©NN)r   r   r   r,   r   r¹   rº   râ   r»   rS   rT   rU   s   @r#   rÐ   rÐ   E  s’   ø€ € € € € ð>ð >ð >ð >ð >ð(
˜uœ|ð 
¸cð 
Èsð 
ð 
ð 
ð 
ð Ø)-ð	&ð &ð   $™;ð	&ð &ð &ð &ð &ð &ð &ð &r"   rÐ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚTvpIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S ©N)r+   r,   r   rÖ   r\   Úintermediate_sizerÜ   Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnr“   s     €r#   r,   zTvpIntermediate.__init__Œ  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r"   r   r•   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rü   )rÜ   r  )r3   r   s     r#   rS   zTvpIntermediate.forward”  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr"   ©r   r   r   r,   r   r¹   rS   rT   rU   s   @r#   rú   rú   ‹  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r"   rú   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚTvpOutputLayerc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _	        t          j
        |j        ¦  «        | _        d S )Nrƒ   )r+   r,   r   rÖ   rý   r\   rÜ   r�   rŽ   r�   r�   r‘   r’   r“   s     €r#   r,   zTvpOutputLayer.__init__›  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ, vÔ'9¸vÔ?TÐUÑUÔUˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr"   r   Úinput_tensorr•   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rü   )rÜ   r’   r�   )r3   r   r  s      r#   rS   zTvpOutputLayer.forward¡  s@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš¨¸Ñ(DÑEÔEˆØÐr"   r  rU   s   @r#   r  r  š  si   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r"   r  c                   ó6   ‡ — e Zd Zˆ fd„Z	 	 ddedz  fd„Zˆ xZS )ÚTvpEncodeLayerc                 óÀ   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        d S rü   )r+   r,   rÐ   Ú	attentionrú   Úintermediater  Úoutputr“   s     €r#   r,   zTvpEncodeLayer.__init__©  sK   ø€ Ý‰Œ×ÒÑÔÐÝ% fÑ-Ô-ˆŒÝ+¨FÑ3Ô3ˆÔÝ$ VÑ,Ô,ˆŒˆˆr"   Nrã   c                 ó¼   — |                       |||¬¦  «        }|d         }|dd …         }|                      |¦  «        }|                      ||¦  «        }|f|z   }|S )N)rã   r   r   )r  r  r  )	r3   r   rí   rã   Úself_attention_outputsÚattention_outputr÷   Úintermediate_outputÚlayer_outputs	            r#   rS   zTvpEncodeLayer.forward¯  s|   € ð "&§¢ØØØ/ð "0ñ "
ô "
Ðð
 2°!Ô4ÐØ(¨¨¨Ô,ˆØ"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØ�/ GÑ+ˆØˆr"   rø   )r   r   r   r,   r»   rS   rT   rU   s   @r#   r
  r
  ¨  sg   ø€ € € € € ð-ð -ð -ð -ð -ð Ø)-ð	ð ð   $™;ð	ð ð ð ð ð ð ð r"   r
  c            
       óX   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddedz  dedz  dedz  deez  fd„Zˆ xZS )	Ú
TvpEncoderc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r!   )r
  )Ú.0Ú_rZ   s     €r#   ú
<listcomp>z'TvpEncoder.__init__.<locals>.<listcomp>Æ  s!   ø€ Ð#dÐ#dÐ#d¸q¥N°6Ñ$:Ô$:Ð#dÐ#dÐ#dr"   F)	r+   r,   rZ   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingr“   s    `€r#   r,   zTvpEncoder.__init__Ã  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#dÐ#dÐ#dÐ#dÅEÈ&ÔJbÑDcÔDcÐ#dÑ#dÔ#dÑeÔeˆŒ
Ø&+ˆÔ#Ð#Ð#r"   Nrã   Úoutput_hidden_statesÚreturn_dictr•   c                 óf  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }d}d}t	          | j        ¦  «        D ]0\  }}	|r||fz   } |	|||¦  «        }
|
d         }|r||
d         fz   }Œ1|r||fz   }|s|f}|r||fz   }|r||fz   }|S t          ||r|nd |r|nd ¬¦  «        S )Nr!   r   r   )Úlast_hidden_stater   r   )rZ   r!  rã   r   Ú	enumerater  r   )r3   r   rí   rã   r   r!  Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputsr÷   s               r#   rS   zTvpEncoder.forwardÉ  sQ  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆØ1BÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð ÐØˆå(¨¬Ñ4Ô4ð 	Fð 	F‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜L¨¸ÐHYÑZÔZˆMà)¨!Ô,ˆMØ ð FØ!/°=ÀÔ3CÐ2EÑ!E�øð  ð 	EØ 1°]Ð4DÑ DÐàð 	Ø$Ð&ˆGØ#ð 9Ø!Ð%6Ð$8Ñ8�Ø ð 6Ø! ^Ð$5Ñ5�ØˆNåØ+Ø/CÐMÐ+Ð+ÈØ):ÐD�~�~Àð
ñ 
ô 
ð 	
r"   rÍ   )	r   r   r   r,   r»   r    r   rS   rT   rU   s   @r#   r  r  Â  s˜   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð Ø)-Ø,0Ø#'ð*
ð *
ð   $™;ð	*
ð
 # T™kð*
ð ˜D‘[ð*
ð 
�Ñ	 ð*
ð *
ð *
ð *
ð *
ð *
ð *
ð *
r"   r  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )Ú	TvpPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rü   )r+   r,   r   rÖ   r\   rÜ   ÚTanhÚ
activationr“   s     €r#   r,   zTvpPooler.__init__ø  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr"   r   r•   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rÜ   r.  )r3   r   Úfirst_token_tensorÚpooled_outputs       r#   rS   zTvpPooler.forwardý  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr"   r  rU   s   @r#   r+  r+  ÷  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r"   r+  c                   óp   ‡ — e Zd ZU eed<   dZdZdZ ej	        ¦   «         de
j        fˆ fd„¦   «         Zˆ xZS )ÚTvpPreTrainedModelrZ   Úmodel)ÚvideoÚtextTÚmodulec                 ó¢  •— t          ¦   «                              |¦  «         t          |t          j        ¦  «        r>t          j        |j        dd¬¦  «         |j        �t          j	        |j        d¦  «         n.t          |t          ¦  «        rt          j        |j        ¦  «         t          |d¦  «        rt          j        |j        ¦  «         t          |d¦  «        rt          j        |j        ¦  «         t          |d¦  «        rt          j        |j        ¦  «         t          |d	¦  «        rt          j        |j        ¦  «         dS dS )
zInitialize the weightsÚfan_outrr   )rš   ÚnonlinearityNr   Úpad_upÚpad_downÚpad_leftÚ	pad_right)r+   Ú_init_weightsrþ   r   re   ÚinitÚkaiming_normal_Úweightra   Ú	constant_ÚTvpModelÚnormal_Útext_promptrd   r;  r<  r=  r>  )r3   r7  r4   s     €r#   r?  z TvpPreTrainedModel._init_weights  s.  ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�bœiÑ(Ô(ð 	-ÝÔ  ¤°YÈVÐTÑTÔTÐTØŒ{Ð&Ý”˜vœ{¨AÑ.Ô.Ð.øÝ˜¥Ñ)Ô)ð 	-ÝŒL˜Ô+Ñ,Ô,Ð,å�6˜8Ñ$Ô$ð 	(ÝŒL˜œÑ'Ô'Ð'Ý�6˜:Ñ&Ô&ð 	*ÝŒL˜œÑ)Ô)Ð)Ý�6˜:Ñ&Ô&ð 	*ÝŒL˜œÑ)Ô)Ð)Ý�6˜;Ñ'Ô'ð 	+ÝŒL˜Ô)Ñ*Ô*Ð*Ð*Ð*ð	+ð 	+r"   )r   r   r   r   r   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingr   Úno_gradr   ÚModuler?  rT   rU   s   @r#   r3  r3    sx   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#à€U„]�_„_ð+ B¤Ið +ð +ð +ð +ð +ñ „_ð+ð +ð +ð +ð +r"   r3  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚTvpFrameDownPadPrompterz>
    Pad frames extracted from videos only at the bottom.
    c           	      óV  •— |j         dvrt          d¦  «        ‚t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        | _        |j         | _         t          j        t          j
        d|j        d|j        |j        g¦  «        ¦  «        | _        d S )N©rB   ÚreplaceÚremoveú9`visual_prompter_apply` must be in (add, replace, remove)r   r   )Úvisual_prompter_applyr1   r+   r,   Úvisual_prompt_sizeÚ	frame_numÚmax_img_sizer   Ú	Parameterr   Úrandnr<  r“   s     €r#   r,   z TvpFrameDownPadPrompter.__init__'  sš   ø€ ØÔ'Ð/KÐKÐKÝÐXÑYÔYÐYå‰Œ×ÒÑÔÐØ"(Ô";ˆÔØÔ)ˆŒØ"Ô/ˆÔØ%+Ô%AˆÔ"åœÝŒK˜˜FÔ,¨a°Ô1JÈFÔL_Ð`ÑaÔañ
ô 
ˆŒˆˆr"   c                 óä  — | j         dk    rOt          j        | j        | j        g|j        |j        ¬¦  «        }d|| j        | j        z
  | j        …d d …f<   ||z  }| j         dk    rŠt          j        |j        d         |j        d         d| j        | j        g|j        ¬¦  «        }| j        | j        z
  }| j	        |d d …d d …d d …|| j        …d d …f<   || 
                    |j        ¦  «        z  }|S )	NrB   r¡   g        rQ  r   r   r   ©r£   )rS  r   ÚonesrV  r¢   r£   rT  r³   rn   r<  Úto)r3   rt   Úvisual_prompt_maskÚpromptÚstart_points        r#   rS   zTvpFrameDownPadPrompter.forward5  s(  € ØÔ%¨Ò.Ð.Ý!&¤ØÔ" DÔ$5Ð6¸lÔ>PÐYeÔYlð"ñ "ô "Ðð fiÐ˜tÔ0°4Ô3JÑJÈTÔM^Ð^Ð`aÐ`aÐ`aÐaÑbØÐ.Ñ.ˆLØÔ%¨Ò1Ð1Ý”[ØÔ# AÔ&¨Ô(:¸1Ô(=¸qÀ$ÔBSÐUYÔUfÐgØ#Ô*ðñ ô ˆFð Ô+¨dÔ.EÑEˆKØBFÄ-ˆF�1�1�1�a�a�a˜˜˜˜K¨$Ô*;Ð;¸Q¸Q¸QÐ>Ñ?Ø˜FŸIšI lÔ&8Ñ9Ô9Ñ9ˆLØÐr"   rÎ   rU   s   @r#   rM  rM  "  sQ   ø€ € € € € ðð ð
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð r"   rM  c                   ó\   ‡ — e Zd ZdZˆ fd„Zdej        dededej        fd„Zdd	e	fd
„Z
ˆ xZS )ÚTvpFramePadPrompterz?
    Pad frames extracted from videos in the surroundings.
    c           
      ó  •— |j         dvrt          d¦  «        ‚t          ¦   «                              ¦   «          |j        | _        |j        | _        |j         | _         |j        |j        dz  z
  | _        t          j	        t          j        d|j        d|j        |j        g¦  «        ¦  «        | _        t          j	        t          j        d|j        d|j        |j        g¦  «        ¦  «        | _        t          j	        t          j        d|j        d|j        |j        dz  z
  |j        g¦  «        ¦  «        | _        t          j	        t          j        d|j        d|j        |j        dz  z
  |j        g¦  «        ¦  «        | _        d S )NrO  rR  rj   r   r   )rS  r1   r+   r,   rv   rV  rT  Ú	base_sizer   rW  r   rX  r;  r<  r=  r>  r“   s     €r#   r,   zTvpFramePadPrompter.__init__L  sx  ø€ ØÔ'Ð/KÐKÐKÝÐXÑYÔYÐYå‰Œ×ÒÑÔÐØ Ô+ˆŒØ"Ô/ˆÔØ%+Ô%AˆÔ"ØÔ,¨vÔ/HÈ1Ñ/LÑLˆŒÝ”lÝŒK˜˜FÔ-¨q°&Ô2KÈVÔM`ÐaÑbÔbñ
ô 
ˆŒõ œÝŒK˜˜FÔ-¨q°&Ô2KÈVÔM`ÐaÑbÔbñ
ô 
ˆŒõ œÝŒKàØÔ%ØØÔ'¨&Ô*CÀaÑ*GÑGØÔ-ðñô ñ

ô 

ˆŒõ œÝŒKàØÔ%ØØÔ'¨&Ô*CÀaÑ*GÑGØÔ-ðñô ñ

ô 

ˆŒˆˆr"   r^  rx   ry   r•   c                 óú   — || j         z  || j         z  }}|j        \  }}}}	}
|                     ||z  ||	|
¦  «        }t          j                             |||fdd¬¦  «        }|                     |||||¦  «        }|S )z·
        This method allows to interpolate the pre-trained pad weights, to be able to use the model on collection of high
        resolution images (high resolution videos).

        r—   Fr˜   )rV  rn   rì   r   rp   rœ   )r3   r^  rx   ry   r�   rž   Úbatchrv   ÚchannelsÚprompt_heightÚprompt_widths              r#   Úinterpolate_pad_encodingz,TvpFramePadPrompter.interpolate_pad_encodingr  s™   € ð ˜$Ô+Ñ+¨U°TÔ5FÑ-FˆBˆàCIÄ<Ñ@ˆˆz˜8 ]°Lð —’ ¨
Ñ 2°H¸mÈ\ÑZÔZˆÝ”×*Ò*ØØ˜b˜ØØð	 +ñ 
ô 
ˆð —’  z°8¸VÀUÑKÔKˆØˆr"   Fri  c                 ó¾  — |r|j         d         |j         d         fn| j        | j        f\  }}| j        dvrt          d| j        › �¦  «        ‚| j        dv r(t	          j        ||g|j        |j        ¬¦  «        }||z  }| j        dv rÕt	          j        d| j	        d	| j
        | j
        |j        ¬
¦  «        }t	          j        | j        || j        gd¬¦  «        }t	          j        | j        || j        gd	¬¦  «        }t	          j        |                     d¦  «        |gz  ¦  «        }|r|                      |||¦  «        }||                     |j        ¦  «        z   }|S )Nrå   rY   )rB   rQ  rP  z$Invalid visual_prompter_apply value )rP  rQ  r¡   )rP  rB   r   r   rZ  rm   ræ   r   )rn   rV  rS  r1   r   r[  r¢   r£   r³   rv   rc  Úcatr=  r>  r;  r<  rÅ   ri  r\  )r3   rt   ri  rx   ry   r]  Úbaser^  s           r#   rS   zTvpFramePadPrompter.forwardŠ  sz  € ð (ð8ˆ\Ô Ô# \Ô%7¸Ô%;Ð<Ð<àÔ# TÔ%6Ð7ñ 	ˆ�ð
 Ô%Ð-IÐIÐIÝÐ`ÀDÔD^Ð`Ð`ÑaÔaÐaØÔ%Ð)>Ð>Ð>Ý!&¤¨V°U¨OÀ<ÔCUÐ^jÔ^qÐ!rÑ!rÔ!rÐØÐ.Ñ.ˆLØÔ%Ð);Ð;Ð;Ý”;˜q $¤/°1°d´nÀdÄnÐ]iÔ]pÐqÑqÔqˆDå”Y ¤¨t°T´^ÐDÈ!ÐLÑLÔLˆFÝ”Y ¤¨V°T´]ÐCÈÐKÑKÔKˆFÝ”Y˜|×0Ò0°Ñ3Ô3°v°hÑ>Ñ?Ô?ˆFØ'ð NØ×6Ò6°v¸vÀuÑMÔM�Ø'¨&¯)ª)°LÔ4FÑ*GÔ*GÑGˆLØÐr"   r¸   )r   r   r   r   r,   r   r¹   rº   ri  r»   rS   rT   rU   s   @r#   ra  ra  G  sš   ø€ € € € € ðð ð$
ð $
ð $
ð $
ð $
ðL¨u¬|ð ÀSð ÐQTð ÐY^ÔYeð ð ð ð ð0ð ¸dð ð ð ð ð ð ð ð r"   ra  )ÚframedownpadÚframepadzw
    The bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top.
    )Úcustom_introc                   óº   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  d	e
dz  d
e
dz  de
dz  de
deez  fd„¦   «         Zˆ xZS )rD  c                 ób  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _
        t          |¦  «        | _        t          j        t          j        dd|j        g¦  «        ¦  «        | _        t          j        |j        ¦  «        | _        |j        t.          vrt1          d¦  «        ‚t/          |j                 |¦  «        | _        |                      ¦   «          d S )Nr   é
   z:`visual_prompter_type` must be in (framedownpad, framepad))r+   r,   rZ   rW   Úvision_modelr½   r·   r�   Úvisual_embeddingsr  Úencoderr+  Úpoolerr   rW  r   rX  r\   rF  r�   r‘   r’   Úvisual_prompter_typeÚTVP_PROMPTER_CLASSES_MAPPINGr1   Úvisual_prompterÚ	post_initr“   s     €r#   r,   zTvpModel.__init__­  sö   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ*¨6Ñ2Ô2ˆÔÝ0°Ñ8Ô8ˆŒÝ!8¸Ñ!@Ô!@ˆÔÝ! &Ñ)Ô)ˆŒÝ Ñ'Ô'ˆŒÝœ<­¬°Q¸¸FÔ<NÐ4OÑ(PÔ(PÑQÔQˆÔÝ”z &Ô"<Ñ=Ô=ˆŒØÔ&Õ.JÐJÐJÝÐYÑZÔZÐZÝ;¸FÔ<WÔXÐY_Ñ`Ô`ˆÔà�ŠÑÔÐÐÐr"   c                 ó   — | j         j        S rü   ©r·   rÂ   )r3   s    r#   Úget_input_embeddingszTvpModel.get_input_embeddings½  s   € ØŒÔ.Ð.r"   c                 ó   — || j         _        d S rü   r|  )r3   rÙ   s     r#   Úset_input_embeddingszTvpModel.set_input_embeddingsÀ  s   € Ø*/ˆŒÔ'Ð'Ð'r"   NFrÈ   rt   rí   rã   r   r!  rŸ   r•   c                 ó¦  — |�|n| j         j        }|                      |                      ||¬¦  «        ¦  «        }|                      |¬¦  «        }	|                      ||¬¦  «        }
|�z|                     |
j        dd…         ¦  «        }t          j	        |j        d         d¦  «         
                    |j        |j        ¬¦  «        }t          j        |||gd	¬
¦  «        }| j                             |	j        d         d	d	¦  «        }t          j        ||	|
gd¬
¦  «        }t!          | j         ||¬¦  «        }|                      |||||¬¦  «        }|r|j        n|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|s||f|dd…         z   S t+          |||j        |j        ¬¦  «        S )a  
        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpModel

        >>> model = TvpModel.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)ri  )rÈ   r±   rj   r   rr  )r£   r¢   rY   ræ   r   )rZ   rÊ   rí   )rí   rã   r   r!  )r#  Úpooler_outputr   r   )rZ   r!  rs  ry  r·   rt  Únew_onesrn   r   r[  r\  r£   r¢   rk  rF  rÇ   r	   ru  r#  rv  r’   r   r   r   )r3   rÈ   rt   rí   rã   r   r!  rŸ   ÚkwargsÚtext_embedding_outputÚvisual_embedding_outputÚvisual_attention_maskÚpt_maskrF  Úembedding_outputÚencoder_outputsr#  r1  s                     r#   rS   zTvpModel.forwardÃ  s)  € ð4 &1Ð%<�k�kÀ$Ä+ÔBYˆà×(Ò(Ø× Ò  ÐH`Ð ÑaÔañ
ô 
ˆð !%§¢¸) Ñ DÔ DÐà"&×"8Ò"8ØÐ3Kð #9ñ #
ô #
Ðð Ð%à$2×$;Ò$;Ð<SÔ<YÐZ\Ð[\ÐZ\Ô<]Ñ$^Ô$^Ð!Ý”j Ô!5°aÔ!8¸"Ñ=Ô=×@Ò@Ø%Ô,°NÔ4Hð Añ ô ˆGõ #œY¨°ÐAVÐ'WÐ]_Ð`Ñ`Ô`ˆNàÔ&×-Ò-Ð.CÔ.IÈ!Ô.LÈbÐRTÑUÔUˆå œ9 kÐ3HÐJaÐ%bÐhiÐjÑjÔjÐå2Ø”;Ø*Ø)ð
ñ 
ô 
ˆð Ÿ,š,ØØ)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð BMÐd˜OÔ=Ð=ÐRaÐbcÔRdÐØŸšÐ$5Ñ6Ô6ˆØ ŸLšLÐ):Ñ;Ô;ÐØŸš ]Ñ3Ô3ˆØð 	LØ% }Ð5¸ÈÈÈÔ8KÑKÐKÝ)Ø/Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r"   )NNNNNNF)r   r   r   r,   r}  r  r   r   Ú
LongTensorr   r»   r    r   rS   rT   rU   s   @r#   rD  rD  §  s  ø€ € € € € ðð ð ð ð ð /ð /ð /ð0ð 0ð 0ð ð .2Ø15Ø26Ø)-Ø,0Ø#'Ø).ðI
ð I
àÔ# dÑ*ðI
ð Ô'¨$Ñ.ðI
ð Ô(¨4Ñ/ð	I
ð
   $™;ðI
ð # T™kðI
ð ˜D‘[ðI
ð #'ðI
ð 
Ð+Ñ	+ðI
ð I
ð I
ñ „^ðI
ð I
ð I
ð I
ð I
r"   rD  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVideoGroundingHeadc                 ó:  •— t          ¦   «                              ¦   «          t          j        |j        |j        dz  ¦  «        | _        t          j        |j        dz  d¦  «        | _        t          j        ¦   «         | _        t          j	        ¦   «         | _
        d S )Nrj   )r+   r,   r   rÖ   r\   Úlayer_0Úlayer_1ÚReLUÚactivation_0ÚSigmoidÚactivation_1r“   s     €r#   r,   zTvpVideoGroundingHead.__init__  st   ø€ Ý‰Œ×ÒÑÔÐÝ”y Ô!3°VÔ5GÈ!Ñ5KÑLÔLˆŒÝ”y Ô!3°aÑ!7¸Ñ;Ô;ˆŒÝœG™IœIˆÔÝœJ™LœLˆÔÐÐr"   c                 ó¦   — |                       |                      |¦  «        ¦  «        }|                      |                      |¦  «        ¦  «        }|S rü   )r‘  rŽ  r“  r�  )r3   r�  r   s      r#   rS   zTvpVideoGroundingHead.forward  sE   € Ø×"Ò" 4§<¢<°Ñ#>Ô#>Ñ?Ô?ˆØ×"Ò" 4§<¢<°Ñ#7Ô#7Ñ8Ô8ˆØˆr"   r   rU   s   @r#   rŒ  rŒ    sG   ø€ € € € € ð)ð )ð )ð )ð )ðð ð ð ð ð ð r"   rŒ  zb
    Tvp Model with a video grounding head on top computing IoU, distance, and duration loss.
    c                   óÐ   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  deej	                 dz  de
dz  d	e
dz  d
e
dz  de
deez  fd„¦   «         Zˆ xZS )ÚTvpForVideoGroundingc                 óÐ   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rü   )r+   r,   rZ   rD  r4  rŒ  Úvideo_grounding_headrz  r“   s     €r#   r,   zTvpForVideoGrounding.__init__$  sW   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ˜fÑ%Ô%ˆŒ
Ý$9¸&Ñ$AÔ$AˆÔ!à�ŠÑÔÐÐÐr"   NFrÈ   rt   rí   rP   rã   r   r!  rŸ   r•   c	           	      óà  — |�|n| j         j        }|                      |||||||¬¦  «        }
|
d         }|                      |¦  «        }d}|�kt	          g d¢¦  «        }|                     | j        ¦  «          |||¦  «        }|d         | j         j        |d         z  z   | j         j        |d         z  z   }|s|f|
dd…         z   }
|�|f|
z   }
|
S t          |||
j
        |
j        ¬	¦  «        S )
aç  
        labels (`torch.FloatTensor` of shape `(batch_size, 3)`, *optional*):
            The labels contains duration, start time, and end time of the video corresponding to the text.

        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpForVideoGrounding

        >>> model = TvpForVideoGrounding.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rã   r   r!  rŸ   r   r'   r(   r)   r*   rj   )r   r   r   r   )rZ   r!  r4  r˜  r%   r\  r£   Údistance_loss_weightÚduration_loss_weightr   r   r   )r3   rÈ   rt   rí   rP   rã   r   r!  rŸ   rƒ  r÷   r�  r   r   Ú	criterionÚ	loss_dicts                   r#   rS   zTvpForVideoGrounding.forward,  sA  € ð< &1Ð%<�k�kÀ$Ä+ÔBYˆØ—*’*ØØØØ/Ø!5Ø#Ø%=ð ñ 
ô 
ˆð   œ
ˆØ×*Ò*¨=Ñ9Ô9ˆàˆØÐÝÐ ?Ð ?Ð ?Ñ@Ô@ˆIØ�LŠL˜œÑ%Ô%Ð%Ø!˜	 &¨&Ñ1Ô1ˆIà˜%Ô Ø”+Ô2°Y¸zÔ5JÑJñKà”+Ô2°Y¸zÔ5JÑJñKð ð
 ð 	Ø�i '¨!¨"¨"¤+Ñ-ˆGØÐØ˜' GÑ+�ØˆNå&ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r"   )NNNNNNNF)r   r   r   r,   r   r   rŠ  r   r    r¹   r»   r   rS   rT   rU   s   @r#   r–  r–    s  ø€ € € € € ðð ð ð ð ð ð .2Ø15Ø26Ø-1Ø)-Ø,0Ø#'Ø).ð?
ð ?
àÔ# dÑ*ð?
ð Ô'¨$Ñ.ð?
ð Ô(¨4Ñ/ð	?
ð
 �e”lÔ# dÑ*ð?
ð   $™;ð?
ð # T™kð?
ð ˜D‘[ð?
ð #'ð?
ð 
Ð(Ñ	(ð?
ð ?
ð ?
ñ „^ð?
ð ?
ð ?
ð ?
ð ?
r"   r–  )rD  r3  r–  )3r   ré   Údataclassesr   r   r   Ú r   r@  Úactivationsr   Úbackbone_utilsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_tvpr   Ú
get_loggerr   Úloggerr   rK  r%   rW   r�   r½   rÐ   rú   r  r
  r  r+  r3  rM  ra  rx  rD  rŒ  r–  Ú__all__r!   r"   r#   ú<module>r«     s½  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø +Ð +Ð +Ð +Ð +Ð +Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XØ -Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø (Ð (Ð (Ð (Ð (Ð (ð 
ˆÔ	˜HÑ	%Ô	%€ð Ø
ð<ð <ð <ð <ð <˜kñ <ô <ñ „ñ „ð<ð$Mð Mð Mð Mð MˆbŒiñ Mô Mð Mð`%ð %ð %ð %ð %�R”Yñ %ô %ð %ðPnð nð nð nð n˜bœiñ nô nð nðb!ð !ð !ð !ð !˜RœYñ !ô !ð !ðHBð Bð Bð Bð B�2”9ñ Bô Bð BðLð ð ð ð �b”iñ ô ð ðð ð ð ð �R”Yñ ô ð ðð ð ð ð Ð/ñ ô ð ð41
ð 1
ð 1
ð 1
ð 1
�”ñ 1
ô 1
ð 1
ðjð ð ð ð �”	ñ ô ð ð ð+ð +ð +ð +ð +˜ñ +ô +ñ „ð+ð6"ð "ð "ð "ð "˜bœiñ "ô "ð "ðJWð Wð Wð Wð W˜"œ)ñ Wô Wð Wðv ,Ø#ð ð  Ð ð €ððñ ô ð
a
ð a
ð a
ð a
ð a
Ð!ñ a
ô a
ñô ð
a
ðHð ð ð ð ˜BœIñ ô ð ð €ððñ ô ð
I
ð I
ð I
ð I
ð I
Ð-ñ I
ô I
ñô ð
I
ðX EÐ
DÐ
D€€€r"   