§
    ‚ŠtjÇä  ã                   ó  — d Z ddlZddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z ddlmZmZ dd	lmZ dd
lmZmZmZ ddlmZ  ej        e¦  «        Z G d„ dej        ¦  «        Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z ed¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z G d„ dej        ¦  «        Z G d„ dej        ¦  «        Z  G d„ dej        ¦  «        Z! G d„ d ej        ¦  «        Z" G d!„ d"ej        ¦  «        Z# G d#„ d$ej        ¦  «        Z$ G d%„ d&ej        ¦  «        Z% G d'„ d(ej        ¦  «        Z& G d)„ d*ej        ¦  «        Z' G d+„ d,ej        ¦  «        Z( G d-„ d.ej        ¦  «        Z) G d/„ d0ej        ¦  «        Z* G d1„ d2ej        ¦  «        Z+ G d3„ d4ej        ¦  «        Z, G d5„ d6ej        ¦  «        Z- G d7„ d8ej        ¦  «        Z. G d9„ d:ej        ¦  «        Z/e G d;„ d<e¦  «        ¦   «         Z0e G d=„ d>e0¦  «        ¦   «         Z1e G d?„ d@e0¦  «        ¦   «         Z2 edA¬¦  «         G dB„ dCe0¦  «        ¦   «         Z3g dD¢Z4dS )EzPyTorch LXMERT model.é    N)Ú	dataclass)Únn)ÚCrossEntropyLossÚSmoothL1Lossé   )Úinitialization)ÚACT2FNÚgelu)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚloggingé   )ÚLxmertConfigc                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚGeLUc                 óH   •— t          ¦   «                              ¦   «          d S ©N)ÚsuperÚ__init__)ÚselfÚ	__class__s    €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/lxmert/modeling_lxmert.pyr   zGeLU.__init__"   s   ø€ Ý‰Œ×ÒÑÔÐÐÐó    c                 ó    — t          |¦  «        S r   )r
   )r   Úxs     r   ÚforwardzGeLU.forward%   s   € Ý�A‰wŒwˆr   ©Ú__name__Ú
__module__Ú__qualname__r   r   Ú__classcell__©r   s   @r   r   r   !   sG   ø€ € € € € ðð ð ð ð ðð ð ð ð ð ð r   r   zÿ
    Lxmert's outputs that contain the last hidden states, pooled outputs, and attention probabilities for the language,
    visual, and, cross-modality encoders. (note: the visual encoder in Lxmert is referred to as the "relation-ship"
    encoder")
    )Úcustom_introc                   ó@  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dS )ÚLxmertModelOutputa
  
    language_output (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the language encoder.
    vision_output (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the visual encoder.
    pooled_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Last layer hidden-state of the first token of the sequence (classification, CLS, token) further processed
        by a Linear layer and a Tanh activation function. The Linear
    language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    NÚlanguage_outputÚvision_outputÚpooled_outputÚlanguage_hidden_statesÚvision_hidden_statesÚlanguage_attentionsÚvision_attentionsÚcross_encoder_attentions)r   r    r!   Ú__doc__r'   ÚtorchÚFloatTensorÚ__annotations__r(   r)   r*   Útupler+   r,   r-   r.   © r   r   r&   r&   )   sü   € € € € € € ðð ð8 15€O�UÔ&¨Ñ-Ð4Ð4Ñ4Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBØ<@Ð˜% Ô 1Ô2°TÑ9Ð@Ð@Ñ@Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ø9=Ð�u˜UÔ.Ô/°$Ñ6Ð=Ð=Ñ=Ø@DÐ˜e EÔ$5Ô6¸Ñ=ÐDÐDÑDÐDÐDr   r&   z8
    Output type of [`LxmertForQuestionAnswering`].
    c                   ó"  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed	<   dS )
Ú LxmertForQuestionAnsweringOutputa	  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the masked language modeling loss and the next sequence prediction
        (classification) loss.k.
    question_answering_score (`torch.FloatTensor` of shape `(batch_size, n_qa_answers)`, *optional*):
        Prediction scores of question answering objective (classification).
    language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    NÚlossÚquestion_answering_scorer*   r+   r,   r-   r.   )r   r    r!   r/   r7   r0   r1   r2   r8   r*   r3   r+   r,   r-   r.   r4   r   r   r6   r6   X   så   € € € € € € ðð ð4 &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø9=Ð˜eÔ/°$Ñ6Ð=Ð=Ñ=Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBØ<@Ð˜% Ô 1Ô2°TÑ9Ð@Ð@Ñ@Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ø9=Ð�u˜UÔ.Ô/°$Ñ6Ð=Ð=Ñ=Ø@DÐ˜e EÔ$5Ô6¸Ñ=ÐDÐDÑDÐDÐDr   r6   z2
    Output type of [`LxmertForPreTraining`].
    c                   ó^  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dZeej                 dz  ed
<   dZeej                 dz  ed<   dS )ÚLxmertForPreTrainingOutputaÂ
  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the masked language modeling loss and the next sequence prediction
        (classification) loss.
    prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    cross_relationship_score (`torch.FloatTensor` of shape `(batch_size, 2)`):
        Prediction scores of the textual matching objective (classification) head (scores of True/False
        continuation before SoftMax).
    question_answering_score (`torch.FloatTensor` of shape `(batch_size, n_qa_answers)`):
        Prediction scores of question answering objective (classification).
    language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
        shape `(batch_size, sequence_length, hidden_size)`.
    language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
        the self-attention heads.
    Nr7   Úprediction_logitsÚcross_relationship_scorer8   r*   r+   r,   r-   r.   )r   r    r!   r/   r7   r0   r1   r2   r;   r<   r8   r*   r3   r+   r,   r-   r.   r4   r   r   r:   r:   ‚   s  € € € € € € ðð ð> &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø9=Ð˜eÔ/°$Ñ6Ð=Ð=Ñ=Ø9=Ð˜eÔ/°$Ñ6Ð=Ð=Ñ=Ø>BÐ˜E %Ô"3Ô4°tÑ;ÐBÐBÑBØ<@Ð˜% Ô 1Ô2°TÑ9Ð@Ð@Ñ@Ø;?Ð˜˜uÔ0Ô1°DÑ8Ð?Ð?Ñ?Ø9=Ð�u˜UÔ.Ô/°$Ñ6Ð=Ð=Ñ=Ø@DÐ˜e EÔ$5Ô6¸Ñ=ÐDÐDÑDÐDÐDr   r:   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚLxmertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó¨  •— t          ¦   «                              ¦   «          t          j        |j        |j        d¬¦  «        | _        t          j        |j        |j        d¬¦  «        | _        t          j        |j	        |j        d¬¦  «        | _
        t          j        |j        d¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   )Úpadding_idxçê-�™—q=©Úeps)r   r   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚDropoutÚhidden_dropout_probÚdropout©r   Úconfigr   s     €r   r   zLxmertEmbeddings.__init__¶   s«   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_`ÐaÑaÔaˆÔÝ#%¤<°Ô0NÐPVÔPbÐpqÐ#rÑ#rÔ#rˆÔ Ý%'¤\°&Ô2HÈ&ÔJ\ÐjkÐ%lÑ%lÔ%lˆÔ"åœ fÔ&8¸eÐDÑDÔDˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr   Nc                 ój  — |�|                      ¦   «         }|j        }n#|                      ¦   «         d d…         }|j        }|d         }t          j        |t          j        |¬¦  «        }|                     d¦  «                             |¦  «        }|€+t          j        |t          j        | j        j        ¬¦  «        }|€|  	                    |¦  «        }|  
                    |¦  «        }|                      |¦  «        }	||z   |	z   }
|                      |
¦  «        }
|                      |
¦  «        }
|
S )Néÿÿÿÿr   ©ÚdtypeÚdevicer   )ÚsizerV   r0   ÚarangeÚlongÚ	unsqueezeÚexpandÚzerosÚposition_idsrG   rI   rK   rL   rO   )r   Ú	input_idsÚtoken_type_idsÚinputs_embedsÚinput_shaperV   Ú
seq_lengthr]   rI   rK   Ú
embeddingss              r   r   zLxmertEmbeddings.forward¿   s#  € ØÐ Ø#Ÿ.š.Ñ*Ô*ˆKØÔ%ˆFˆFà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKØ"Ô)ˆFØ  ”^ˆ
å”| Jµe´jÈÐPÑPÔPˆØ#×-Ò-¨aÑ0Ô0×7Ò7¸ÑDÔDˆàÐ!Ý"œ[¨½E¼JÈtÔO`ÔOgÐhÑhÔhˆNàÐ Ø ×0Ò0°Ñ;Ô;ˆMØ"×6Ò6°|ÑDÔDÐØ $× :Ò :¸>Ñ JÔ JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr   ©NN)r   r    r!   r/   r   r   r"   r#   s   @r   r>   r>   ³   sR   ø€ € € € € ØQÐQð>ð >ð >ð >ð >ðð ð ð ð ð ð ð r   r>   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚLxmertAttentionNc                 óB  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r t	          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        |€|j        }t          j	        |j        | j        ¦  «        | _
        t          j	        || j        ¦  «        | _        t          j	        || j        ¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú))r   r   rF   Únum_attention_headsÚ
ValueErrorÚintÚattention_head_sizeÚ	head_sizer   ÚLinearÚqueryÚkeyÚvaluerM   Úattention_probs_dropout_probrO   )r   rQ   Úctx_dimr   s      €r   r   zLxmertAttention.__init__Ú   s  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ ØÔ1°DÔ4LÑLˆŒð ˆ?ØÔ(ˆGÝ”Y˜vÔ1°4´>ÑBÔBˆŒ
Ý”9˜W d¤nÑ5Ô5ˆŒÝ”Y˜w¨¬Ñ7Ô7ˆŒ
å”z &Ô"EÑFÔFˆŒˆˆr   Fc                 óæ  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }g |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	|                      |¦  «                             |¦  «                             dd¦  «        }
t          j        ||	                     dd¦  «        ¦  «        }|t          j
        | j        ¦  «        z  }|�||z   }t          j                             |d¬¦  «        }|                      |¦  «        }t          j        ||
¦  «        }|                     dddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   }|                     |¦  «        }|r||fn|f}|S )NrS   r   é   éþÿÿÿ)Údimr   r   )Úshaperl   ro   ÚviewÚ	transposerp   rq   r0   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxrO   ÚpermuteÚ
contiguousrW   rm   )r   Úhidden_statesÚcontextÚattention_maskÚoutput_attentionsra   Úhidden_shapeÚquery_layerÚkv_shapeÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                   r   r   zLxmertAttention.forwardî   sè  € Ø#Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆØ—j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆØF�W”] 3 B 3Ô'ÐF¨ÐF¨TÔ-EÐFÐFˆØ—H’H˜WÑ%Ô%×*Ò*¨8Ñ4Ô4×>Ò>¸qÀ!ÑDÔDˆ	Ø—j’j Ñ)Ô)×.Ò.¨xÑ8Ô8×BÒBÀ1ÀaÑHÔHˆõ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐØ+­d¬i¸Ô8PÑ.QÔ.QÑQÐàÐ%Ø/°.Ñ@Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆØ%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸t¼~Ð>OÑ"OÐØ%×*Ò*Ð+BÑCÔCˆà6GÐ]�= /Ð2Ð2ÈmÐM]ˆØˆr   r   ©NFr   r#   s   @r   rf   rf   Ù   sW   ø€ € € € € ðGð Gð Gð Gð Gð Gð(ð ð ð ð ð ð ð r   rf   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertAttentionOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        d¬¦  «        | _        t          j        |j        ¦  «        | _	        d S ©NrA   rB   )
r   r   r   rn   rF   ÚdenserL   rM   rN   rO   rP   s     €r   r   zLxmertAttentionOutput.__init__  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸eÐDÑDÔDˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r   ©r•   rO   rL   ©r   r‚   Úinput_tensors      r   r   zLxmertAttentionOutput.forward  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr   r   r#   s   @r   r’   r’     óG   ø€ € € € € ð>ð >ð >ð >ð >ðð ð ð ð ð ð r   r’   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚLxmertCrossAttentionLayerc                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S r   )r   r   rf   Úattr’   ÚoutputrP   s     €r   r   z"LxmertCrossAttentionLayer.__init__  s;   ø€ Ý‰Œ×ÒÑÔÐÝ" 6Ñ*Ô*ˆŒÝ+¨FÑ3Ô3ˆŒˆˆr   NFc                 ó–   — |                       ||||¬¦  «        }|r|d         }|                      |d         |¦  «        }|r||fn|f}|S ©N©r…   r   r   )rŸ   r    )	r   r™   Ú
ctx_tensorÚctx_att_maskr…   r    rŒ   Úattention_outputr�   s	            r   r   z!LxmertCrossAttentionLayer.forward!  sc   € Ø—’˜,¨
°LÐTe�ÑfÔfˆØð 	(Ø$ QœiˆOØŸ;š; v¨a¤y°,Ñ?Ô?ÐØ9JÐcÐ# _Ð5Ð5ÐQaÐPcˆØˆr   r�   r   r#   s   @r   r�   r�     sL   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð
ð ð ð ð ð ð ð r   r�   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚLxmertSelfAttentionLayerc                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S r   )r   r   rf   r   r’   r    rP   s     €r   r   z!LxmertSelfAttentionLayer.__init__+  s;   ø€ Ý‰Œ×ÒÑÔÐÝ# FÑ+Ô+ˆŒ	Ý+¨FÑ3Ô3ˆŒˆˆr   Fc                 ó–   — |                       ||||¬¦  «        }|r|d         }|                      |d         |¦  «        }|r||fn|f}|S r¢   )r   r    )r   r™   r„   r…   r    rŒ   r¦   r�   s           r   r   z LxmertSelfAttentionLayer.forward0  so   € à—’ØØØØ/ð	 ñ 
ô 
ˆð ð 	(Ø$ QœiˆOØŸ;š; v¨a¤y°,Ñ?Ô?ÐØ9JÐcÐ# _Ð5Ð5ÐQaÐPcˆØˆr   ©Fr   r#   s   @r   r¨   r¨   *  sL   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð
ð ð ð ð ð ð ð r   r¨   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertIntermediatec                 ó¾   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _	        d S r   )
r   r   r   rn   rF   Úintermediate_sizer•   r	   Ú
hidden_actÚintermediate_act_fnrP   s     €r   r   zLxmertIntermediate.__init__@  sH   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý#)¨&Ô*;Ô#<ˆÔ Ð Ð r   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r   )r•   r±   ©r   r‚   s     r   r   zLxmertIntermediate.forwardE  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr   r   r#   s   @r   r­   r­   ?  sG   ø€ € € € € ð=ð =ð =ð =ð =ð
ð ð ð ð ð ð r   r­   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        d¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S r”   )r   r   r   rn   r¯   rF   r•   rL   rM   rN   rO   rP   s     €r   r   zLxmertOutput.__init__L  sc   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸eÐDÑDÔDˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r   r—   r˜   s      r   r   zLxmertOutput.forwardR  rš   r   r   r#   s   @r   rµ   rµ   K  r›   r   rµ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚLxmertLayerc                 óÀ   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        d S r   )r   r   r¨   Ú	attentionr­   Úintermediaterµ   r    rP   s     €r   r   zLxmertLayer.__init__Z  sK   ø€ Ý‰Œ×ÒÑÔÐÝ1°&Ñ9Ô9ˆŒÝ.¨vÑ6Ô6ˆÔÝ" 6Ñ*Ô*ˆŒˆˆr   NFc                 ó¸   — |                       |||¬¦  «        }|d         }|                      |¦  «        }|                      ||¦  «        }|f|dd …         z   }|S )Nr£   r   r   )r»   r¼   r    )r   r‚   r„   r…   r�   r¦   Úintermediate_outputÚlayer_outputs           r   r   zLxmertLayer.forward`  sg   € Ø—.’. °ÐRc�.ÑdÔdˆØ" 1œ:ÐØ"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØ�/ G¨A¨B¨B¤KÑ/ˆØˆr   r�   r   r#   s   @r   r¹   r¹   Y  sL   ø€ € € € € ð+ð +ð +ð +ð +ðð ð ð ð ð ð ð r   r¹   c                   ó>   ‡ — e Zd Zˆ fd„Z	 dd„Zd„ Zd„ Z	 dd„Zˆ xZS )ÚLxmertXLayerc                 ó`  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _
        t          |¦  «        | _        t          |¦  «        | _        d S r   )r   r   r�   Úvisual_attentionr¨   Úlang_self_attÚvisn_self_attr­   Ú
lang_interrµ   Úlang_outputÚ
visn_interÚvisn_outputrP   s     €r   r   zLxmertXLayer.__init__j  s‘   ø€ Ý‰Œ×ÒÑÔÐå 9¸&Ñ AÔ AˆÔõ 6°fÑ=Ô=ˆÔÝ5°fÑ=Ô=ˆÔõ -¨VÑ4Ô4ˆŒÝ'¨Ñ/Ô/ˆÔÝ,¨VÑ4Ô4ˆŒÝ'¨Ñ/Ô/ˆÔÐÐr   Fc                 ón   — |                       ||||¬¦  «        }|                       |||d¬¦  «        }||fS )N)r¥   r…   F)rÃ   )r   Ú
lang_inputÚlang_attention_maskÚvisual_inputÚvisual_attention_maskÚoutput_x_attentionsÚlang_att_outputÚvisual_att_outputs           r   Ú	cross_attzLxmertXLayer.cross_atty  sa   € ð ×/Ò/ØØØ.Ø1ð	 0ñ 
ô 
ˆð !×1Ò1ØØØ,Ø#ð	 2ñ 
ô 
Ðð Ð 1Ð1Ð1r   c                 ó‚   — |                       ||d¬¦  «        }|                      ||d¬¦  «        }|d         |d         fS )NFr£   r   )rÄ   rÅ   )r   rË   rÌ   rÍ   rÎ   rÐ   rÑ   s          r   Úself_attzLxmertXLayer.self_att�  sQ   € à×,Ò,¨ZÐ9LÐ`eÐ,ÑfÔfˆØ ×.Ò.¨|Ð=RÐfkÐ.ÑlÔlÐØ˜qÔ!Ð#4°QÔ#7Ð7Ð7r   c                 ó¶   — |                       |¦  «        }|                      |¦  «        }|                      ||¦  «        }|                      ||¦  «        }||fS r   )rÆ   rÈ   rÇ   rÉ   )r   rË   rÍ   Úlang_inter_outputÚvisual_inter_outputrÇ   Úvisual_outputs          r   Ú	output_fczLxmertXLayer.output_fc–  s_   € à ŸOšO¨JÑ7Ô7ÐØ"Ÿošo¨lÑ;Ô;Ðð ×&Ò&Ð'8¸*ÑEÔEˆØ×(Ò(Ð)<¸lÑKÔKˆà˜MÐ)Ð)r   c                 óò   — |                       |||||¬¦  «        \  }}|dd …         }|                      |d         ||d         |¦  «        \  }}|                      ||¦  «        \  }	}
|r|	|
|d         fn|	|
fS )N)rË   rÌ   rÍ   rÎ   rÏ   r   r   )rÒ   rÔ   rÙ   )r   Ú
lang_featsrÌ   Úvisual_featsrÎ   r…   rÐ   rÑ   rŒ   rÇ   rØ   s              r   r   zLxmertXLayer.forward¡  s¼   € ð .2¯^ª^Ø!Ø 3Ø%Ø"7Ø 1ð .<ñ .
ô .
Ñ*ˆÐ*ð *¨!¨"¨"Ô-ˆØ-1¯]ª]Ø˜AÔØØ˜aÔ Ø!ñ	.
ô .
Ñ*ˆÐ*ð &*§^¢^°OÐEVÑ%WÔ%WÑ"ˆ�]ð !ð.ØØØ Ô"ðð ð ˜}Ð-ð	
r   r«   )	r   r    r!   r   rÒ   rÔ   rÙ   r   r"   r#   s   @r   rÁ   rÁ   i  sˆ   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð* "ð2ð 2ð 2ð 2ð.8ð 8ð 8ð	*ð 	*ð 	*ð"  ð 
ð  
ð  
ð  
ð  
ð  
ð  
ð  
r   rÁ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertVisualFeatureEncoderc                 óœ  •— t          ¦   «                              ¦   «          |j        }|j        }t	          j        ||j        ¦  «        | _        t	          j        |j        d¬¦  «        | _	        t	          j        ||j        ¦  «        | _
        t	          j        |j        d¬¦  «        | _        t	          j        |j        ¦  «        | _        d S r”   )r   r   Úvisual_feat_dimÚvisual_pos_dimr   rn   rF   Úvisn_fcrL   Úvisn_layer_normÚbox_fcÚbox_layer_normrM   rN   rO   )r   rQ   Úfeat_dimÚpos_dimr   s       €r   r   z#LxmertVisualFeatureEncoder.__init__Å  s¥   ø€ Ý‰Œ×ÒÑÔÐØÔ)ˆØÔ'ˆõ ”y ¨6Ô+=Ñ>Ô>ˆŒÝ!œ|¨FÔ,>ÀEÐJÑJÔJˆÔõ ”i ¨Ô);Ñ<Ô<ˆŒÝ œl¨6Ô+=À5ÐIÑIÔIˆÔå”z &Ô"<Ñ=Ô=ˆŒˆˆr   c                 óè   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   dz  }|                      |¦  «        }|S ©Nru   )râ   rã   rä   rå   rO   )r   rÜ   Ú
visual_posr   Úyr    s         r   r   z"LxmertVisualFeatureEncoder.forwardÔ  sm   € Ø�LŠL˜Ñ&Ô&ˆØ× Ò  Ñ#Ô#ˆØ�KŠK˜
Ñ#Ô#ˆØ×Ò Ñ"Ô"ˆØ�a‘%˜1‘ˆà—’˜fÑ%Ô%ˆØˆr   r   r#   s   @r   rÞ   rÞ   Ä  sG   ø€ € € € € ð>ð >ð >ð >ð >ðð ð ð ð ð ð r   rÞ   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚLxmertEncoderc                 ó  •‡— t          ¦   «                              ¦   «          t          ‰¦  «        | _        ‰| _        ‰j        | _        ‰j        | _        ‰j	        | _
        t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _        t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _        t          j        ˆfd„t          | j
        ¦  «        D ¦   «         ¦  «        | _	        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4   ©r¹   ©Ú.0Ú_rQ   s     €r   ú
<listcomp>z*LxmertEncoder.__init__.<locals>.<listcomp>î  s!   ø€ Ð#ZÐ#ZÐ#Z¸A¥K°Ñ$7Ô$7Ð#ZÐ#ZÐ#Zr   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4   )rÁ   rñ   s     €r   rô   z*LxmertEncoder.__init__.<locals>.<listcomp>ï  s!   ø€ Ð&^Ð&^Ð&^À¥|°FÑ';Ô';Ð&^Ð&^Ð&^r   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r4   rð   rñ   s     €r   rô   z*LxmertEncoder.__init__.<locals>.<listcomp>ð  s!   ø€ Ð&]Ð&]Ð&]¸q¥{°6Ñ':Ô':Ð&]Ð&]Ð&]r   )r   r   rÞ   râ   rQ   Úl_layersÚnum_l_layersÚx_layersÚnum_x_layersÚr_layersÚnum_r_layersr   Ú
ModuleListÚrangeÚlayerrP   s    `€r   r   zLxmertEncoder.__init__à  së   øø€ Ý‰Œ×ÒÑÔÐõ 2°&Ñ9Ô9ˆŒØˆŒð #œOˆÔØ"œOˆÔØ"œOˆÔõ ”]Ð#ZÐ#ZÐ#ZÐ#ZÅÀtÔGXÑAYÔAYÐ#ZÑ#ZÔ#ZÑ[Ô[ˆŒ
ÝœÐ&^Ð&^Ð&^Ð&^ÅUÈ4ÔK\ÑE]ÔE]Ð&^Ñ&^Ô&^Ñ_Ô_ˆŒÝœÐ&]Ð&]Ð&]Ð&]ÅEÈ$ÔJ[ÑD\ÔD\Ð&]Ñ&]Ô&]Ñ^Ô^ˆŒˆˆr   Nc                 ó&  — d}d}|s| j         j        rdnd }	|s| j         j        rdnd }
|s| j         j        rdnd }|                      ||¦  «        }| j        D ],} ||||¬¦  «        }|d         }||fz   }|
�|
|d         fz   }
Œ-| j        D ],} ||||¬¦  «        }|d         }||fz   }|	�|	|d         fz   }	Œ-| j        D ]9} ||||||¬¦  «        }|d d…         \  }}||fz   }||fz   }|�||d         fz   }Œ:||r|	nd f}||r|
nd f}|||r|nd fS )Nr4   r£   r   r   ru   )rQ   r…   râ   rÿ   rû   rù   )r   rÛ   rÌ   rÜ   rê   rÎ   r…   r+   r*   r-   r,   r.   Úlayer_moduleÚ	l_outputsÚ	v_outputsÚ	x_outputsÚvisual_encoder_outputsÚlang_encoder_outputss                     r   r   zLxmertEncoder.forwardò  sü  € ð  "ÐØ!#ÐØ"3Ð^°t´{Ô7TÐ^˜B˜BÐZ^ÐØ$5Ð`¸¼Ô9VÐ`˜b˜bÐ\`ÐØ):Ð#e¸d¼kÔ>[Ð#e 2 2ÐaeÐ à—|’| L°*Ñ=Ô=ˆð !œJð 	Lð 	LˆLØ$˜ ZÐ1DÐXiÐjÑjÔjˆIØ" 1œˆJØ%;¸z¸mÑ%KÐ"Ø"Ð.Ø&9¸YÀq¼\¸OÑ&KÐ#øð !œMð 	Hð 	HˆLØ$˜ \Ð3HÐ\mÐnÑnÔnˆIØ$ Qœ<ˆLØ#7¸<¸/Ñ#IÐ Ø Ð,Ø$5¸À1¼¸Ñ$GÐ!øð !œMð 	Vð 	VˆLØ$˜ØØ#ØØ%Ø"3ðñ ô ˆIð (1°°!°¤}Ñ$ˆJ˜Ø#7¸<¸/Ñ#IÐ Ø%;¸z¸mÑ%KÐ"Ø'Ð3Ø+CÀyÐQRÄ|ÀoÑ+UÐ(øà Ø!2Ð<ÐÐ¸ð"
Ðð
 #Ø#4Ð>ÐÐ¸$ð 
Ðð
 #Ø Ø(9ÐCÐ$Ð$¸tð
ð 	
r   rd   r   r#   s   @r   rí   rí   ß  sY   ø€ € € € € ð_ð _ð _ð _ð _ð0 #Øð;
ð ;
ð ;
ð ;
ð ;
ð ;
ð ;
ð ;
r   rí   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S r   )r   r   r   rn   rF   r•   ÚTanhÚ
activationrP   s     €r   r   zLxmertPooler.__init__1  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )r•   r  )r   r‚   Úfirst_token_tensorr)   s       r   r   zLxmertPooler.forward6  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr   r   r#   s   @r   r  r  0  sG   ø€ € € € € ð$ð $ð $ð $ð $ð
ð ð ð ð ð ð r   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertPredictionHeadTransformc                 óþ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j                 | _        t          j	        |j        d¬¦  «        | _	        d S r”   )
r   r   r   rn   rF   r•   r	   r°   Útransform_act_fnrL   rP   s     €r   r   z&LxmertPredictionHeadTransform.__init__@  s_   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý & vÔ'8Ô 9ˆÔÝœ fÔ&8¸eÐDÑDÔDˆŒˆˆr   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r   )r•   r  rL   r³   s     r   r   z%LxmertPredictionHeadTransform.forwardF  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐr   r   r#   s   @r   r  r  ?  sL   ø€ € € € € ðEð Eð Eð Eð Eðð ð ð ð ð ð r   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertLMPredictionHeadc                 ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        t	          j	        t          j        |j        ¦  «        ¦  «        | _        d S )NF©Úbias)r   r   r  Ú	transformr   rn   rF   rE   ÚdecoderÚ	Parameterr0   r\   r  rP   s     €r   r   zLxmertLMPredictionHead.__init__N  sh   ø€ Ý‰Œ×ÒÑÔÐÝ6°vÑ>Ô>ˆŒÝ”y Ô!3°VÔ5FÈUÐSÑSÔSˆŒÝ”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r   c                 ój   — |                       |¦  «        }|                      |¦  «        | j        z   }|S r   )r  r  r  r³   s     r   r   zLxmertLMPredictionHead.forwardT  s1   € ØŸš }Ñ5Ô5ˆØŸš ]Ñ3Ô3°d´iÑ?ˆØÐr   r   r#   s   @r   r  r  M  sL   ø€ € € € € ðAð Að Að Að Aðð ð ð ð ð ð r   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertVisualAnswerHeadc           	      ó,  •— t          ¦   «                              ¦   «          |j        }t          j        t          j        ||dz  ¦  «        t          ¦   «         t          j        |dz  d¬¦  «        t          j        |dz  |¦  «        ¦  «        | _        d S )Nru   rA   rB   )	r   r   rF   r   Ú
Sequentialrn   r   rL   Úlogit_fc)r   rQ   Ú
num_labelsÚhid_dimr   s       €r   r   zLxmertVisualAnswerHead.__init__[  s{   ø€ Ý‰Œ×ÒÑÔÐØÔ$ˆÝœÝŒI�g˜w¨™{Ñ+Ô+Ý‰FŒFÝŒL˜ 1™¨%Ð0Ñ0Ô0ÝŒI�g ‘k :Ñ.Ô.ñ	
ô 
ˆŒˆˆr   c                 ó,   — |                       |¦  «        S r   )r   r³   s     r   r   zLxmertVisualAnswerHead.forwarde  s   € Ø�}Š}˜]Ñ+Ô+Ð+r   r   r#   s   @r   r  r  Z  sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð,ð ,ð ,ð ,ð ,ð ,ð ,r   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertVisualObjHeadc                 ób  •‡ ‡— t          ¦   «                              ¦   «          t          ‰¦  «        ‰ _        i }‰j        rd‰j        dœ|d<   ‰j        rd‰j        dœ|d<   ‰j        rd‰j	        f‰j	        dœ|d<   |‰ _
        t          j        ˆˆ fd„‰ j
        D ¦   «         ¦  «        ‰ _        d S )N©rS   )rx   ÚnumÚobjÚattrrS   Úfeatc                 óh   •— i | ].}|t          j        ‰j        ‰j        |         d          ¦  «        “Œ/S )r(  )r   rn   rF   Úvisual_losses)rò   rp   rQ   r   s     €€r   ú
<dictcomp>z0LxmertVisualObjHead.__init__.<locals>.<dictcomp>}  s8   ø€ ÐnÐnÐnÐTWˆS•"”)˜FÔ.°Ô0BÀ3Ô0GÈÔ0NÑOÔOÐnÐnÐnr   )r   r   r  r  Úvisual_obj_lossÚnum_object_labelsÚvisual_attr_lossÚnum_attr_labelsÚvisual_feat_lossrà   r-  r   Ú
ModuleDictÚdecoder_dict©r   rQ   r-  r   s   `` €r   r   zLxmertVisualObjHead.__init__j  sâ   øøø€ Ý‰Œ×ÒÑÔÐÝ6°vÑ>Ô>ˆŒàˆØÔ!ð 	UØ-2¸6Ô;SÐ#TÐ#TˆM˜%Ñ ØÔ"ð 	TØ.3¸FÔ<RÐ$SÐ$SˆM˜&Ñ!ØÔ"ð 	à˜fÔ4Ð5ØÔ-ð%ð %ˆM˜&Ñ!ð +ˆÔõ œMØnÐnÐnÐnÐnÐ[_Ô[mÐnÑnÔnñ
ô 
ˆÔÐÐr   c                 óz   — |                       |¦  «        }i }| j        D ]} | j        |         |¦  «        ||<   Œ|S r   )r  r-  r5  )r   r‚   r    rp   s       r   r   zLxmertVisualObjHead.forward€  sN   € ØŸš }Ñ5Ô5ˆØˆØÔ%ð 	@ð 	@ˆCØ0˜$Ô+¨CÔ0°Ñ?Ô?ˆF�3‰KˆKØˆr   r   r#   s   @r   r%  r%  i  sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð,ð ð ð ð ð ð r   r%  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLxmertPreTrainingHeadsc                 ó®   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        d¦  «        | _        d S ré   )r   r   r  Úpredictionsr   rn   rF   Úseq_relationshiprP   s     €r   r   zLxmertPreTrainingHeads.__init__‰  sF   ø€ Ý‰Œ×ÒÑÔÐÝ1°&Ñ9Ô9ˆÔÝ "¤	¨&Ô*<¸aÑ @Ô @ˆÔÐÐr   c                 ó^   — |                       |¦  «        }|                      |¦  «        }||fS r   )r;  r<  )r   Úsequence_outputr)   Úprediction_scoresÚseq_relationship_scores        r   r   zLxmertPreTrainingHeads.forwardŽ  s6   € Ø ×,Ò,¨_Ñ=Ô=ÐØ!%×!6Ò!6°}Ñ!EÔ!EÐØ Ð"8Ð8Ð8r   r   r#   s   @r   r9  r9  ˆ  sL   ø€ € € € € ðAð Að Að Að Að
9ð 9ð 9ð 9ð 9ð 9ð 9r   r9  c                   ó\   ‡ — e Zd ZU eed<   dZdZ ej        ¦   «         ˆ fd„¦   «         Z	ˆ xZ
S )ÚLxmertPreTrainedModelrQ   Úlxmert)ÚimageÚtextc                 óª   •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS dS )zInitialize the weightsN)r   Ú_init_weightsÚ
isinstancer  ÚinitÚzeros_r  )r   Úmoduler   s     €r   rG  z#LxmertPreTrainedModel._init_weightsš  sR   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ4Ñ5Ô5ð 	%ÝŒK˜œÑ$Ô$Ð$Ð$Ð$ð	%ð 	%r   )r   r    r!   r   r2   Úbase_model_prefixÚinput_modalitiesr0   Úno_gradrG  r"   r#   s   @r   rB  rB  ”  sd   ø€ € € € € € àÐÐÑØ ÐØ(Ðà€U„]�_„_ð%ð %ð %ð %ñ „_ð%ð %ð %ð %ð %r   rB  c                   ó"  ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej	        dz  dej	        dz  d	ej	        dz  d
ej        dz  dej	        dz  de
dz  de
dz  de
dz  deeej	                 z  fd„¦   «         Zˆ xZS )ÚLxmertModelc                 óê   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S r   )	r   r   r>   rc   rí   Úencoderr  ÚpoolerÚ	post_initrP   s     €r   r   zLxmertModel.__init__¤  s^   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý*¨6Ñ2Ô2ˆŒÝ$ VÑ,Ô,ˆŒÝ" 6Ñ*Ô*ˆŒà�ŠÑÔÐÐÐr   c                 ó   — | j         j        S r   ©rc   rG   ©r   s    r   Úget_input_embeddingsz LxmertModel.get_input_embeddings¬  s   € ØŒÔ.Ð.r   c                 ó   — || j         _        d S r   rV  )r   Únew_embeddingss     r   Úset_input_embeddingsz LxmertModel.set_input_embeddings¯  s   € Ø*8ˆŒÔ'Ð'Ð'r   Nr^   rÜ   rê   r„   rÎ   r_   r`   r…   Úoutput_hidden_statesÚreturn_dictÚreturnc           
      óh  — |�|n| j         j        }|	�|	n| j         j        }	|
�|
n| j         j        }
|�|�t	          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }n.|�|                     ¦   «         dd…         }nt	          d¦  «        ‚|€t	          d¦  «        ‚|€t	          d¦  «        ‚|�|j        n|j        }|€t          j	        ||¬¦  «        }|€!t          j
        |t          j        |¬¦  «        }|                     d	¦  «                             d
¦  «        }|                     | j        ¬¦  «        }d|z
  t          j        | j        ¦  «        j        z  }|�h|                     d	¦  «                             d
¦  «        }|                     | j        ¬¦  «        }d|z
  t          j        | j        ¦  «        j        z  }nd}|                      |||¦  «        }|                      ||||||¬¦  «        }|dd
…         \  }}|d         }|d         }d}|r|d	         }|d	         }|d
         }|||f}|	r||fnd}|d         }|d         }|                      |¦  «        }|
s|||f|z   |z   S t)          ||||	r|nd|	r|nd|r|nd|r|nd|r|nd¬¦  «        S )aw  
        visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`):
            This input represents visual features. They ROI pooled object features from bounding boxes using a
            faster-RCNN model)

            These are currently not provided by the transformers library.
        visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`):
            This input represents spatial features corresponding to their relative (via index) visual features. The
            pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to
            1.

            These are currently not provided by the transformers library.
        visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        NzDYou cannot specify both input_ids and inputs_embeds at the same timerS   z5You have to specify either input_ids or inputs_embedsz`visual_feats` cannot be `None`z`visual_pos` cannot be `None`©rV   rT   r   ru   )rU   g      ð?)rÜ   rê   rÎ   r…   r   r4   )r)   r'   r(   r*   r+   r,   r-   r.   )rQ   r…   r\  r]  rj   Ú%warn_if_padding_and_no_attention_maskrW   rV   r0   Úonesr\   rY   rZ   ÚtorU   ÚfinfoÚminrc   rR  rS  r&   )r   r^   rÜ   rê   r„   rÎ   r_   r`   r…   r\  r]  Úkwargsra   rV   Úextended_attention_maskÚextended_visual_attention_maskÚembedding_outputÚencoder_outputsr  r  r+   r*   Úall_attentionsr,   r-   r.   r‚   rØ   rÇ   r)   s                                 r   r   zLxmertModel.forward²  sˆ  € ðF 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUàÐÝÐ>Ñ?Ô?Ð?ØÐÝÐ<Ñ=Ô=Ð=à%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨¸FÐCÑCÔCˆNØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNð #1×":Ò":¸1Ñ"=Ô"=×"GÒ"GÈÑ"JÔ"JÐð #:×"<Ò"<À4Ä:Ð"<Ñ"NÔ"NÐØ#&Ð)@Ñ#@ÅEÄKÐPTÔPZÑD[ÔD[ÔD_Ñ"_Ðð !Ð,Ø-B×-LÒ-LÈQÑ-OÔ-O×-YÒ-YÐZ[Ñ-\Ô-\Ð*Ø-K×-NÒ-NÐUYÔU_Ð-NÑ-`Ô-`Ð*Ø.1Ð4RÑ.RÕV[ÔVaÐbfÔblÑVmÔVmÔVqÑ-qÐ*Ð*à-1Ð*ð  Ÿ?š?¨9°nÀmÑTÔTÐð Ÿ,š,ØØ#Ø%Ø!Ø"@Ø/ð 'ñ 
ô 
ˆð 8GÀrÈÀrÔ7JÑ4ÐÐ 4Ø5°aÔ8ÐØ!5°aÔ!8ÐàˆØð 	Ø"6°qÔ"9ÐØ 6°qÔ 9ÐØ'6°qÔ'9Ð$à#Ø!Ø(ðˆNð K_ÐfÐ/Ð1EÐFÐFÐdfˆà,¨RÔ0ˆØ,¨RÔ0ˆØŸš KÑ0Ô0ˆàð 	`Ø °Ð>ÀÑNÐQ_Ñ_Ð_å Ø'Ø'Ø'Ø=QÐ#[Ð#9Ð#9ÐW[Ø9MÐ!WÐ!5Ð!5ÐSWØ7HÐ RÐ 3Ð 3ÈdØ3DÐNÐ/Ð/È$ØARÐ%\Ð%=Ð%=ÐX\ð	
ñ 	
ô 	
ð 		
r   )
NNNNNNNNNN)r   r    r!   r   rX  r[  r   r0   Ú
LongTensorr1   Úboolr&   r3   r   r"   r#   s   @r   rP  rP  ¢  sl  ø€ € € € € ðð ð ð ð ð/ð /ð /ð9ð 9ð 9ð ð .2Ø15Ø/3Ø37Ø:>Ø26Ø26Ø)-Ø,0Ø#'ðC
ð C
àÔ# dÑ*ðC
ð Ô'¨$Ñ.ðC
ð Ô%¨Ñ,ð	C
ð
 Ô)¨DÑ0ðC
ð  %Ô0°4Ñ7ðC
ð Ô(¨4Ñ/ðC
ð Ô(¨4Ñ/ðC
ð   $™;ðC
ð # T™kðC
ð ˜D‘[ðC
ð 
˜U 5Ô#4Ô5Ñ	5ðC
ð C
ð C
ñ „^ðC
ð C
ð C
ð C
ð C
r   rP  c            "       ó
  ‡ — e Zd ZddiZˆ fd„Z	 d dededz  ded	ej        fˆ fd
„Z	defd„Z
d„ Zd„ Zd	ej        fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  deeeej        ej        f         f         dz  dej        dz  dej        dz  dedz  dedz  dedz  d	eeej                 z  fd„¦   «         Zˆ xZS )"ÚLxmertForPreTrainingzcls.predictions.decoder.weightz(lxmert.embeddings.word_embeddings.weightc                 óÜ  •— t          ¦   «                              |¦  «         || _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        |j        | _        t          |¦  «        | _
        t          |¦  «        | _        | j        rt          |¦  «        | _        | j        rt          || j        ¦  «        | _        |                      ¦   «          t%          d¬¦  «        t'          d¬¦  «        t'          ¦   «         dœ| _        i }|j        rd|j        ddœ|d<   |j        rd|j        ddœ|d<   |j        rd	|j        f|j        d
dœ|d<   || _        d S )NÚnone)Ú	reduction)Úl2Ú	visual_ceÚcer'  rt  )rx   r(  r7   r)  r*  rS   rs  r+  )r   r   rQ   Únum_qa_labelsÚvisual_loss_normalizerÚtask_mask_lmÚtask_obj_predictÚtask_matchedÚtask_qarP  rC  r9  Úclsr%  Úobj_predict_headr  Úanswer_headrT  r   r   Ú	loss_fctsr/  r0  r1  r2  r3  rà   r-  r6  s      €r   r   zLxmertForPreTraining.__init__@  s¤  ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒØ#Ô1ˆÔØ&,Ô&CˆÔ#ð #Ô/ˆÔØ &Ô 7ˆÔØ"Ô/ˆÔØ”~ˆŒõ " &Ñ)Ô)ˆŒõ *¨&Ñ1Ô1ˆŒØÔ ð 	@Ý$7¸Ñ$?Ô$?ˆDÔ!ØŒ<ð 	RÝ5°f¸dÔ>PÑQÔQˆDÔð 	�ŠÑÔÐõ ¨Ð0Ñ0Ô0Ý)°FÐ;Ñ;Ô;Ý"Ñ$Ô$ð
ð 
ˆŒð ˆØÔ!ð 	àØÔ/Ø#ð$ð $ˆM˜%Ñ ð
 Ô"ð 	àØÔ-Ø#ð%ð %ˆM˜&Ñ!ð
 Ô"ð 	à˜fÔ4Ð5ØÔ-Øð%ð %ˆM˜&Ñ!ð
 +ˆÔÐÐr   NTÚnew_num_tokensÚpad_to_multiple_ofÚmean_resizingr^  c                 ó¶   •— t          ¦   «                              |||¦  «        }|                      | j        j        j        |¦  «        | j        j        _        |S r   )r   Úresize_token_embeddingsÚ_resize_biasr|  r;  r  )r   r€  r�  r‚  rZ  r   s        €r   r„  z,LxmertForPreTraining.resize_token_embeddingsw  sN   ø€ õ ™œ×8Ò8¸ÐI[Ð]jÑkÔkˆØ$(×$5Ò$5°d´hÔ6JÔ6OÐQ_Ñ$`Ô$`ˆŒÔÔ!ØÐr   c                 óÒ   — |j         d         }||k    r|d |…         }n4t          j        ||z
  |j        ¬¦  «        }t          j        ||g¦  «        }t          j        |¦  «        }|S )Nr   r`  )rx   r0   r\   rV   Úcatr   r  )r   r  r€  Úold_num_tokensÚnew_biasÚ
extra_biass         r   r…  z!LxmertForPreTraining._resize_bias  sk   € Øœ AœˆØ˜^Ò+Ð+Ø˜O˜^˜OÔ,ˆHˆHåœ ^°nÑ%DÈTÌ[ÐYÑYÔYˆJÝ”y $¨
Ð!3Ñ4Ô4ˆHÝ”< Ñ)Ô)ˆØˆr   c                 óŠ   — |                       ¦   «         }|�|€dS |                      |¦  «        }|| j        _        || _        |S ©aç  
        Build a resized question answering linear layer Module from a provided new linear layer. Increasing the size
        will add newly initialized weights. Reducing the size will remove weights from the end

        Args:
            num_labels (`int`, *optional*):
                New number of labels in the linear layer weight matrix. Increasing the size will add newly initialized
                weights at the end. Reducing the size will remove weights from the end. If not provided or `None`, just
                returns a pointer to the qa labels ``torch.nn.Linear``` module of the model without doing anything.

        Return:
            `torch.nn.Linear`: Pointer to the resized Linear layer or the old Linear layer
        N©Úget_qa_logit_layerÚ_resize_qa_labelsrQ   rv  ©r   r!  Úcur_qa_logit_layerÚnew_qa_logit_layers       r   Úresize_num_qa_labelsz)LxmertForPreTraining.resize_num_qa_labels‰  óS   € ð "×4Ò4Ñ6Ô6ÐØÐÐ!3Ð!;ØˆFØ!×3Ò3°JÑ?Ô?ÐØ$.ˆŒÔ!Ø'ˆÔà!Ð!r   c                 ó¨   — |                       ¦   «         }|                      ||¦  «        }|                      |¦  «         |                       ¦   «         S r   ©rŽ  Ú_get_resized_qa_labelsÚ_set_qa_logit_layerr�  s       r   r�  z&LxmertForPreTraining._resize_qa_labels¡  óR   € Ø!×4Ò4Ñ6Ô6ÐØ!×8Ò8Ð9KÈZÑXÔXÐØ× Ò Ð!3Ñ4Ô4Ð4Ø×&Ò&Ñ(Ô(Ð(r   c                 óJ   — t          | d¦  «        r| j        j        d         S dS )a  
        Returns the linear layer that produces question answering logits.

        Returns:
            `nn.Module`: A torch module mapping the question answering prediction hidden states or `None` if LXMERT
            does not have a visual answering head.
        r~  rS   N©Úhasattrr~  r   rW  s    r   rŽ  z'LxmertForPreTraining.get_qa_logit_layer§  s1   € õ �4˜Ñ'Ô'ð 	1ØÔ#Ô,¨RÔ0Ð0ð	1ð 	1r   c                 ó$   — || j         j        d<   d S ©NrS   ©r~  r   ©r   Úqa_logit_layers     r   r˜  z(LxmertForPreTraining._set_qa_logit_layer²  ó   € Ø(6ˆÔÔ! "Ñ%Ð%Ð%r   c                 ó  — |€|S |j                              ¦   «         \  }}||k    r|S t          |dd ¦  «        �t          j        ||¦  «        }nt          j        ||d¬¦  «        }|                     |j         j        ¦  «         |                      |¦  «         t          ||¦  «        }|j         j	        d |…d d …f         |j         j	        d |…d d …f<   t          |dd ¦  «        �#|j
        j	        d |…         |j
        j	        d |…<   |S ©Nr  Fr  ©ÚweightrW   Úgetattrr   rn   rc  rV   rG  re  Údatar  ©r   r‘  r!  Úcur_qa_labelsÚ
hidden_dimr’  Únum_labels_to_copys          r   r—  z+LxmertForPreTraining._get_resized_qa_labelsµ  ó?  € ØÐØ%Ð%à$6Ô$=×$BÒ$BÑ$DÔ$DÑ!ˆ�zØ˜JÒ&Ð&Ø%Ð%õ Ð% v¨tÑ4Ô4Ð@Ý!#¤¨:°zÑ!BÔ!BÐÐå!#¤¨:°zÈÐ!NÑ!NÔ!NÐà×ÒÐ0Ô7Ô>Ñ?Ô?Ð?ð 	×ÒÐ-Ñ.Ô.Ð.õ ! °
Ñ;Ô;ÐØASÔAZÔA_Ð`sÐasÐ`sÐuvÐuvÐuvÐ`vÔAwÐÔ!Ô&Ð':Ð(:Ð':¸A¸A¸AÐ'=Ñ>ÝÐ% v¨tÑ4Ô4Ð@Ø@RÔ@WÔ@\Ð]pÐ^pÐ]pÔ@qÐÔ#Ô(Ð)<Ð*<Ð)<Ñ=à!Ð!r   r^   rÜ   rê   r„   rÎ   r_   r`   ÚlabelsÚ
obj_labelsÚmatched_labelÚansr…   r\  r]  c                 óX  — |�|n| j         j        }|�|j        n|j        }|                      ||||||||||¬¦
  «
        }|d         |d         |d         }}}|                      ||¦  «        \  }}| j        r|                      |¦  «        }n|d         d         }|€|
€|	€|€dnt          j        d|¬¦  «        }|�T| j	        rM | j
        d         |                     d	| j         j        ¦  «        |                     d	¦  «        ¦  «        }||z  }|
�J| j        rC | j
        d         |                     d	d¦  «        |
                     d	¦  «        ¦  «        }||z  }|	��-| j        �r%t          j        d|j        ¬¦  «        }|                      |¦  «        }| j                             ¦   «         D ]Ö\  }}|	|         \  }} |d
         }!|d         }"|d         }#| j        }$| j
        |"         }%||         }& |%|&                     d	|!¦  «        |                     |#¦  «        ¦  «        }'|'                     ¦   «         dk    r|'                     d¦  «        }'|'|                      d	¦  «        z                       ¦   «         |$z  }'||'z  }Œ×||z  }|�O| j        rH | j
        d         |                     d	| j        ¦  «        |                     d	¦  «        ¦  «        }(||(z  }|s|||f|dd…         z   })|�|f|)z   n|)S t-          |||||j        |j        |j        |j        |j        ¬¦	  «	        S )a²	  
        visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`):
            This input represents visual features. They ROI pooled object features from bounding boxes using a
            faster-RCNN model)

            These are currently not provided by the transformers library.
        visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`):
            This input represents spatial features corresponding to their relative (via index) visual features. The
            pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to
            1.

            These are currently not provided by the transformers library.
        visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        obj_labels (`dict[Str: tuple[Torch.FloatTensor, Torch.FloatTensor]]`, *optional*):
            each key is named after each one of the visual losses and each element of the tuple is of the shape
            `(batch_size, num_features)` and `(batch_size, num_features, visual_feature_dim)` for each the label id and
            the label score respectively
        matched_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the whether or not the text input matches the image (classification) loss. Input
            should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`:

            - 0 indicates that the sentence does not match the image,
            - 1 indicates that the sentence does match the image.
        ans (`Torch.Tensor` of shape `(batch_size)`, *optional*):
            a one hot representation hof the correct answer *optional*
        N©
r^   rÜ   rê   r_   r„   rÎ   r`   r\  r…   r]  r   r   ru   g        r`  ru  rS   r(  r7   rx   r   )	r7   r;   r<   r8   r*   r+   r,   r-   r.   )rQ   r]  rV   rC  r|  r{  r~  r0   Útensorrx  r  ry   rE   rz  ry  r}  r-  Úitemsrw  rw   Úmeanrv  r:   r*   r+   r,   r-   r.   )*r   r^   rÜ   rê   r„   rÎ   r_   r`   r®  r¯  r°  r±  r…   r\  r]  rf  rV   Úlxmert_outputrÇ   rØ   r)   Úlang_prediction_scoresr<   Úanswer_scoreÚ
total_lossÚmasked_lm_lossÚmatched_lossÚtotal_visual_lossÚvisual_prediction_scores_dictrp   Úkey_infoÚlabelÚ	mask_confÚ
output_dimÚloss_fct_nameÚlabel_shaper¦  Úvisual_loss_fctÚvisual_prediction_scoresÚvisual_lossÚanswer_lossr    s*                                             r   r   zLxmertForPreTraining.forwardÐ  sË  € ðp &1Ð%<�k�kÀ$Ä+ÔBYˆà%.Ð%:�Ô!Ð!ÀÔ@TˆØŸšØØ%Ø!Ø)Ø)Ø"7Ø'Ø!5Ø/Ø#ð $ñ 
ô 
ˆð ˜!ÔØ˜!ÔØ˜!Ôð %2�]ˆð
 <@¿8º8ÀKÐQ^Ñ;_Ô;_Ñ8ÐÐ 8ØŒ<ð 	/Ø×+Ò+¨MÑ:Ô:ˆLˆLà(¨Ô+¨AÔ.ˆLð � =Ð#8¸ZÐ=OÐTWÐT_ð ˆDå”˜c¨&Ð1Ñ1Ô1ð 	ð
 Ð $Ô"3ÐØ1˜Tœ^¨DÔ1Ø&×+Ò+¨B°´Ô0FÑGÔGØ—’˜B‘”ñô ˆNð ˜.Ñ(ˆJØÐ$¨Ô):Ð$Ø/˜4œ>¨$Ô/Ð0H×0MÒ0MÈbÐRSÑ0TÔ0TÐVc×VhÒVhÐikÑVlÔVlÑmÔmˆLØ˜,Ñ&ˆJØÑ! dÔ&;Ñ!Ý %¤¨S¸Ô9IÐ JÑ JÔ JÐØ,0×,AÒ,AÀ-Ñ,PÔ,PÐ)Ø!%Ô!3×!9Ò!9Ñ!;Ô!;ð 1ð 1‘��XØ#-¨c¤?Ñ ��yØ% eœ_�
Ø (¨Ô 0�Ø& wÔ/�ØÔ4�Ø"&¤.°Ô"?�Ø+HÈÔ+MÐ(Ø-˜oØ,×1Ò1°"°jÑAÔAØ—J’J˜{Ñ+Ô+ñô �ð —?’?Ñ$Ô$ qÒ(Ð(Ø"-×"2Ò"2°1Ñ"5Ô"5�KØ*¨Y¯^ª^¸BÑ-?Ô-?Ñ?×EÒEÑGÔGÈ&ÑP�Ø! [Ñ0Ð!Ð!ØÐ+Ñ+ˆJØˆ?˜tœ|ˆ?Ø.˜$œ.¨Ô.¨|×/@Ò/@ÀÀTÔEWÑ/XÔ/XÐZ]×ZbÒZbÐceÑZfÔZfÑgÔgˆKØ˜+Ñ%ˆJàð 	Rà&Ø(Øðð ˜a˜b˜bÔ!ñ	"ˆFð
 0:Ð/E�Z�M FÑ*Ð*È6ÐQå)ØØ4Ø%=Ø%1Ø#0Ô#GØ!.Ô!CØ -Ô AØ+Ô=Ø%2Ô%Kð

ñ 

ô 

ð 
	
r   )NT)NNNNNNNNNNNNNN)r   r    r!   Ú_tied_weights_keysr   rk   rm  r   rD   r„  r…  r“  r�  ÚModulerŽ  r˜  r—  r   r0   rl  r1   ÚdictÚstrr3   ÚTensorr:   r   r"   r#   s   @r   ro  ro  9  s‘  ø€ € € € € ð 	)Ð*TðÐð5+ð 5+ð 5+ð 5+ð 5+ðp aeðð Ø!ðØ7:¸T±zðØY]ðà	Œðð ð ð ð ð ð°ð ð ð ð ð"ð "ð "ð0)ð )ð )ð	1 B¤Ið 	1ð 	1ð 	1ð 	1ð7ð 7ð 7ð"ð "ð "ð6 ð .2Ø15Ø/3Ø37Ø:>Ø26Ø26Ø*.ØTXØ15Ø#'Ø)-Ø,0Ø#'ðJ
ð J
àÔ# dÑ*ðJ
ð Ô'¨$Ñ.ðJ
ð Ô%¨Ñ,ð	J
ð
 Ô)¨DÑ0ðJ
ð  %Ô0°4Ñ7ðJ
ð Ô(¨4Ñ/ðJ
ð Ô(¨4Ñ/ðJ
ð Ô  4Ñ'ðJ
ð ˜˜e EÔ$5°uÔ7HÐ$HÔIÐIÔJÈTÑQðJ
ð Ô'¨$Ñ.ðJ
ð Œ\˜DÑ ðJ
ð   $™;ðJ
ð # T™kðJ
ð ˜D‘[ðJ
ð" 
$ e¨EÔ,=Ô&>Ñ	>ð#J
ð J
ð J
ñ „^ðJ
ð J
ð J
ð J
ð J
r   ro  zR
    Lxmert Model with a visual-answering head on top for downstream QA tasks
    c                   óZ  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej        fd„Zd„ Z	d„ Z
e	 	 	 	 	 	 	 	 	 	 	 dd	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dedz  deeej                 z  fd„¦   «         Zˆ xZS )ÚLxmertForQuestionAnsweringc                 ó2  •— t          ¦   «                              |¦  «         || _        |j        | _        |j        | _        t          |¦  «        | _        t          || j        ¦  «        | _        |  	                    ¦   «          t          ¦   «         | _        d S r   )r   r   rQ   rv  rw  rP  rC  r  r~  rT  r   r7   rP   s     €r   r   z#LxmertForQuestionAnswering.__init__d  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆŒØ#Ô1ˆÔØ&,Ô&CˆÔ#õ " &Ñ)Ô)ˆŒå1°&¸$Ô:LÑMÔMˆÔð 	�ŠÑÔÐõ %Ñ&Ô&ˆŒ	ˆ	ˆ	r   c                 óŠ   — |                       ¦   «         }|�|€dS |                      |¦  «        }|| j        _        || _        |S rŒ  r�  r�  s       r   r“  z/LxmertForQuestionAnswering.resize_num_qa_labelsw  r”  r   c                 ó¨   — |                       ¦   «         }|                      ||¦  «        }|                      |¦  «         |                       ¦   «         S r   r–  r�  s       r   r�  z,LxmertForQuestionAnswering._resize_qa_labels�  r™  r   r^  c                 óJ   — t          | d¦  «        r| j        j        d         S dS )a  
        Returns the linear layer that produces question answering logits

        Returns:
            `nn.Module`: A torch module mapping the question answering prediction hidden states. `None`: A NoneType
            object if Lxmert does not have the visual answering head.
        r~  rS   Nr›  rW  s    r   rŽ  z-LxmertForQuestionAnswering.get_qa_logit_layer•  s1   € õ �4˜Ñ'Ô'ð 	1ØÔ#Ô,¨RÔ0Ð0ð	1ð 	1r   c                 ó$   — || j         j        d<   d S rž  rŸ  r   s     r   r˜  z.LxmertForQuestionAnswering._set_qa_logit_layer¡  r¢  r   c                 ó  — |€|S |j                              ¦   «         \  }}||k    r|S t          |dd ¦  «        �t          j        ||¦  «        }nt          j        ||d¬¦  «        }|                     |j         j        ¦  «         |                      |¦  «         t          ||¦  «        }|j         j	        d |…d d …f         |j         j	        d |…d d …f<   t          |dd ¦  «        �#|j
        j	        d |…         |j
        j	        d |…<   |S r¤  r¥  r©  s          r   r—  z1LxmertForQuestionAnswering._get_resized_qa_labels¤  r­  r   Nr^   rÜ   rê   r„   rÎ   r_   r`   r®  r…   r\  r]  c                 ó¸  — |�|n| j         j        }|                      ||||||||
|	|¬¦
  «
        }|d         }|                      |¦  «        }d}|�B|                      |                     d| j        ¦  «        |                     d¦  «        ¦  «        }|s|f|dd…         z   }|�|f|z   n|S t          |||j        |j	        |j
        |j        |j        ¬¦  «        S )a÷  
        visual_feats (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`):
            This input represents visual features. They ROI pooled object features from bounding boxes using a
            faster-RCNN model)

            These are currently not provided by the transformers library.
        visual_pos (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_pos_dim)`):
            This input represents spatial features corresponding to their relative (via index) visual features. The
            pre-trained LXMERT model expects these spatial features to be normalized bounding boxes on a scale of 0 to
            1.

            These are currently not provided by the transformers library.
        visual_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        labels (`Torch.Tensor` of shape `(batch_size)`, *optional*):
            A one-hot representation of the correct answer
        Nr³  ru   rS   r   )r7   r8   r*   r+   r,   r-   r.   )rQ   r]  rC  r~  r7   ry   rv  r6   r*   r+   r,   r-   r.   )r   r^   rÜ   rê   r„   rÎ   r_   r`   r®  r…   r\  r]  rf  r·  r)   r¹  r7   r    s                     r   r   z"LxmertForQuestionAnswering.forward¿  s  € ðL &1Ð%<�k�kÀ$Ä+ÔBYˆàŸšØØ%Ø!Ø)Ø)Ø"7Ø'Ø!5Ø/Ø#ð $ñ 
ô 
ˆð & aÔ(ˆØ×'Ò'¨Ñ6Ô6ˆØˆØÐØ—9’9˜\×.Ò.¨r°4Ô3EÑFÔFÈÏÊÐTVÉÌÑXÔXˆDàð 	DØ"�_ }°Q°R°RÔ'8Ñ8ˆFØ'+Ð'7�D�7˜VÑ#Ð#¸VÐCå/ØØ%1Ø#0Ô#GØ!.Ô!CØ -Ô AØ+Ô=Ø%2Ô%Kð
ñ 
ô 
ð 	
r   )NNNNNNNNNNN)r   r    r!   r   r“  r�  r   rÊ  rŽ  r˜  r—  r   r0   rl  r1   rÍ  rm  r6   r3   r   r"   r#   s   @r   rÏ  rÏ  ^  s¼  ø€ € € € € ð'ð 'ð 'ð 'ð 'ð&"ð "ð "ð0)ð )ð )ð
1 B¤Ið 
1ð 
1ð 
1ð 
1ð7ð 7ð 7ð"ð "ð "ð6 ð .2Ø15Ø/3Ø37Ø:>Ø26Ø26Ø&*Ø)-Ø,0Ø#'ðF
ð F
àÔ# dÑ*ðF
ð Ô'¨$Ñ.ðF
ð Ô%¨Ñ,ð	F
ð
 Ô)¨DÑ0ðF
ð  %Ô0°4Ñ7ðF
ð Ô(¨4Ñ/ðF
ð Ô(¨4Ñ/ðF
ð ”˜tÑ#ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
*¨E°%Ô2CÔ,DÑ	DðF
ð F
ð F
ñ „^ðF
ð F
ð F
ð F
ð F
r   rÏ  )rí   ro  rÏ  rP  rB  rÞ   rÁ   )5r/   r|   Údataclassesr   r0   r   Útorch.nnr   r   Ú r   rI  Úactivationsr	   r
   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_lxmertr   Ú
get_loggerr   ÚloggerrÊ  r   r&   r6   r:   r>   rf   r’   r�   r¨   r­   rµ   r¹   rÁ   rÞ   rí   r  r  r  r  r%  r9  rB  rP  ro  rÏ  Ú__all__r4   r   r   ú<module>rá     sW  ðð Ð à €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3à &Ð &Ð &Ð &Ð &Ð &Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ð 'Ø -Ð -Ð -Ð -Ð -Ð -Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð ˆ2Œ9ñ ô ð ð €ððñ ô ð ð$Eð $Eð $Eð $Eð $E˜ñ $Eô $Eñ „ñô ð$EðN €ððñ ô ð
 ð!Eð !Eð !Eð !Eð !E {ñ !Eô !Eñ „ñô ð!EðH €ððñ ô ð
 ð(Eð (Eð (Eð (Eð (E ñ (Eô (Eñ „ñô ð(EðV#ð #ð #ð #ð #�r”yñ #ô #ð #ðL1ð 1ð 1ð 1ð 1�b”iñ 1ô 1ð 1ðhð ð ð ð ˜BœIñ ô ð ðð ð ð ð  ¤	ñ ô ð ðð ð ð ð ˜rœyñ ô ð ð*	ð 	ð 	ð 	ð 	˜œñ 	ô 	ð 	ðð ð ð ð �2”9ñ ô ð ðð ð ð ð �"”)ñ ô ð ð X
ð X
ð X
ð X
ð X
�2”9ñ X
ô X
ð X
ðvð ð ð ð  ¤ñ ô ð ð6N
ð N
ð N
ð N
ð N
�B”Iñ N
ô N
ð N
ðbð ð ð ð �2”9ñ ô ð ðð ð ð ð  B¤Iñ ô ð ð
ð 
ð 
ð 
ð 
˜RœYñ 
ô 
ð 
ð,ð ,ð ,ð ,ð ,˜RœYñ ,ô ,ð ,ðð ð ð ð ˜"œ)ñ ô ð ð>	9ð 	9ð 	9ð 	9ð 	9˜RœYñ 	9ô 	9ð 	9ð ð
%ð 
%ð 
%ð 
%ð 
%˜Oñ 
%ô 
%ñ „ð
%ð ðS
ð S
ð S
ð S
ð S
Ð'ñ S
ô S
ñ „ðS
ðl ða
ð a
ð a
ð a
ð a
Ð0ñ a
ô a
ñ „ða
ðH	 €ððñ ô ð
c
ð c
ð c
ð c
ð c
Ð!6ñ c
ô c
ñô ð
c
ðLð ð €€€r   