§
    ‚ŠtjG) ã                   óà  — d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4  e*j5        e6¦  «        Z7dZ8 e)d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z9 e)d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z: G d„ d ej;        ¦  «        Z< G d!„ d"ej;        ¦  «        Z= G d#„ d$ej;        ¦  «        Z> G d%„ d&ej;        ¦  «        Z? G d'„ d(ej;        ¦  «        Z@ G d)„ d*ej;        ¦  «        ZA G d+„ d,ej;        ¦  «        ZB G d-„ d.ej;        ¦  «        ZC G d/„ d0ej;        ¦  «        ZD	 	 ded2ej;        d3ejE        d4ejE        d5ejE        d6ejE        dz  d7eFdz  d8eFd9e$e(         fd:„ZG G d;„ d<ej;        ¦  «        ZH G d=„ d>ej;        ¦  «        ZI G d?„ d@ej;        ¦  «        ZJ G dA„ dBej;        ¦  «        ZK G dC„ dDe¦  «        ZL G dE„ dFej;        ¦  «        ZM G dG„ dHej;        ¦  «        ZNe) G dI„ dJe"¦  «        ¦   «         ZO G dK„ dLeO¦  «        ZP e)dM¬¦  «         G dN„ dOeO¦  «        ¦   «         ZQ e)dP¬¦  «         G dQ„ dReO¦  «        ¦   «         ZR G dS„ dTej;        ¦  «        ZS G dU„ dVej;        ¦  «        ZT G dW„ dXej;        ¦  «        ZU e)dY¬¦  «         G dZ„ d[eO¦  «        ¦   «         ZV e)d\¬¦  «         G d]„ d^eO¦  «        ¦   «         ZW G d_„ d`ej;        ¦  «        ZX e)da¬¦  «         G db„ dceO¦  «        ¦   «         ZYg dd¢ZZdS )fzPyTorch BridgeTower Modelé    )ÚOrderedDict)ÚCallable)Ú	dataclassN)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FNÚQuickGELUActivation)ÚCacheÚDynamicCacheÚEncoderDecoderCache)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚModelOutputÚSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚloggingÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚBridgeTowerConfigÚBridgeTowerTextConfigÚBridgeTowerVisionConfigÚRobertaTokenizerz.
    Output type of [`BridgeTowerModel`].
    )Úcustom_introc                   óÂ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
eej                 dz  ed<   dZeej                 dz  ed<   dS )ÚBridgeTowerModelOutputa«  
    text_features (`torch.FloatTensor` of shape `(batch_size, text_sequence_length, hidden_size)`):
        Sequence of hidden-states at the text output of the last layer of the model.
    image_features (`torch.FloatTensor` of shape `(batch_size, image_sequence_length, hidden_size)`):
        Sequence of hidden-states at the image output of the last layer of the model.
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size x 2)`):
        Concatenation of last layer hidden-state of the first token of the text and image sequence (classification
        token), respectively, after further processing through layers used for auxiliary pretraining tasks.
    NÚtext_featuresÚimage_featuresÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r*   ÚtorchÚFloatTensorÚ__annotations__r+   r,   r-   Útupler.   © ó    úr/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/bridgetower/modeling_bridgetower.pyr)   r)   2   s    € € € € € € ðð ð /3€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø/3€N�EÔ%¨Ñ,Ð3Ð3Ñ3Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r8   r)   z>
    Output type of ['BridgeTowerForContrastiveLearning']
    c                   ó"  — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed<   dZe
ej                 dz  ed	<   dS )
ÚBridgeTowerContrastiveOutputaö  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Image-text contrastive loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    text_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The text embeddings obtained by applying the projection layer to the pooler_output.
    image_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The image embeddings obtained by applying the projection layer to the pooler_output.
    cross_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
        The text-image cross-modal embeddings obtained by applying the projection layer to the pooler_output.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
    NÚlossÚlogitsÚtext_embedsÚimage_embedsÚcross_embedsr-   r.   )r/   r0   r1   r2   r<   r3   r4   r5   r=   r>   r6   r?   r@   r-   r.   r7   r8   r9   r;   r;   J   sß   € € € € € € ðð ð  &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø37€K��uÔ(Ô)¨DÑ0Ð7Ð7Ñ7Ø48€L�%˜Ô)Ô*¨TÑ1Ð8Ð8Ñ8Ø48€L�%˜Ô)Ô*¨TÑ1Ð8Ð8Ñ8Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r8   r;   c                   ón   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zddej        dej        dz  fd„Zˆ xZS )ÚBridgeTowerResidualAttentionc                 ó.  •— t          ¦   «                              ¦   «          t          j        |j        |j        dz  ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        t          dt          j        |j        |j        dz  ¦  «        fdt          ¦   «         fdt          j        |j        dz  |j        ¦  «        fg¦  «        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        d | _        d S )Né@   ©ÚepsÚc_fcé   ÚgeluÚc_proj)ÚsuperÚ__init__r   ÚMultiheadAttentionÚhidden_sizeÚattnÚ	LayerNormÚlayer_norm_epsÚln_1Ú
ModuleDictr   ÚLinearr   ÚmlpÚln_2Ú	attn_mask©ÚselfÚconfigÚ	__class__s     €r9   rL   z%BridgeTowerResidualAttention.__init__k   sñ   ø€ Ý‰Œ×ÒÑÔÐåÔ)¨&Ô*<¸fÔ>PÐTVÑ>VÑWÔWˆŒ	Ý”L Ô!3¸Ô9NÐOÑOÔOˆŒ	Ý”=Ýà�RœY vÔ'9¸6Ô;MÐPQÑ;QÑRÔRÐSØÕ0Ñ2Ô2Ð3Ø�rœy¨Ô);¸aÑ)?ÀÔASÑTÔTÐUðñô ñ
ô 
ˆŒõ ”L Ô!3¸Ô9NÐOÑOÔOˆŒ	ØˆŒˆˆr8   Úhidden_stateÚattention_maskc                 ó  — |�&|                      t          j        |j        ¬¦  «        }| j        �&| j                              |j        |j        ¬¦  «        nd | _        |                      |||d| j        |¬¦  «        d         S )N©ÚdtypeÚdeviceF)Úneed_weightsrW   Úkey_padding_maskr   )Útor3   Úboolra   rW   r`   rO   )rY   r\   r]   s      r9   Ú	attentionz&BridgeTowerResidualAttention.attention|   s—   € ØÐ%Ø+×.Ò.µU´ZÈÔH[Ð.Ñ\Ô\ˆNð Œ~Ð)ð ŒN×Ò LÔ$6¸|Ô?RÐÑSÔSÐSàð 	Œð
 �yŠyØØØØØ”nØ+ð ñ 
ô 
ð ôð 	r8   Nc                 óà   — ||                       |                      |¦  «        |¦  «        z   }|                      |¦  «        }| j                             ¦   «         D ]} ||¦  «        }Œ||z   }|S ©N)rf   rR   rV   rU   Úvalues)rY   r\   r]   Úresidual_stateÚlayers        r9   Úforwardz$BridgeTowerResidualAttention.forward�   st   € Ø%¨¯ª°t·y²yÀÑ7NÔ7NÐP^Ñ(_Ô(_Ñ_ˆØ—y’y Ñ0Ô0ˆØ”X—_’_Ñ&Ô&ð 	/ð 	/ˆEØ ˜5 Ñ.Ô.ˆLˆLØ%¨Ñ4ˆØÐr8   rh   )	r/   r0   r1   rL   r3   ÚTensorrf   rl   Ú__classcell__©r[   s   @r9   rB   rB   j   s�   ø€ € € € € ðð ð ð ð ð" e¤lð ÀEÄLð ð ð ð ð"ð  E¤Lð À%Ä,ÐQUÑBUð ð ð ð ð ð ð ð r8   rB   c                   óJ   ‡ — e Zd Zˆ fd„Zddej        dej        dz  fd„Zˆ xZS )ÚBridgeTowerTransformerc                 ó„  •‡— t          ¦   «                              ¦   «          ‰j        | _        ‰j        | _        ‰j        r;t          j        ˆfd„t          | j        dz
  ¦  «        D ¦   «         ¦  «        | _        n7t          j        ˆfd„t          | j        ¦  «        D ¦   «         ¦  «        | _        ‰j	        | _	        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   ©rB   ©Ú.0Ú_rZ   s     €r9   ú
<listcomp>z3BridgeTowerTransformer.__init__.<locals>.<listcomp>�   s"   ø€ ÐaÐaÐa¸!Õ-¨fÑ5Ô5ÐaÐaÐar8   r"   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   rt   ru   s     €r9   rx   z3BridgeTowerTransformer.__init__.<locals>.<listcomp>¡   s"   ø€ Ð]Ð]Ð]¸!Õ-¨fÑ5Ô5Ð]Ð]Ð]r8   )
rK   rL   rN   Únum_hidden_layersÚremove_last_layerr   Ú
ModuleListÚrangeÚ	resblocksÚstop_gradientrX   s    `€r9   rL   zBridgeTowerTransformer.__init__—   sÂ   øø€ Ý‰Œ×ÒÑÔÐØ!Ô-ˆÔØ!'Ô!9ˆÔØÔ#ð 	Ýœ]ØaÐaÐaÐa½uÀTÔE[Ð^_ÑE_Ñ?`Ô?`ÐaÑaÔañô ˆDŒNˆNõ  œ]Ø]Ð]Ð]Ð]½uÀTÔE[Ñ?\Ô?\Ð]Ñ]Ô]ñô ˆDŒNð $Ô1ˆÔÐÐr8   Nr\   r]   c                 ó¾   — g }| j         D ]R} |||¦  «        }| j        r(|                     |                     ¦   «         ¦  «         Œ=|                     |¦  «         ŒS|S rh   )r~   r   ÚappendÚdetach)rY   r\   r]   r-   Úblocks        r9   rl   zBridgeTowerTransformer.forward¥   st   € ØˆØ”^ð 	3ð 	3ˆEØ ˜5 ¨~Ñ>Ô>ˆLØÔ!ð 3Ø×$Ò$ \×%8Ò%8Ñ%:Ô%:Ñ;Ô;Ð;Ð;à×$Ò$ \Ñ2Ô2Ð2Ð2ØÐr8   rh   ©r/   r0   r1   rL   r3   rm   rl   rn   ro   s   @r9   rq   rq   –   si   ø€ € € € € ð2ð 2ð 2ð 2ð 2ðð  E¤Lð À%Ä,ÐQUÑBUð ð ð ð ð ð ð ð r8   rq   c                   óv   ‡ — e Zd Zdefˆ fd„Zdej        dededej        fd„Zdd	ej	        dej        fd
„Z
ˆ xZS )ÚBridgeTowerVisionEmbeddingsrZ   c                 óz  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        |j        | _        t          j        t          j
        | j        ¦  «        ¦  «        | _        t          j        |j        | j        | j        | j        d¬¦  «        | _        | j        | j        z  dz  | _        | j        dz   | _        t          j        | j        | j        ¦  «        | _        |                      dt          j        | j        ¦  «                             d¦  «        d¬¦  «         d S )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasé   r"   Úposition_ids©r"   éÿÿÿÿ©Ú
persistent)rK   rL   rZ   rN   Ú	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr3   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferÚarangeÚexpandrX   s     €r9   rL   z$BridgeTowerVisionEmbeddings.__init__²   s  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØ Ô+ˆŒØ Ô+ˆŒå!œ|­E¬K¸¼Ñ,GÔ,GÑHÔHˆÔå!œyØÔ+ØœØœØ”?Øð 
ñ  
ô  
ˆÔð !œO¨t¬Ñ>À1ÑDˆÔØ!Ô-°Ñ1ˆÔÝ"$¤,¨tÔ/AÀ4Ä>Ñ"RÔ"RˆÔØ×Ò˜^­U¬\¸$Ô:LÑ-MÔ-M×-TÒ-TÐU\Ñ-]Ô-]ÐjoÐÑpÔpÐpÐpÐpr8   Ú
embeddingsÚheightÚwidthÚreturnc                 óÚ  — |j         d         dz
  }| j        j                             d¦  «        }|j         d         dz
  }t          j                             ¦   «         s&||k    r ||k    r|                      | j        ¦  «        S |dd…dd…f         }|dd…dd…f         }|j         d         }	|| j        z  }
|| j        z  }t          |dz  ¦  «        }| 
                    d|||	¦  «        }|                     dddd¦  «        }t          j                             ||
|fdd	¬
¦  «        }|                     dddd¦  «                             dd|	¦  «        }t	          j        ||fd¬¦  «        S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r"   r   Nr�   ç      à?r   r�   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)ÚshaperŸ   ÚweightÚ	unsqueezer3   ÚjitÚ
is_tracingrŽ   r•   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)rY   r£   r¤   r¥   rœ   rŸ   r�   Úclass_pos_embedÚpatch_pos_embedr®   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r9   Úinterpolate_pos_encodingz4BridgeTowerVisionEmbeddings.interpolate_pos_encodingÈ   s‘  € ð !Ô& qÔ)¨AÑ-ˆØ!Ô4Ô;×EÒEÀaÑHÔHÐØ*Ô0°Ô3°aÑ7ˆõ Œy×#Ò#Ñ%Ô%ð 	>¨+¸Ò*FÐ*FÈ6ÐUZÊ?È?Ø×*Ò*¨4Ô+<Ñ=Ô=Ð=à,¨Q¨Q¨Q°°°¨UÔ3ˆØ,¨Q¨Q¨Q°°°¨UÔ3ˆàÔ˜rÔ"ˆà˜tœÑ.ˆ
Ø˜Tœ_Ñ,ˆ	å& }°cÑ'9Ñ:Ô:ÐØ)×1Ò1°!Ð5GÐI[Ð]`ÑaÔaˆØ)×1Ò1°!°Q¸¸1Ñ=Ô=ˆåœ-×3Ò3ØØ˜iÐ(ØØð	 4ñ 
ô 
ˆð *×1Ò1°!°Q¸¸1Ñ=Ô=×BÒBÀ1ÀbÈ#ÑNÔNˆåŒy˜/¨?Ð;ÀÐCÑCÔCÐCr8   FÚpixel_valuesc                 ó<  — |j         \  }}}}|s<|| j        k    s|| j        k    r&t          d|› d|› d| j        › d| j        › d�	¦  «        ‚| j        j        j        }|                      |                     |¬¦  «        ¦  «        }|                     d¦  «                             dd¦  «        }| j	         
                    |dd¦  «        }	t          j        |	|gd¬	¦  «        }
|r|
|                      |
||¦  «        z   }
n|
|                      | j        ¦  «        z   }
|
S )
NzInput image size (Ú*z) doesn't match model (z).©r`   r�   r"   r�   r­   )r¯   r”   Ú
ValueErrorr›   r°   r`   rd   ÚflattenÚ	transposer˜   r¢   r3   r¹   r¿   rŸ   rŽ   )rY   rÀ   r¿   Ú
batch_sizerw   r¤   r¥   Útarget_dtypeÚpatch_embedsÚclass_embedsr£   s              r9   rl   z#BridgeTowerVisionEmbeddings.forwardñ   sD  € Ø'3Ô'9Ñ$ˆ
�A�v˜uØ'ð 	¨V°t´Ò-FÐ-FÈ%ÐSWÔSbÒJbÐJbÝØq VÐqÐq¨eÐqÐqÈDÌOÐqÐqÐ^bÔ^mÐqÐqÐqñô ð ð Ô+Ô2Ô8ˆØ×+Ò+¨L¯OªOÀ,¨OÑ,OÔ,OÑPÔPˆØ#×+Ò+¨AÑ.Ô.×8Ò8¸¸AÑ>Ô>ˆàÔ+×2Ò2°:¸qÀ"ÑEÔEˆÝ”Y ¨lÐ;ÀÐCÑCÔCˆ
Ø#ð 	QØ# d×&CÒ&CÀJÐPVÐX]Ñ&^Ô&^Ñ^ˆJˆJà# d×&=Ò&=¸dÔ>OÑ&PÔ&PÑPˆJØÐr8   ©F)r/   r0   r1   r%   rL   r3   rm   Úintr¿   r4   rl   rn   ro   s   @r9   r†   r†   ±   sº   ø€ € € € € ðqÐ6ð qð qð qð qð qð qð,'D°5´<ð 'DÈð 'DÐUXð 'DÐ]bÔ]ið 'Dð 'Dð 'Dð 'DðRð  EÔ$5ð ÐZ_ÔZfð ð ð ð ð ð ð ð r8   r†   c                   óp   ‡ — e Zd Zˆ fd„Z	 d	dej        defd„Z	 d	dej        defd„Zdej        fd„Z	ˆ xZ
S )
ÚBridgeTowerVisionTransformerc                 óÆ  •‡— t          ¦   «                              ¦   «          t          ‰¦  «        | _        t	          j        ‰j        ‰j        ¬¦  «        | _        t          ‰¦  «        | _
        t	          j        ‰j        ‰j        ¬¦  «        | _        ‰j        | _        ‰j        s9t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S d S )NrE   c                 óP   •— g | ]"}t          j        ‰j        ‰j        ¬ ¦  «        ‘Œ#S )rE   )r   rP   rN   rQ   ru   s     €r9   rx   z9BridgeTowerVisionTransformer.__init__.<locals>.<listcomp>  s/   ø€ ÐvÐvÐvÐQR•”˜fÔ0°fÔ6KÐLÑLÔLÐvÐvÐvr8   )rK   rL   r†   r£   r   rP   rN   rQ   Úln_prerq   ÚtransformerÚln_postÚshare_layernormr|   r}   rz   Úln_separaterX   s    `€r9   rL   z%BridgeTowerVisionTransformer.__init__  sÏ   øø€ Ý‰Œ×ÒÑÔÐå5°fÑ=Ô=ˆŒÝ”l 6Ô#5¸6Ô;PÐQÑQÔQˆŒÝ1°&Ñ9Ô9ˆÔÝ”| FÔ$6¸FÔ<QÐRÑRÔRˆŒØ%Ô5ˆÔØÔ%ð 	Ý!œ}ØvÐvÐvÐvÕV[Ð\bÔ\tÑVuÔVuÐvÑvÔvñ ô  ˆDÔÐÐð	ð 	r8   FrÀ   r¿   c                 óò  — |                       ||¦  «        }|                      |¦  «        }|                     ddd¦  «        }|                      ||¦  «        }t	          j        |d¬¦  «        }|                     dddd¦  «        }| j        r|                      |¦  «        }nSg }t          || j	        ¦  «        D ]%\  }} ||¦  «        }| 
                    |¦  «         Œ&t	          j        |d¬¦  «        }|S )Nr"   r   r�   r­   r   )r£   rÑ   rµ   rÒ   r3   ÚstackrÔ   rÓ   ÚziprÕ   r�   )rY   rÀ   r]   r¿   r-   Úhidden_states_stackÚlns          r9   rl   z$BridgeTowerVisionTransformer.forward  s	  € ð Ÿš¨Ð6NÑOÔOˆØŸš MÑ2Ô2ˆà%×-Ò-¨a°°AÑ6Ô6ˆà×(Ò(¨¸ÑGÔGˆåœ M°qÐ9Ñ9Ô9ˆà%×-Ò-¨a°°A°qÑ9Ô9ˆØÔð 	DØ ŸLšL¨Ñ7Ô7ˆMˆMà"$ÐÝ%(¨¸Ô8HÑ%IÔ%Ið :ð :Ñ!�˜rØ "  =Ñ 1Ô 1�Ø#×*Ò*¨=Ñ9Ô9Ð9Ð9å!œKÐ(;ÀÐCÑCÔCˆMØÐr8   c                 óŒ   — |                       ||¬¦  «        }|                      |¦  «        }|                     ddd¦  «        }|S )N©r¿   r"   r   r�   )r£   rÑ   rµ   )rY   rÀ   r¿   r-   s       r9   Úforward_prez(BridgeTowerVisionTransformer.forward_pre-  sH   € ð
 Ÿš¨ÐOg˜ÑhÔhˆØŸš MÑ2Ô2ˆà%×-Ò-¨a°°AÑ6Ô6ˆØÐr8   r\   c                 ó^   — |                      ddd¦  «        }|                      |¦  «        }|S )Nr"   r   r�   )rµ   rÓ   )rY   r\   Úvisual_output_posts      r9   Úforward_postz)BridgeTowerVisionTransformer.forward_post8  s3   € Ø)×1Ò1°!°Q¸Ñ:Ô:ÐØ!Ÿ\š\Ð*<Ñ=Ô=ÐØ!Ð!r8   rË   )r/   r0   r1   rL   r3   rm   re   rl   rÝ   rà   rn   ro   s   @r9   rÎ   rÎ     s¹   ø€ € € € € ðð ð ð ð ð" */ð	ð à”lðð #'ð	ð ð ð ð< */ð	ð 	à”lð	ð #'ð	ð 	ð 	ð 	ð"¨¬ð "ð "ð "ð "ð "ð "ð "ð "r8   rÎ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerLinkTowerc                 óÞ  •— t          ¦   «                              ¦   «          |j        | _        |j        | _        |j        dv r”|j        dk    r,t	          j        t          j        d¦  «        ¦  «        | _        n6|j        dk    r+t	          j        t          j        d¦  «        ¦  «        | _	        t	          j
        | j        |j        ¬¦  «        | _
        d S t          d|j        › d�¦  «        ‚)	N)ÚaddÚ
scaled_addr·   rå   g      ð?r·   r¨   rE   úlink_tower_type ú is not implemented)rK   rL   Úlink_tower_typerN   r   r–   r3   ÚtensorÚscaled_factorÚbetarP   rQ   ÚNotImplementedErrorrX   s     €r9   rL   zBridgeTowerLinkTower.__init__?  sÑ   ø€ Ý‰Œ×ÒÑÔÐØ%Ô5ˆÔØ!Ô-ˆÔØÔ!Ð%IÐIÐIØÔ%¨Ò5Ð5Ý%'¤\µ%´,¸sÑ2CÔ2CÑ%DÔ%D�Ô"Ð"ØÔ'¨=Ò8Ð8ÝœL­¬°cÑ):Ô):Ñ;Ô;�”	Ýœ\¨$Ô*:ÀÔ@UÐVÑVÔVˆDŒNˆNˆNå%Ð&d¸Ô9OÐ&dÐ&dÐ&dÑeÔeÐer8   c                 ó:  — | j         dk    r|                      ||z   ¦  «        S | j         dk    r |                      || j        z  |z   ¦  «        S | j         dk    r+|                      |d| j        z
  z  || j        z  z   ¦  «        S t	          d| j         › d�¦  «        ‚)Nrä   rå   r·   r"   ræ   rç   )rè   rP   rê   rë   rì   )rY   r-   Úcross_modal_hidden_statesr]   s       r9   rl   zBridgeTowerLinkTower.forwardL  s­   € ØÔ 5Ò(Ð(Ø—>’> -Ð2KÑ"KÑLÔLÐLØÔ! \Ò1Ð1Ø—>’> -°$Ô2DÑ"DÐG`Ñ"`ÑaÔaÐaØÔ! ]Ò2Ð2Ø—>’> -°1°t´y±=Ñ"AÐD]Ð`dÔ`iÑDiÑ"iÑjÔjÐjå%Ð&b¸Ô9MÐ&bÐ&bÐ&bÑcÔcÐcr8   ©r/   r0   r1   rL   rl   rn   ro   s   @r9   râ   râ   >  sS   ø€ € € € € ðfð fð fð fð fðdð dð dð dð dð dð dr8   râ   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚBridgeTowerSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©NrE   )rK   rL   r   rT   rN   ÚdenserP   rQ   ÚDropoutÚhidden_dropout_probÚdropoutrX   s     €r9   rL   zBridgeTowerSelfOutput.__init__Y  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr8   r-   Úinput_tensorr¦   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rh   ©rô   r÷   rP   ©rY   r-   rø   s      r9   rl   zBridgeTowerSelfOutput.forward_  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr8   r„   ro   s   @r9   rñ   rñ   X  ói   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r8   rñ   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚBridgeTowerIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S rh   )rK   rL   r   rT   rN   Úintermediate_sizerô   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnrX   s     €r9   rL   z BridgeTowerIntermediate.__init__h  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r8   r-   r¦   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rh   )rô   r  ©rY   r-   s     r9   rl   zBridgeTowerIntermediate.forwardp  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr8   r„   ro   s   @r9   rÿ   rÿ   g  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r8   rÿ   c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚBridgeTowerOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S ró   )rK   rL   r   rT   r  rN   rô   rP   rQ   rõ   rö   r÷   rX   s     €r9   rL   zBridgeTowerOutput.__init__x  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr8   r-   rø   r¦   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rh   rú   rû   s      r9   rl   zBridgeTowerOutput.forward~  rü   r8   r„   ro   s   @r9   r	  r	  w  rý   r8   r	  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚBridgeTowerPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rh   )rK   rL   r   rT   rN   rô   ÚTanhÚ
activationrX   s     €r9   rL   zBridgeTowerPooler.__init__‡  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr8   r-   r¦   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rô   r  )rY   r-   Úfirst_token_tensorÚpooled_outputs       r9   rl   zBridgeTowerPooler.forwardŒ  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr8   r„   ro   s   @r9   r  r  †  s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r8   r  ç        ÚmoduleÚqueryÚkeyÚvaluer]   Úscalingr÷   Úkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Nr�   ç      à¿r�   r   r­   )ÚpÚtrainingr"   )
rª   r3   ÚmatmulrÆ   r   r¶   Úsoftmaxr÷   r  Ú
contiguous)
r  r  r  r  r]   r  r÷   r  Úattn_weightsÚattn_outputs
             r9   Úeager_attention_forwardr$  –  sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r8   c                   ó„   ‡ — e Zd Zd
ˆ fd„	Z	 	 ddej        dej        dz  dedz  dee	         de
ej                 f
d	„Zˆ xZS )ÚBridgeTowerSelfAttentionFNc                 óÄ  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        | j        dz  | _
        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        || _        || _        d S ©Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r  )rK   rL   rN   Únum_attention_headsÚhasattrrÄ   rZ   rÌ   Úattention_head_sizeÚall_head_sizer  r   rT   r  r  r  rõ   Úattention_probs_dropout_probr÷   Ú
is_decoderÚ	is_causalÚ	layer_idx©rY   rZ   r1  r2  r[   s       €r9   rL   z!BridgeTowerSelfAttention.__init__´  sG  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð ˆŒà#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØÔ/°Ñ5ˆŒå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà Ô+ˆŒØ"ˆŒØ"ˆŒˆˆr8   r-   r]   Úpast_key_valuesr  r¦   c                 óÈ  — |j         d d…         }g |¢d‘| j        ‘R } |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        } |                      |¦  «        j        |Ž                      dd¦  «        }	|�=|}
t          |t          ¦  «        r|j	        }
|
 
                    ||	| j        ¦  «        \  }}	t          j        | j        j        t           ¦  «        } || |||	|f| j        sdn| j        j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )Nr�   r"   r�   r  ©r÷   r  )r¯   r-  r  r¸   rÆ   r  r  r  r   Úself_attention_cacheÚupdater2  r   Úget_interfacerZ   Ú_attn_implementationr$  r  r÷   r  r  r´   r!  )rY   r-   r]   r4  r  Úinput_shapeÚhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer#  r"  s                 r9   rl   z BridgeTowerSelfAttention.forwardÌ  s¨  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆð 5�d—j’j Ñ/Ô/Ô4°lÐC×MÒMÈaÐQRÑSÔSˆØ0�D—H’H˜]Ñ+Ô+Ô0°,Ð?×IÒIÈ!ÈQÑOÔOˆ	Ø4�d—j’j Ñ/Ô/Ô4°lÐC×MÒMÈaÐQRÑSÔSˆàÐ&à&5Ð#Ý˜/Õ+>Ñ?Ô?ð OØ*9Ô*NÐ'ð &=×%CÒ%CÀIÈ{Ð\`Ô\jÑ%kÔ%kÑ"ˆI�{å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð@�C�C°$´,´.Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r8   ©FN)NN©r/   r0   r1   rL   r3   rm   r4   r   r   r   r6   rl   rn   ro   s   @r9   r&  r&  ³  s©   ø€ € € € € ð#ð #ð #ð #ð #ð #ð6 48Ø(,ð	')ð ')à”|ð')ð Ô)¨DÑ0ð')ð  ™ð	')ð
 Ð+Ô,ð')ð 
ˆuŒ|Ô	ð')ð ')ð ')ð ')ð ')ð ')ð ')ð ')r8   r&  c                   óš   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddej        dej        dz  dej        dz  dedz  dee	         d	e
ej                 fd
„Zˆ xZS )ÚBridgeTowerCrossAttentionFNc                 ó¬  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        | j        dz  | _
        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        ¦  «        | _        || _        || _        d S r(  )rK   rL   rN   r+  r,  rÄ   rZ   rÌ   r-  r.  r  r   rT   r  r  r  rõ   r/  r÷   r1  r2  r3  s       €r9   rL   z"BridgeTowerCrossAttention.__init__ø  s=  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð ˆŒà#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔØÔ/°Ñ5ˆŒå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒà"ˆŒØ"ˆŒˆˆr8   r-   Úencoder_hidden_statesr]   r4  r  r¦   c                 óì  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|�|j                             | j        ¦  «        nd}	|�;|	r9|j        j	        | j                 j
        }
|j        j	        | j                 j        }nÈg |j         d d…         ¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }
|                      |¦  «                             |¦  «                             dd¦  «        }|�3|j                             |
|| j        ¦  «        \  }
}d|j        | j        <   t          j        | j        j        t&          ¦  «        } || ||
||f| j        sdn| j        j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }||fS )Nr�   r"   r�   FTr  r6  )r¯   r-  r  r¸   rÆ   Ú
is_updatedÚgetr2  Úcross_attention_cacheÚlayersÚkeysri   r  r  r8  r   r9  rZ   r:  r$  r  r÷   r  r  r´   r!  )rY   r-   rG  r]   r4  r  r;  r<  r=  rI  r>  r?  Úkv_shaperA  r#  r"  s                   r9   rl   z!BridgeTowerCrossAttention.forward  s-  € ð $Ô)¨#¨2¨#Ô.ˆàC˜ÐC bÐC¨$Ô*BÐCÐCˆð —j’j Ñ/Ô/×4Ò4°\ÑBÔB×LÒLÈQÐPQÑRÔRˆàGVÐGb�_Ô/×3Ò3°D´NÑCÔCÐCÐhmˆ
ØÐ&¨:Ð&à'Ô=ÔDÀTÄ^ÔTÔYˆIØ)Ô?ÔFÀtÄ~ÔVÔ]ˆKˆKàXÐ.Ô4°S°b°SÔ9ÐX¸2ÐX¸tÔ?WÐXÐXˆHØŸšÐ!6Ñ7Ô7×<Ò<¸XÑFÔF×PÒPÐQRÐTUÑVÔVˆIØŸ*š*Ð%:Ñ;Ô;×@Ò@ÀÑJÔJ×TÒTÐUVÐXYÑZÔZˆKàÐ*à)8Ô)N×)UÒ)UØ˜{¨D¬Nñ*ô *Ñ&�	˜;ð >B�Ô*¨4¬>Ñ:å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð@�C�C°$´,´.Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r8   rB  ©NNN)r/   r0   r1   rL   r3   rm   r4   r   r   r   r6   rl   rn   ro   s   @r9   rE  rE  ÷  s¿   ø€ € € € € ð#ð #ð #ð #ð #ð #ð4 ;?Ø37Ø6:ð1)ð 1)à”|ð1)ð  %Ô0°4Ñ7ð1)ð Ô)¨DÑ0ð	1)ð
 -¨tÑ3ð1)ð Ð+Ô,ð1)ð 
ˆuŒ|Ô	ð1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)ð 1)r8   rE  c                   ó°   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dedz  d	ee	         d
e
ej                 fd„Zˆ xZS )ÚBridgeTowerAttentionFNc                 óÄ   •— t          ¦   «                              ¦   «          || _        |rt          nt          } ||||¬¦  «        | _        t          |¦  «        | _        d S )N©r1  r2  )rK   rL   Úis_cross_attentionrE  r&  rY   rñ   Úoutput)rY   rZ   r1  r2  rT  Úattention_classr[   s         €r9   rL   zBridgeTowerAttention.__init__E  s]   ø€ Ý‰Œ×ÒÑÔÐØ"4ˆÔØ7IÐgÕ3Ð3ÕOgˆØ#�O F°iÈ9ÐUÑUÔUˆŒ	Ý+¨FÑ3Ô3ˆŒˆˆr8   r-   r]   rG  Úencoder_attention_maskr4  r  r¦   c                 óv   — | j         s|n|} | j        |f|||dœ|¤Ž\  }}|                      ||¦  «        }||fS )N)rG  r]   r4  )rT  rY   rU  )	rY   r-   r]   rG  rW  r4  r  Úattention_outputr"  s	            r9   rl   zBridgeTowerAttention.forwardL  sq   € ð 04Ô/FÐb˜˜ÐLbˆØ)2¨¬Øð*
à"7Ø)Ø+ð	*
ð *
ð
 ð*
ð *
Ñ&Ð˜,ð  Ÿ;š;Ð'7¸ÑGÔGÐØ Ð-Ð-r8   )FNF©NNNNrC  ro   s   @r9   rQ  rQ  D  sÓ   ø€ € € € € ð4ð 4ð 4ð 4ð 4ð 4ð 48Ø:>Ø;?Ø(,ð.ð .à”|ð.ð Ô)¨DÑ0ð.ð  %Ô0°4Ñ7ð	.ð
 !&Ô 1°DÑ 8ð.ð  ™ð.ð Ð+Ô,ð.ð 
ˆuŒ|Ô	ð.ð .ð .ð .ð .ð .ð .ð .r8   rQ  c                   óF   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddee         fd„Zd„ Zˆ xZS )ÚBridgeTowerBertCrossLayerNc                 óL  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |d|¬¦  «        | _        |j        | _        |j        | _        t	          |d|d¬¦  «        | _        t          |¦  «        | _
        t          |¦  «        | _        d S )Nr"   TrS  F©r1  r2  rT  )rK   rL   Úchunk_size_feed_forwardÚseq_len_dimrQ  rf   r0  Úadd_cross_attentionÚcrossattentionrÿ   Úintermediater	  rU  ©rY   rZ   r2  r[   s      €r9   rL   z"BridgeTowerBertCrossLayer.__init__b  s    ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ-¨fÀÐPYÐZÑZÔZˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ Ý2ØØØØ#ð	
ñ 
ô 
ˆÔõ 4°FÑ;Ô;ˆÔÝ'¨Ñ/Ô/ˆŒˆˆr8   r  c                 óª   —  | j         |f|d dœ|¤Ž\  }}|}	 | j        |	f||||dœ|¤Ž\  }
}|
}	t          | j        | j        | j        |	¦  «        }|||fS )N)r]   r4  )r]   rG  rW  r4  )rf   rb  r   Úfeed_forward_chunkr_  r`  )rY   r-   rG  r]   rW  r4  r  Úself_attention_outputÚself_attn_weightsrY  Úcross_attention_outputÚcross_attn_weightsÚlayer_outputs                r9   rl   z!BridgeTowerBertCrossLayer.forwardr  sÁ   € ð 4B°4´>Øð4
à)Ø ð4
ð 4
ð ð	4
ð 4
Ñ0ÐÐ0ð 1Ðà5H°TÔ5HØð6
à)Ø"7Ø#9Ø+ð6
ð 6
ð ð6
ð 6
Ñ2ÐÐ 2ð 2Ðå0ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð ØØð
ð 	
r8   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rh   ©rc  rU  ©rY   rY  Úintermediate_outputrk  s       r9   rf  z,BridgeTowerBertCrossLayer.feed_forward_chunk–  ó2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr8   rh   rO  )	r/   r0   r1   rL   r   r   rl   rf  rn   ro   s   @r9   r\  r\  a  s�   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ð( Ø#Øð"
ð "
ð Ð+Ô,ð"
ð "
ð "
ð "
ðHð ð ð ð ð ð r8   r\  c                   óª   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dedz  dee	         d	ej        fd
„Z
d„ Zˆ xZS )ÚBridgeTowerTextLayerNc                 ó–  •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          ||j        |¬¦  «        | _        |j        | _        |j        | _        | j        r1| j        st          | › d�¦  «        ‚t	          |d|d¬¦  «        | _	        t          |¦  «        | _        t          |¦  «        | _        d S )Nr"   rS  z> should be used as a decoder model if cross attention is addedFTr^  )rK   rL   r_  r`  rQ  r0  rf   ra  rÄ   rb  rÿ   rc  r	  rU  rd  s      €r9   rL   zBridgeTowerTextLayer.__init__�  sÐ   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ-¨fÀÔ@QÐ]fÐgÑgÔgˆŒØ Ô+ˆŒØ#)Ô#=ˆÔ ØÔ#ð 	Ø”?ð jÝ  DÐ!hÐ!hÐ!hÑiÔiÐiÝ"6ØØØ#Ø#'ð	#ñ #ô #ˆDÔõ 4°FÑ;Ô;ˆÔÝ'¨Ñ/Ô/ˆŒˆˆr8   r-   r]   rG  rW  r4  r  r¦   c                 óü   —  | j         ||fd|i|¤Ž\  }}|}	| j        r=|�;t          | d¦  «        st          d| › d�¦  «        ‚ | j        |d ||fd|i|¤Ž\  }
}|
}	t          | j        | j        | j        |	¦  «        }|S )Nr4  rb  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	rf   r0  r,  rÄ   rb  r   rf  r_  r`  )rY   r-   r]   rG  rW  r4  r  rg  rw   rY  ri  rk  s               r9   rl   zBridgeTowerTextLayer.forward±  s  € ð $2 4¤>ØØð$
ð $
ð ,ð$
ð ð	$
ð $
Ñ Ð˜qð 1ÐàŒ?ð 	6Ð4Ð@Ý˜4Ð!1Ñ2Ô2ð Ý ðD¸dð Dð Dð Dñô ð ð
 )<¨Ô(;Ø%ØØ%Ø&ð	)ð )ð
 !0ð)ð ð)ð )Ñ%Ð" Að  6Ðå0ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð Ðr8   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rh   rm  rn  s       r9   rf  z'BridgeTowerTextLayer.feed_forward_chunkØ  rp  r8   rh   rZ  )r/   r0   r1   rL   r3   rm   r4   r   r   r   rl   rf  rn   ro   s   @r9   rr  rr  œ  sÞ   ø€ € € € € ð0ð 0ð 0ð 0ð 0ð 0ð. 48Ø:>Ø;?Ø(,ð%ð %à”|ð%ð Ô)¨DÑ0ð%ð  %Ô0°4Ñ7ð	%ð
 !&Ô 1°DÑ 8ð%ð  ™ð%ð Ð+Ô,ð%ð 
Œð%ð %ð %ð %ðNð ð ð ð ð ð r8   rr  c                   ó¤   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej        dej        dz  dej        dz  dej        dz  dedz  dedz  d	e	e
         d
efd„Zˆ xZS )ÚBridgeTowerTextEncoderc                 óÆ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS )©r2  )rr  )rv   ÚirZ   s     €r9   rx   z3BridgeTowerTextEncoder.__init__.<locals>.<listcomp>ä  s'   ø€ Ð`Ð`Ð`¸1Õ! &°AÐ6Ñ6Ô6Ð`Ð`Ð`r8   )rK   rL   rZ   r   r|   r}   rz   rk   rX   s    `€r9   rL   zBridgeTowerTextEncoder.__init__à  sZ   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ø`Ð`Ð`Ð`ÅÀfÔF^Ñ@_Ô@_Ð`Ñ`Ô`ñ
ô 
ˆŒ
ˆ
ˆ
r8   Nr-   r]   rG  rW  r4  Ú	use_cacher  r¦   c                 ó\   — | j         D ]} ||||f||dœ|¤Ž}Œt          ||r|nd ¬¦  «        S )N)rW  r4  )Úlast_hidden_stater4  )rk   r   )	rY   r-   r]   rG  rW  r4  r|  r  Úlayer_modules	            r9   rl   zBridgeTowerTextEncoder.forwardç  su   € ð !œJð 	ð 	ˆLØ(˜LØØØ%ðð (>Ø /ðð ð ðð ˆMˆMõ 9Ø+Ø/8ÐB˜O˜O¸dð
ñ 
ô 
ð 	
r8   )NNNNN)r/   r0   r1   rL   r3   rm   r4   r   re   r   r   r   rl   rn   ro   s   @r9   rw  rw  ß  sØ   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð 48Ø:>Ø;?Ø(,Ø!%ð
ð 
à”|ð
ð Ô)¨DÑ0ð
ð  %Ô0°4Ñ7ð	
ð
 !&Ô 1°DÑ 8ð
ð  ™ð
ð ˜$‘;ð
ð Ð+Ô,ð
ð 
3ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r8   rw  c                   óÆ   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ed
ej	        fd„Z
ed„ ¦   «         Zedd„¦   «         Zˆ xZS )ÚBridgeTowerTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óø  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j
        |j        |j        ¬¦  «        | _
        t          j        |j        ¦  «        | _        |                      dt!          j        |j        ¦  «                             d¦  «        d¬¦  «         |                      dt!          j        | j                             ¦   «         t           j        ¬¦  «        d¬¦  «         |j        | _        t          j        |j        |j        | j        ¬¦  «        | _        d S )	N)Úpadding_idxrE   rŽ   r�   Fr‘   Útoken_type_idsrÃ   )rK   rL   r   rž   Ú
vocab_sizerN   Úpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsrP   rQ   rõ   rö   r÷   r    r3   r¡   Úmax_position_embeddingsr¢   ÚzerosrŽ   rª   Úlongrƒ  Úposition_embeddingsrX   s     €r9   rL   z"BridgeTowerTextEmbeddings.__init__  sJ  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒà×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	×ÒØ�eœk¨$Ô*;×*@Ò*@Ñ*BÔ*BÍ%Ì*ÐUÑUÔUÐbgð 	ñ 	
ô 	
ð 	
ð "Ô.ˆÔÝ#%¤<ØÔ*¨FÔ,>ÈDÔL\ð$
ñ $
ô $
ˆÔ Ð Ð r8   Nr   Ú	input_idsr„  rŽ   Úinputs_embedsÚpast_key_values_lengthr¦   c                 ó*  — |€:|�|                       || j        |¦  «        }n|                      || j        ¦  «        }|�|                     ¦   «         }n|                     ¦   «         d d…         }|\  }}|€§t	          | d¦  «        rl| j                             |j        ¦  «                             |j	        d         d¦  «        }	t          j        |	d|¬¦  «        }	|	                     ||¦  «        }n+t          j        |t          j        | j        j        ¬¦  «        }|€|                      |¦  «        }|                      |¦  «        }
||
z   }|                      |¦  «        }||z   }|                      |¦  «        }|                      |¦  «        }|S )Nr�   r„  r   r"   )r®   Úindexr_   )Ú"create_position_ids_from_input_idsrƒ  Ú&create_position_ids_from_inputs_embedsrª   r,  r„  rd   ra   r¢   r¯   r3   Úgatherr‹  rŒ  rŽ   r‡  r‰  r�  rP   r÷   )rY   rŽ  r„  rŽ   r�  r�  r;  rÇ   Ú
seq_lengthÚbuffered_token_type_idsr‰  r£   r�  s                r9   rl   z!BridgeTowerTextEmbeddings.forward  s¨  € ð ÐØÐ$à#×FÒFØ˜tÔ/Ð1Gñ ô  ��ð  $×JÒJÈ=ÐZ^ÔZjÑkÔk�àÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà!,Ñˆ
�Jð
 Ð!Ý�tÐ-Ñ.Ô.ð mà*.Ô*=×*@Ò*@ÀÔATÑ*UÔ*U×*\Ò*\Ð]iÔ]oÐpqÔ]rÐtvÑ*wÔ*wÐ'Ý*/¬,Ð7NÐTUÐ]iÐ*jÑ*jÔ*jÐ'Ø!8×!?Ò!?À
ÈJÑ!WÔ!W��å!&¤¨[ÅÄ
ÐSWÔSdÔSkÐ!lÑ!lÔ!l�àÐ Ø ×0Ò0°Ñ;Ô;ˆMØ $× :Ò :¸>Ñ JÔ JÐØ"Ð%:Ñ:ˆ
à"×6Ò6°|ÑDÔDÐØÐ"5Ñ5ˆ
à—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr8   c                 óú   — |                       ¦   «         dd…         }|d         }t          j        |dz   ||z   dz   t          j        | j        ¬¦  «        }|                     d¦  «                             |¦  «        S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr�   r"   r_   r   )rª   r3   r¡   rŒ  ra   r±   r¢   )r�  rƒ  r;  Úsequence_lengthrŽ   s        r9   r”  z@BridgeTowerTextEmbeddings.create_position_ids_from_inputs_embedsI  s~   € ð $×(Ò(Ñ*Ô*¨3¨B¨3Ô/ˆØ% aœ.ˆå”|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÅeÄjÐYfÔYmð
ñ 
ô 
ˆð ×%Ò% aÑ(Ô(×/Ò/°Ñ<Ô<Ð<r8   c                 óÜ   — |                       |¦  «                             ¦   «         }t          j        |d¬¦  «                             |¦  «        |z   |z  }|                     ¦   «         |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r"   r­   )ÚnerÌ   r3   ÚcumsumÚtype_asrŒ  )rŽ  rƒ  r�  ÚmaskÚincremental_indicess        r9   r“  z<BridgeTowerTextEmbeddings.create_position_ids_from_input_ids[  sg   € ð �|Š|˜KÑ(Ô(×,Ò,Ñ.Ô.ˆÝ$œ|¨D°aÐ8Ñ8Ô8×@Ò@ÀÑFÔFÐI_Ñ_ÐcgÑgÐØ"×'Ò'Ñ)Ô)¨KÑ7Ð7r8   )NNNNr   )r   )r/   r0   r1   r2   rL   r3   Ú
LongTensorr4   rÌ   rm   rl   Ústaticmethodr”  r“  rn   ro   s   @r9   r�  r�    s   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð, .2Ø26Ø04Ø26Ø&'ð.ð .àÔ# dÑ*ð.ð Ô(¨4Ñ/ð.ð Ô&¨Ñ-ð	.ð
 Ô(¨4Ñ/ð.ð !$ð.ð 
Œð.ð .ð .ð .ð` ð=ð =ñ „\ð=ð" ð8ð 8ð 8ñ „\ð8ð 8ð 8ð 8ð 8r8   r�  c                   óŠ   ‡ — e Zd ZU eed<   dZdZdZddgZdgZ	e
eedœZ ej        ¦   «         d	ej        fˆ fd
„¦   «         Zˆ xZS )ÚBridgeTowerPreTrainedModelrZ   Úbridgetower)ÚimageÚtextFr&  rB   r4  )r-   r.   Úcross_attentionsr  c                 óŽ  •— t          ¦   «                              |¦  «         | j        j        }t	          |t
          ¦  «        �rX| j        j        dz  d| j        j        z  dz  z  }| j        j        dz  }d| j        j        z  dz  }|j        j	        D ]»}t          j        |j        j        ||z  ¬¦  «         t          j        |j        j        ¦  «         t          j        |j        j        j        ||z  ¬¦  «         t          j        |j        j        j        ||z  ¬¦  «         t          j        |j        j        j        ||z  ¬¦  «         Œ¼t          j        |j        j        ||z  ¬¦  «         t          j        |j        j        j        ||z  ¬¦  «         �nct	          |t0          j        t0          j        t0          j        f¦  «        r!t          j        |j        dd|z  ¬¦  «         �nt	          |t8          ¦  «        r%t          j        |j        | j        j        ¦  «         n×t	          |t@          ¦  «        rEt          j!        |j"        tG          j$        |j%        ¦  «         &                    d¦  «        ¦  «         n}t	          |tN          ¦  «        rht          j!        |j"        tG          j$        |j"        j(        d         ¦  «         &                    d¦  «        ¦  «         t          j        |j)        ¦  «         t	          |t0          j        tT          f¦  «        r"|j+        �t          j        |j+        ¦  «         d S d S d S )	Nr  r�   )Ústdr  gš™™™™™©?)Úmeanr©  r�   r�   ),rK   Ú_init_weightsrZ   Úinitializer_factorr  rÎ   rN   rz   rÒ   r~   ÚinitÚnormal_rO   Úin_proj_weightÚzeros_Úin_proj_biasÚout_projr°   rU   rG   rJ   r£   r˜   rŸ   r   rT   r™   rž   Ú!BridgeTowerForContrastiveLearningÚ	constant_Úlogit_scaleÚlogit_scale_init_valuer†   Úcopy_rŽ   r3   r¡   r�   r¢   r�  r¯   r„  ÚBridgeTowerMLMHeadrŒ   )rY   r  r©  Úproj_stdÚattn_stdÚfc_stdrƒ   r[   s          €r9   r«  z(BridgeTowerPreTrainedModel._init_weightsz  sÒ  ø€ å‰Œ×Ò˜fÑ%Ô%Ð%ØŒkÔ,ˆÝ�fÕ:Ñ;Ô;ñ 	/ØœÔ/°Ñ5¸1¸t¼{Ô?\Ñ;\ÐaeÑ:eÑfˆHØ”{Ô.°Ñ4ˆHØ˜$œ+Ô1Ñ1°dÑ:ˆFØÔ+Ô5ð Jð J�Ý”˜UœZÔ6¸HÀs¹NÐKÑKÔKÐKÝ”˜EœJÔ3Ñ4Ô4Ð4Ý”˜UœZÔ0Ô7¸XÈ¹^ÐLÑLÔLÐLÝ”˜UœYœ^Ô2¸À¹ÐEÑEÔEÐEÝ”˜UœYÔ-Ô4¸(ÀS¹.ÐIÑIÔIÐIÐIåŒL˜Ô*Ô:ÀÈ3ÁÐOÑOÔOÐOÝŒL˜Ô*Ô=ÔDÈ(ÐUXÉ.ÐYÑYÔYÐYÑYÝ˜¥¤­B¬Iµr´|Ð DÑEÔEð 	/ÝŒL˜œ¨S°d¸S±jÐAÑAÔAÐAÑAÝ˜Õ AÑBÔBð 	/ÝŒN˜6Ô-¨t¬{Ô/QÑRÔRÐRÐRÝ˜Õ ;Ñ<Ô<ð 	/ÝŒJ�vÔ*­E¬L¸Ô9MÑ,NÔ,N×,UÒ,UÐV]Ñ,^Ô,^Ñ_Ô_Ð_Ð_Ý˜Õ 9Ñ:Ô:ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÝŒK˜Ô-Ñ.Ô.Ð.å�f�rœyÕ*<Ð=Ñ>Ô>ð 	%À6Ä;ÐCZÝŒK˜œÑ$Ô$Ð$Ð$Ð$ð	%ð 	%ÐCZÐCZr8   )r/   r0   r1   r#   r5   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementrr  r&  rE  Ú_can_record_outputsr3   Úno_gradr   ÚModuler«  rn   ro   s   @r9   r£  r£  l  s¥   ø€ € € € € € àÐÐÑØ%ÐØ(ÐØ&+Ð#Ø3Ð5SÐTÐØ#4Ð"5Ðà-Ø.Ø5ðð Ðð €U„]�_„_ð% B¤Ið %ð %ð %ð %ð %ñ „_ð%ð %ð %ð %ð %r8   r£  c                   óL   ‡ — e Zd ZU eed<   dZˆ fd„Zed„ ¦   «         Zdd„Z	ˆ xZ
S )	ÚBridgeTowerVisionModelrZ   )r¥  c                 óš   •— t          ¦   «                              |¦  «         t          |¦  «        | _        |                      ¦   «          d S rh   )rK   rL   rÎ   ÚvisualÚ	post_initrX   s     €r9   rL   zBridgeTowerVisionModel.__init__�  s@   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý2°6Ñ:Ô:ˆŒØ�ŠÑÔÐÐÐr8   c                 ó8   — | j         j        j        j        j        S rh   )rÇ  r£   r›   r°   r`   ©rY   s    r9   r`   zBridgeTowerVisionModel.dtype¢  s   € àŒ{Ô%Ô5Ô<ÔBÐBr8   NFc                 ó`   — |                       |                     | j        ¦  «        ||¦  «        S rh   )rÇ  Útyper`   )rY   r¥  Ú
image_maskr¿   r  s        r9   rl   zBridgeTowerVisionModel.forward¦  s'   € Ø�{Š{˜5Ÿ:š: d¤jÑ1Ô1°:Ð?WÑXÔXÐXr8   )NF)r/   r0   r1   r%   r5   r½  rL   Úpropertyr`   rl   rn   ro   s   @r9   rÅ  rÅ  ™  s†   ø€ € € € € € Ø#Ð#Ð#Ñ#Ø!Ððð ð ð ð ð
 ðCð Cñ „XðCðYð Yð Yð Yð Yð Yð Yð Yr8   rÅ  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    c                   óB  ‡ — e Zd ZU eed<   dZdˆ fd„	Zd„ Zd„ Ze	e
e	 	 	 	 	 	 	 	 	 ddej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dee         defd„¦   «         ¦   «         ¦   «         Zd„ Zˆ xZS )ÚBridgeTowerTextModelrZ   )r¦  Tc                 ó  •— t          ¦   «                              |¦  «         || _        d| _        t	          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _	        |  
                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)rK   rL   rZ   Úgradient_checkpointingr�  r£   rw  Úencoderr  ÚpoolerrÈ  )rY   rZ   Úadd_pooling_layerr[   s      €r9   rL   zBridgeTowerTextModel.__init__¼  s|   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒØ&+ˆÔ#å3°FÑ;Ô;ˆŒÝ-¨fÑ5Ô5ˆŒà3DÐNÕ'¨Ñ/Ô/Ð/È$ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         j        S rh   ©r£   r‡  rÊ  s    r9   Úget_input_embeddingsz)BridgeTowerTextModel.get_input_embeddingsÍ  s   € ØŒÔ.Ð.r8   c                 ó   — || j         _        d S rh   r×  ©rY   r  s     r9   Úset_input_embeddingsz)BridgeTowerTextModel.set_input_embeddingsÐ  s   € Ø*/ˆŒÔ'Ð'Ð'r8   NrŽ  r]   r„  rŽ   r�  rG  rW  r4  r|  r  r¦   c
           
      ó  — |d u |d uz  rt          d¦  «        ‚| j        j        sd}	|	r8|€6t          t	          | j        ¬¦  «        t	          | j        ¬¦  «        ¦  «        }|�|                     ¦   «         nd}|                      |||||¬¦  «        }|                      |||||¬¦  «        \  }} | j        |f|||||	|dœ|
¤Ž}|d         }| j	        �|  	                    |¦  «        nd }t          |||j        ¬¦  «        S )	Nz:You must specify exactly one of input_ids or inputs_embedsF)rZ   r   )rŽ  rŽ   r„  r�  r�  )r]   rW  Úembedding_outputrG  r4  )r]   rG  rW  r4  r|  rŽ   )r~  r,   r4  )rÄ   rZ   r0  r   r   Úget_seq_lengthr£   Ú_create_attention_masksrÓ  rÔ  r   r4  )rY   rŽ  r]   r„  rŽ   r�  rG  rW  r4  r|  r  r�  rÝ  Úencoder_outputsÚsequence_outputr  s                   r9   rl   zBridgeTowerTextModel.forwardÓ  sy  € ð& ˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàŒ{Ô%ð 	ØˆIàð 	v˜Ð0Ý1µ,ÀdÄkÐ2RÑ2RÔ2RÕT`ÐhlÔhsÐTtÑTtÔTtÑuÔuˆOàETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐàŸ?š?ØØ%Ø)Ø'Ø#9ð +ñ 
ô 
Ðð 26×1MÒ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nñ 2
ô 2
Ñ.ˆÐ.ð '˜$œ,Øð	
à)Ø"7Ø#9Ø+ØØ%ð	
ð 	
ð ð	
ð 	
ˆð *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå;Ø-Ø'Ø+Ô;ð
ñ 
ô 
ð 	
r8   c                 ó¶   — | j         j        rt          | j         |||¬¦  «        }nt          | j         ||¬¦  «        }|�t          | j         |||¬¦  «        }||fS )N)rZ   r�  r]   r4  ©rZ   r�  r]   )rZ   r�  r]   rG  )rZ   r0  r   r   )rY   r]   rW  rÝ  rG  r4  s         r9   rß  z,BridgeTowerTextModel._create_attention_masks  s�   € ð Œ;Ô!ð 	Ý/Ø”{Ø.Ø-Ø /ð	ñ ô ˆNˆNõ 7Ø”{Ø.Ø-ðñ ô ˆNð "Ð-Ý%>Ø”{Ø.Ø5Ø&;ð	&ñ &ô &Ð"ð Ð5Ð5Ð5r8   )T)	NNNNNNNNN)r/   r0   r1   r$   r5   r½  rL   rØ  rÛ  r    r!   r   r3   rm   r   re   r   r   r   rl   rß  rn   ro   s   @r9   rÐ  rÐ  ª  sƒ  ø€ € € € € € ð "Ð!Ð!Ñ!Ø Ððð ð ð ð ð ð"/ð /ð /ð0ð 0ð 0ð  ØØð *.Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø(,Ø!%ð9
ð 9
à”< $Ñ&ð9
ð œ tÑ+ð9
ð œ tÑ+ð	9
ð
 ”l TÑ)ð9
ð ”| dÑ*ð9
ð  %œ|¨dÑ2ð9
ð !&¤¨tÑ 3ð9
ð  ™ð9
ð ˜$‘;ð9
ð Ð+Ô,ð9
ð 
6ð9
ð 9
ð 9
ñ	 „^ñ „_ñ  Ôð9
ðx6ð 6ð 6ð 6ð 6ð 6ð 6r8   rÐ  zv
    The bare BridgeTower Model transformer outputting BridgeTowerModelOutput object without any specific head on
    c                   óœ  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej        dedej        fd„Z	dej        dedej        fd„Z
ee	 	 	 	 	 	 	 	 	 	 ddej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  dej        d	z  ded	z  dej        d	z  dedee         deej                 ez  fd„¦   «         ¦   «         Zd„ Zˆ xZS )ÚBridgeTowerModelc                 ó�  •‡‡‡— t          ¦   «                              ‰¦  «         ‰| _        ‰j        Š‰j        Š‰j        rIt          j        ‰j        ‰j        ¦  «        | _	        t          j        ‰j        ‰j        ¦  «        | _
        npt          j        ˆˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _	        t          j        ˆˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _
        t          j        d‰j        ¦  «        | _        t!          ‰¦  «        | _        t%          ‰¦  «        | _        ‰j        se‰j        r^| j        j        j        D ]L}| j        j        j        j        j        |j        _        | j        j        j        j        j        |j        _        ŒMt          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t=          ‰¦  «        | _        t=          ‰¦  «        | _         t          j!        ‰j        ‰j"        ¬¦  «        | _#        t          j!        ‰j        ‰j"        ¬¦  «        | _$        ‰j%        r)tM          ‰¦  «        | _'        tM          ‰¦  «        | _(        ntt          j        ˆfd„t          ‰j        dz
  ¦  «        D ¦   «         ¦  «        | _'        t          j        ˆfd	„t          ‰j        dz
  ¦  «        D ¦   «         ¦  «        | _(        |  )                    ¦   «          d S )
Nc                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S r7   ©r   rT   rN   )rv   rw   rZ   Útext_configs     €€r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>G  s+   ø€ ÐqÐqÐqÈA•”˜;Ô2°FÔ4FÑGÔGÐqÐqÐqr8   c                 óN   •— g | ]!}t          j        ‰j        ‰j        ¦  «        ‘Œ"S r7   rè  )rv   rw   rZ   Úvision_configs     €€r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>J  s+   ø€ ÐsÐsÐsÈa•”˜=Ô4°fÔ6HÑIÔIÐsÐsÐsr8   r�   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   ©r\  ©rv   rw   ré  s     €r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>Y  ó"   ø€ Ð]Ð]Ð]¸Õ& {Ñ3Ô3Ð]Ð]Ð]r8   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   rí  rî  s     €r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>\  rï  r8   rE   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   ©râ   ru   s     €r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>l  ó"   ø€ Ð[Ð[Ð[°!Õ% fÑ-Ô-Ð[Ð[Ð[r8   r"   c                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r7   rò  ru   s     €r9   rx   z-BridgeTowerModel.__init__.<locals>.<listcomp>o  ró  r8   )*rK   rL   rZ   rë  ré  Ú$share_cross_modal_transformer_layersr   rT   rN   Úcross_modal_text_transformÚcross_modal_image_transformr|   r}   rz   rž   r‰  rÅ  Úvision_modelrÐ  Ú
text_modelrÔ   Ú"init_layernorm_from_vision_encoderrÇ  Úcross_modal_ln_separaterÓ   r°   ÚdatarŒ   Úcross_modal_image_layersÚcross_modal_text_layersr  Úcross_modal_image_poolerÚcross_modal_text_poolerrP   rQ   Úcross_modal_text_layernormÚcross_modal_image_layernormÚshare_link_tower_layersrâ   Úcross_modal_text_link_towerÚcross_modal_image_link_towerrÈ  )rY   rZ   rÚ   ré  rë  r[   s    ` @@€r9   rL   zBridgeTowerModel.__init__<  s  øøøø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ,ˆØÔ(ˆàÔ6ð 		Ý.0¬i¸Ô8OÐQWÔQcÑ.dÔ.dˆDÔ+Ý/1¬y¸Ô9RÐTZÔTfÑ/gÔ/gˆDÔ,Ð,å.0¬mØqÐqÐqÐqÐqÕQVÐW]ÔWoÑQpÔQpÐqÑqÔqñ/ô /ˆDÔ+õ 02¬}ØsÐsÐsÐsÐsÕSXÐY_ÔYqÑSrÔSrÐsÑsÔsñ0ô 0ˆDÔ,õ &(¤\°!°VÔ5GÑ%HÔ%HˆÔ"å2°=ÑAÔAˆÔå.¨{Ñ;Ô;ˆŒàÔ,ð 	J°Ô1Zð 	JØÔ'Ô.ÔFð Jð J�Ø!%Ô!2Ô!9Ô!AÔ!HÔ!M�”	”Ø#Ô0Ô7Ô?ÔDÔI�””�å(*¬Ø]Ð]Ð]Ð]½UÀ6ÔC[Ñ=\Ô=\Ð]Ñ]Ô]ñ)
ô )
ˆÔ%õ (*¤}Ø]Ð]Ð]Ð]½UÀ6ÔC[Ñ=\Ô=\Ð]Ñ]Ô]ñ(
ô (
ˆÔ$õ
 ):¸&Ñ(AÔ(AˆÔ%Ý'8¸Ñ'@Ô'@ˆÔ$õ +-¬,°vÔ7IÈvÔOdÐ*eÑ*eÔ*eˆÔ'Ý+-¬<¸Ô8JÐPVÔPeÐ+fÑ+fÔ+fˆÔ(àÔ)ð 		Ý/CÀFÑ/KÔ/KˆDÔ,Ý0DÀVÑ0LÔ0LˆDÔ-Ð-å/1¬}Ø[Ð[Ð[Ð[µu¸VÔ=UÐXYÑ=YÑ7ZÔ7ZÐ[Ñ[Ô[ñ0ô 0ˆDÔ,õ 13´Ø[Ð[Ð[Ð[µu¸VÔ=UÐXYÑ=YÑ7ZÔ7ZÐ[Ñ[Ô[ñ1ô 1ˆDÔ-ð 	�ŠÑÔÐÐÐr8   c                 ó4   — | j                              ¦   «         S rh   )rù  rØ  rÊ  s    r9   rØ  z%BridgeTowerModel.get_input_embeddingst  s   € ØŒ×3Ò3Ñ5Ô5Ð5r8   c                 ó:   — | j                              |¦  «         d S rh   )rù  rÛ  rÚ  s     r9   rÛ  z%BridgeTowerModel.set_input_embeddingsw  s   € ØŒ×,Ò,¨UÑ3Ô3Ð3Ð3Ð3r8   r-   r2  r¦   c                 óp   — | j         j        r|                      |¦  «        S  | j        |         |¦  «        S rh   )rZ   rõ  rö  ©rY   r-   r2  s      r9   Ú_apply_text_transformz&BridgeTowerModel._apply_text_transformz  s<   € ØŒ;Ô;ð 	BØ×2Ò2°=ÑAÔAÐAØ9ˆtÔ.¨yÔ9¸-ÑHÔHÐHr8   c                 óp   — | j         j        r|                      |¦  «        S  | j        |         |¦  «        S rh   )rZ   rõ  r÷  r	  s      r9   Ú_apply_image_transformz'BridgeTowerModel._apply_image_transform  s<   € ØŒ;Ô;ð 	CØ×3Ò3°MÑBÔBÐBØ:ˆtÔ/°	Ô:¸=ÑIÔIÐIr8   NFrŽ  r]   r„  rÀ   Ú
pixel_maskr�  r?   Úimage_token_type_idxÚlabelsr¿   r  c           
      ó4  — g }g }g }g }|�|€t          d¦  «        ‚|pd}|                     ¦   «         }| j                             |¬¦  «        }|                     |¦  «         |€&t          j        |t
          j        |j        ¬¦  «        }t          | j
        |dd…dd…dd…f         |¬¦  «        }t          | j        j        j        ¦  «        | j
        j        z
  dz   }| j        j        j        d|…         D ]#} |||¦  «        }|                     |¦  «         Œ$|€?| j        j                             |                     | j        j        ¦  «        |
¬¦  «        }n|                     ddd	¦  «        }|                     |¦  «         | j        j        j        j        d|…         D ]"} ||¦  «        }|                     |¦  «         Œ#| j        j                             |                     | j        j        ¦  «        ¦  «        }|                      |d¬
¦  «        }|                      t          j        dt
          j        |j        ¬¦  «        ¦  «                             |¦  «        }|                      ||z   ¦  «        }|                      |d¬
¦  «        }|                      t          j        d|t
          j        |j        ¬¦  «        ¦  «                             |¦  «        }||z   }|                      |¦  «        }t          j        |                     d¦  «        |                     d¦  «        ft
          j        |j        ¬¦  «        }t          | j
        ||¬¦  «        } | j         d         ||||¬¦  «        }|d         } | j!        d         ||||¬¦  «        }|d         }|                     ||f¦  «         |                     |d         |d         f¦  «         d} tE          |t          | j        j        j        ¦  «        ¦  «        D �]¢}! | j        j        j        |!         ||¦  «        } | j        j        j        j        |!         |¦  «                             | j        j        ¦  «        }|                      | j        j                             |¦  «        | dz   ¦  «        |z   }| j#        |          }"| j$        |          }#|                      || dz   ¦  «        }$ |"|$|z   ||¦  «        }% |#|||¦  «        }& | j         | dz            |%|&||¬¦  «        }|d         } | j!        | dz            |&|%||¬¦  «        }|d         }| dz  } |                     |¦  «         |                     |¦  «         |                     ||f¦  «         |                     |d         |d         f¦  «         �Œ¤||}(}'|  %                    |'|(¦  «        })tM          |'|(|)tO          |¦  «        tO          |¦  «        tO          |¦  «        ftO          |¦  «        ¬¦  «        S )aˆ  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        image_token_type_idx (`int`, *optional*):
            - The token type ids for images.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels are currently not supported.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerModel
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> # prepare image and text
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> text = "hello world"
        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base")
        >>> model = BridgeTowerModel.from_pretrained("BridgeTower/bridgetower-base")

        >>> inputs = processor(image, text, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.keys()
        odict_keys(['text_features', 'image_features', 'pooler_output'])
        ```NzYBridgeTowerModel does not use `inputs_embeds`.  Make sure to pass in `input_ids` instead.r"   )rŽ  r_   r   rã  rÜ   r�   rz  ©r"   )r]   rW  )r*   r+   r,   r-   r.   )(rì   rª   rù  r£   r�   r3   ÚonesrŒ  ra   r   rZ   ÚlenrÓ  rk   rz   rø  rÇ  rÝ   rÌ  r`   rµ   rÒ   r~   rà   r
  r‰  r‹  Ú	expand_asr  r  Úfullr  rþ  rý  r}   r  r  Úget_cls_featuresr)   r6   )*rY   rŽ  r]   r„  rÀ   r  r�  r?   r  r  r¿   r  Úall_hidden_states_textÚall_hidden_states_imageÚall_hidden_states_crossÚall_self_attentionsr;  r>   Úextend_text_masksÚsplit_indexrk   rƒ   Úimage_embeds_with_lnÚcross_modal_textÚtext_token_type_embeddingsÚimage_token_type_embeddingsÚcross_modal_imageÚextend_image_masksÚlayer_outputs_textÚcross_text_featuresÚlayer_outputs_imageÚcross_image_featuresÚlink_layer_indexr{  Útext_link_towerÚimage_link_towerÚtransformed_text_embedsÚcross_text_features_Úcross_image_features_r*   r+   Úcls_featuress*                                             r9   rl   zBridgeTowerModel.forward„  sý  € ð\ "$ÐØ"$ÐØ"$ÐØ ÐàÐ$¨Ð):Ý%Økñô ð ð  4Ð8°qÐØ—n’nÑ&Ô&ˆØ”o×0Ò0¸9Ð0ÑEÔEˆØ×%Ò% kÑ2Ô2Ð2àÐ!Ý"œZ¨½5¼:ÈiÔN^Ð_Ñ_Ô_ˆNå5Ø”;Ø% a a a¨¨1¨¨a¨a¨a iÔ0Ø)ð
ñ 
ô 
Ðõ ˜$œ/Ô1Ô7Ñ8Ô8¸4¼;Ô;XÑXÐ[\Ñ\ˆð ”_Ô,Ô2°<°K°<Ô@ð 	7ð 	7ˆEØ˜% Ð->Ñ?Ô?ˆKØ"×)Ò)¨+Ñ6Ô6Ð6Ð6àÐØÔ,Ô3×?Ò?Ø×!Ò! $Ô"3Ô"9Ñ:Ô:ÐUmð @ñ ô ˆLˆLð
 (×/Ò/°°1°aÑ8Ô8ˆLà×&Ò& |Ñ4Ô4Ð4ð Ô&Ô-Ô9ÔCÀLÀ[ÀLÔQð 	9ð 	9ˆEØ ˜5 Ñ.Ô.ˆLØ#×*Ò*¨<Ñ8Ô8Ð8Ð8à#Ô0Ô7×DÒDÀ\×EVÒEVÐW[ÔWhÔWnÑEoÔEoÑpÔpÐð  ×5Ò5°kÈQÐ5ÑOÔOÐà%)×%?Ò%?ÝŒK˜¥¤°IÔ4DÐEÑEÔEñ&
ô &
ç
Š)Ð$Ñ
%Ô
%ð 	#ð  ×:Ò:Ð;KÐNhÑ;hÑiÔiÐà#×:Ò:Ð;OÐ[\Ð:Ñ]Ô]ÐØ&*×&@Ò&@ÝŒJ�tÐ1½¼ÈIÔL\Ð]Ñ]Ô]ñ'
ô '
ç
Š)Ð(Ñ
)Ô
)ð 	$ð  4Ð6QÑQÐØ ×<Ò<Ð=QÑRÔRÐå”ZØ×#Ò# AÑ&Ô&Ð(9×(>Ò(>¸qÑ(AÔ(AÐBÝ”*ØÔ#ð
ñ 
ô 
ˆ
õ
 7Ø”;Ø+Ø%ð
ñ 
ô 
Ðð =˜TÔ9¸!Ô<ØØØ,Ø#5ð	
ñ 
ô 
Ðð 1°Ô3Ðà>˜dÔ;¸AÔ>ØØØ-Ø#4ð	
ñ 
ô 
Ðð  3°1Ô5Ðà×&Ò&Ð(;Ð=QÐ'RÑSÔSÐSà×"Ò"Ð$6°qÔ$9Ð;NÈqÔ;QÐ#RÑSÔSÐSàÐõ �{¥C¨¬Ô(?Ô(EÑ$FÔ$FÑGÔGð -	Xñ -	XˆAØ:˜$œ/Ô1Ô7¸Ô:¸;ÐHYÑZÔZˆKØL˜4Ô,Ô3Ô?ÔIÈ!ÔLÈ\ÑZÔZ×_Ò_ØÔ!Ô'ñô ˆLð ×+Ò+¨DÔ,=Ô,D×,QÒ,QÐR^Ñ,_Ô,_ÐaqÐtuÑauÑvÔvØ-ñ.ð !ð
 #Ô>Ð?OÔPˆOØ#Ô@ÐAQÔRÐð '+×&@Ò&@ÀÐN^ÐabÑNbÑ&cÔ&cÐ#Ø#2 ?Ø'Ð*DÑDØ#Ø!ñ$ô $Ð ð
 %5Ð$4Ð5IÐK_ÐasÑ$tÔ$tÐ!ð "T Ô!=Ð>NÐQRÑ>RÔ!SØ$Ø%Ø0Ø'9ð	"ñ "ô "Ðð #5°QÔ"7Ðà"U $Ô"?Ð@PÐSTÑ@TÔ"UØ%Ø$Ø1Ø'8ð	#ñ #ô #Ðð $7°qÔ#9Ð à Ñ!Ðà"×)Ò)¨+Ñ6Ô6Ð6Ø#×*Ò*¨<Ñ8Ô8Ð8Ø#×*Ò*Ð,?ÐAUÐ+VÑWÔWÐWà×&Ò&Ð(:¸1Ô(=Ð?RÐSTÔ?UÐ'VÑWÔWÐWÑWð )<Ð=Q�~ˆØ×,Ò,¨]¸NÑKÔKˆå%Ø'Ø)Ø&åÐ,Ñ-Ô-ÝÐ-Ñ.Ô.ÝÐ-Ñ.Ô.ðõ
 Ð0Ñ1Ô1ð

ñ 

ô 

ð 
	
r8   c                 ó†   — |                       |¦  «        }|                      |¦  «        }t          j        ||gd¬¦  «        S )Nr�   r­   )r   rÿ  r3   r¹   )rY   r*   r+   Úcls_features_textÚcls_features_images        r9   r  z!BridgeTowerModel.get_cls_featuresV  sF   € Ø ×8Ò8¸ÑGÔGÐØ!×:Ò:¸>ÑJÔJÐÝŒyÐ+Ð-?Ð@ÀbÐIÑIÔIÐIr8   )
NNNNNNNNNF)r/   r0   r1   rL   rØ  rÛ  r3   rm   rÌ   r
  r  r   r   r   r4   re   r   r   r6   r)   rl   r  rn   ro   s   @r9   rå  rå  6  s  ø€ € € € € ð6ð 6ð 6ð 6ð 6ðp6ð 6ð 6ð4ð 4ð 4ðI°5´<ð IÈCð IÐTYÔT`ð Ið Ið Ið Ið
J°E´Lð JÈSð JÐUZÔUað Jð Jð Jð Jð
 Øð .2Ø37Ø26Ø15Ø.2Ø26Ø15Ø+/Ø*.Ø).ðN
ð N
àÔ# dÑ*ðN
ð Ô)¨DÑ0ðN
ð Ô(¨4Ñ/ð	N
ð
 Ô'¨$Ñ.ðN
ð Ô$ tÑ+ðN
ð Ô(¨4Ñ/ðN
ð Ô'¨$Ñ.ðN
ð " D™jðN
ð Ô  4Ñ'ðN
ð #'ðN
ð Ð+Ô,ðN
ð 
ˆuŒ|Ô	Ð5Ñ	5ðN
ð N
ð N
ñ „^ñ ÔðN
ð`Jð Jð Jð Jð Jð Jð Jr8   rå  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú"BridgeTowerPredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S ró   )rK   rL   r   rT   rN   rô   r  r  r  r
   Útransform_act_fnrP   rQ   rX   s     €r9   rL   z+BridgeTowerPredictionHeadTransform.__init__^  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr8   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rh   )rô   r4  rP   r  s     r9   rl   z*BridgeTowerPredictionHeadTransform.forwardg  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐr8   rï   ro   s   @r9   r2  r2  ]  sL   ø€ € € € € ðUð Uð Uð Uð Uðð ð ð ð ð ð r8   r2  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )r¸  Nc                 ó^  •— t          ¦   «                              ¦   «          || _        t          |¦  «        | _        t          j        |j        |j        j	        d¬¦  «        | _
        t          j        t          j        |j        j	        ¦  «        ¦  «        | _        |�|| j
        _        d S d S )NF)rŒ   )rK   rL   rZ   r2  Ú	transformr   rT   rN   ré  r…  Údecoderr–   r3   r‹  rŒ   r°   )rY   rZ   r°   r[   s      €r9   rL   zBridgeTowerMLMHead.__init__o  s�   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ;¸FÑCÔCˆŒÝ”y Ô!3°VÔ5GÔ5RÐY^Ð_Ñ_Ô_ˆŒÝ”L¥¤¨VÔ-?Ô-JÑ!KÔ!KÑLÔLˆŒ	ØÐØ"(ˆDŒLÔÐÐð Ðr8   c                 ój   — |                       |¦  «        }|                      |¦  «        | j        z   }|S rh   )r8  r9  rŒ   )rY   ÚxÚ	mlm_scores      r9   rl   zBridgeTowerMLMHead.forwardx  s1   € Ø—N’N 1Ñ%Ô%ˆ	Ø—L’L Ñ+Ô+¨d¬iÑ7ˆ	ØÐr8   rh   rï   ro   s   @r9   r¸  r¸  n  sL   ø€ € € € € ð)ð )ð )ð )ð )ð )ðð ð ð ð ð ð r8   r¸  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerITMHeadc                 ó|   •— t          ¦   «                              ¦   «          t          j        |d¦  «        | _        d S ©Nr�   ©rK   rL   r   rT   Úfc)rY   rN   r[   s     €r9   rL   zBridgeTowerITMHead.__init__  s0   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜K¨Ñ+Ô+ˆŒˆˆr8   c                 ó0   — |                       |¦  «        }|S rh   ©rB  )rY   r;  Ú	itm_scores      r9   rl   zBridgeTowerITMHead.forwardƒ  s   € Ø—G’G˜A‘J”Jˆ	ØÐr8   rï   ro   s   @r9   r>  r>  ~  sG   ø€ € € € € ð,ð ,ð ,ð ,ð ,ðð ð ð ð ð ð r8   r>  z\
    BridgeTower Model with a language modeling head on top as done during pretraining.
    c                   ó   ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 dde	j
        dz  de	j        dz  d	e	j
        dz  d
e	j        dz  de	j
        dz  de	j        dz  de	j        dz  de	j
        dz  dee         defd„¦   «         ¦   «         Zˆ xZS )ÚBridgeTowerForMaskedLMzmlm_score.decoder.weightz8bridgetower.text_model.embeddings.word_embeddings.weightc                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S rh   )rK   rL   rå  r¤  r¸  r<  rÈ  rX   s     €r9   rL   zBridgeTowerForMaskedLM.__init__�  sR   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆÔÝ+¨FÑ3Ô3ˆŒð 	�ŠÑÔÐÐÐr8   c                 ó   — | j         j        S rh   ©r<  r9  rÊ  s    r9   Úget_output_embeddingsz,BridgeTowerForMaskedLM.get_output_embeddings™  s   € ØŒ~Ô%Ð%r8   c                 ó   — || j         _        d S rh   rJ  )rY   Únew_embeddingss     r9   Úset_output_embeddingsz,BridgeTowerForMaskedLM.set_output_embeddingsœ  s   € Ø!/ˆŒÔÐÐr8   NrŽ  r]   r„  rÀ   r  r�  r?   r  r  r¦   c	                 óv  —  | j         d|||||||dœ|	¤Ž}
|                      |
j        ¦  «        }d}|�jt          ¦   «         }|                     |j        ¦  «        } ||                     d| j        j        j	        ¦  «        |                     d¦  «        ¦  «        }t          |||
j        |
j        ¬¦  «        S )aà  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForMaskedLM
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO

        >>> url = "http://images.cocodataset.org/val2017/000000360943.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read())).convert("RGB")
        >>> text = "a <mask> looking out of the window"

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForMaskedLM.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # prepare inputs
        >>> encoding = processor(image, text, return_tensors="pt")

        >>> # forward pass
        >>> outputs = model(**encoding)

        >>> results = processor.decode(outputs.logits.argmax(dim=-1).squeeze(0).tolist())

        >>> print(results)
        .a cat looking out of the window.
        ```©rŽ  r]   r„  rÀ   r  r�  r?   Nr�   ©r<   r=   r-   r.   r7   )r¤  r<  r*   r   rd   ra   r¸   rZ   ré  r…  r   r-   r.   )rY   rŽ  r]   r„  rÀ   r  r�  r?   r  r  ÚoutputsÚ
mlm_logitsÚmasked_lm_lossÚloss_fcts                 r9   rl   zBridgeTowerForMaskedLM.forwardŸ  sâ   € ðd #�$Ô"ð 	
ØØ)Ø)Ø%Ø!Ø'Ø%ð	
ð 	
ð ð	
ð 	
ˆð —^’^ GÔ$9Ñ:Ô:ˆ
ØˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜zÔ0Ñ1Ô1ˆFØ%˜X j§o¢o°b¸$¼+Ô:QÔ:\Ñ&]Ô&]Ð_e×_jÒ_jÐkmÑ_nÔ_nÑoÔoˆNåØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r8   ©NNNNNNNN)r/   r0   r1   Ú_tied_weights_keysrL   rK  rN  r   r   r3   r   r4   r   r   r   rl   rn   ro   s   @r9   rG  rG  ˆ  si  ø€ € € € € ð 5Ð6pÐqÐðð ð ð ð ð&ð &ð &ð0ð 0ð 0ð Øð .2Ø37Ø26Ø15Ø.2Ø26Ø15Ø*.ðH
ð H
àÔ# dÑ*ðH
ð Ô)¨DÑ0ðH
ð Ô(¨4Ñ/ð	H
ð
 Ô'¨$Ñ.ðH
ð Ô$ tÑ+ðH
ð Ô(¨4Ñ/ðH
ð Ô'¨$Ñ.ðH
ð Ô  4Ñ'ðH
ð Ð+Ô,ðH
ð 
ðH
ð H
ð H
ñ „^ñ ÔðH
ð H
ð H
ð H
ð H
r8   rG  zª
    BridgeTower Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the
    [CLS] token) for image-to-text matching.
    c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  de	e
         defd„¦   «         ¦   «         Zˆ xZS )Ú#BridgeTowerForImageAndTextRetrievalc                 óÒ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        dz  ¦  «        | _        |                      ¦   «          d S r@  )rK   rL   rå  r¤  r>  rN   rE  rÈ  rX   s     €r9   rL   z,BridgeTowerForImageAndTextRetrieval.__init__ó  sZ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆÔå+¨FÔ,>ÀÑ,BÑCÔCˆŒð 	�ŠÑÔÐÐÐr8   NrŽ  r]   r„  rÀ   r  r�  r?   r  r  r¦   c	                 ó  —  | j         d|||||||dœ|	¤Ž}
|
j        }|                      |¦  «        }d}|�4t          ¦   «         }|                     |j        ¦  «        } |||¦  «        }t          |||
j        |
j        ¬¦  «        S )a^  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*):
            Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match.
            The pairs with 0 will be skipped for calculation.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForImageAndTextRetrieval
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))
        >>> texts = ["An image of two cats chilling on a couch", "A football player scoring a goal"]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForImageAndTextRetrieval.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # forward pass
        >>> scores = dict()
        >>> for text in texts:
        ...     # prepare inputs
        ...     encoding = processor(image, text, return_tensors="pt")
        ...     outputs = model(**encoding)
        ...     scores[text] = outputs.logits[0, 1].item()
        ```rP  NrQ  r7   )	r¤  r,   rE  r   rd   ra   r   r-   r.   )rY   rŽ  r]   r„  rÀ   r  r�  r?   r  r  rR  r,   r=   Úitm_lossrU  s                  r9   rl   z+BridgeTowerForImageAndTextRetrieval.forwardý  sÀ   € ð\ #�$Ô"ð 	
ØØ)Ø)Ø%Ø!Ø'Ø%ð	
ð 	
ð ð	
ð 	
ˆð  Ô-ˆà—’ Ñ.Ô.ˆàˆØÐÝ'Ñ)Ô)ˆHà—Y’Y˜vœ}Ñ-Ô-ˆFØ�x ¨Ñ/Ô/ˆHå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r8   rV  )r/   r0   r1   rL   r   r   r3   r   r4   r   r   r   rl   rn   ro   s   @r9   rY  rY  ì  s=  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø15Ø.2Ø26Ø15Ø*.ðG
ð G
àÔ# dÑ*ðG
ð Ô)¨DÑ0ðG
ð Ô(¨4Ñ/ð	G
ð
 Ô'¨$Ñ.ðG
ð Ô$ tÑ+ðG
ð Ô(¨4Ñ/ðG
ð Ô'¨$Ñ.ðG
ð Ô  4Ñ'ðG
ð Ð+Ô,ðG
ð 
"ðG
ð G
ð G
ñ „^ñ ÔðG
ð G
ð G
ð G
ð G
r8   rY  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerContrastiveHeadc                 ó|   •— t          ¦   «                              ¦   «          t          j        ||¦  «        | _        d S rh   rA  )rY   rN   Ú
embed_sizer[   s      €r9   rL   z#BridgeTowerContrastiveHead.__init__J  s0   ø€ Ý‰Œ×ÒÑÔÐÝ”)˜K¨Ñ4Ô4ˆŒˆˆr8   c                 ó0   — |                       |¦  «        }|S rh   rD  )rY   r;  s     r9   rl   z"BridgeTowerContrastiveHead.forwardN  s   € Ø�GŠG�A‰JŒJˆØˆr8   rï   ro   s   @r9   r^  r^  I  sG   ø€ € € € € ð5ð 5ð 5ð 5ð 5ðð ð ð ð ð ð r8   r^  zl
    BridgeTower Model with a image-text contrastive head on top computing image-text contrastive loss.
    c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  de
e         defd„¦   «         ¦   «         Zˆ xZS )r³  c                 óÄ  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |j        |j        ¦  «        | _        t	          |j        |j        ¦  «        | _        t	          |j        dz  |j        ¦  «        | _	        t          j        t          j        | j        j        ¦  «        ¦  «        | _        |                      ¦   «          d S r@  )rK   rL   rå  r¤  r^  rN   Úcontrastive_hidden_sizeÚitc_text_headÚitc_image_headÚitc_cross_modal_headr   r–   r3   ré   rZ   r¶  rµ  rÈ  rX   s     €r9   rL   z*BridgeTowerForContrastiveLearning.__init__Y  s±   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆÔå7¸Ô8JÈFÔLjÑkÔkˆÔÝ8¸Ô9KÈVÔMkÑlÔlˆÔÝ$>¸vÔ?QÐTUÑ?UÐW]ÔWuÑ$vÔ$vˆÔ!åœ<­¬°T´[Ô5WÑ(XÔ(XÑYÔYˆÔà�ŠÑÔÐÐÐr8   NrŽ  r]   r„  rÀ   r  r�  r?   Úreturn_lossr  r¦   c	                 ó¶  — |	                      dd¦  «          | j        d|||||||dœ|	¤Ž}
|
j        }|
j        \  }}}|d         }|d         }| j        j        j                             |¦  «        }| j                             t          j	        ddt          j
        | j        j        j        j        ¬¦  «        ¦  «                             |¦  «        }| j                             |¦  «        |z   }t          j                             |                      |dd…d	dd…f         ¦  «        dd
¬¦  «        }t          j                             |                      |dd…d	dd…f         ¦  «        dd
¬¦  «                             |j        ¬¦  «        }t          j                             |                      |¦  «        dd
¬¦  «                             |j        ¬¦  «        }t          j        |||gd¬¦  «        }| j                             ¦   «                              |j        ¬¦  «        }t          j        ||                     ¦   «         ¦  «        |z  }t          j        ||                     ¦   «         ¦  «        |z  }t          j        ||                     ¦   «         ¦  «        |z  }d}|r“t          j        t9          |¦  «        |j        ¬¦  «        }t          j                             ||¦  «        }t          j                             ||¦  «        }t          j                             ||¦  «        }||z   |z   dz  }t=          ||||||
j        |
j        ¬¦  «        S )a¥  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image
        >>> import torch

        >>> image_urls = [
        ...     "https://farm4.staticflickr.com/3395/3428278415_81c3e27f15_z.jpg",
        ...     "http://images.cocodataset.org/val2017/000000039769.jpg",
        ... ]
        >>> texts = ["two dogs in a car", "two cats sleeping on a couch"]

        >>> with httpx.stream("GET", urls[0]) as response:
        ...     image1 = Image.open(BytesIO(response.read()))

        >>> with httpx.stream("GET", urls[1]) as response:
        ...     image2 = Image.open(BytesIO(response.read()))

        >>> images = [image1, image2]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
        >>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")

        >>> inputs = processor(images, texts, padding=True, return_tensors="pt")
        >>> loss = model(**inputs, return_loss=True).loss

        >>> inputs = processor(images, texts[::-1], padding=True, return_tensors="pt")
        >>> loss_swapped = model(**inputs, return_loss=True).loss

        >>> print("Loss", round(loss.item(), 4))
        Loss 0.0019

        >>> print("Loss with swapped images", round(loss_swapped.item(), 4))
        Loss with swapped images 2.126
        ```Úoutput_hidden_statesTrP  r�   r  r"   r_   Nr   r�   )r®   r  )ra   éþÿÿÿr­   g      @)r<   r=   r>   r?   r@   r-   r.   r7   ) Ú
setdefaultr¤  r,   r-   rø  rÇ  rà   r‰  r3   r  rŒ  r°   ra   r  r÷  r   r¶   Ú	normalizere  rf  rd   rg  r×   rµ  Úexpr  Útr¡   r  Úcross_entropyr;   r.   )rY   rŽ  r]   r„  rÀ   r  r�  r?   rh  r  rR  r,   Úhidden_states_txtÚhidden_states_imgÚhidden_states_cross_modalr>   r  r   r@   r=   rµ  Úlogits_text_to_imageÚlogits_text_to_crossÚlogits_image_to_crossÚitc_lossr  Útext_to_image_lossÚtext_to_cross_lossÚimage_to_cross_losss                                r9   rl   z)BridgeTowerForContrastiveLearning.forwardf  sn  € ðv 	×ÒÐ0°$Ñ7Ô7Ð7Ø"�$Ô"ð 	
ØØ)Ø)Ø%Ø!Ø'Ø%ð	
ð 	
ð ð	
ð 	
ˆð  Ô-ˆØJQÔJ_ÑGÐÐ,Ð.Gà'¨Ô+ˆØ(¨Ô,ˆà#Ô/Ô<ÔC×PÒPÐQ]Ñ^Ô^ÐØ&*Ô&6×&LÒ&LÝŒJ�t˜Q¥e¤j¸Ô9IÔ9_Ô9fÔ9mÐnÑnÔnñ'
ô '
ç
Š)Ð(Ñ
)Ô
)ð 	$ð Ô'×CÒCÐDXÑYÔYÐ\wÑwˆõ ”m×-Ò-¨d×.@Ò.@ÀÈQÈQÈQÐPQÐSTÐSTÐSTÈWÔAUÑ.VÔ.VÐ\^ÐbcÐ-ÑdÔdˆÝ”}×.Ò.¨t×/BÒ/BÀ<ÐPQÐPQÐPQÐSTÐVWÐVWÐVWÐPWÔCXÑ/YÔ/YÐ_aÐefÐ.ÑgÔg×jÒjØÔ%ð kñ 
ô 
ˆõ ”}×.Ò.¨t×/HÒ/HÈÑ/WÔ/WÐ]_ÐcdÐ.ÑeÔe×hÒhØÔ%ð iñ 
ô 
ˆõ ”˜k¨<¸ÐFÈBÐOÑOÔOˆàÔ&×*Ò*Ñ,Ô,×/Ò/°{Ô7IÐ/ÑJÔJˆÝ$œ|¨K¸¿ºÑ9IÔ9IÑJÔJÈ[ÑXÐÝ$œ|¨K¸¿ºÑ9IÔ9IÑJÔJÈ[ÑXÐÝ %¤¨\¸<¿>º>Ñ;KÔ;KÑ LÔ LÈ{Ñ ZÐàˆàð 	]Ý”\¥# f¡+¤+°f´mÐDÑDÔDˆFÝ!#¤×!<Ò!<Ð=QÐSYÑ!ZÔ!ZÐÝ!#¤×!<Ò!<Ð=QÐSYÑ!ZÔ!ZÐÝ"$¤-×"=Ò"=Ð>SÐU[Ñ"\Ô"\ÐØ*Ð-?Ñ?ÐBUÑUÐY\Ñ\ˆHå+ØØØ#Ø%Ø%Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r8   rV  )r/   r0   r1   rL   r   r   r3   r   r4   re   r   r   r;   rl   rn   ro   s   @r9   r³  r³  S  s9  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø15Ø.2Ø26Ø15Ø#'ðs
ð s
àÔ# dÑ*ðs
ð Ô)¨DÑ0ðs
ð Ô(¨4Ñ/ð	s
ð
 Ô'¨$Ñ.ðs
ð Ô$ tÑ+ðs
ð Ô(¨4Ñ/ðs
ð Ô'¨$Ñ.ðs
ð ˜D‘[ðs
ð Ð+Ô,ðs
ð 
&ðs
ð s
ð s
ñ „^ñ Ôðs
ð s
ð s
ð s
ð s
r8   r³  )r³  rY  rG  rå  r£  )Nr  )[r2   Úcollectionsr   Úcollections.abcr   Údataclassesr   r3   r   Útorch.nnr   Ú r	   r­  Úactivationsr
   r   Úcache_utilsr   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr   r    Úutils.output_capturingr!   Úconfiguration_bridgetowerr#   r$   r%   Ú
get_loggerr/   ÚloggerÚ_TOKENIZER_FOR_DOCr)   r;   rÃ  rB   rq   r†   rÎ   râ   rñ   rÿ   r	  r  rm   Úfloatr$  r&  rE  rQ  r\  rr  rw  r�  r£  rÅ  rÐ  rå  r2  r¸  r>  rG  rY  r^  r³  Ú__all__r7   r8   r9   ú<module>r‘     s	  ðð  Ð à #Ð #Ð #Ð #Ð #Ð #Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %à &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KÐ KØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hÐ hð 
ˆÔ	˜HÑ	%Ô	%€à'Ð ð €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7˜[ñ 7ô 7ñ „ñô ð7ð$ €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 ;ñ 7ô 7ñ „ñô ð7ð4)ð )ð )ð )ð ) 2¤9ñ )ô )ð )ðXð ð ð ð ˜RœYñ ô ð ð6Pð Pð Pð Pð P "¤)ñ Pô Pð Pðf7"ð 7"ð 7"ð 7"ð 7" 2¤9ñ 7"ô 7"ð 7"ðtdð dð dð dð d˜2œ9ñ dô dð dð4ð ð ð ð ˜BœIñ ô ð ðð ð ð ð ˜bœiñ ô ð ð ð ð ð ð ˜œ	ñ ô ð ðð ð ð ð ˜œ	ñ ô ð ð, !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð:@)ð @)ð @)ð @)ð @)˜rœyñ @)ô @)ð @)ðHI)ð I)ð I)ð I)ð I) ¤	ñ I)ô I)ð I)ðZ.ð .ð .ð .ð .˜2œ9ñ .ô .ð .ð:8ð 8ð 8ð 8ð 8 ¤	ñ 8ô 8ð 8ðv?ð ?ð ?ð ?ð ?Ð5ñ ?ô ?ð ?ðF
ð 
ð 
ð 
ð 
˜RœYñ 
ô 
ð 
ðFg8ð g8ð g8ð g8ð g8 ¤	ñ g8ô g8ð g8ðT ð)%ð )%ð )%ð )%ð )% ñ )%ô )%ñ „ð)%ðXYð Yð Yð Yð YÐ7ñ Yô Yð Yð" €ððñ ô ð{6ð {6ð {6ð {6ð {6Ð5ñ {6ô {6ñô ð{6ð| €ððñ ô ð
^Jð ^Jð ^Jð ^Jð ^JÐ1ñ ^Jô ^Jñô ð
^JðD	ð ð ð ð ¨¬ñ ô ð ð"ð ð ð ð ˜œñ ô ð ð ð ð ð ð ˜œñ ô ð ð €ððñ ô ð
\
ð \
ð \
ð \
ð \
Ð7ñ \
ô \
ñô ð
\
ð~ €ððñ ô ðT
ð T
ð T
ð T
ð T
Ð*Dñ T
ô T
ñô ðT
ðnð ð ð ð  ¤ñ ô ð ð €ððñ ô ð
C
ð C
ð C
ð C
ð C
Ð(Bñ C
ô C
ñô ð
C
ðLð ð €€€r8   