§
    ‚Štj «  ã                   ó*  — d Z ddlZddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,  e#j-        e.¦  «        Z/ e"d¬¦  «        e G d„ de ¦  «        ¦   «         ¦   «         Z0 e"d¬¦  «        e G d„ de ¦  «        ¦   «         ¦   «         Z1e"e G d„ de ¦  «        ¦   «         ¦   «         Z2dej3        dej3        fd „Z4d!ej3        dej3        fd"„Z5d#e,d$e6fd%„Z7d`d'e6e8z  d(e9fd)„Z: G d*„ d+e	j;        ¦  «        Z< G d,„ d-e	j=        ¦  «        Z> G d.„ d/e	j;        ¦  «        Z? G d0„ d1e	j;        ¦  «        Z@ G d2„ d3e	j;        ¦  «        ZA G d4„ d5e	j;        ¦  «        ZB G d6„ d7e	j;        ¦  «        ZC G d8„ d9e	j;        ¦  «        ZD G d:„ d;e	j;        ¦  «        ZE	 dad=e	j;        d>ej3        d?ej3        d@ej3        dAej3        dz  dBeFdCeFfdD„ZG G dE„ dFe	j;        ¦  «        ZH G dG„ dHe	j;        ¦  «        ZI G dI„ dJe	j;        ¦  «        ZJ G dK„ dLe	j;        ¦  «        ZK G dM„ dNe	j;        ¦  «        ZL G dO„ dPe¦  «        ZM G dQ„ dRe	j;        ¦  «        ZN G dS„ dTe	j;        ¦  «        ZOe" G dU„ dVe¦  «        ¦   «         ZP e"dW¬¦  «         G dX„ dYeP¦  «        ¦   «         ZQ e"dZ¬¦  «         G d[„ d\eP¦  «        ¦   «         ZRe" G d]„ d^eP¦  «        ¦   «         ZSg d_¢ZTdS )bzPyTorch ALIGN model.é    N)ÚCallable)Ú	dataclass)ÚAny)Únné   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithNoAttentionÚBaseModelOutputWithPoolingÚ(BaseModelOutputWithPoolingAndNoAttention)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚAlignConfigÚAlignTextConfigÚAlignVisionConfigz}
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
    )Úcustom_introc                   óz   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dS )ÚAlignVisionModelOutputzø
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
        The image embeddings obtained by applying the projection layer to the pooler_output.
    NÚimage_embedsÚlast_hidden_stateÚhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   ÚtorchÚFloatTensorÚ__annotations__r#   r$   Útuple© ó    úf/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/align/modeling_align.pyr!   r!   .   sl   € € € € € € ðð ð
 .2€L�%Ô# dÑ*Ð1Ð1Ñ1Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ð9Ð9r.   r!   ze
    Base class for text model's outputs that also contains a pooling of the last hidden states.
    c                   ó¤   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dS )ÚAlignTextModelOutputzö
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
        The text embeddings obtained by applying the projection layer to the pooler_output.
    NÚtext_embedsr#   r$   Ú
attentions)r%   r&   r'   r(   r2   r)   r*   r+   r#   r$   r,   r3   r-   r.   r/   r1   r1   ?   s‰   € € € € € € ðð ð
 -1€K�Ô" TÑ)Ð0Ð0Ñ0Ø26Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r.   r1   c                   óÞ   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZej        dz  ed<   dZeed<   dZeed	<   d
ee         fd„ZdS )ÚAlignOutputar  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for image-text similarity.
    logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
        The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
        The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
        similarity scores.
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`AlignTextModel`].
    image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
        The output of [`AlignVisionModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`AlignTextModel`].
    vision_model_output (`BaseModelOutputWithPoolingAndNoAttention`):
        The output of the [`AlignVisionModel`].
    NÚlossÚlogits_per_imageÚlogits_per_textr2   r"   Útext_model_outputÚvision_model_outputÚreturnc                 ó^   ‡ — t          ˆ fd„‰                      ¦   «         D ¦   «         ¦  «        S )Nc              3   ót   •K  — | ]2}|d vr‰|         n!t          ‰|¦  «                             ¦   «         V — Œ3dS ))r9   r:   N)ÚgetattrÚto_tuple)Ú.0ÚkÚselfs     €r/   ú	<genexpr>z'AlignOutput.to_tuple.<locals>.<genexpr>p   sc   øè è € ð 
ð 
àð Ð LÐLÐLˆD�ŒGˆGÕRYÐZ^Ð`aÑRbÔRb×RkÒRkÑRmÔRmð
ð 
ð 
ð 
ð 
ð 
r.   )r,   Úkeys©rB   s   `r/   r?   zAlignOutput.to_tupleo   sC   ø€ Ýð 
ð 
ð 
ð 
à—Y’Y‘[”[ð
ñ 
ô 
ñ 
ô 
ð 	
r.   )r%   r&   r'   r(   r6   r)   r*   r+   r7   r8   r2   r"   r9   r   r:   r   r,   r   r?   r-   r.   r/   r5   r5   Q   sÞ   € € € € € € ðð ð& &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø15Ð�eÔ'¨$Ñ.Ð5Ð5Ñ5Ø04€O�UÔ&¨Ñ-Ð4Ð4Ñ4Ø,0€K�Ô" TÑ)Ð0Ð0Ñ0Ø-1€L�%Ô# dÑ*Ð1Ð1Ñ1Ø48ÐÐ1Ð8Ð8Ñ8ØDHÐÐAÐHÐHÑHð
˜% œ*ð 
ð 
ð 
ð 
ð 
ð 
r.   r5   Úlogitsr;   c                 ó’   — t           j                             | t          j        t          | ¦  «        | j        ¬¦  «        d¬¦  «        S )N©Údevicegš™™™™™¹?)Úlabel_smoothing)r   Ú
functionalÚcross_entropyr)   ÚarangeÚlenrI   )rF   s    r/   Úcontrastive_lossrO   x   s9   € ÝŒ=×&Ò& v­u¬|½CÀ¹K¼KÐPVÔP]Ð/^Ñ/^Ô/^ÐpsÐ&ÑtÔtÐtr.   Ú
similarityc                 ór   — t          | ¦  «        }t          |                      ¦   «         ¦  «        }||z   dz  S )Ng       @)rO   Út)rP   Úcaption_lossÚ
image_losss      r/   Ú
align_lossrU   |   s4   € Ý# JÑ/Ô/€LÝ! *§,¢,¡.¤.Ñ1Ô1€JØ˜:Ñ%¨Ñ,Ð,r.   ÚconfigÚnum_channelsc                 ó°   — | j         }|| j        z  }t          |t          ||dz  z   ¦  «        |z  |z  ¦  «        }|d|z  k     r||z  }t          |¦  «        S )z<
    Round number of filters based on depth multiplier.
    é   gÍÌÌÌÌÌì?)Údepth_divisorÚwidth_coefficientÚmaxÚint)rV   rW   ÚdivisorÚnew_dims       r/   Úround_filtersr`   ƒ   sk   € ð Ô"€GØ�FÔ,Ñ,€LÝ�'�3˜|¨g¸©kÑ9Ñ:Ô:¸gÑEÈÑOÑPÔP€Gð ��|Ñ#Ò#Ð#Ø�7Ñˆåˆw‰<Œ<Ðr.   TÚkernel_sizeÚadjustc                 óè   — t          | t          ¦  «        r| | f} | d         dz  | d         dz  f}|r$|d         dz
  |d         |d         dz
  |d         fS |d         |d         |d         |d         fS )aJ  
    Utility function to get the tuple padding value for the depthwise convolution.

    Args:
        kernel_size (`int` or `tuple`):
            Kernel size of the convolution layers.
        adjust (`bool`, *optional*, defaults to `True`):
            Adjusts padding value to apply to right and bottom sides of the input.
    r   rY   r   )Ú
isinstancer]   )ra   rb   Úcorrects      r/   Úcorrect_padrf   “   s‹   € õ �+�sÑ#Ô#ð 1Ø" KÐ0ˆà˜1Œ~ Ñ" K°¤N°aÑ$7Ð8€GØð @Ø˜”
˜Q‘ ¨¤
¨G°A¬J¸©N¸GÀA¼JÐGÐGà˜”
˜G AœJ¨°¬
°G¸A´JÐ?Ð?r.   c                   óL   ‡ — e Zd ZdZdefˆ fd„Zdej        dej        fd„Zˆ xZ	S )ÚAlignVisionEmbeddingszL
    A module that corresponds to the stem module of the original work.
    rV   c                 ó|  •— t          ¦   «                              ¦   «          t          |d¦  «        | _        t	          j        d¬¦  «        | _        t	          j        |j        | j        dddd¬¦  «        | _	        t	          j
        | j        |j        |j        ¬	¦  «        | _        t          |j                 | _        d S )
Né    )r   r   r   r   ©Úpaddingr   rY   ÚvalidF©ra   Ústriderl   Úbias)ÚepsÚmomentum)ÚsuperÚ__init__r`   Úout_dimr   Ú	ZeroPad2drl   ÚConv2drW   ÚconvolutionÚBatchNorm2dÚbatch_norm_epsÚbatch_norm_momentumÚ	batchnormr	   Ú
hidden_actÚ
activation©rB   rV   Ú	__class__s     €r/   rt   zAlignVisionEmbeddings.__init__­   s¡   ø€ Ý‰Œ×ÒÑÔÐå$ V¨RÑ0Ô0ˆŒÝ”|¨LÐ9Ñ9Ô9ˆŒÝœ9ØÔ ¤¸1ÀQÐPWÐ^cð
ñ 
ô 
ˆÔõ œ¨¬¸&Ô:OÐZ`ÔZtÐuÑuÔuˆŒÝ  Ô!2Ô3ˆŒˆˆr.   Úpixel_valuesr;   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S ©N)rl   rx   r|   r~   )rB   r�   Úfeaturess      r/   ÚforwardzAlignVisionEmbeddings.forward¸   sM   € Ø—<’< Ñ-Ô-ˆØ×#Ò# HÑ-Ô-ˆØ—>’> (Ñ+Ô+ˆØ—?’? 8Ñ,Ô,ˆàˆr.   )
r%   r&   r'   r(   r   rt   r)   ÚTensorr…   Ú__classcell__©r€   s   @r/   rh   rh   ¨   su   ø€ € € € € ðð ð	4Ð0ð 	4ð 	4ð 	4ð 	4ð 	4ð 	4ð E¤Lð °U´\ð ð ð ð ð ð ð ð r.   rh   c                   ó.   ‡ — e Zd Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚAlignVisionDepthwiseConv2dr   r   r   TÚzerosc	                 óf   •— ||z  }	t          ¦   «                              ||	|||||||¬¦	  «	         d S )N)	Úin_channelsÚout_channelsra   ro   rl   ÚdilationÚgroupsrp   Úpadding_mode)rs   rt   )rB   r�   Údepth_multiplierra   ro   rl   r�   rp   r‘   rŽ   r€   s             €r/   rt   z#AlignVisionDepthwiseConv2d.__init__Ã   sV   ø€ ð #Ð%5Ñ5ˆÝ‰Œ×ÒØ#Ø%Ø#ØØØØØØ%ð 	ñ 
	
ô 
	
ð 
	
ð 
	
ð 
	
r.   )r   r   r   r   r   Tr‹   )r%   r&   r'   rt   r‡   rˆ   s   @r/   rŠ   rŠ   Â   sT   ø€ € € € € ð ØØØØØØð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r.   rŠ   c                   óX   ‡ — e Zd ZdZdedededefˆ fd„Zdej        dej	        fd	„Z
ˆ xZS )
ÚAlignVisionExpansionLayerz_
    This corresponds to the expansion phase of each block in the original implementation.
    rV   Úin_dimru   ro   c                 óò   •— t          ¦   «                              ¦   «          t          j        ||ddd¬¦  «        | _        t          j        ||j        ¬¦  «        | _        t          |j	                 | _
        d S )Nr   ÚsameF©r�   rŽ   ra   rl   rp   )Únum_featuresrq   )rs   rt   r   rw   Úexpand_convry   rz   Ú	expand_bnr	   r}   Ú
expand_act)rB   rV   r•   ru   ro   r€   s        €r/   rt   z"AlignVisionExpansionLayer.__init__â   so   ø€ Ý‰Œ×ÒÑÔÐÝœ9ØØ ØØØð
ñ 
ô 
ˆÔõ œ°WÀ&ÔBWÐXÑXÔXˆŒÝ  Ô!2Ô3ˆŒˆˆr.   r$   r;   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rƒ   )rš   r›   rœ   ©rB   r$   s     r/   r…   z!AlignVisionExpansionLayer.forwardî   s=   € à×(Ò(¨Ñ7Ô7ˆØŸš }Ñ5Ô5ˆØŸš¨Ñ6Ô6ˆàÐr.   )r%   r&   r'   r(   r   r]   rt   r)   r*   r†   r…   r‡   rˆ   s   @r/   r”   r”   Ý   sŒ   ø€ € € € € ðð ð
4Ð0ð 
4¸#ð 
4Èð 
4ÐUXð 
4ð 
4ð 
4ð 
4ð 
4ð 
4ð UÔ%6ð ¸5¼<ð ð ð ð ð ð ð ð r.   r”   c            
       ó\   ‡ — e Zd ZdZdededededef
ˆ fd„Zdej	        d	ej
        fd
„Zˆ xZS )ÚAlignVisionDepthwiseLayerzk
    This corresponds to the depthwise convolution phase of each block in the original implementation.
    rV   r•   ro   ra   Úadjust_paddingc                 óv  •— t          ¦   «                              ¦   «          || _        | j        dk    rdnd}t          ||¬¦  «        }t	          j        |¬¦  «        | _        t          ||||d¬¦  «        | _        t	          j	        ||j
        |j        ¬¦  «        | _        t          |j                 | _        d S )	NrY   rm   r—   )rb   rk   Frn   ©r™   rq   rr   )rs   rt   ro   rf   r   rv   Údepthwise_conv_padrŠ   Údepthwise_convry   rz   r{   Údepthwise_normr	   r}   Údepthwise_act)	rB   rV   r•   ro   ra   r¡   Úconv_padrl   r€   s	           €r/   rt   z"AlignVisionDepthwiseLayer.__init__ý   sÁ   ø€ õ 	‰Œ×ÒÑÔÐØˆŒØ"œk¨QÒ.Ð.�7�7°FˆÝ˜k°.ÐAÑAÔAˆå"$¤,°wÐ"?Ñ"?Ô"?ˆÔÝ8Ø °FÀHÐSXð
ñ 
ô 
ˆÔõ !œnØ VÔ%:ÀVÔE_ð
ñ 
ô 
ˆÔõ $ FÔ$5Ô6ˆÔÐÐr.   r$   r;   c                 óÄ   — | j         dk    r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )NrY   )ro   r¤   r¥   r¦   r§   rž   s     r/   r…   z!AlignVisionDepthwiseLayer.forward  sa   € àŒ;˜!ÒÐØ ×3Ò3°MÑBÔBˆMà×+Ò+¨MÑ:Ô:ˆØ×+Ò+¨MÑ:Ô:ˆØ×*Ò*¨=Ñ9Ô9ˆàÐr.   ©r%   r&   r'   r(   r   r]   Úboolrt   r)   r*   r†   r…   r‡   rˆ   s   @r/   r    r    ø   sž   ø€ € € € € ðð ð7à!ð7ð ð7ð ð	7ð
 ð7ð ð7ð 7ð 7ð 7ð 7ð 7ð,	 UÔ%6ð 	¸5¼<ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r.   r    c            	       óZ   ‡ — e Zd ZdZddedededefˆ fd„Zdej	        d	ej
        fd
„Zˆ xZS )ÚAlignVisionSqueezeExciteLayerzl
    This corresponds to the Squeeze and Excitement phase of each block in the original implementation.
    FrV   r•   Ú
expand_dimÚexpandc                 óà  •— t          ¦   «                              ¦   «          |r|n|| _        t          dt	          ||j        z  ¦  «        ¦  «        | _        t          j        d¬¦  «        | _	        t          j
        | j        | j        dd¬¦  «        | _        t          j
        | j        | j        dd¬¦  «        | _        t          |j                 | _        t          j        ¦   «         | _        d S )Nr   )Úoutput_sizer—   )r�   rŽ   ra   rl   )rs   rt   Údimr\   r]   Úsqueeze_expansion_ratioÚdim_ser   ÚAdaptiveAvgPool2dÚsqueezerw   Úreducer¯   r	   r}   Ú
act_reduceÚSigmoidÚ
act_expand)rB   rV   r•   r®   r¯   r€   s        €r/   rt   z&AlignVisionSqueezeExciteLayer.__init__%  sÓ   ø€ Ý‰Œ×ÒÑÔÐØ!'Ð3�:�:¨VˆŒÝ˜!�S ¨&Ô*HÑ!HÑIÔIÑJÔJˆŒåÔ+¸Ð:Ñ:Ô:ˆŒÝ”iØœØœØØð	
ñ 
ô 
ˆŒõ ”iØœØœØØð	
ñ 
ô 
ˆŒõ ! Ô!2Ô3ˆŒÝœ*™,œ,ˆŒˆˆr.   r$   r;   c                 ó  — |}|                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }t          j        ||¦  «        }|S rƒ   )r¶   r·   r¸   r¯   rº   r)   Úmul)rB   r$   Úinputss      r/   r…   z%AlignVisionSqueezeExciteLayer.forward:  ss   € ØˆØŸš ]Ñ3Ô3ˆØŸš MÑ2Ô2ˆØŸš¨Ñ6Ô6ˆàŸš MÑ2Ô2ˆØŸš¨Ñ6Ô6ˆÝœ	 &¨-Ñ8Ô8ˆàÐr.   )Frª   rˆ   s   @r/   r­   r­      s‘   ø€ € € € € ðð ð'ð 'Ð0ð '¸#ð 'È3ð 'ÐX\ð 'ð 'ð 'ð 'ð 'ð 'ð*
 UÔ%6ð 
¸5¼<ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r.   r­   c                   ón   ‡ — e Zd ZdZdedededededefˆ fd„Zd	e	j
        d
e	j
        de	j        fd„Zˆ xZS )ÚAlignVisionFinalBlockLayerz[
    This corresponds to the final phase of each block in the original implementation.
    rV   r•   ru   ro   Ú	drop_rateÚid_skipc                 ó   •— t          ¦   «                              ¦   «          |dk    o| | _        t          j        ||ddd¬¦  «        | _        t          j        ||j        |j        ¬¦  «        | _	        t          j
        |¬¦  «        | _        d S )Nr   r—   Fr˜   r£   )Úp)rs   rt   Úapply_dropoutr   rw   Úproject_convry   rz   r{   Ú
project_bnÚDropoutÚdropout)rB   rV   r•   ru   ro   rÀ   rÁ   r€   s          €r/   rt   z#AlignVisionFinalBlockLayer.__init__L  s—   ø€ õ 	‰Œ×ÒÑÔÐØ# qš[Ð8°¨[ˆÔÝœIØØ ØØØð
ñ 
ô 
ˆÔõ œ.Ø  fÔ&;ÀfÔF`ð
ñ 
ô 
ˆŒõ ”z IÐ.Ñ.Ô.ˆŒˆˆr.   Ú
embeddingsr$   r;   c                 óœ   — |                       |¦  «        }|                      |¦  «        }| j        r|                      |¦  «        }||z   }|S rƒ   )rÅ   rÆ   rÄ   rÈ   )rB   rÉ   r$   s      r/   r…   z"AlignVisionFinalBlockLayer.forward]  sR   € Ø×)Ò)¨-Ñ8Ô8ˆØŸš¨Ñ6Ô6ˆàÔð 	7Ø ŸLšL¨Ñ7Ô7ˆMØ)¨JÑ6ˆMàÐr.   ©r%   r&   r'   r(   r   r]   Úfloatr«   rt   r)   r*   r†   r…   r‡   rˆ   s   @r/   r¿   r¿   G  sª   ø€ € € € € ðð ð/Ø'ð/Ø14ð/Ø?Bð/ØLOð/Ø\að/Ølpð/ð /ð /ð /ð /ð /ð" %Ô"3ð ÀEÔDUð ÐZ_ÔZfð ð ð ð ð ð ð ð r.   r¿   c                   ól   ‡ — e Zd ZdZdededededededed	ed
efˆ fd„Zde	j
        de	j        fd„Zˆ xZS )ÚAlignVisionBlocka�  
    This corresponds to the block module of original the EfficientNet vision encoder implementation.

    Args:
        config ([`AlignVisionConfig`]):
            Model configuration class.
        in_dim (`int`):
            Number of input channels.
        out_dim (`int`):
            Number of output channels.
        stride (`int`):
            Stride size to be used in convolution layers.
        expand_ratio (`int`):
            Expand ratio to set the output dimensions for the expansion and squeeze-excite layers.
        kernel_size (`int`):
            Kernel size for the depthwise convolution layer.
        drop_rate (`float`):
            Dropout rate to be used in the final phase of each block.
        id_skip (`bool`):
            Whether to apply dropout and sum the final hidden states with the input embeddings during the final phase
            of each block. Set to `True` for the first block of each stage.
        adjust_padding (`bool`):
            Whether to apply padding to only right and bottom side of the input kernel before the depthwise convolution
            operation, set to `True` for inputs with odd input sizes.
    rV   r•   ru   ro   Úexpand_ratiora   rÀ   rÁ   r¡   c
                 ó‚  •— t          ¦   «                              ¦   «          || _        | j        dk    | _        ||z  }
| j        rt	          |||
|¬¦  «        | _        t          || j        r|
n||||	¬¦  «        | _        t          |||
| j        ¬¦  «        | _	        t          || j        r|
n|||||¬¦  «        | _        d S )Nr   )rV   r•   ru   ro   )rV   r•   ro   ra   r¡   )rV   r•   r®   r¯   )rV   r•   ru   ro   rÀ   rÁ   )rs   rt   rÏ   r¯   r”   Ú	expansionr    r¥   r­   Úsqueeze_exciter¿   Ú
projection)rB   rV   r•   ru   ro   rÏ   ra   rÀ   rÁ   r¡   Úexpand_in_dimr€   s              €r/   rt   zAlignVisionBlock.__init__ƒ  sõ   ø€ õ 	‰Œ×ÒÑÔÐØ(ˆÔØÔ'¨1Ò,ˆŒØ Ñ-ˆàŒ;ð 	Ý6Ø f°mÈFðñ ô ˆDŒNõ 8ØØ$(¤KÐ;�=�=°VØØ#Ø)ð
ñ 
ô 
ˆÔõ <Ø &°]È4Ì;ð
ñ 
ô 
ˆÔõ 5ØØ$(¤KÐ;�=�=°VØØØØð
ñ 
ô 
ˆŒˆˆr.   r$   r;   c                 óÊ   — |}| j         dk    r|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      ||¦  «        }|S ©Nr   )rÏ   rÑ   r¥   rÒ   rÓ   )rB   r$   rÉ   s      r/   r…   zAlignVisionBlock.forward¬  sg   € Ø"ˆ
àÔ Ò!Ð!Ø ŸNšN¨=Ñ9Ô9ˆMØ×+Ò+¨MÑ:Ô:ˆð ×+Ò+¨MÑ:Ô:ˆØŸš¨
°MÑBÔBˆØÐr.   rË   rˆ   s   @r/   rÎ   rÎ   h  sÇ   ø€ € € € € ðð ð4'
à!ð'
ð ð'
ð ð	'
ð
 ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð ð'
ð '
ð '
ð '
ð '
ð '
ðR
 UÔ%6ð 
¸5¼<ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r.   rÎ   c                   óR   ‡ — e Zd ZdZdefˆ fd„Zdej        dee	         de
fd„Zˆ xZS )ÚAlignVisionEncoderz·
    Forward propagates the embeddings through each vision encoder (EfficientNet) block.

    Args:
        config ([`AlignVisionConfig`]):
            Model configuration class.
    rV   c                 óþ  •‡ ‡— t          ¦   «                              ¦   «          |j        ‰ _        ˆ fd„Št          |j        ¦  «        }t          ˆfd„|j        D ¦   «         ¦  «        }d}g }t          |¦  «        D ]ç}t          ||j        |         ¦  «        }t          ||j	        |         ¦  «        }|j
        |         }	|j        |         }
|j        |         }t           ‰|j        |         ¦  «        ¦  «        D ]d}|dk    }|dk    rdn|	}	|dk    r|n|}||j        v}|j        |z  |z  }t          ||||	|
||||¬¦	  «	        }|                     |¦  «         |dz  }ŒeŒèt#          j        |¦  «        ‰ _        d S )Nc                 óV   •— t          t          j        ‰j        | z  ¦  «        ¦  «        S rƒ   )r]   ÚmathÚceilÚdepth_coefficient)ÚrepeatsrB   s    €r/   Úround_repeatsz2AlignVisionEncoder.__init__.<locals>.round_repeatsÆ  s#   ø€ å•t”y Ô!7¸'Ñ!AÑBÔBÑCÔCÐCr.   c              3   ó.   •K  — | ]} ‰|¦  «        V — Œd S rƒ   r-   )r@   Únrß   s     €r/   rC   z.AlignVisionEncoder.__init__.<locals>.<genexpr>Ë  s-   øè è € ÐLÐL¨a˜˜ qÑ)Ô)ÐLÐLÐLÐLÐLÐLr.   r   r   )	rV   r•   ru   ro   ra   rÏ   rÀ   rÁ   r¡   )rs   rt   rÝ   rN   r�   ÚsumÚnum_block_repeatsÚranger`   rŽ   ÚstridesÚkernel_sizesÚexpand_ratiosÚdepthwise_paddingÚdrop_connect_raterÎ   Úappendr   Ú
ModuleListÚblocks)rB   rV   Únum_base_blocksÚ
num_blocksÚcurr_block_numrì   Úir•   ru   ro   ra   rÏ   ÚjrÁ   r¡   rÀ   Úblockrß   r€   s   `                @€r/   rt   zAlignVisionEncoder.__init__Â  sÎ  øøø€ Ý‰Œ×ÒÑÔÐØ!'Ô!9ˆÔð	Dð 	Dð 	Dð 	Dð 	Dõ ˜fÔ0Ñ1Ô1ˆÝÐLÐLÐLÐL°6Ô3KÐLÑLÔLÑLÔLˆ
àˆØˆÝ�Ñ'Ô'ð 	$ð 	$ˆAÝ" 6¨6Ô+=¸aÔ+@ÑAÔAˆFÝ# F¨FÔ,?ÀÔ,BÑCÔCˆGØ”^ AÔ&ˆFØ Ô-¨aÔ0ˆKØ!Ô/°Ô2ˆLå˜=˜=¨Ô)AÀ!Ô)DÑEÔEÑFÔFð $ð $�Ø˜qš&�Ø !še˜e˜˜¨�Ø$%¨¢E E˜˜¨v�Ø!/°vÔ7OÐ!O�Ø"Ô4°~ÑEÈ
ÑR�	å(Ø!Ø!Ø#Ø!Ø +Ø!-Ø'Ø#Ø#1ð
ñ 
ô 
�ð —’˜eÑ$Ô$Ð$Ø !Ñ#��ð'$õ* ”m FÑ+Ô+ˆŒˆˆr.   r$   Úkwargsr;   c                 óL   — | j         D ]} ||¦  «        }Œt          |¬¦  «        S ©N)r#   )rì   r   )rB   r$   ró   rò   s       r/   r…   zAlignVisionEncoder.forwardí  s@   € ð
 ”[ð 	1ð 	1ˆEØ!˜E -Ñ0Ô0ˆMˆMå-Ø+ð
ñ 
ô 
ð 	
r.   )r%   r&   r'   r(   r   rt   r)   r*   r   r   r   r…   r‡   rˆ   s   @r/   rØ   rØ   ¹  s‰   ø€ € € € € ðð ð),Ð0ð ),ð ),ð ),ð ),ð ),ð ),ðV

àÔ(ð

ð Ð+Ô,ð

ð 
(ð	

ð 

ð 

ð 

ð 

ð 

ð 

ð 

r.   rØ   c                   ó’   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 d
dej        dz  dej        dz  dej        dz  dej        dz  dej        f
d	„Z	ˆ xZ
S )ÚAlignTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óÒ  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt%          j        |j        ¦  «                             d¦  «        d¬¦  «         |                      dt%          j        | j                             ¦   «         t$          j        ¬¦  «        d¬¦  «         d S )	N)Úpadding_idx©rq   Úposition_ids©r   éÿÿÿÿF)Ú
persistentÚtoken_type_ids)Údtype)rs   rt   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsrÇ   Úhidden_dropout_probrÈ   Úregister_bufferr)   rM   r¯   r‹   rû   ÚsizeÚlongr   s     €r/   rt   zAlignTextEmbeddings.__init__ý  s3  ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?QÐ_eÔ_rÐsÑsÔsˆÔÝ#%¤<°Ô0NÐPVÔPbÑ#cÔ#cˆÔ Ý%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒà×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	×ÒØ�eœk¨$Ô*;×*@Ò*@Ñ*BÔ*BÍ%Ì*ÐUÑUÔUÐbgð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r.   NÚ	input_idsrÿ   rû   Úinputs_embedsr;   c                 ón  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€mt          | d¦  «        r2| j        d d …d |…f         }|                     |d         |¦  «        }|}n+t          j        |t
          j        | j        j        ¬¦  «        }|€|  	                    |¦  «        }|  
                    |¦  «        }	||	z   }
|                      |¦  «        }|
|z  }
|                      |
¦  «        }
|                      |
¦  «        }
|
S )Nrý   r   rÿ   r   )r   rI   )r  rû   Úhasattrrÿ   r¯   r)   r‹   r  rI   r  r	  r  r
  rÈ   )rB   r  rÿ   rû   r  Úinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedr	  rÉ   r  s               r/   r…   zAlignTextEmbeddings.forward  sV  € ð Ð Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLð
 Ð!Ý�tÐ-Ñ.Ô.ð mØ*.Ô*=¸a¸a¸aÀÀ*À¸nÔ*MÐ'Ø3J×3QÒ3QÐR]Ð^_ÔR`ÐblÑ3mÔ3mÐ0Ø!A��å!&¤¨[ÅÄ
ÐSWÔSdÔSkÐ!lÑ!lÔ!l�àÐ Ø ×0Ò0°Ñ;Ô;ˆMØ $× :Ò :¸>Ñ JÔ JÐØ"Ð%:Ñ:ˆ
à"×6Ò6°|ÑDÔDÐØÐ)Ñ)ˆ
à—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr.   )NNNN)r%   r&   r'   r(   rt   r)   Ú
LongTensorr*   r†   r…   r‡   rˆ   s   @r/   r÷   r÷   ú  s·   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ð$ .2Ø26Ø04Ø26ð&ð &àÔ# dÑ*ð&ð Ô(¨4Ñ/ð&ð Ô&¨Ñ-ð	&ð
 Ô(¨4Ñ/ð&ð 
Œð&ð &ð &ð &ð &ð &ð &ð &r.   r÷   ç        ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrÈ   c                 óÀ  — t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |dt           j        ¬¦  «                             |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «                             ¦   «         }	|	|fS )NrY   r   rý   )r²   r   )rÃ   Útrainingr   )r)   ÚmatmulÚ	transposer   rK   ÚsoftmaxÚfloat32Útor   rÈ   r!  Ú
contiguous)
r  r  r  r  r  r  rÈ   ró   Úattn_weightsÚattn_outputs
             r/   Úeager_attention_forwardr*  6  sÃ   € õ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LØÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2ÅUÄ]Ð(ÑSÔS×VÒVÐW\ÔWbÑcÔc€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€KØ˜Ð$Ð$r.   c                   óŠ   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         de	ej        ej        dz  f         fd„Z
ˆ xZS )	ÚAlignTextSelfAttentionc                 ó¨  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|| _        |j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _	        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        | j	        ¦  «        | _        t          j        |j        ¦  «        | _        |j        | _        | j        dz  | _        d S )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)g      à¿)rs   rt   r  Únum_attention_headsr  Ú
ValueErrorrV   r]   Úattention_head_sizeÚall_head_sizer   ÚLinearr  r  r  rÇ   Úattention_probs_dropout_probrÈ   Úattention_dropoutr  r   s     €r/   rt   zAlignTextSelfAttention.__init__M  s:  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 ˆŒØ#)Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒØ!'Ô!DˆÔØÔ/°Ñ5ˆŒˆˆr.   Nr$   r  ró   r;   c                 ó‚  — |j         d d…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }|                      |¦  «                             |¦  «                             dd¦  «        }t          j        | j	        j
        t          ¦  «        }	 |	| ||||f| j        sdn| j        | j        dœ|¤Ž\  }
} |
j        g |¢d‘R Ž                      ¦   «         }
|
|fS )Nrý   r   rY   r  )rÈ   r  )Úshaper2  r  Úviewr#  r  r  r   Úget_interfacerV   Ú_attn_implementationr*  r!  r6  r  Úreshaper'  )rB   r$   r  ró   r  Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacer)  r(  s               r/   r…   zAlignTextSelfAttention.forwardb  sf  € ð $Ô)¨#¨2¨#Ô.ˆØC˜ÐC bÐC¨$Ô*BÐCÐCˆà—z’z -Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆØ—X’X˜mÑ,Ô,×1Ò1°,Ñ?Ô?×IÒIÈ!ÈQÑOÔOˆ
Ø—z’z -Ñ0Ô0×5Ò5°lÑCÔC×MÒMÈaÐQRÑSÔSˆå(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}ÐH�C�C°$Ô2HØ”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ˜LÐ(Ð(r.   rƒ   )r%   r&   r'   rt   r)   r†   r*   r   r   r,   r…   r‡   rˆ   s   @r/   r,  r,  L  sŸ   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð0 48ð)ð )à”|ð)ð Ô)¨DÑ0ð)ð Ð+Ô,ð	)ð
 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð)ð )ð )ð )ð )ð )ð )ð )r.   r,  c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚAlignTextSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©Nrú   )rs   rt   r   r4  r  Údenser
  r  rÇ   r  rÈ   r   s     €r/   rt   zAlignTextSelfOutput.__init__„  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr.   r$   Úinput_tensorr;   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rƒ   ©rF  rÈ   r
  ©rB   r$   rG  s      r/   r…   zAlignTextSelfOutput.forwardŠ  ó@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr.   ©r%   r&   r'   rt   r)   r†   r…   r‡   rˆ   s   @r/   rC  rC  ƒ  ói   ø€ € € € € ð>ð >ð >ð >ð >ð U¤\ð ÀÄð ÐRWÔR^ð ð ð ð ð ð ð ð r.   rC  c            	       ój   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         dej        fd„Z	ˆ xZ
S )	ÚAlignTextAttentionc                 ó˜   •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        d S rƒ   )rs   rt   r,  rB   rC  Úoutputr   s     €r/   rt   zAlignTextAttention.__init__’  s;   ø€ Ý‰Œ×ÒÑÔÐÝ*¨6Ñ2Ô2ˆŒ	Ý)¨&Ñ1Ô1ˆŒˆˆr.   Nr$   r  ró   r;   c                 ó\   — |} | j         |fd|i|¤Ž\  }}|                      ||¦  «        }|S ©Nr  )rB   rQ  )rB   r$   r  ró   ÚresidualÚ_s         r/   r…   zAlignTextAttention.forward—  sV   € ð !ˆØ$˜4œ9Øð
ð 
à)ð
ð ð
ð 
Ñˆ�qð
 Ÿš M°8Ñ<Ô<ˆØÐr.   rƒ   )r%   r&   r'   rt   r)   r†   r*   r   r   r…   r‡   rˆ   s   @r/   rO  rO  ‘  sŽ   ø€ € € € € ð2ð 2ð 2ð 2ð 2ð 48ðð à”|ðð Ô)¨DÑ0ðð Ð+Ô,ð	ð
 
Œðð ð ð ð ð ð ð r.   rO  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAlignTextIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S rƒ   )rs   rt   r   r4  r  Úintermediate_sizerF  rd   r}   Ústrr	   Úintermediate_act_fnr   s     €r/   rt   zAlignTextIntermediate.__init__©  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r.   r$   r;   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rƒ   )rF  r[  rž   s     r/   r…   zAlignTextIntermediate.forward±  s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr.   rL  rˆ   s   @r/   rW  rW  ¨  s^   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð U¤\ð °e´lð ð ð ð ð ð ð ð r.   rW  c                   óP   ‡ — e Zd Zˆ fd„Zdej        dej        dej        fd„Zˆ xZS )ÚAlignTextOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S rE  )rs   rt   r   r4  rY  r  rF  r
  r  rÇ   r  rÈ   r   s     €r/   rt   zAlignTextOutput.__init__¹  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr.   r$   rG  r;   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S rƒ   rI  rJ  s      r/   r…   zAlignTextOutput.forward¿  rK  r.   rL  rˆ   s   @r/   r^  r^  ¸  rM  r.   r^  c            	       óp   ‡ — e Zd Zˆ fd„Z	 d	dej        dej        dz  dee         dej        fd„Z	d„ Z
ˆ xZS )
ÚAlignTextLayerc                 óæ   •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _	        d S rÖ   )
rs   rt   Úchunk_size_feed_forwardÚseq_len_dimrO  Ú	attentionrW  Úintermediater^  rQ  r   s     €r/   rt   zAlignTextLayer.__init__Ç  s^   ø€ Ý‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ+¨FÑ3Ô3ˆŒÝ1°&Ñ9Ô9ˆÔÝ% fÑ-Ô-ˆŒˆˆr.   Nr$   r  ró   r;   c                 óh   —  | j         |fd|i|¤Ž}t          | j        | j        | j        |¦  «        }|S rS  )rf  r   Úfeed_forward_chunkrd  re  )rB   r$   r  ró   s       r/   r…   zAlignTextLayer.forwardÏ  s]   € ð '˜œØð
ð 
à)ð
ð ð
ð 
ˆõ 2ØÔ# TÔ%AÀ4ÔCSÐUbñ
ô 
ˆð Ðr.   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S rƒ   )rg  rQ  )rB   Úattention_outputÚintermediate_outputÚlayer_outputs       r/   ri  z!AlignTextLayer.feed_forward_chunká  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr.   rƒ   )r%   r&   r'   rt   r)   r†   r*   r   r   r…   ri  r‡   rˆ   s   @r/   rb  rb  Æ  s�   ø€ € € € € ð.ð .ð .ð .ð .ð 48ðð à”|ðð Ô)¨DÑ0ðð Ð+Ô,ð	ð
 
Œðð ð ð ð$ð ð ð ð ð ð r.   rb  c            	       ó`   ‡ — e Zd Zˆ fd„Z	 ddej        dej        dz  dee         de	fd„Z
ˆ xZS )	ÚAlignTextEncoderc                 óÔ   •‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó.   •— g | ]}t          ‰¦  «        ‘ŒS r-   )rb  )r@   rð   rV   s     €r/   ú
<listcomp>z-AlignTextEncoder.__init__.<locals>.<listcomp>ë  s!   ø€ Ð#dÐ#dÐ#d¸q¥N°6Ñ$:Ô$:Ð#dÐ#dÐ#dr.   F)	rs   rt   rV   r   rë   rä   Únum_hidden_layersÚlayerÚgradient_checkpointingr   s    `€r/   rt   zAlignTextEncoder.__init__è  s`   øø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”]Ð#dÐ#dÐ#dÐ#dÅEÈ&ÔJbÑDcÔDcÐ#dÑ#dÔ#dÑeÔeˆŒ
Ø&+ˆÔ#Ð#Ð#r.   Nr$   r  ró   r;   c                 óJ   — | j         D ]} |||fi |¤Ž}Œt          |¬¦  «        S rõ   )rt  r   )rB   r$   r  ró   Úlayer_modules        r/   r…   zAlignTextEncoder.forwardî  sY   € ð !œJð 	ð 	ˆLØ(˜LØØðð ð ðð ˆMˆMõ Ø+ð
ñ 
ô 
ð 	
r.   rƒ   )r%   r&   r'   rt   r)   r†   r*   r   r   r   r…   r‡   rˆ   s   @r/   ro  ro  ç  sŒ   ø€ € € € € ð,ð ,ð ,ð ,ð ,ð 48ð
ð 
à”|ð
ð Ô)¨DÑ0ð
ð Ð+Ô,ð	
ð
 
ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r.   ro  c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚAlignTextPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rƒ   )rs   rt   r   r4  r  rF  ÚTanhr~   r   s     €r/   rt   zAlignTextPooler.__init__  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr.   r$   r;   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rF  r~   )rB   r$   Úfirst_token_tensorÚpooled_outputs       r/   r…   zAlignTextPooler.forward  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr.   rL  rˆ   s   @r/   ry  ry    s^   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¤\ð °e´lð ð ð ð ð ð ð ð r.   ry  c                   óp   ‡ — e Zd ZU eed<   dZdZdZ ej	        ¦   «         de
j        fˆ fd„¦   «         Zˆ xZS )ÚAlignPreTrainedModelrV   Úalign)ÚimageÚtextTr  c                 ó6  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rbt	          j        |j        j        ¦  «         t	          j        |j        j	        ¦  «         t	          j
        |j        | j        j        ¦  «         dS t          |t          ¦  «        rjt	          j        |j        t#          j        |j        j        d         ¦  «                             d¦  «        ¦  «         t	          j        |j        ¦  «         dS dS )zInitialize the weightsrý   rü   N)rs   Ú_init_weightsrd   Ú
AlignModelÚinitÚxavier_uniform_Útext_projectionÚweightÚzeros_rp   Ú	constant_ÚtemperaturerV   Útemperature_init_valuer÷   Úcopy_rû   r)   rM   r8  r¯   rÿ   )rB   r  r€   s     €r/   r…  z"AlignPreTrainedModel._init_weights  sì   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�jÑ)Ô)ð 	/ÝÔ  Ô!7Ô!>Ñ?Ô?Ð?ÝŒK˜Ô.Ô3Ñ4Ô4Ð4ÝŒN˜6Ô-¨t¬{Ô/QÑRÔRÐRÐRÐRÝ˜Õ 3Ñ4Ô4ð 	/ÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÝŒK˜Ô-Ñ.Ô.Ð.Ð.Ð.ð	/ð 	/r.   )r%   r&   r'   r   r+   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingr)   Úno_gradr   ÚModuler…  r‡   rˆ   s   @r/   r€  r€    sx   ø€ € € € € € àÐÐÑØÐØ(ÐØ&*Ð#à€U„]�_„_ð	/ B¤Ið 	/ð 	/ð 	/ð 	/ð 	/ñ „_ð	/ð 	/ð 	/ð 	/ð 	/r.   r€  zJ
    The text model from ALIGN without any head or projection on top.
    c                   ó  ‡ — e Zd ZU eed<   dZdgZeedœZ	ddede
fˆ fd„Zd„ Zd	„ Zeee	 	 	 	 	 ddej        d
z  dej        d
z  dej        d
z  dej        d
z  dej        d
z  dee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚAlignTextModelrV   )rƒ  r÷   )r$   r3   TÚadd_pooling_layerc                 ó   •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |rt          |¦  «        nd| _        |  	                    ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
rs   rt   rV   r÷   rÉ   ro  Úencoderry  ÚpoolerÚ	post_init)rB   rV   r—  r€   s      €r/   rt   zAlignTextModel.__init__2  ss   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒå-¨fÑ5Ô5ˆŒÝ'¨Ñ/Ô/ˆŒà1BÐL•o fÑ-Ô-Ð-ÈˆŒð 	�ŠÑÔÐÐÐr.   c                 ó   — | j         j        S rƒ   ©rÉ   r  rE   s    r/   Úget_input_embeddingsz#AlignTextModel.get_input_embeddingsB  s   € ØŒÔ.Ð.r.   c                 ó   — || j         _        d S rƒ   r�  )rB   r  s     r/   Úset_input_embeddingsz#AlignTextModel.set_input_embeddingsE  s   € Ø*/ˆŒÔ'Ð'Ð'r.   Nr  r  rÿ   rû   r  ró   r;   c                 ó*  — |�|�t          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }n.|�|                     ¦   «         dd…         }nt          d¦  «        ‚|\  }}	|�|j        n|j        }
|€t	          j        ||	f|
¬¦  «        }|                      ||||¬¦  «        }t          | j        ||¬¦  «        } | j	        |fd|i|¤Ž}|d	         }| j
        �|  
                    |¦  «        nd}t          ||¬
¦  «        S )a-  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, AlignTextModel

        >>> model = AlignTextModel.from_pretrained("kakaobrain/align-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("kakaobrain/align-base")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timerý   z5You have to specify either input_ids or inputs_embedsrH   )r  rû   rÿ   r  )rV   r  r  r  r   ©r#   Úpooler_output)r1  Ú%warn_if_padding_and_no_attention_maskr  rI   r)   ÚonesrÉ   r
   rV   r™  rš  r   )rB   r  r  rÿ   rû   r  ró   r  Ú
batch_sizer  rI   Úembedding_outputÚencoder_outputsÚsequence_outputr~  s                  r/   r…   zAlignTextModel.forwardH  sv  € ð6 Ð  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNàŸ?š?ØØ%Ø)Ø'ð	 +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð '˜$œ,Øð
ð 
à)ð
ð ð
ð 
ˆð
 *¨!Ô,ˆØ8<¼Ð8O˜Ÿš OÑ4Ô4Ð4ÐUYˆå)Ø-Ø'ð
ñ 
ô 
ð 	
r.   ©T©NNNNN)r%   r&   r'   r   r+   r‘  Ú_no_split_modulesrb  r,  Ú_can_record_outputsr«   rt   rž  r   r   r   r   r)   r†   r   r   r,   r   r…   r‡   rˆ   s   @r/   r–  r–  $  sn  ø€ € € € € € ð ÐÐÑØ ÐØ.Ð/Ðà'Ø,ðð Ðð
ð ˜ð À4ð ð ð ð ð ð ð /ð /ð /ð0ð 0ð 0ð  ØØð *.Ø.2Ø.2Ø,0Ø-1ð@
ð @
à”< $Ñ&ð@
ð œ tÑ+ð@
ð œ tÑ+ð	@
ð
 ”l TÑ)ð@
ð ”| dÑ*ð@
ð Ð+Ô,ð@
ð 
Ð+Ñ	+ð@
ð @
ð @
ñ „^ñ „_ñ  Ôð@
ð @
ð @
ð @
ð @
r.   r–  zL
    The vision model from ALIGN without any head or projection on top.
    c                   ó¸   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	de
iZdefˆ fd„Zeee	 ddej        d	z  d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚAlignVisionModelrV   r�   )r‚  Frx   rÎ   r$   c                 ó®  •— t          ¦   «                              |¦  «         || _        t          |¦  «        | _        t          |¦  «        | _        |j        dk    r!t          j	        |j
        d¬¦  «        | _        nC|j        dk    r!t          j        |j
        d¬¦  «        | _        nt          d|j        › �¦  «        ‚|                      ¦   «          d S )NÚmeanT)Ú	ceil_moder\   z2config.pooling must be one of ['mean', 'max'] got )rs   rt   rV   rh   rÉ   rØ   r™  Úpooling_typer   Ú	AvgPool2dÚ
hidden_dimrš  Ú	MaxPool2dr1  Úpoolingr›  r   s     €r/   rt   zAlignVisionModel.__init__ž  sÇ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒÝ/°Ñ7Ô7ˆŒÝ)¨&Ñ1Ô1ˆŒð Ô &Ò(Ð(Ýœ, vÔ'8ÀDÐIÑIÔIˆDŒKˆKØÔ  EÒ)Ð)Ýœ, vÔ'8ÀDÐIÑIÔIˆDŒKˆKåÐbÐRXÔR`ÐbÐbÑcÔcÐcð 	�ŠÑÔÐÐÐr.   Nró   r;   c                 ó
  — |€t          d¦  «        ‚|                      |¦  «        } | j        |fi |¤Ž}|d         }|                      |¦  «        }|                     |j        dd…         ¦  «        }t          ||¬¦  «        S )a  
        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, AlignVisionModel

        >>> model = AlignVisionModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```Nz You have to specify pixel_valuesr   rY   r¢  )r1  rÉ   r™  rš  r<  r8  r   )rB   r�   ró   r§  r¨  r#   r~  s          r/   r…   zAlignVisionModel.forward¯  s©   € ð< ÐÝÐ?Ñ@Ô@Ð@àŸ?š?¨<Ñ8Ô8ÐØ&˜$œ,Øð
ð 
àð
ð 
ˆð ,¨AÔ.ÐØŸšÐ$5Ñ6Ô6ˆØ%×-Ò-¨mÔ.AÀ"À1À"Ô.EÑFÔFˆå7Ø/Ø'ð
ñ 
ô 
ð 	
r.   rƒ   )r%   r&   r'   r   r+   Úmain_input_namer‘  r’  Ú_input_embed_layerr¬  rÎ   r­  rt   r   r   r   r)   r*   r   r   r,   r   r…   r‡   rˆ   s   @r/   r¯  r¯  Ž  sò   ø€ € € € € € ð ÐÐÑØ$€OØ!ÐØ&+Ð#Ø&ÐØ+Ð,ÐàÐ)ðÐðÐ0ð ð ð ð ð ð ð"  ØØð 26ð*
ð *
àÔ'¨$Ñ.ð*
ð Ð+Ô,ð*
ð 
Ð9Ñ	9ð	*
ð *
ð *
ñ „^ñ „_ñ  Ôð*
ð *
ð *
ð *
ð *
r.   r¯  c                   ó  ‡ — e Zd ZU eed<   defˆ fd„Zee	 	 	 	 	 ddej	        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  d	e
e         d
eez  fd„¦   «         ¦   «         Zeedej        d	e
e         d
eez  fd„¦   «         ¦   «         Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej	        dz  dej	        dz  dej	        dz  dej	        dz  dedz  d	e
e         d
eez  fd„¦   «         ¦   «         Zˆ xZS )r†  rV   c                 ó¼  •— t          ¦   «                              |¦  «         t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚t          |j        t          ¦  «        s%t          dt          |j        ¦  «        › d�¦  «        ‚|j        }|j        }|j	        | _	        |j
        | _        t          |¦  «        | _        t          |¦  «        | _        t!          j        | j        | j	        ¦  «        | _        t!          j        t)          j        | j        j        ¦  «        ¦  «        | _        |                      ¦   «          d S )NzLconfig.text_config is expected to be of type AlignTextConfig but is of type ú.zPconfig.vision_config is expected to be of type AlignVisionConfig but is of type )rs   rt   rd   Útext_configr   Ú	TypeErrorÚtypeÚvision_configr   Úprojection_dimr  Útext_embed_dimr–  Ú
text_modelr¯  Úvision_modelr   r4  r‰  Ú	Parameterr)   ÚtensorrV   rŽ  r�  r›  )rB   rV   r¾  rÁ  r€   s       €r/   rt   zAlignModel.__init__ã  sJ  ø€ Ý‰Œ×Ò˜Ñ Ô Ð å˜&Ô,­oÑ>Ô>ð 	Ýð0Ý˜Ô+Ñ,Ô,ð0ð 0ð 0ñô ð õ
 ˜&Ô.Õ0AÑBÔBð 	Ýð2Ý˜Ô-Ñ.Ô.ð2ð 2ð 2ñô ð ð
 Ô(ˆØÔ,ˆà$Ô3ˆÔØ)Ô5ˆÔå(¨Ñ5Ô5ˆŒÝ,¨]Ñ;Ô;ˆÔå!œy¨Ô)<¸dÔ>QÑRÔRˆÔÝœ<­¬°T´[Ô5WÑ(XÔ(XÑYÔYˆÔð 	�ŠÑÔÐÐÐr.   Nr  r  rÿ   rû   r  ró   r;   c           	      óŠ   —  | j         d|||||dœ|¤Ž}|d         dd…ddd…f         }|                      |¦  «        |_        |S )aù  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoTokenizer, AlignModel

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("kakaobrain/align-base")

        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     text_features = model.get_text_features(**inputs)
        ```©r  r  rÿ   rû   r  r   Nr-   )rÄ  r‰  r£  )	rB   r  r  rÿ   rû   r  ró   Útext_outputsr#   s	            r/   Úget_text_featureszAlignModel.get_text_features  sw   € ð2 4C°4´?ð 4
ØØ)Ø)Ø%Ø'ð4
ð 4
ð ð4
ð 4
ˆð )¨œO¨A¨A¨A¨q°!°!°!¨GÔ4ÐØ%)×%9Ò%9Ð:KÑ%LÔ%LˆÔ"àÐr.   r�   c                 ó    —  | j         dd|i|¤ŽS )a}  
        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AlignModel
        >>> from transformers.image_utils import load_image

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(images=image, return_tensors="pt")
        >>> with torch.inference_mode():
        ...     image_features = model.get_image_features(**inputs)
        ```r�   r-   )rÅ  )rB   r�   ró   s      r/   Úget_image_featureszAlignModel.get_image_features'  s"   € ð. !ˆtÔ ÐEÐE¨lÐE¸fÐEÐEÐEr.   Úreturn_lossc           	      óî  —  | j         dd|i|¤Ž}	 | j        d|||||dœ|¤Ž}
|	d         }|
d         dd…ddd…f         }|                      |¦  «        }||                     ddd¬	¦  «        z  }||                     ddd¬	¦  «        z  }t	          j        ||                     ¦   «         ¦  «        | j        z  }|                     ¦   «         }d}|rt          |¦  «        }t          ||||||
|	¬
¦  «        S )aö  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.

        Examples:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, AlignModel
        >>> from transformers.image_utils import load_image

        >>> model = AlignModel.from_pretrained("kakaobrain/align-base")
        >>> processor = AutoProcessor.from_pretrained("kakaobrain/align-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = load_image(url)

        >>> inputs = processor(
        ...     images=image, text=["a photo of a cat", "a photo of a dog"], return_tensors="pt", padding=True
        ... )

        >>> with torch.inference_mode():
        ...     outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```r�   rÉ  r   r   NrY   rý   T)rÃ   r²   Úkeepdim)r6   r7   r8   r2   r"   r9   r:   r-   )
rÅ  rÄ  r‰  Únormr)   r"  rR   r�  rU   r5   )rB   r  r�   r  rÿ   rû   r  rÎ  ró   Úvision_outputsrÊ  r"   r2   r8   r7   r6   s                   r/   r…   zAlignModel.forward@  s^  € ðN +˜Ô*ð 
ð 
Ø%ð
àð
ð 
ˆð
 '�t”ð 
ØØ)Ø)Ø%Ø'ð
ð 
ð ð
ð 
ˆð & aÔ(ˆØ" 1”o a a a¨¨A¨A¨A gÔ.ˆØ×*Ò*¨;Ñ7Ô7ˆð $ l×&7Ò&7¸!ÀÈTÐ&7Ñ&RÔ&RÑRˆØ! K×$4Ò$4°q¸bÈ$Ð$4Ñ$OÔ$OÑOˆõ  œ, {°L·N²NÑ4DÔ4DÑEÔEÈÔHXÑXˆØ*×,Ò,Ñ.Ô.ÐàˆØð 	/Ý˜oÑ.Ô.ˆDåØØ-Ø+Ø#Ø%Ø*Ø .ð
ñ 
ô 
ð 	
r.   r«  )NNNNNNN)r%   r&   r'   r   r+   rt   r   r   r)   r†   r   r   r,   r   rË  r*   rÍ  r  r«   r5   r…   r‡   rˆ   s   @r/   r†  r†  ß  sL  ø€ € € € € € àÐÐÑð˜{ð ð ð ð ð ð ð< Øð *.Ø.2Ø.2Ø,0Ø-1ð"ð "à”< $Ñ&ð"ð œ tÑ+ð"ð œ tÑ+ð	"ð
 ”l TÑ)ð"ð ”| dÑ*ð"ð Ð+Ô,ð"ð 
Ð+Ñ	+ð"ð "ð "ñ „^ñ Ôð"ðH ØðFØ!Ô-ðFØ9?Ð@RÔ9SðFà	Ð+Ñ	+ðFð Fð Fñ „^ñ ÔðFð. Øð .2Ø15Ø.2Ø.2Ø,0Ø-1Ø#'ðK
ð K
àÔ# dÑ*ðK
ð Ô'¨$Ñ.ðK
ð œ tÑ+ð	K
ð
 œ tÑ+ðK
ð ”l TÑ)ðK
ð ”| dÑ*ðK
ð ˜D‘[ðK
ð Ð+Ô,ðK
ð 
�Ñ	ðK
ð K
ð K
ñ „^ñ ÔðK
ð K
ð K
ð K
ð K
r.   r†  )r€  r–  r¯  r†  rª  )r  )Ur(   rÛ   Úcollections.abcr   Údataclassesr   Útypingr   r)   r   Ú r   r‡  Úactivationsr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_alignr   r   r   Ú
get_loggerr%   Úloggerr!   r1   r5   r†   rO   rU   r]   r`   r,   r«   rf   r”  rh   rw   rŠ   r”   r    r­   r¿   rÎ   rØ   r÷   rÌ   r*  r,  rC  rO  rW  r^  rb  ro  ry  r€  r–  r¯  r†  Ú__all__r-   r.   r/   ú<module>rå     sì  ðð Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ IÐ IÐ IÐ IÐ IÐ IÐ IÐ IØ 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ Pð 
ˆÔ	˜HÑ	%Ô	%€ð €ððñ ô ð
 ð:ð :ð :ð :ð :˜[ñ :ô :ñ „ñô ð:ð €ððñ ô ð
 ð	7ð 	7ð 	7ð 	7ð 	7˜;ñ 	7ô 	7ñ „ñô ð	7ð Ø
ð 
ð  
ð  
ð  
ð  
�+ñ  
ô  
ñ „ñ „ð 
ðJu˜Uœ\ð u¨e¬lð uð uð uð uð-˜5œ<ð -¨E¬Lð -ð -ð -ð -ðÐ+ð ¸3ð ð ð ð ð @ð @˜S 5™[ð @°$ð @ð @ð @ð @ð*ð ð ð ð ˜BœIñ ô ð ð4
ð 
ð 
ð 
ð 
 ¤ñ 
ô 
ð 
ð6ð ð ð ð  ¤	ñ ô ð ð6$ð $ð $ð $ð $ ¤	ñ $ô $ð $ðP$ð $ð $ð $ð $ B¤Iñ $ô $ð $ðNð ð ð ð  ¤ñ ô ð ðBNð Nð Nð Nð N�r”yñ Nô Nð Nðb>
ð >
ð >
ð >
ð >
˜œñ >
ô >
ð >
ðB9ð 9ð 9ð 9ð 9˜"œ)ñ 9ô 9ð 9ðF ð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð ð%ð ð%ð %ð %ð %ð,3)ð 3)ð 3)ð 3)ð 3)˜RœYñ 3)ô 3)ð 3)ðnð ð ð ð ˜"œ)ñ ô ð ðð ð ð ð ˜œñ ô ð ð.ð ð ð ð ˜BœIñ ô ð ð ð ð ð ð �b”iñ ô ð ðð ð ð ð Ð/ñ ô ð ðB
ð 
ð 
ð 
ð 
�r”yñ 
ô 
ð 
ð4ð ð ð ð �b”iñ ô ð ð ð/ð /ð /ð /ð /˜?ñ /ô /ñ „ð/ð& €ððñ ô ð
b
ð b
ð b
ð b
ð b
Ð)ñ b
ô b
ñô ð
b
ðJ €ððñ ô ð
I
ð I
ð I
ð I
ð I
Ð+ñ I
ô I
ñô ð
I
ðX ðm
ð m
ð m
ð m
ð m
Ð%ñ m
ô m
ñ „ðm
ð` WÐ
VÐ
V€€€r.   