§
    ‚ŠtjuŽ  ã                   óâ  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ  ej        e¦  «        Z  G d„ dej!        ¦  «        Z" G d„ dej!        ¦  «        Z# G d„ dej$        ¦  «        Z% G d„ dej!        ¦  «        Z& G d„ dej!        ¦  «        Z' G d„ dej!        ¦  «        Z( G d„ dej!        ¦  «        Z) G d„ dej!        ¦  «        Z* G d„ dej!        ¦  «        Z+ G d „ d!ej!        ¦  «        Z, G d"„ d#ej!        ¦  «        Z- G d$„ d%ej!        ¦  «        Z.e G d&„ d'e¦  «        ¦   «         Z/e G d(„ d)e/¦  «        ¦   «         Z0e G d*„ d+e/¦  «        ¦   «         Z1 ed,¬-¦  «         G d.„ d/e/¦  «        ¦   «         Z2e G d0„ d1e/¦  «        ¦   «         Z3e G d2„ d3e/¦  «        ¦   «         Z4e G d4„ d5e/¦  «        ¦   «         Z5g d6¢Z6dS )7zPyTorch SqueezeBert model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚSqueezeBertConfigc                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚSqueezeBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó8  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        ¬¦  «        | _        t          j        |j        |j        ¦  «        | _	        t          j        |j
        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt'          j        |j        ¦  «                             d¦  «        d¬¦  «         d S )N)Úpadding_idx©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚhidden_sizeÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚexpand©ÚselfÚconfigÚ	__class__s     €úr/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/squeezebert/modeling_squeezebert.pyr"   zSqueezeBertEmbeddings.__init__0   sî   ø€ Ý‰Œ×ÒÑÔÐÝ!œ|¨FÔ,=¸vÔ?TÐbhÔbuÐvÑvÔvˆÔÝ#%¤<°Ô0NÐPVÔPeÑ#fÔ#fˆÔ Ý%'¤\°&Ô2HÈ&ÔJ_Ñ%`Ô%`ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒð 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	
ð 	
ó    Nc                 óæ  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€+t          j        |t          j        | j        j        ¬¦  «        }|€|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }||z   |z   }	|  	                    |	¦  «        }	|  
                    |	¦  «        }	|	S )Nr   r   ©ÚdtypeÚdevice)Úsizer   r3   ÚzerosÚlongr?   r'   r)   r+   r,   r1   )
r7   Ú	input_idsÚtoken_type_idsr   Úinputs_embedsÚinput_shapeÚ
seq_lengthr)   r+   Ú
embeddingss
             r:   ÚforwardzSqueezeBertEmbeddings.forward>   sù   € ØÐ Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLàÐ!Ý"œ[¨½E¼JÈtÔO`ÔOgÐhÑhÔhˆNàÐ Ø ×0Ò0°Ñ;Ô;ˆMØ"×6Ò6°|ÑDÔDÐØ $× :Ò :¸>Ñ JÔ JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr;   )NNNN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   rI   Ú__classcell__©r9   s   @r:   r   r   -   sR   ø€ € € € € ØQÐQð
ð 
ð 
ð 
ð 
ðð ð ð ð ð ð ð r;   r   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚMatMulWrapperzÓ
    Wrapper for torch.matmul(). This makes flop-counting easier to implement. Note that if you directly call
    torch.matmul() in your code, the flop counter will typically ignore the flops of the matmul.
    c                 óH   •— t          ¦   «                              ¦   «          d S ©N)r!   r"   )r7   r9   s    €r:   r"   zMatMulWrapper.__init__]   s   ø€ Ý‰Œ×ÒÑÔÐÐÐr;   c                 ó,   — t          j        ||¦  «        S )a0  

        :param inputs: two torch tensors :return: matmul of these tensors

        Here are the typical dimensions found in BERT (the B is optional) mat1.shape: [B, <optional extra dims>, M, K]
        mat2.shape: [B, <optional extra dims>, K, N] output shape: [B, <optional extra dims>, M, N]
        )r3   Úmatmul)r7   Úmat1Úmat2s      r:   rI   zMatMulWrapper.forward`   s   € õ Œ|˜D $Ñ'Ô'Ð'r;   rJ   rP   s   @r:   rR   rR   W   sQ   ø€ € € € € ðð ð
ð ð ð ð ð(ð (ð (ð (ð (ð (ð (r;   rR   c                   ó    — e Zd ZdZdd„Zd„ ZdS )ÚSqueezeBertLayerNormz¥
    This is a nn.LayerNorm subclass that accepts NCW data layout and performs normalization in the C dimension.

    N = batch C = channels W = sequence length
    çê-�™—q=c                 óJ   — t           j                             | ||¬¦  «         d S )N)Únormalized_shaper   )r   r,   r"   )r7   r-   r   s      r:   r"   zSqueezeBertLayerNorm.__init__r   s%   € Ý
Œ×Ò˜d°[ÀcÐÑJÔJÐJÐJÐJr;   c                 óž   — |                      ddd¦  «        }t          j                             | |¦  «        }|                      ddd¦  «        S )Nr   é   r   )Úpermuter   r,   rI   )r7   Úxs     r:   rI   zSqueezeBertLayerNorm.forwardu   sD   € Ø�IŠI�a˜˜AÑÔˆÝŒL× Ò   qÑ)Ô)ˆØ�yŠy˜˜A˜qÑ!Ô!Ð!r;   N)r[   )rK   rL   rM   rN   r"   rI   © r;   r:   rZ   rZ   k   sE   € € € € € ðð ðKð Kð Kð Kð"ð "ð "ð "ð "r;   rZ   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚConvDropoutLayerNormz8
    ConvDropoutLayerNorm: Conv, Dropout, LayerNorm
    c                 óÜ   •— t          ¦   «                              ¦   «          t          j        ||d|¬¦  «        | _        t          |¦  «        | _        t          j        |¦  «        | _        d S ©Nr   ©Úin_channelsÚout_channelsÚkernel_sizeÚgroups)	r!   r"   r   ÚConv1dÚconv1drZ   Ú	layernormr/   r1   )r7   ÚcinÚcoutrk   Údropout_probr9   s        €r:   r"   zConvDropoutLayerNorm.__init__€   sY   ø€ Ý‰Œ×ÒÑÔÐå”i¨C¸dÐPQÐZ`ÐaÑaÔaˆŒÝ-¨dÑ3Ô3ˆŒÝ”z ,Ñ/Ô/ˆŒˆˆr;   c                 óŽ   — |                       |¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }|S rT   )rm   r1   rn   )r7   Úhidden_statesÚinput_tensorra   s       r:   rI   zConvDropoutLayerNorm.forward‡   sB   € Ø�KŠK˜Ñ&Ô&ˆØ�LŠL˜‰OŒOˆØ�ÑˆØ�NŠN˜1ÑÔˆØˆr;   rJ   rP   s   @r:   rd   rd   {   sQ   ø€ € € € € ðð ð0ð 0ð 0ð 0ð 0ðð ð ð ð ð ð r;   rd   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚConvActivationz*
    ConvActivation: Conv, Activation
    c                 ó¦   •— t          ¦   «                              ¦   «          t          j        ||d|¬¦  «        | _        t
          |         | _        d S rf   )r!   r"   r   rl   rm   r	   Úact)r7   ro   rp   rk   rx   r9   s        €r:   r"   zConvActivation.__init__”   sD   ø€ Ý‰Œ×ÒÑÔÐÝ”i¨C¸dÐPQÐZ`ÐaÑaÔaˆŒÝ˜#”;ˆŒˆˆr;   c                 óV   — |                       |¦  «        }|                      |¦  «        S rT   )rm   rx   )r7   ra   Úoutputs      r:   rI   zConvActivation.forward™   s#   € Ø—’˜Q‘”ˆØ�xŠx˜ÑÔÐr;   rJ   rP   s   @r:   rv   rv   �   sQ   ø€ € € € € ðð ðð ð ð ð ð
 ð  ð  ð  ð  ð  ð  r;   rv   c                   ó8   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚSqueezeBertSelfAttentionr   c                 ó|  •— t          ¦   «                              ¦   «          ||j        z  dk    rt          d|› d|j        › d�¦  «        ‚|j        | _        t	          ||j        z  ¦  «        | _        | j        | j        z  | _        t          j        ||d|¬¦  «        | _	        t          j        ||d|¬¦  «        | _
        t          j        ||d|¬¦  «        | _        t          j        |j        ¦  «        | _        t          j        d¬¦  «        | _        t#          ¦   «         | _        t#          ¦   «         | _        d	S )
zº
        config = used for some things; ignored for others (work in progress...) cin = input channels = output channels
        groups = number of groups to use in conv1d layers
        r   zcin (z6) is not a multiple of the number of attention heads (ú)r   rg   r   ©ÚdimN)r!   r"   Únum_attention_headsÚ
ValueErrorÚintÚattention_head_sizeÚall_head_sizer   rl   ÚqueryÚkeyÚvaluer/   Úattention_probs_dropout_probr1   ÚSoftmaxÚsoftmaxrR   Ú	matmul_qkÚ
matmul_qkv)r7   r8   ro   Úq_groupsÚk_groupsÚv_groupsr9   s         €r:   r"   z!SqueezeBertSelfAttention.__init__Ÿ   s%  ø€ õ
 	‰Œ×ÒÑÔÐØ�Ô+Ñ+¨qÒ0Ð0ÝØp˜ÐpÐpÐSYÔSmÐpÐpÐpñô ð ð $*Ô#=ˆÔ Ý#& s¨VÔ-GÑ'GÑ#HÔ#HˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y¨3¸SÈaÐX`ÐaÑaÔaˆŒ
Ý”9¨¸3ÈAÐV^Ð_Ñ_Ô_ˆŒÝ”Y¨3¸SÈaÐX`ÐaÑaÔaˆŒ
å”z &Ô"EÑFÔFˆŒÝ”z bÐ)Ñ)Ô)ˆŒå&™œˆŒÝ'™/œ/ˆŒˆˆr;   c                 óÆ   — |                      ¦   «         d         | j        | j        |                      ¦   «         d         f} |j        |Ž }|                     dddd¦  «        S )z
        - input: [N, C, W]
        - output: [N, C1, W, C2] where C1 is the head index, and C2 is one head's contents
        r   r   r   r   r_   )r@   r�   r„   Úviewr`   ©r7   ra   Únew_x_shapes      r:   Útranspose_for_scoresz-SqueezeBertSelfAttention.transpose_for_scores·   s]   € ð
 —v’v‘x”x ”{ DÔ$<¸dÔ>VÐXY×X^ÒX^ÑX`ÔX`ÐacÔXdÐeˆØˆAŒF�KÐ ˆØ�yŠy˜˜A˜q !Ñ$Ô$Ð$r;   c                 óš   — |                      ¦   «         d         | j        | j        |                      ¦   «         d         f} |j        |Ž }|S )z
        - input: [N, C, W]
        - output: [N, C1, C2, W] where C1 is the head index, and C2 is one head's contents
        r   r   )r@   r�   r„   r’   r“   s      r:   Útranspose_key_for_scoresz1SqueezeBertSelfAttention.transpose_key_for_scoresÀ   sJ   € ð
 —v’v‘x”x ”{ DÔ$<¸dÔ>VÐXY×X^ÒX^ÑX`ÔX`ÐacÔXdÐeˆØˆAŒF�KÐ ˆàˆr;   c                 óâ   — |                      dddd¦  «                             ¦   «         }|                     ¦   «         d         | j        |                     ¦   «         d         f} |j        |Ž }|S )zE
        - input: [N, C1, W, C2]
        - output: [N, C, W]
        r   r   r   r_   )r`   Ú
contiguousr@   r…   r’   r“   s      r:   Útranspose_outputz)SqueezeBertSelfAttention.transpose_outputÊ   sa   € ð
 �IŠI�a˜˜A˜qÑ!Ô!×,Ò,Ñ.Ô.ˆØ—v’v‘x”x ”{ DÔ$6¸¿º¹¼À¼ÐDˆØˆAŒF�KÐ ˆØˆr;   c                 ó4  — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }	|                      ||¦  «        }
|
t          j        | j        ¦  «        z  }
|�|
|z   }
|  	                    |
¦  «        }|  
                    |¦  «        }|                      ||	¦  «        }|                      |¦  «        }d|i}|r|
|d<   |S )z›
        expects hidden_states in [N, C, W] data layout.

        The attention_mask data layout is [N, W], and it does not need to be transposed.
        NÚcontext_layerÚattention_score)r†   r‡   rˆ   r•   r—   rŒ   ÚmathÚsqrtr„   r‹   r1   r�   rš   )r7   rs   Úattention_maskÚoutput_attentionsÚmixed_query_layerÚmixed_key_layerÚmixed_value_layerÚquery_layerÚ	key_layerÚvalue_layerr�   Úattention_probsrœ   Úresults                 r:   rI   z SqueezeBertSelfAttention.forwardÔ   s  € ð !ŸJšJ }Ñ5Ô5ÐØŸ(š( =Ñ1Ô1ˆØ ŸJšJ }Ñ5Ô5Ðà×/Ò/Ð0AÑBÔBˆØ×1Ò1°/ÑBÔBˆ	Ø×/Ò/Ð0AÑBÔBˆð Ÿ.š.¨°iÑ@Ô@ˆØ)­D¬I°dÔ6NÑ,OÔ,OÑOˆàÐ%Ø-°Ñ>ˆOð Ÿ,š, Ñ7Ô7ˆð Ÿ,š, Ñ7Ô7ˆàŸš¨¸ÑEÔEˆØ×-Ò-¨mÑ<Ô<ˆà! =Ð1ˆØð 	8Ø(7ˆFÐ$Ñ%Øˆr;   )r   r   r   )	rK   rL   rM   r"   r•   r—   rš   rI   rO   rP   s   @r:   r|   r|   ž   sy   ø€ € € € € ð*ð *ð *ð *ð *ð *ð0%ð %ð %ðð ð ðð ð ð"ð "ð "ð "ð "ð "ð "r;   r|   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSqueezeBertModulec                 óœ  •— t          ¦   «                              ¦   «          |j        }|j        }|j        }|j        }t	          |||j        |j        |j        ¬¦  «        | _        t          |||j
        |j        ¬¦  «        | _        t          |||j        |j        ¬¦  «        | _        t          |||j        |j        ¬¦  «        | _        dS )a€  
        - hidden_size = input chans = output chans for Q, K, V (they are all the same ... for now) = output chans for
          the module
        - intermediate_size = output chans for intermediate layer
        - groups = number of groups for all layers in the BertModule. (eventually we could change the interface to
          allow different groups for different layers)
        )r8   ro   rŽ   r�   r�   )ro   rp   rk   rq   )ro   rp   rk   rx   N)r!   r"   r-   Úintermediate_sizer|   rŽ   r�   r�   Ú	attentionrd   Úpost_attention_groupsr0   Úpost_attentionrv   Úintermediate_groupsÚ
hidden_actÚintermediateÚoutput_groupsrz   )r7   r8   Úc0Úc1Úc2Úc3r9   s         €r:   r"   zSqueezeBertModule.__init__ú   s×   ø€ õ 	‰Œ×ÒÑÔÐàÔˆØÔˆØÔ%ˆØÔˆå1Ø˜r¨F¬OÀfÄoÐ`fÔ`oð
ñ 
ô 
ˆŒõ 3Ø˜ FÔ$@ÈvÔOið
ñ 
ô 
ˆÔõ +¨r¸À6ÔC]ÐciÔctÐuÑuÔuˆÔÝ*Ø˜ FÔ$8ÀvÔGað
ñ 
ô 
ˆŒˆˆr;   c                 óè   — |                       |||¦  «        }|d         }|                      ||¦  «        }|                      |¦  «        }|                      ||¦  «        }d|i}	|r|d         |	d<   |	S )Nrœ   Úfeature_mapr�   )r®   r°   r³   rz   )
r7   rs   r    r¡   ÚattÚattention_outputÚpost_attention_outputÚintermediate_outputÚlayer_outputÚoutput_dicts
             r:   rI   zSqueezeBertModule.forward  sŽ   € Ø�nŠn˜]¨NÐ<MÑNÔNˆØ˜Ô/Ðà $× 3Ò 3Ð4DÀmÑ TÔ TÐØ"×/Ò/Ð0EÑFÔFÐØ—{’{Ð#6Ð8MÑNÔNˆà$ lÐ3ˆØð 	DØ-0Ð1BÔ-CˆKÐ)Ñ*àÐr;   ©rK   rL   rM   r"   rI   rO   rP   s   @r:   r«   r«   ù   sG   ø€ € € € € ð
ð 
ð 
ð 
ð 
ð4ð ð ð ð ð ð r;   r«   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚSqueezeBertEncoderc                 óì   •‡— t          ¦   «                              ¦   «          ‰j        ‰j        k    s
J d¦   «         ‚t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d S )Nz™If you want embedding_size != intermediate hidden_size, please insert a Conv1d layer to adjust the number of channels before the first SqueezeBertModule.c              3   ó6   •K  — | ]}t          ‰¦  «        V — Œd S rT   )r«   )Ú.0Ú_r8   s     €r:   ú	<genexpr>z.SqueezeBertEncoder.__init__.<locals>.<genexpr>-  s,   øè è € Ð#gÐ#gÀ!Õ$5°fÑ$=Ô$=Ð#gÐ#gÐ#gÐ#gÐ#gÐ#gr;   )	r!   r"   r%   r-   r   Ú
ModuleListÚrangeÚnum_hidden_layersÚlayersr6   s    `€r:   r"   zSqueezeBertEncoder.__init__$  su   øø€ Ý‰Œ×ÒÑÔÐàÔ$¨Ô(:Ò:Ð:Ð:ð2ñ ;Ô:Ð:õ ”mÐ#gÐ#gÐ#gÐ#gÅuÈVÔMeÑGfÔGfÐ#gÑ#gÔ#gÑgÔgˆŒˆˆr;   NFTc                 óÀ  — |                      ddd¦  «        }|rdnd }|rdnd }| j        D ]e}|r4|                      ddd¦  «        }||fz  }|                      ddd¦  «        }|                     |||¦  «        }	|	d         }|r||	d         fz  }Œf|                      ddd¦  «        }|r||fz  }|st          d„ |||fD ¦   «         ¦  «        S t	          |||¬¦  «        S )	Nr   r_   r   rb   rº   r�   c              3   ó   K  — | ]}|®|V — Œ	d S rT   rb   )rÆ   Úvs     r:   rÈ   z-SqueezeBertEncoder.forward.<locals>.<genexpr>Q  s(   è è € ÐhÐh˜qÐZ[ÐZg˜ÐZgÐZgÐZgÐZgÐhÐhr;   )Úlast_hidden_staters   Ú
attentions)r`   rÌ   rI   Útupler   )
r7   rs   r    r¡   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_attentionsÚlayerr¿   s
             r:   rI   zSqueezeBertEncoder.forward/  sI  € ð &×-Ò-¨a°°AÑ6Ô6ˆà"6Ð@˜B˜B¸DÐØ0Ð:˜˜°dˆà”[ð 	Eð 	EˆEØ#ð ?Ø -× 5Ò 5°a¸¸AÑ >Ô >�Ø! mÐ%5Ñ5Ð!Ø -× 5Ò 5°a¸¸AÑ >Ô >�à Ÿ=š=¨¸ÐHYÑZÔZˆLà(¨Ô7ˆMà ð EØ <Ð0AÔ#BÐ"DÑD�øð &×-Ò-¨a°°AÑ6Ô6ˆàð 	2Ø -Ð!1Ñ1Ðàð 	iÝÐhÐh ]Ð4EÀ~Ð$VÐhÑhÔhÑhÔhÐhÝØ+Ð;LÐYgð
ñ 
ô 
ð 	
r;   )NFFTrÁ   rP   s   @r:   rÃ   rÃ   #  s_   ø€ € € € € ð	hð 	hð 	hð 	hð 	hð ØØ"Øð%
ð %
ð %
ð %
ð %
ð %
ð %
ð %
r;   rÃ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSqueezeBertPoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S rT   )r!   r"   r   ÚLinearr-   ÚdenseÚTanhÚ
activationr6   s     €r:   r"   zSqueezeBertPooler.__init__X  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr;   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rÜ   rÞ   )r7   rs   Úfirst_token_tensorÚpooled_outputs       r:   rI   zSqueezeBertPooler.forward]  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr;   rÁ   rP   s   @r:   rÙ   rÙ   W  sG   ø€ € € € € ð$ð $ð $ð $ð $ð
ð ð ð ð ð ð r;   rÙ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú"SqueezeBertPredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S )Nr   )r!   r"   r   rÛ   r-   rÜ   Ú
isinstancer²   Ústrr	   Útransform_act_fnr,   r.   r6   s     €r:   r"   z+SqueezeBertPredictionHeadTransform.__init__g  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr;   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rT   )rÜ   rç   r,   ©r7   rs   s     r:   rI   z*SqueezeBertPredictionHeadTransform.forwardp  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐr;   rÁ   rP   s   @r:   rã   rã   f  sL   ø€ € € € € ðUð Uð Uð Uð Uðð ð ð ð ð ð r;   rã   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSqueezeBertLMPredictionHeadc                 ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        t	          j	        t          j        |j        ¦  «        ¦  «        | _        d S )NT)Úbias)r!   r"   rã   Ú	transformr   rÛ   r-   r$   ÚdecoderÚ	Parameterr3   rA   rí   r6   s     €r:   r"   z$SqueezeBertLMPredictionHead.__init__x  sj   ø€ Ý‰Œ×ÒÑÔÐÝ;¸FÑCÔCˆŒõ ”y Ô!3°VÔ5FÈTÐRÑRÔRˆŒå”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r;   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S rT   )rî   rï   ré   s     r:   rI   z#SqueezeBertLMPredictionHead.forward„  s*   € ØŸš }Ñ5Ô5ˆØŸš ]Ñ3Ô3ˆØÐr;   rÁ   rP   s   @r:   rë   rë   w  sL   ø€ € € € € ðAð Að Að Að Aðð ð ð ð ð ð r;   rë   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚSqueezeBertOnlyMLMHeadc                 óp   •— t          ¦   «                              ¦   «          t          |¦  «        | _        d S rT   )r!   r"   rë   Úpredictionsr6   s     €r:   r"   zSqueezeBertOnlyMLMHead.__init__‹  s/   ø€ Ý‰Œ×ÒÑÔÐÝ6°vÑ>Ô>ˆÔÐÐr;   c                 ó0   — |                       |¦  «        }|S rT   )rõ   )r7   Úsequence_outputÚprediction_scoress      r:   rI   zSqueezeBertOnlyMLMHead.forward�  s   € Ø ×,Ò,¨_Ñ=Ô=ÐØ Ð r;   rÁ   rP   s   @r:   ró   ró   Š  sG   ø€ € € € € ð?ð ?ð ?ð ?ð ?ð!ð !ð !ð !ð !ð !ð !r;   ró   c                   óX   ‡ — e Zd ZU eed<   dZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZ	S )ÚSqueezeBertPreTrainedModelr8   Útransformerc                 óv  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rt	          j        |j        ¦  «         dS t          |t          ¦  «        rQt	          j        |j	        t          j        |j	        j        d         ¦  «                             d¦  «        ¦  «         dS dS )zInitialize the weightsr   r   N)r!   Ú_init_weightsrå   rë   ÚinitÚzeros_rí   r   Úcopy_r   r3   r4   Úshaper5   )r7   Úmoduler9   s     €r:   rý   z(SqueezeBertPreTrainedModel._init_weights™  s©   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ9Ñ:Ô:ð 	iÝŒK˜œÑ$Ô$Ð$Ð$Ð$Ý˜Õ 5Ñ6Ô6ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	ir;   )
rK   rL   rM   r   Ú__annotations__Úbase_model_prefixr3   Úno_gradrý   rO   rP   s   @r:   rú   rú   ”  sg   ø€ € € € € € àÐÐÑØ%Ðà€U„]�_„_ðið ið ið iñ „_ðið ið ið ið ir;   rú   c                   óà   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ej	        dz  d
e
dz  de
dz  de
dz  deez  fd„¦   «         Zˆ xZS )ÚSqueezeBertModelc                 óê   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        t          |¦  «        | _        |                      ¦   «          d S rT   )	r!   r"   r   rH   rÃ   ÚencoderrÙ   ÚpoolerÚ	post_initr6   s     €r:   r"   zSqueezeBertModel.__init__¥  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å/°Ñ7Ô7ˆŒÝ)¨&Ñ1Ô1ˆŒÝ'¨Ñ/Ô/ˆŒð 	�ŠÑÔÐÐÐr;   c                 ó   — | j         j        S rT   ©rH   r'   ©r7   s    r:   Úget_input_embeddingsz%SqueezeBertModel.get_input_embeddings¯  s   € ØŒÔ.Ð.r;   c                 ó   — || j         _        d S rT   r  ©r7   Únew_embeddingss     r:   Úset_input_embeddingsz%SqueezeBertModel.set_input_embeddings²  s   € Ø*8ˆŒÔ'Ð'Ð'r;   NrC   r    rD   r   rE   r¡   rÓ   rÔ   Úreturnc	                 óþ  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|�|�t	          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }
n.|�|                     ¦   «         d d…         }
nt	          d¦  «        ‚|�|j        n|j        }|€t          j	        |
|¬¦  «        }|€!t          j
        |
t          j        |¬¦  «        }|                      ||||¬¦  «        }t          | j         ||¬¦  «        }|                      |||||¬¦  «        }|d	         }|                      |¦  «        }|s||f|d
d …         z   S t!          |||j        |j        ¬¦  «        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embeds)r?   r=   )rC   r   rD   rE   )r8   rE   r    )rs   r    r¡   rÓ   rÔ   r   r   )rÐ   Úpooler_outputrs   rÑ   )r8   r¡   rÓ   rÔ   r‚   Ú%warn_if_padding_and_no_attention_maskr@   r?   r3   ÚonesrA   rB   rH   r
   r	  r
  r   rs   rÑ   )r7   rC   r    rD   r   rE   r¡   rÓ   rÔ   ÚkwargsrF   r?   Úembedding_outputÚencoder_outputsr÷   rá   s                   r:   rI   zSqueezeBertModel.forwardµ  sä  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨¸FÐCÑCÔCˆNØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNàŸ?š?Ø¨lÈ>Ðivð +ñ 
ô 
Ðõ 3Ø”;Ø*Ø)ð
ñ 
ô 
ˆð Ÿ,š,Ø*Ø)Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð *¨!Ô,ˆØŸš OÑ4Ô4ˆàð 	JØ# ]Ð3°oÀaÀbÀbÔ6IÑIÐIå)Ø-Ø'Ø)Ô7Ø&Ô1ð	
ñ 
ô 
ð 	
r;   )NNNNNNNN)rK   rL   rM   r"   r  r  r   r3   ÚTensorÚFloatTensorÚboolrÒ   r   rI   rO   rP   s   @r:   r  r  £  s#  ø€ € € € € ðð ð ð ð ð/ð /ð /ð9ð 9ð 9ð ð *.Ø.2Ø.2Ø,0Ø26Ø)-Ø,0Ø#'ð?
ð ?
à”< $Ñ&ð?
ð œ tÑ+ð?
ð œ tÑ+ð	?
ð
 ”l TÑ)ð?
ð Ô(¨4Ñ/ð?
ð   $™;ð?
ð # T™kð?
ð ˜D‘[ð?
ð 
Ð+Ñ	+ð?
ð ?
ð ?
ñ „^ð?
ð ?
ð ?
ð ?
ð ?
r;   r  c                   ó   ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 ddej	        dz  d	ej	        dz  d
ej	        dz  dej	        dz  dej	        dz  dej	        dz  de
dz  de
dz  de
dz  deez  fd„¦   «         Zˆ xZS )ÚSqueezeBertForMaskedLMzcls.predictions.biasz-transformer.embeddings.word_embeddings.weight)zcls.predictions.decoder.biaszcls.predictions.decoder.weightc                 óÂ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          |¦  «        | _        |                      ¦   «          d S rT   )r!   r"   r  rû   ró   Úclsr  r6   s     €r:   r"   zSqueezeBertForMaskedLM.__init__ÿ  sR   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆÔÝ)¨&Ñ1Ô1ˆŒð 	�ŠÑÔÐÐÐr;   c                 ó$   — | j         j        j        S rT   )r"  rõ   rï   r  s    r:   Úget_output_embeddingsz,SqueezeBertForMaskedLM.get_output_embeddings  s   € ØŒxÔ#Ô+Ð+r;   c                 óT   — || j         j        _        |j        | j         j        _        d S rT   )r"  rõ   rï   rí   r  s     r:   Úset_output_embeddingsz,SqueezeBertForMaskedLM.set_output_embeddings  s%   € Ø'5ˆŒÔÔ$Ø$2Ô$7ˆŒÔÔ!Ð!Ð!r;   NrC   r    rD   r   rE   Úlabelsr¡   rÓ   rÔ   r  c
           
      ó¢  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }d}|�Kt	          ¦   «         } ||                     d| j         j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j        |j	        ¬¦  «        S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N©r    rD   r   rE   r¡   rÓ   rÔ   r   r   r_   ©ÚlossÚlogitsrs   rÑ   )
r8   rÔ   rû   r"  r   r’   r$   r   rs   rÑ   )r7   rC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  Úoutputsr÷   rø   Úmasked_lm_lossÚloss_fctrz   s                    r:   rI   zSqueezeBertForMaskedLM.forward  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð " !œ*ˆØ ŸHšH _Ñ5Ô5ÐàˆØÐÝ'Ñ)Ô)ˆHØ%˜XÐ&7×&<Ò&<¸RÀÄÔAWÑ&XÔ&XÐZ`×ZeÒZeÐfhÑZiÔZiÑjÔjˆNàð 	ZØ'Ð)¨G°A°B°B¬KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.Ð.ÐSYÐYåØØ$Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r;   ©	NNNNNNNNN)rK   rL   rM   Ú_tied_weights_keysr"   r$  r&  r   r3   r  r  rÒ   r   rI   rO   rP   s   @r:   r   r   ø  sJ  ø€ € € € € ð )?Ø*Yðð Ðð
ð ð ð ð ð,ð ,ð ,ð8ð 8ð 8ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ð1
ð 1
à”< $Ñ&ð1
ð œ tÑ+ð1
ð œ tÑ+ð	1
ð
 ”l TÑ)ð1
ð ”| dÑ*ð1
ð ”˜tÑ#ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
�Ñ	ð1
ð 1
ð 1
ñ „^ð1
ð 1
ð 1
ð 1
ð 1
r;   r   z£
    SqueezeBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )Ú$SqueezeBertForSequenceClassificationc                 óN  •— t          ¦   «                              |¦  «         |j        | _        || _        t	          |¦  «        | _        t          j        |j        ¦  «        | _	        t          j
        |j        | j        j        ¦  «        | _        |                      ¦   «          d S rT   )r!   r"   Ú
num_labelsr8   r  rû   r   r/   r0   r1   rÛ   r-   Ú
classifierr  r6   s     €r:   r"   z-SqueezeBertForSequenceClassification.__init__K  sƒ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒØˆŒå+¨FÑ3Ô3ˆÔÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸¼Ô8NÑOÔOˆŒð 	�ŠÑÔÐÐÐr;   NrC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  c
           
      óì  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t!          |||j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr)  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr   r_   r*  )r8   rÔ   rû   r1   r7  Úproblem_typer6  r>   r3   rB   rƒ   r   Úsqueezer   r’   r   r   rs   rÑ   )r7   rC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  r-  rá   r,  r+  r/  rz   s                    r:   rI   z,SqueezeBertForSequenceClassification.forwardW  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�àð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r;   r0  )rK   rL   rM   r"   r   r3   r  r  rÒ   r   rI   rO   rP   s   @r:   r4  r4  D  s+  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ðE
ð E
à”< $Ñ&ðE
ð œ tÑ+ðE
ð œ tÑ+ð	E
ð
 ”l TÑ)ðE
ð ”| dÑ*ðE
ð ”˜tÑ#ðE
ð   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
Ð)Ñ	)ðE
ð E
ð E
ñ „^ðE
ð E
ð E
ð E
ð E
r;   r4  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚSqueezeBertForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S )Nr   )r!   r"   r  rû   r   r/   r0   r1   rÛ   r-   r7  r  r6   s     €r:   r"   z%SqueezeBertForMultipleChoice.__init__¢  sm   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å+¨FÑ3Ô3ˆÔÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr;   NrC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  c
           
      ó¸  — |	�|	n| j         j        }	|�|j        d         n|j        d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
            *input_ids* above)
        Nr   r   éþÿÿÿr)  r_   r*  )r8   rÔ   r  r’   r@   rû   r1   r7  r   r   rs   rÑ   )r7   rC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  Únum_choicesr-  rá   r,  Úreshaped_logitsr+  r/  rz   s                      r:   rI   z$SqueezeBertForMultipleChoice.forward¬  s*  € ðX &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð ×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r;   r0  )rK   rL   rM   r"   r   r3   r  r  rÒ   r   rI   rO   rP   s   @r:   r?  r?     s+  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ðW
ð W
à”< $Ñ&ðW
ð œ tÑ+ðW
ð œ tÑ+ð	W
ð
 ”l TÑ)ðW
ð ”| dÑ*ðW
ð ”˜tÑ#ðW
ð   $™;ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*ðW
ð W
ð W
ñ „^ðW
ð W
ð W
ð W
ð W
r;   r?  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )Ú!SqueezeBertForTokenClassificationc                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S rT   )r!   r"   r6  r  rû   r   r/   r0   r1   rÛ   r-   r7  r  r6   s     €r:   r"   z*SqueezeBertForTokenClassification.__init__	  sz   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå+¨FÑ3Ô3ˆÔÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr;   NrC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  c
           
      óÂ  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j	        |j
        ¬¦  «        S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr)  r   r   r_   r*  )r8   rÔ   rû   r1   r7  r   r’   r6  r   rs   rÑ   )r7   rC   r    rD   r   rE   r'  r¡   rÓ   rÔ   r  r-  r÷   r,  r+  r/  rz   s                    r:   rI   z)SqueezeBertForTokenClassification.forward  s  € ð$ &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r;   r0  )rK   rL   rM   r"   r   r3   r  r  rÒ   r   rI   rO   rP   s   @r:   rF  rF    s  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ð1
ð 1
à”< $Ñ&ð1
ð œ tÑ+ð1
ð œ tÑ+ð	1
ð
 ”l TÑ)ð1
ð ”| dÑ*ð1
ð ”˜tÑ#ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
Ð&Ñ	&ð1
ð 1
ð 1
ñ „^ð1
ð 1
ð 1
ð 1
ð 1
r;   rF  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dedz  dee	z  fd„¦   «         Z
ˆ xZS )ÚSqueezeBertForQuestionAnsweringc                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rT   )
r!   r"   r6  r  rû   r   rÛ   r-   Ú
qa_outputsr  r6   s     €r:   r"   z(SqueezeBertForQuestionAnswering.__init__K  sf   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå+¨FÑ3Ô3ˆÔÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr;   NrC   r    rD   r   rE   Ústart_positionsÚend_positionsr¡   rÓ   rÔ   r  c           
      ó®  — |
�|
n| j         j        }
|                      |||||||	|
¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }| 	                    d|¦  «        }| 	                    d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|
s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	Nr)  r   r   r   r   )Úignore_indexr_   )r+  Ústart_logitsÚ
end_logitsrs   rÑ   )r8   rÔ   rû   rL  Úsplitr=  r™   Úlenr@   Úclampr   r   rs   rÑ   )r7   rC   r    rD   r   rE   rM  rN  r¡   rÓ   rÔ   r  r-  r÷   r,  rQ  rR  Ú
total_lossÚignored_indexr/  Ú
start_lossÚend_lossrz   s                          r:   rI   z'SqueezeBertForQuestionAnswering.forwardU  s  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r;   )
NNNNNNNNNN)rK   rL   rM   r"   r   r3   r  r  rÒ   r   rI   rO   rP   s   @r:   rJ  rJ  I  s,  ø€ € € € € ðð ð ð ð ð ð *.Ø.2Ø.2Ø,0Ø-1Ø/3Ø-1Ø)-Ø,0Ø#'ð=
ð =
à”< $Ñ&ð=
ð œ tÑ+ð=
ð œ tÑ+ð	=
ð
 ”l TÑ)ð=
ð ”| dÑ*ð=
ð œ¨Ñ,ð=
ð ”| dÑ*ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
Ð-Ñ	-ð=
ð =
ð =
ñ „^ð=
ð =
ð =
ð =
ð =
r;   rJ  )r   r?  rJ  r4  rF  r  r«   rú   )7rN   rž   r3   r   Útorch.nnr   r   r   Ú r   rþ   Úactivationsr	   Úmasking_utilsr
   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_squeezebertr   Ú
get_loggerrK   ÚloggerÚModuler   rR   r,   rZ   rd   rv   r|   r«   rÃ   rÙ   rã   rë   ró   rú   r  r   r4  r?  rF  rJ  Ú__all__rb   r;   r:   ú<module>rf     sC  ðð !Ð  à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð 9Ð 8Ð 8Ð 8Ð 8Ð 8ð 
ˆÔ	˜HÑ	%Ô	%€ð'ð 'ð 'ð 'ð '˜BœIñ 'ô 'ð 'ðT(ð (ð (ð (ð (�B”Iñ (ô (ð (ð("ð "ð "ð "ð "˜2œ<ñ "ô "ð "ð ð ð ð ð ˜2œ9ñ ô ð ð( ð  ð  ð  ð  �R”Yñ  ô  ð  ðXð Xð Xð Xð X˜rœyñ Xô Xð Xðv'ð 'ð 'ð 'ð '˜œ	ñ 'ô 'ð 'ðT1
ð 1
ð 1
ð 1
ð 1
˜œñ 1
ô 1
ð 1
ðhð ð ð ð ˜œ	ñ ô ð ðð ð ð ð ¨¬ñ ô ð ð"ð ð ð ð  "¤)ñ ô ð ð&!ð !ð !ð !ð !˜RœYñ !ô !ð !ð ðið ið ið ið i ñ iô iñ „ðið ðQ
ð Q
ð Q
ð Q
ð Q
Ð1ñ Q
ô Q
ñ „ðQ
ðh ðH
ð H
ð H
ð H
ð H
Ð7ñ H
ô H
ñ „ðH
ðV €ððñ ô ðS
ð S
ð S
ð S
ð S
Ð+Eñ S
ô S
ñô ðS
ðl ðc
ð c
ð c
ð c
ð c
Ð#=ñ c
ô c
ñ „ðc
ðL ð>
ð >
ð >
ð >
ð >
Ð(Bñ >
ô >
ñ „ð>
ðB ðI
ð I
ð I
ð I
ð I
Ð&@ñ I
ô I
ñ „ðI
ðX	ð 	ð 	€€€r;   