§
    ‚ŠtjÉ|  ã                   ó(  — d Z ddlZddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZmZmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$  e"j%        e&¦  «        Z' e	j(        ¦   «         eeedœZ) G d„ de	j*        ¦  «        Z+ G d„ de	j*        ¦  «        Z, G d„ de	j*        ¦  «        Z- G d„ de	j*        ¦  «        Z.e! G d„ de¦  «        ¦   «         Z/ e!d¬¦  «        e G d„ d e ¦  «        ¦   «         ¦   «         Z0e! G d!„ d"e/¦  «        ¦   «         Z1 e!d#¬¦  «         G d$„ d%e/e¦  «        ¦   «         Z2 e!d&¬¦  «         G d'„ d(e/¦  «        ¦   «         Z3 e!d)¬¦  «         G d*„ d+e/¦  «        ¦   «         Z4g d,¢Z5dS )-zPyTorch OpenAI GPT model.é    N)ÚCallable)Ú	dataclass)ÚAny)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úgelu_newÚget_activationÚsilu)ÚGenerationMixin)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutput)ÚPreTrainedModel)ÚConv1D)ÚModelOutputÚauto_docstringÚloggingé   )ÚOpenAIGPTConfig)Úrelur   ÚgeluÚswishc                   ó>   ‡ — e Zd Zdˆ fd„	Zd	d„Zd„ Zdd„Zd	d„Zˆ xZS )
Ú	AttentionFc           	      óV  •— t          ¦   «                              ¦   «          || _        |}||j        z  dk    rt	          d|› d|j        › �¦  «        ‚|                      dt          j        t          j        ||¦  «        ¦  «         	                    dd||¦  «        d¬¦  «         |j        | _        || _
        || _        t          |dz  |¦  «        | _        t          ||¦  «        | _        t          j        |j        ¦  «        | _        t          j        |j        ¦  «        | _        d S )	Nr   zAttention n_state shape: z$ must be divisible by config.n_head Úbiasr   F©Ú
persistentr
   )ÚsuperÚ__init__Ún_positionsÚn_headÚ
ValueErrorÚregister_bufferÚtorchÚtrilÚonesÚviewÚ
split_sizeÚscaler   Úc_attnÚc_projr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropout)ÚselfÚnxr%   Úconfigr.   Ún_stateÚ	__class__s         €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/openai/modeling_openai.pyr$   zAttention.__init__/   s  ø€ Ý‰Œ×ÒÑÔÐØ&ˆÔØˆØ�V”]Ñ" aÒ'Ð'ÝÐu¸ÐuÐuÐflÔfsÐuÐuÑvÔvÐvØ×ÒØÝŒJ•u”z +¨{Ñ;Ô;Ñ<Ô<×AÒAÀ!ÀQÈÐU`ÑaÔaØð 	ñ 	
ô 	
ð 	
ð
 ”mˆŒØ!ˆŒØˆŒ
å˜W q™[¨"Ñ-Ô-ˆŒÝ˜W bÑ)Ô)ˆŒÝœJ vÔ'8Ñ9Ô9ˆÔÝœZ¨Ô(:Ñ;Ô;ˆÔÐÐó    Nc                 ó   — t          j        ||¦  «        }| j        r*|t          j        |                     d¦  «        ¦  «        z  }| j        d d …d d …d |                     d¦  «        …d |                     d¦  «        …f         }||z  dd|z
  z  z   }|�||z   }t          j         	                    |d¬¦  «        }|  
                    |¦  «        }t          j        ||¦  «        g}|r|                     |¦  «         |S )Néÿÿÿÿéþÿÿÿg     ˆÃÀr   ©Údim)r)   Úmatmulr.   ÚmathÚsqrtÚsizer    r   Ú
functionalÚsoftmaxr3   Úappend)	r6   ÚqÚkÚvÚattention_maskÚoutput_attentionsÚwÚbÚoutputss	            r;   Ú_attnzAttention._attnC   sú   € ÝŒL˜˜AÑÔˆØŒ:ð 	*Ø•D”I˜aŸfšf R™jœjÑ)Ô)Ñ)ˆAàŒI�a�a�a˜˜˜˜L˜aŸfšf R™jœj˜L¨,¨A¯FªF°2©J¬J¨,Ð6Ô7ˆØ�‰E�D˜A ™E‘NÑ"ˆàÐ%à�NÑ"ˆAåŒM×!Ò! !¨Ð!Ñ,Ô,ˆØ×Ò˜aÑ Ô ˆå”<  1Ñ%Ô%Ð&ˆØð 	Ø�NŠN˜1ÑÔÐØˆr<   c                 óü   — |                      dddd¦  «                             ¦   «         }|                     ¦   «         d d…         |                     d¦  «        |                     d¦  «        z  fz   } |j        |Ž S )Nr   é   r   r
   r?   r>   )ÚpermuteÚ
contiguousrE   r,   )r6   ÚxÚnew_x_shapes      r;   Úmerge_headszAttention.merge_headsW   sj   € Ø�IŠI�a˜˜A˜qÑ!Ô!×,Ò,Ñ.Ô.ˆØ—f’f‘h”h˜s ˜s”m q§v¢v¨b¡z¤z°A·F²F¸2±J´JÑ'>Ð&@Ñ@ˆØˆqŒv�{Ð#Ð#r<   c                 óü   — |                      ¦   «         d d…         | j        |                      d¦  «        | j        z  fz   } |j        |Ž }|r|                     dddd¦  «        S |                     dddd¦  «        S )Nr>   r   rS   r
   r   )rE   r&   r,   rT   )r6   rV   rJ   rW   s       r;   Úsplit_headszAttention.split_heads\   sz   € Ø—f’f‘h”h˜s ˜s”m t¤{°A·F²F¸2±J´JÀ$Ä+Ñ4MÐ&NÑNˆØˆAŒF�KÐ ˆØð 	)Ø—9’9˜Q  1 aÑ(Ô(Ð(à—9’9˜Q  1 aÑ(Ô(Ð(r<   c                 óÎ  — |                       |¦  «        }|                     | j        d¬¦  «        \  }}}|                      |¦  «        }|                      |d¬¦  «        }|                      |¦  «        }|                      |||||¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|g|dd …         z   }	|	S )NrS   r@   T)rJ   r   r   )r/   Úsplitr-   rZ   rQ   rX   r0   r5   )
r6   rV   rL   rM   ÚqueryÚkeyÚvalueÚattn_outputsÚarP   s
             r;   ÚforwardzAttention.forwardd   sß   € Ø�KŠK˜‰NŒNˆØŸGšG D¤O¸˜GÑ;Ô;Ñˆˆs�EØ× Ò  Ñ'Ô'ˆØ×Ò˜s dÐÑ+Ô+ˆØ× Ò  Ñ'Ô'ˆà—z’z %¨¨e°^ÐEVÑWÔWˆØ˜ŒOˆà×Ò˜QÑÔˆØ�KŠK˜‰NŒNˆØ×Ò˜qÑ!Ô!ˆà�#˜ Q R RÔ(Ñ(ˆØˆr<   ©F©NF)	Ú__name__Ú
__module__Ú__qualname__r$   rQ   rX   rZ   rb   Ú__classcell__©r:   s   @r;   r   r   .   sˆ   ø€ € € € € ð<ð <ð <ð <ð <ð <ð(ð ð ð ð($ð $ð $ð
)ð )ð )ð )ðð ð ð ð ð ð ð r<   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMLPc                 ó  •— t          ¦   «                              ¦   «          |j        }t          ||¦  «        | _        t          ||¦  «        | _        t          |j                 | _        t          j
        |j        ¦  «        | _        d S ©N)r#   r$   Ún_embdr   Úc_fcr0   ÚACT_FNSÚafnÚactr   r1   r4   Údropout)r6   r9   r8   r7   r:   s       €r;   r$   zMLP.__init__w   sh   ø€ Ý‰Œ×ÒÑÔÐØŒ]ˆÝ˜7 BÑ'Ô'ˆŒ	Ý˜R Ñ)Ô)ˆŒÝ˜6œ:Ô&ˆŒÝ”z &Ô"4Ñ5Ô5ˆŒˆˆr<   c                 ó¦   — |                       |                      |¦  «        ¦  «        }|                      |¦  «        }|                      |¦  «        S rm   )rr   ro   r0   rs   )r6   rV   ÚhÚh2s       r;   rb   zMLP.forward   s>   € Ø�HŠH�T—Y’Y˜q‘\”\Ñ"Ô"ˆØ�[Š[˜‰^Œ^ˆØ�|Š|˜BÑÔÐr<   ©re   rf   rg   r$   rb   rh   ri   s   @r;   rk   rk   v   sG   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð ð  ð  ð  ð  ð  ð  r<   rk   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚBlockFc                 ó4  •— t          ¦   «                              ¦   «          |j        }t          ||||¦  «        | _        t          j        ||j        ¬¦  «        | _        t          d|z  |¦  «        | _
        t          j        ||j        ¬¦  «        | _        d S )N)Úepsé   )r#   r$   rn   r   Úattnr   Ú	LayerNormÚlayer_norm_epsilonÚln_1rk   ÚmlpÚln_2)r6   r%   r8   r.   r7   r:   s        €r;   r$   zBlock.__init__†   s€   ø€ Ý‰Œ×ÒÑÔÐØŒ]ˆÝ˜b +¨v°uÑ=Ô=ˆŒ	Ý”L ¨Ô)BÐCÑCÔCˆŒ	Ý�q˜2‘v˜vÑ&Ô&ˆŒÝ”L ¨Ô)BÐCÑCÔCˆŒ	ˆ	ˆ	r<   Nc                 óì   — |                       |||¬¦  «        }|d         }|                      ||z   ¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|g|dd …         z   }	|	S )N)rL   rM   r   r   )r}   r€   r�   r‚   )
r6   rV   rL   rM   r`   ra   ÚnÚmru   rP   s
             r;   rb   zBlock.forwardŽ   s�   € Ø—y’yØØ)Ø/ð !ñ 
ô 
ˆð
 ˜ŒOˆà�IŠI�a˜!‘eÑÔˆØ�HŠH�Q‰KŒKˆØ�IŠI�a˜!‘eÑÔˆà�#˜ Q R RÔ(Ñ(ˆØˆr<   rc   rd   rw   ri   s   @r;   ry   ry   …   sW   ø€ € € € € ðDð Dð Dð Dð Dð Dðð ð ð ð ð ð ð r<   ry   c                   ód   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dej        fd„Z	ˆ xZ
S )
ÚOpenAIGPTSequenceSummaryaÑ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`OpenAIGPTConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    r8   c                 óV  •— t          ¦   «                              ¦   «          t          |dd¦  «        | _        | j        dk    rt          ‚t          j        ¦   «         | _        t          |d¦  «        rW|j	        rPt          |d¦  «        r|j
        r|j        dk    r|j        }n|j        }t          j        |j        |¦  «        | _        t          |dd ¦  «        }|rt          |¦  «        nt          j        ¦   «         | _        t          j        ¦   «         | _        t          |d¦  «        r)|j        dk    rt          j        |j        ¦  «        | _        t          j        ¦   «         | _        t          |d	¦  «        r+|j        dk    r"t          j        |j        ¦  «        | _        d S d S d S )
NÚsummary_typeÚlastr}   Úsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r#   r$   Úgetattrr‰   ÚNotImplementedErrorr   ÚIdentityÚsummaryÚhasattrr‹   rŒ   Ú
num_labelsÚhidden_sizeÚLinearr   Ú
activationÚfirst_dropoutrŽ   r1   Úlast_dropoutr�   )r6   r8   Únum_classesÚactivation_stringr:   s       €r;   r$   z!OpenAIGPTSequenceSummary.__init__¹   sœ  ø€ Ý‰Œ×ÒÑÔÐå# F¨N¸FÑCÔCˆÔØÔ Ò&Ð&õ &Ð%å”{‘}”}ˆŒÝ�6Ð-Ñ.Ô.ð 	F°6Ô3Jð 	FÝ�vÐ7Ñ8Ô8ð 1¸VÔ=Zð 1Ð_eÔ_pÐstÒ_tÐ_tØ$Ô/��à$Ô0�Ýœ9 VÔ%7¸ÑEÔEˆDŒLå# FÐ,@À$ÑGÔGÐØIZÐ$m¥NÐ3DÑ$EÔ$EÐ$EÕ`bÔ`kÑ`mÔ`mˆŒåœ[™]œ]ˆÔÝ�6Ð2Ñ3Ô3ð 	J¸Ô8TÐWXÒ8XÐ8XÝ!#¤¨FÔ,HÑ!IÔ!IˆDÔåœK™MœMˆÔÝ�6Ð1Ñ2Ô2ð 	H°vÔ7RÐUVÒ7VÐ7VÝ "¤
¨6Ô+FÑ GÔ GˆDÔÐÐð	Hð 	HÐ7VÐ7Vr<   NÚhidden_statesÚ	cls_indexÚreturnc                 ó:  — | j         dk    r|dd…df         }�n-| j         dk    r|dd…df         }�n| j         dk    r|                     d¬¦  «        }nò| j         d	k    rÕ|€=t          j        |d
dd…dd…f         |j        d         dz
  t          j        ¬¦  «        }nl|                     d¦  «                             d¦  «        }|                     d|                     ¦   «         dz
  z  | 	                    d¦  «        fz   ¦  «        }| 
                    d|¦  «                             d¦  «        }n| j         dk    rt          ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        rŠ   Nr>   Úfirstr   Úmeanr   r@   rž   .r?   ©Údtype)r>   r}   )r‰   r¢   r)   Ú	full_likeÚshapeÚlongÚ	unsqueezeÚexpandrA   rE   ÚgatherÚsqueezer‘   r™   r“   r˜   rš   )r6   r�   rž   Úoutputs       r;   rb   z OpenAIGPTSequenceSummary.forwardÖ   sª  € ð Ô Ò&Ð&Ø" 1 1 1 b 5Ô)ˆF‰FØÔ 'Ò)Ð)Ø" 1 1 1 a 4Ô(ˆF‰FØÔ &Ò(Ð(Ø"×'Ò'¨AÐ'Ñ.Ô.ˆFˆFØÔ +Ò-Ð-ØÐ Ý!œOØ! # r¨ r¨1¨1¨1 *Ô-Ø!Ô'¨Ô+¨aÑ/Ýœ*ðñ ô �	�	ð &×/Ò/°Ñ3Ô3×=Ò=¸bÑAÔA�	Ø%×,Ò,¨U°i·m²m±o´oÈÑ6IÑ-JÈm×N`ÒN`ÐacÑNdÔNdÐMfÑ-fÑgÔg�	à"×)Ò)¨"¨iÑ8Ô8×@Ò@ÀÑDÔDˆFˆFØÔ &Ò(Ð(Ý%Ð%à×#Ò# FÑ+Ô+ˆØ—’˜fÑ%Ô%ˆØ—’ Ñ(Ô(ˆØ×"Ò" 6Ñ*Ô*ˆàˆr<   rm   )re   rf   rg   Ú__doc__r   r$   r)   ÚFloatTensorÚ
LongTensorrb   rh   ri   s   @r;   r‡   r‡   Ÿ   s›   ø€ € € € € ðð ð2H˜ð Hð Hð Hð Hð Hð Hð< VZð)ð )Ø"Ô.ð)Ø;@Ô;KÈdÑ;Rð)à	Ô	ð)ð )ð )ð )ð )ð )ð )ð )r<   r‡   c                   ó.   ‡ — e Zd ZU eed<   dZˆ fd„Zˆ xZS )ÚOpenAIGPTPreTrainedModelr8   Útransformerc           	      óÊ  •— t          ¦   «                              |¦  «         t          |t          ¦  «        r^|j        }t          j        |j        t          j	        t          j
        ||¦  «        ¦  «                             dd||¦  «        ¦  «         d S t          |t          ¦  «        r8t          j        |j        t          j        |j        j        ¦  «        ¦  «         d S d S )Nr   )r#   Ú_init_weightsÚ
isinstancer   r%   ÚinitÚcopy_r    r)   r*   r+   r,   ÚOpenAIGPTModelÚposition_idsÚaranger8   )r6   Úmoduler%   r:   s      €r;   r´   z&OpenAIGPTPreTrainedModel._init_weights  sÔ   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�iÑ(Ô(ð 	UØ Ô,ˆKÝŒJØ”�UœZ­¬
°;ÀÑ(LÔ(LÑMÔM×RÒRÐSTÐVWÐYdÐfqÑrÔrñô ð ð ð õ ˜¥Ñ/Ô/ð 	UÝŒJ�vÔ*­E¬L¸¼Ô9RÑ,SÔ,SÑTÔTÐTÐTÐTð	Uð 	Ur<   )re   rf   rg   r   Ú__annotations__Úbase_model_prefixr´   rh   ri   s   @r;   r±   r±     sU   ø€ € € € € € àÐÐÑØ%ÐðUð Uð Uð Uð Uð Uð Uð Uð Ur<   r±   z^
    Base class for outputs of models predicting if two sentences are consecutive or not.
    )Úcustom_introc                   óà   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed<   dS )	ÚOpenAIGPTDoubleHeadsModelOutputa¦  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    mc_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `mc_labels` is provided):
        Multiple choice classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    mc_logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
    NÚlossÚmc_lossÚlogitsÚ	mc_logitsr�   Ú
attentions)re   rf   rg   r­   rÁ   r)   r®   r¼   rÂ   rÃ   rÄ   r�   ÚtuplerÅ   © r<   r;   rÀ   rÀ     s¸   € € € € € € ð	ð 	ð &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø(,€GˆUÔ Ñ%Ð,Ð,Ñ,Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø*.€IˆuÔ  4Ñ'Ð.Ð.Ñ.Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r<   rÀ   c                   óö   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  dej        dz  dej        dz  d	ej	        dz  d
e
dz  de
dz  de
dz  deej                 ez  fd„¦   «         Zˆ xZS )r¸   c                 ó  •‡— t          ¦   «                              ‰¦  «         t          j        ‰j        ‰j        ¦  «        | _        t          j        ‰j        ‰j        ¦  «        | _        t          j	        ‰j
        ¦  «        | _        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        |                      dt#          j        ‰j        ¦  «        d¬¦  «         |                      ¦   «          d S )Nc                 ó>   •— g | ]}t          ‰j        ‰d ¬¦  «        ‘ŒS )T)r.   )ry   r%   )Ú.0Ú_r8   s     €r;   ú
<listcomp>z+OpenAIGPTModel.__init__.<locals>.<listcomp>4  s,   ø€ ÐmÐmÐmÐRS¥ fÔ&8¸&ÈÐ MÑ MÔ MÐmÐmÐmr<   r¹   Fr!   )r#   r$   r   Ú	EmbeddingÚ
vocab_sizern   Útokens_embedr%   Úpositions_embedr1   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerru   r(   r)   rº   Ú	post_init©r6   r8   r:   s    `€r;   r$   zOpenAIGPTModel.__init__.  sÔ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð åœL¨Ô):¸F¼MÑJÔJˆÔÝ!œ|¨FÔ,>ÀÄÑNÔNˆÔÝ”J˜vÔ0Ñ1Ô1ˆŒ	Ý”ÐmÐmÐmÐmÕW\Ð]cÔ]kÑWlÔWlÐmÑmÔmÑnÔnˆŒà×Ò˜^­U¬\¸&Ô:LÑ-MÔ-MÐZ_ÐÑ`Ô`Ð`à�ŠÑÔÐÐÐr<   c                 ó   — | j         S rm   ©rÐ   )r6   s    r;   Úget_input_embeddingsz#OpenAIGPTModel.get_input_embeddings:  s   € ØÔ Ð r<   c                 ó   — || _         d S rm   rÚ   )r6   Únew_embeddingss     r;   Úset_input_embeddingsz#OpenAIGPTModel.set_input_embeddings=  s   € Ø*ˆÔÐÐr<   NÚ	input_idsrL   Útoken_type_idsr¹   Úinputs_embedsrM   Úoutput_hidden_statesÚreturn_dictrŸ   c	                 ó*  — |�|n| j         j        }|�|n| j         j        }|�|n| j         j        }|�|�t	          d¦  «        ‚|�G|                      ||¦  «         |                     ¦   «         }
|                     d|
d         ¦  «        }n.|�|                     ¦   «         d d…         }
nt	          d¦  «        ‚|€| j        d d |
d         …f         }|�†| 	                    d¦  «         	                    d¦  «        }| 
                    t          |                      ¦   «         ¦  «        j        ¬¦  «        }d|z
  t          j        | j        ¦  «        j        z  }|€|                      |¦  «        }|                      |¦  «        }|�?|                     d|                     d¦  «        ¦  «        }|                      |¦  «        }nd}||z   |z   }|                      |¦  «        }|
|                     d¦  «        fz   }|rd	nd }|rd	nd }t)          | j        ¦  «        D ]1\  }}|r||fz   } ||||¬
¦  «        }|d         }|r||d         fz   }Œ2 |j        |Ž }|r||fz   }|st-          d„ |||fD ¦   «         ¦  «        S t/          |||¬¦  «        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer>   z5You have to specify either input_ids or inputs_embedsr   rS   r£   g      ð?r   rÇ   )rM   c              3   ó   K  — | ]}|®|V — Œ	d S rm   rÇ   )rË   rK   s     r;   ú	<genexpr>z)OpenAIGPTModel.forward.<locals>.<genexpr>‘  s(   è è € ÐhÐh˜qÐZ[ÐZg˜ÐZgÐZgÐZgÐZgÐhÐhr<   )Úlast_hidden_stater�   rÅ   )r8   rM   râ   rã   r'   Ú%warn_if_padding_and_no_attention_maskrE   r,   r¹   r¨   ÚtoÚnextÚ
parametersr¤   r)   ÚfinfoÚminrÐ   rÑ   rÓ   Ú	enumerateru   rÆ   r   )r6   rß   rL   rà   r¹   rá   rM   râ   rã   ÚkwargsÚinput_shapeÚposition_embedsÚtoken_type_embedsr�   Úoutput_shapeÚall_attentionsÚall_hidden_statesÚiÚblockrP   s                       r;   rb   zOpenAIGPTModel.forward@  s   € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð &1Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIˆIØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUàÐàÔ,¨TÐ3D°[À´_Ð3DÐ-DÔEˆLð Ð%ð ,×5Ò5°aÑ8Ô8×BÒBÀ1ÑEÔEˆNð ,×.Ò.µT¸$¿/º/Ñ:KÔ:KÑ5LÔ5LÔ5RÐ.ÑSÔSˆNØ! NÑ2µe´kÀ$Ä*Ñ6MÔ6MÔ6QÑQˆNàÐ Ø ×-Ò-¨iÑ8Ô8ˆMØ×.Ò.¨|Ñ<Ô<ˆØÐ%Ø+×0Ò0°°^×5HÒ5HÈÑ5LÔ5LÑMÔMˆNØ $× 1Ò 1°.Ñ AÔ AÐÐà !ÐØ%¨Ñ7Ð:KÑKˆØŸ	š	 -Ñ0Ô0ˆà" m×&8Ò&8¸Ñ&<Ô&<Ð%>Ñ>ˆà0Ð:˜˜°dˆØ"6Ð@˜B˜B¸DÐÝ! $¤&Ñ)Ô)ð 	@ð 	@‰HˆAˆuØ#ð IØ$5¸Ð8HÑ$HÐ!à�e˜M¨>ÐM^Ð_Ñ_Ô_ˆGØ# AœJˆMØ ð @Ø!/°7¸1´:°-Ñ!?�øà*˜Ô*¨LÐ9ˆàð 	EØ 1°]Ð4DÑ DÐàð 	iÝÐhÐh ]Ð4EÀ~Ð$VÐhÑhÔhÑhÔhÐhåØ+Ø+Ø%ð
ñ 
ô 
ð 	
r<   )NNNNNNNN)re   rf   rg   r$   rÛ   rÞ   r   r)   r¯   r®   ÚboolrÆ   ÚTensorr   rb   rh   ri   s   @r;   r¸   r¸   ,  s?  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð!ð !ð !ð+ð +ð +ð ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ðV
ð V
àÔ# dÑ*ðV
ð Ô)¨DÑ0ðV
ð Ô(¨4Ñ/ð	V
ð
 Ô&¨Ñ-ðV
ð Ô(¨4Ñ/ðV
ð   $™;ðV
ð # T™kðV
ð ˜D‘[ðV
ð 
ˆuŒ|Ô	˜Ñ	.ðV
ð V
ð V
ñ „^ðV
ð V
ð V
ð V
ð V
r<   r¸   z‰
    OpenAI GPT Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                   óH  ‡ — e Zd ZddiZˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  de	dz  de	dz  de	dz  de
ej        z  deej                 ez  fd„¦   «         Zdej        deeef         fd„Zˆ xZS )ÚOpenAIGPTLMHeadModelúlm_head.weightútransformer.tokens_embed.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NF©r    )
r#   r$   r¸   r²   r   r—   rn   rÏ   Úlm_headr×   rØ   s     €r;   r$   zOpenAIGPTLMHeadModel.__init__£  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý)¨&Ñ1Ô1ˆÔÝ”y ¤°Ô0AÈÐNÑNÔNˆŒð 	�ŠÑÔÐÐÐr<   Nr   rß   rL   rà   r¹   rá   ÚlabelsrM   râ   rã   Úlogits_to_keeprŸ   c           
      ó¬  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }t          |
t          ¦  «        rt          |
 d¦  «        n|
}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j         j        dœ|¤Ž}|	s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N©rL   rà   r¹   rá   rM   râ   rã   r   )rÃ   r  rÏ   r   ©rÁ   rÃ   r�   rÅ   rÇ   )r8   rã   r²   rµ   ÚintÚslicer  Úloss_functionrÏ   r   r�   rÅ   )r6   rß   rL   rà   r¹   rá   r  rM   râ   rã   r  rï   Útransformer_outputsr�   Úslice_indicesrÃ   rÁ   r¬   s                     r;   rb   zOpenAIGPTLMHeadModel.forward«  s3  € ð* &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð /ñ 	
ô 	
Ðð ,¨AÔ.ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�YÐ!4°Q°R°RÔ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEåØØØ-Ô;Ø*Ô5ð	
ñ 
ô 
ð 	
r<   c                 óT   — d|i}|                      ¦   «         D ]\  }}||vr|||<   Œ|S )Nrß   )Úitems)r6   rß   rï   Úmodel_inputsr^   r_   s         r;   Úprepare_inputs_for_generationz2OpenAIGPTLMHeadModel.prepare_inputs_for_generationá  sE   € à# YÐ/ˆð !Ÿ,š,™.œ.ð 	*ð 	*‰JˆC�Ø˜,Ð&Ð&Ø$)�˜SÑ!øàÐr<   )
NNNNNNNNNr   )re   rf   rg   Ú_tied_weights_keysr$   r   r)   r¯   r®   rø   r  rù   rÆ   r   rb   ÚdictÚstrr   r  rh   ri   s   @r;   rû   rû   š  sy  ø€ € € € € ð +Ð,MÐNÐðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'Ø-.ð3
ð 3
àÔ# dÑ*ð3
ð Ô)¨DÑ0ð3
ð Ô(¨4Ñ/ð	3
ð
 Ô&¨Ñ-ð3
ð Ô(¨4Ñ/ð3
ð Ô  4Ñ'ð3
ð   $™;ð3
ð # T™kð3
ð ˜D‘[ð3
ð ˜eœlÑ*ð3
ð 
ˆuŒ|Ô	˜~Ñ	-ð3
ð 3
ð 3
ñ „^ð3
ðj	°uÔ7Gð 	ÐVZÐ[^Ð`cÐ[cÔVdð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	r<   rû   a�  
        OpenAI GPT Model transformer with a language modeling and a multiple-choice classification head on top e.g. for
    RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the
    input embeddings, the classification head takes as input the input of a specified classification token index in the
    input sequence).
    c                   ó4  ‡ — e Zd ZddiZˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
ej        dz  dej        dz  dej        dz  de	dz  de	dz  de	dz  de
ej                 ez  fd„¦   «         Zˆ xZS )ÚOpenAIGPTDoubleHeadsModelrý   rü   c                 ó  •— t          ¦   «                              |¦  «         d|_        t          |¦  «        | _        t          j        |j        |j        d¬¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S )Nr   Fr   )r#   r$   r•   r¸   r²   r   r—   rn   rÏ   r  r‡   Úmultiple_choice_headr×   rØ   s     €r;   r$   z"OpenAIGPTDoubleHeadsModel.__init__ø  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð àˆÔÝ)¨&Ñ1Ô1ˆÔÝ”y ¤°Ô0AÈÐNÑNÔNˆŒÝ$<¸VÑ$DÔ$DˆÔ!ð 	�ŠÑÔÐÐÐr<   Nrß   rL   rà   r¹   rá   Úmc_token_idsr  Ú	mc_labelsrM   râ   rã   rŸ   c           
      óL  — |�|n| j         j        }|                      ||||||	|
|¬¦  «        }|d         }|                      |¦  «        }|                      ||¦  «                             d¦  «        }d\  }}|�Tt          ¦   «         } ||                     d|                     d¦  «        ¦  «        |                     d¦  «        ¦  «        }|�“|ddd…dd…f          	                    ¦   «         }|ddd…f          	                    ¦   «         }t          ¦   «         } ||                     d|                     d¦  «        ¦  «        |                     d¦  «        ¦  «        }|s!||f|dd…         z   }|�|f|z   }|�|f|z   n|S t          |||||j        |j        ¬¦  «        S )	aè  
        mc_token_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
            Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
            1]`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-1, 0, ..., config.vocab_size]` All labels set to `-100` are
            ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        mc_labels (`torch.LongTensor` of shape `(batch_size)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, OpenAIGPTDoubleHeadsModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/openai-gpt")
        >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-community/openai-gpt")
        >>> tokenizer.add_special_tokens(
        ...     {"cls_token": "[CLS]"}
        ... )  # Add a [CLS] to the vocabulary (we should train it also!)
        >>> model.resize_token_embeddings(len(tokenizer))

        >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"]
        >>> input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0)  # Batch size 1, 2 choices
        >>> mc_token_ids = torch.tensor([input_ids.size(-1) - 1, input_ids.size(-1) - 1]).unsqueeze(0)  # Batch size 1

        >>> outputs = model(input_ids, mc_token_ids=mc_token_ids)
        >>> lm_logits = outputs.logits
        >>> mc_logits = outputs.mc_logits
        ```Nr  r   r>   )NN.r   )rÁ   rÂ   rÃ   rÄ   r�   rÅ   )r8   rã   r²   r  r  r«   r   r,   rE   rU   rÀ   r�   rÅ   )r6   rß   rL   rà   r¹   rá   r  r  r  rM   râ   rã   rï   r
  r�   Ú	lm_logitsrÄ   Úlm_lossrÂ   Úloss_fctÚshift_logitsÚshift_labelsr¬   s                          r;   rb   z!OpenAIGPTDoubleHeadsModel.forward  sé  € ðb &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð /ñ 	
ô 	
Ðð ,¨AÔ.ˆà—L’L Ñ/Ô/ˆ	Ø×-Ò-¨m¸\ÑJÔJ×RÒRÐSUÑVÔVˆ	à%Ñˆ�ØÐ Ý'Ñ)Ô)ˆHØ�h˜yŸ~š~¨b°)·.².ÀÑ2DÔ2DÑEÔEÀyÇ~Â~ÐVXÑGYÔGYÑZÔZˆGØÐØ$ S¨#¨2¨#¨q¨q¨q [Ô1×<Ò<Ñ>Ô>ˆLØ! # q r r 'œ?×5Ò5Ñ7Ô7ˆLÝ'Ñ)Ô)ˆHØ�h˜|×0Ò0°°\×5FÒ5FÀrÑ5JÔ5JÑKÔKÈ\×M^ÒM^Ð_aÑMbÔMbÑcÔcˆGàð 	LØ Ð+Ð.AÀ!À"À"Ô.EÑEˆFØÐ"Ø!˜ fÑ,�Ø,3Ð,?�W�J Ñ'Ð'ÀVÐKå.ØØØØØ-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r<   )NNNNNNNNNNN)re   rf   rg   r  r$   r   r)   r¯   r®   rø   rÆ   rù   rÀ   rb   rh   ri   s   @r;   r  r  í  sr  ø€ € € € € ð <Ð=MÐNÐð	ð 	ð 	ð 	ð 	ð ð .2Ø37Ø26Ø04Ø26Ø04Ø*.Ø-1Ø)-Ø,0Ø#'ðX
ð X
àÔ# dÑ*ðX
ð Ô)¨DÑ0ðX
ð Ô(¨4Ñ/ð	X
ð
 Ô&¨Ñ-ðX
ð Ô(¨4Ñ/ðX
ð Ô&¨Ñ-ðX
ð Ô  4Ñ'ðX
ð Ô# dÑ*ðX
ð   $™;ðX
ð # T™kðX
ð ˜D‘[ðX
ð 
ˆuŒ|Ô	Ð>Ñ	>ðX
ð X
ð X
ñ „^ðX
ð X
ð X
ð X
ð X
r<   r  aâ  
    The Original OpenAI GPT Model transformer with a sequence classification head on top (linear layer).
    [`OpenAIGPTForSequenceClassification`] uses the last token in order to do the classification, as other causal
    models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the
    last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding
    token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since
    it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
    the last value in each row of the batch).
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	ej
                 ez  fd„¦   «         Zˆ xZS )Ú"OpenAIGPTForSequenceClassificationc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S rÿ   )
r#   r$   r•   r¸   r²   r   r—   rn   Úscorer×   rØ   s     €r;   r$   z+OpenAIGPTForSequenceClassification.__init__k  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ)¨&Ñ1Ô1ˆÔÝ”Y˜vœ}¨d¬oÀEÐJÑJÔJˆŒ
ð 	�ŠÑÔÐÐÐr<   Nrß   rL   rà   r¹   rá   r  rM   râ   rã   rŸ   c
           
      ó  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|�|j        dd…         \  }}n|j        dd…         \  }}| j         j        €|dk    rt          d¦  «        ‚| j         j        €d}n¨|�}|| j         j        k                         |j        t          j
        ¦  «        }t          j        |j        d         |j        t          j
        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d	�¦  «         |t          j        ||j        ¬
¦  «        |f         }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j         _        nd| j         _        | j         j        dk    rWt-          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt1          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt5          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t7          |||j        |j        ¬¦  «        S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   rS   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r>   )Údevicer¤   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r$  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr  )r8   rã   r²   r"  r¦   Úpad_token_idr'   ré   r$  r)   Úint32rº   ÚargmaxÚloggerÚwarning_oncer:   re   Úproblem_typer•   r¤   r§   r  r	   r«   r   r,   r   r   r�   rÅ   )r6   rß   rL   rà   r¹   rá   r  rM   râ   rã   rï   r
  r�   rÃ   Ú
batch_sizeÚsequence_lengthÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsrÁ   r  r¬   s                          r;   rb   z*OpenAIGPTForSequenceClassification.forwardt  sD  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð /ñ 	
ô 	
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø*3¬/¸"¸1¸"Ô*=Ñ'ˆJ˜˜à*7Ô*=¸b¸q¸bÔ*AÑ'ˆJ˜ð Œ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 M×$9Ò$9Ñ$;Ô$;¸V¿^º^Ñ=MÔ=MÑNÔN�D�Dà#˜8 M°6Ñ:Ô:�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x × 2Ò 2°2°t´Ñ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨vÑ6Ô6�Øð 	FØ#Ð%Ð(;¸A¸B¸BÔ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØ Ø-Ô;Ø*Ô5ð	
ñ 
ô 
ð 	
r<   )	NNNNNNNNN)re   rf   rg   r$   r   r)   r¯   r®   rø   rÆ   rù   r   rb   rh   ri   s   @r;   r   r   _  s8  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðZ
ð Z
àÔ# dÑ*ðZ
ð Ô)¨DÑ0ðZ
ð Ô(¨4Ñ/ð	Z
ð
 Ô&¨Ñ-ðZ
ð Ô(¨4Ñ/ðZ
ð Ô  4Ñ'ðZ
ð   $™;ðZ
ð # T™kðZ
ð ˜D‘[ðZ
ð 
ˆuŒ|Ô	Ð7Ñ	7ðZ
ð Z
ð Z
ñ „^ðZ
ð Z
ð Z
ð Z
ð Z
r<   r   )r  r   rû   r¸   r±   )6r­   rC   Úcollections.abcr   Údataclassesr   Útypingr   r)   r   Útorch.nnr   r   r	   Ú r   r¶   Úactivationsr   r   r   Ú
generationr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úconfiguration_openair   Ú
get_loggerre   r+  ÚReLUrp   ÚModuler   rk   ry   r‡   r±   rÀ   r¸   rû   r  r   Ú__all__rÇ   r<   r;   ú<module>rD     s;  ðð  Ð à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø )Ð )Ð )Ð )Ð )Ð )Ø YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YÐ YØ -Ð -Ð -Ð -Ð -Ð -Ø #Ð #Ð #Ð #Ð #Ð #ðð ð ð ð ð ð ð ð ð ð
 2Ð 1Ð 1Ð 1Ð 1Ð 1ð 
ˆÔ	˜HÑ	%Ô	%€ð �2”7‘9”9 d°HÀtÐ
LÐ
L€ðEð Eð Eð Eð E�”	ñ Eô Eð EðP ð  ð  ð  ð  ˆ"Œ)ñ  ô  ð  ðð ð ð ð ˆBŒIñ ô ð ð4`ð `ð `ð `ð `˜rœyñ `ô `ð `ðF ðUð Uð Uð Uð U˜ñ Uô Uñ „ðUð €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 kñ 7ô 7ñ „ñô ð7ð( ðj
ð j
ð j
ð j
ð j
Ð-ñ j
ô j
ñ „ðj
ðZ €ððñ ô ðJð Jð Jð Jð JÐ3°_ñ Jô Jñô ðJðZ €ððñ ô ðg
ð g
ð g
ð g
ð g
Ð 8ñ g
ô g
ñô ðg
ðT €ðð
ñ 
ô 
ðe
ð e
ð e
ð e
ð e
Ð)Añ e
ô e
ñ
ô 
ðe
ðPð ð €€€r<   