§
    ‚Štj”Ç  ã                   ó¨  — d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ dd	lmZmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4  e,j5        e6¦  «        Z7d6d„Z8 G d„ dej9        ¦  «        Z: G d„ dej9        ¦  «        Z; G d„ de¦  «        Z< G d„ dej9        ¦  «        Z=e* G d „ d!e%¦  «        ¦   «         Z> e*d"¬#¦  «        e G d$„ d%e)¦  «        ¦   «         ¦   «         Z?e* G d&„ d'e>¦  «        ¦   «         Z@ e*d(¬#¦  «         G d)„ d*e>e¦  «        ¦   «         ZA e*d+¬#¦  «         G d,„ d-e>e¦  «        ¦   «         ZB e*d.¬#¦  «         G d/„ d0e>¦  «        ¦   «         ZCe* G d1„ d2e>¦  «        ¦   «         ZDe* G d3„ d4e>¦  «        ¦   «         ZEg d5¢ZFdS )7zPyTorch OpenAI GPT-2 model.é    N)ÚCallable)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚConv1D)ÚModelOutputÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
GPT2Configç        c                 ó¾  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }|                     |j        ¦  «        }t          j         	                    ||| j
        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «        }	|	|fS )Néÿÿÿÿç      à¿éþÿÿÿ©Údim)ÚpÚtrainingr$   é   )ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚtypeÚdtypeÚdropoutr.   )
ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr8   ÚkwargsÚattn_weightsÚattn_outputs
             úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gpt2/modeling_gpt2.pyÚeager_attention_forwardrC   6   sÏ   € Ø€Ø—*’*˜R‘.”. DÑ(ˆå”<  s§}¢}°R¸Ñ'<Ô'<Ñ=Ô=ÀÑG€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€Lð  ×$Ò$ U¤[Ñ1Ô1€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-€Kà˜Ð$Ð$ó    c                   óæ   ‡ — e Zd Zdˆ fd„	Zdd„Z	 	 	 	 	 ddeej                 dz  dedz  dej        dz  dej	        dz  d	ej        dz  d
e
dz  deej	        eej	                 z  df         fd„Zˆ xZS )ÚGPT2AttentionFNc                 óÈ  •— t          ¦   «                              ¦   «          || _        |j        | _        |j        | _        | j        | j        z  | _        | j        | _        | j        | j        z  | j        k    r t          d| j        › d| j        › d�¦  «        ‚|j
        | _
        |j        | _        |j        | _        || _        || _        d| _        | j
        r| j        dz  | _        | j        r%| xj        t!          | j        dz   ¦  «        z  c_        | j        rBt#          d| j        z  | j        ¦  «        | _        t#          | j        | j        ¦  «        | _        n"t#          d| j        z  | j        ¦  «        | _        t#          | j        | j        ¦  «        | _        t+          j        |j        ¦  «        | _        t+          j        |j        ¦  «        | _        | | _        d S )	Nz=`embed_dim` must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      ð?r)   r$   r/   r	   )ÚsuperÚ__init__ÚconfigÚhidden_sizeÚ	embed_dimÚnum_attention_headsÚ	num_headsÚhead_dimÚ
split_sizeÚ
ValueErrorÚscale_attn_weightsÚscale_attn_by_inverse_layer_idxÚreorder_and_upcast_attnÚis_cross_attentionÚ	layer_idxr>   Úfloatr   Úc_attnÚq_attnÚc_projr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	is_causal)ÚselfrJ   rU   rV   Ú	__class__s       €rB   rI   zGPT2Attention.__init__L   sÀ  ø€ Ý‰Œ×ÒÑÔÐØˆŒØÔ+ˆŒØÔ3ˆŒØœ¨$¬.Ñ8ˆŒØœ.ˆŒØŒ=˜4œ>Ñ)¨T¬^Ò;Ð;Ýð'ÐPTÔP^ð 'ð 'Ø”Nð'ð 'ð 'ñô ð ð
 #)Ô";ˆÔØ/5Ô/UˆÔ,Ø'-Ô'EˆÔ$Ø"4ˆÔØ"ˆŒð ˆŒØÔ"ð 	/Øœ=¨$Ñ.ˆDŒLØÔ/ð 	6ØˆLŒL�E $¤.°1Ñ"4Ñ5Ô5Ñ5ˆLŒLàÔ"ð 	EÝ   T¤^Ñ!3°T´^ÑDÔDˆDŒKÝ  ¤°´Ñ@Ô@ˆDŒKˆKå   T¤^Ñ!3°T´^ÑDÔDˆDŒKÝ˜Tœ^¨T¬^Ñ<Ô<ˆŒåœJ vÔ'8Ñ9Ô9ˆÔÝœZ¨Ô(:Ñ;Ô;ˆÔØ/Ð/ˆŒˆˆrD   c                 ó®  — |                      ¦   «         \  }}}}|                      ¦   «         \  }	}	}
}	t          j        ||z  ||
t          j        |j        ¬¦  «        }t          |j        j        d¬¦  «        5  |                     d||¦  «        |                     dd¦  «                             d||
¦  «        }}t          j	        || 
                    ¦   «         | 
                    ¦   «         d| j        ¬¦  «        }|                     ||||
¦  «        }d d d ¦  «         n# 1 swxY w Y   |�||z   }t          j                             |d¬¦  «        }|j        t          j        k    rt!          d	¦  «        ‚|                     |j        ¦  «        }|                      |¦  «        }t          j        ||¦  «        }|                     d
d¦  «        }||fS )N)r7   ÚdeviceF)Úenabledr(   r*   r   )ÚbetaÚalphar+   zDError with upcasting, attn_weights does not have dtype torch.float32r$   r/   )r0   r1   ÚemptyÚfloat32rd   r    r6   Úreshaper3   ÚbaddbmmrW   r>   r   r4   r5   r7   ÚRuntimeErrorr]   r2   )ra   r:   r;   r<   r=   ÚbszrN   Ú	q_seq_lenÚdkÚ_Ú	k_seq_lenr@   ÚqÚkrA   s                  rB   Ú_upcast_and_reordered_attnz(GPT2Attention._upcast_and_reordered_attnq   s   € à(-¯
ª
©¬Ñ%ˆˆY˜	 2Ø ŸXšX™ZœZÑˆˆ1ˆi˜õ ”{ 3¨¡?°I¸yÕPUÔP]ÐfkÔfrÐsÑsÔsˆõ ˜EœLÔ-°uÐ=Ñ=Ô=ð 	Vð 	VØ—=’=  Y°Ñ3Ô3°S·]²]À2ÀrÑ5JÔ5J×5RÒ5RÐSUÐWYÐ[dÑ5eÔ5eˆqˆAÝ œ=¨°q·w²w±y´yÀ!Ç'Â'Á)Ä)ÐRSÐ[_Ô[gÐhÑhÔhˆLØ'×/Ò/°°YÀ	È9ÑUÔUˆLð	Vð 	Vð 	Vñ 	Vô 	Vð 	Vð 	Vð 	Vð 	Vð 	Vð 	Vøøøð 	Vð 	Vð 	Vð 	Vð
 Ð%à'¨.Ñ8ˆLå”}×,Ò,¨\¸rÐ,ÑBÔBˆð Ô¥¤Ò.Ð.ÝÐeÑfÔfÐfØ#×(Ò(¨¬Ñ5Ô5ˆØ×(Ò(¨Ñ6Ô6ˆå”l <°Ñ7Ô7ˆØ!×+Ò+¨A¨qÑ1Ô1ˆà˜LÐ(Ð(s   Á9BD"Ä"D&Ä)D&Úhidden_statesÚpast_key_valuesr=   Úencoder_hidden_statesÚencoder_attention_maskÚoutput_attentionsÚreturn.c                 ó´  — |d u}|�Ht          |t          ¦  «        r1|j                             | j        ¦  «        }	|r|j        }
n
|j        }
n|}
|�r
t          | d¦  «        st          d¦  «        ‚|  	                    |¦  «        }|}|�2|	r0|
j
        | j                 j        }|
j
        | j                 j        }�n@|                      |¦  «                             | j        d¬¦  «        \  }}g |j        d d…         ¢d‘| j        ‘R }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }n |                      |¦  «                             | j        d¬¦  «        \  }}}g |j        d d…         ¢d‘| j        ‘R }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }g |j        d d…         ¢d‘| j        ‘R }|                     |¦  «                             dd¦  «        }|�|r|�4|r2|	s0|
                     ||| j        ¦  «        \  }}|rd|j        | j        <   | j        j        dk    }t/          j        | j        j        t2          ¦  «        }|r#| j        r|                      ||||¦  «        \  }}n+ || ||||f| j        r| j        j        nd	| j        d
œ|¤Ž\  }} |j         g |j        d d…         ¢d‘R Ž  !                    ¦   «         }|  "                    |¦  «        }|  #                    |¦  «        }||fS )NrY   z§If class is used as cross attention, the weights `q_attn` have to be defined. Please make sure to instantiate class with `GPT2Attention(..., is_cross_attention=True)`.r/   r+   r(   r$   TÚeagerr&   )r8   r>   r*   )$Ú
isinstancer   Ú
is_updatedÚgetrV   Úcross_attention_cacheÚself_attention_cacheÚhasattrrQ   rY   ÚlayersÚkeysÚvaluesrX   ÚsplitrP   ÚshaperO   Úviewr3   ÚupdaterJ   Ú_attn_implementationr   Úget_interfacerC   rT   rt   r.   r]   r-   r>   rj   Ú
contiguousrZ   r_   )ra   ru   rv   r=   rw   rx   ry   r?   rU   r~   Úcurr_past_key_valuesÚquery_statesÚ
key_statesÚvalue_statesÚshape_kvÚshape_qÚusing_eagerÚattention_interfacerA   r@   s                       rB   ÚforwardzGPT2Attention.forward�   sÙ  € ð 3¸$Ð>ÐØÐ&Ý˜/Õ+>Ñ?Ô?ð 7Ø,Ô7×;Ò;¸D¼NÑKÔK�
Ø%ð Pà+:Ô+PÐ(Ð(à+:Ô+OÐ(Ð(à'6Ð$àñ 	GÝ˜4 Ñ*Ô*ð Ý ðpñô ð ð  Ÿ;š; }Ñ5Ô5ˆLØ3ˆNð Ð*¨zÐ*Ø1Ô8¸¼ÔHÔM�
Ø3Ô:¸4¼>ÔJÔQ�‘à+/¯;ª;Ð7LÑ+MÔ+M×+SÒ+SÐTXÔTcÐijÐ+SÑ+kÔ+kÑ(�
˜LØF˜ZÔ-¨c¨r¨cÔ2ÐF°BÐF¸¼ÐFÐF�Ø'Ÿ_š_¨XÑ6Ô6×@Ò@ÀÀAÑFÔF�
Ø+×0Ò0°Ñ:Ô:×DÒDÀQÈÑJÔJ��à59·[²[ÀÑ5OÔ5O×5UÒ5UÐVZÔVeÐklÐ5UÑ5mÔ5mÑ2ˆL˜* lØB˜Ô)¨#¨2¨#Ô.ÐB°ÐB°D´MÐBÐBˆHØ#Ÿš¨Ñ2Ô2×<Ò<¸QÀÑBÔBˆJØ'×,Ò,¨XÑ6Ô6×@Ò@ÀÀAÑFÔFˆLà?�LÔ& s¨ sÔ+Ð?¨RÐ?°´Ð?Ð?ˆØ#×(Ò(¨Ñ1Ô1×;Ò;¸A¸qÑAÔAˆàÐ'Ð0BÐ'ØÐ'Ð,>Ð'ÀzÐ'à';×'BÒ'BÀ:È|Ð]aÔ]kÑ'lÔ'lÑ$ˆJ˜à!ð BØ=A�Ô*¨4¬>Ñ:à”kÔ6¸'ÒAˆÝ(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð ð 	˜4Ô7ð 	Ø(,×(GÒ(GØ˜j¨,¸ñ)ô )Ñ%ˆK˜˜ð )<Ð(;ØØØØØð	)ð 04¬}ÐE˜Ô)Ô+Ð+À#Øœð	)ð 	)ð ð	)ð 	)Ñ%ˆK˜ð *�kÔ)ÐF¨;Ô+<¸S¸b¸SÔ+AÐFÀ2ÐFÐFÐF×QÒQÑSÔSˆØ—k’k +Ñ.Ô.ˆØ×(Ò(¨Ñ5Ô5ˆà˜LÐ(Ð(rD   )FN©N©NNNNF)Ú__name__Ú
__module__Ú__qualname__rI   rt   Útupler1   ÚFloatTensorr   ÚTensorÚboolr•   Ú__classcell__©rb   s   @rB   rF   rF   K   s  ø€ € € € € ð#0ð #0ð #0ð #0ð #0ð #0ðJ)ð )ð )ð )ðD )-Ø37Ø59Ø;?Ø).ðR)ð R)à˜UÔ.Ô/°$Ñ6ðR)ð  ™ðR)ð Ô)¨DÑ0ð	R)ð
  %œ|¨dÑ2ðR)ð !&Ô 1°DÑ 8ðR)ð   $™;ðR)ð 
ˆuŒ|˜e E¤LÔ1Ñ1°3Ð6Ô	7ðR)ð R)ð R)ð R)ð R)ð R)ð R)ð R)rD   rF   c                   óT   ‡ — e Zd Zˆ fd„Zdeej                 dz  dej        fd„Zˆ xZS )ÚGPT2MLPc                 ó  •— t          ¦   «                              ¦   «          |j        }t          ||¦  «        | _        t          ||¦  «        | _        t          |j                 | _        t          j
        |j        ¦  «        | _        d S r–   )rH   rI   rK   r   Úc_fcrZ   r   Úactivation_functionÚactr   r[   r^   r8   )ra   Úintermediate_sizerJ   rL   rb   s       €rB   rI   zGPT2MLP.__init__æ   sl   ø€ Ý‰Œ×ÒÑÔÐØÔ&ˆ	ÝÐ,¨iÑ8Ô8ˆŒ	Ý˜YÐ(9Ñ:Ô:ˆŒÝ˜&Ô4Ô5ˆŒÝ”z &Ô"4Ñ5Ô5ˆŒˆˆrD   ru   Nrz   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r–   )r¤   r¦   rZ   r8   )ra   ru   s     rB   r•   zGPT2MLP.forwardî   sL   € ØŸ	š	 -Ñ0Ô0ˆØŸš Ñ/Ô/ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐrD   )	r˜   r™   rš   rI   r›   r1   rœ   r•   rŸ   r    s   @rB   r¢   r¢   å   sj   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð U¨5Ô+<Ô%=ÀÑ%Dð ÈÔIZð ð ð ð ð ð ð ð rD   r¢   c                   ó²   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddeej                 dz  dedz  dej        dz  dej        dz  dej        dz  d	e	dz  d
ej        fd„Z
ˆ xZS )Ú	GPT2BlockNc                 óÎ  •— t          ¦   «                              ¦   «          |j        }|j        �|j        nd|z  }t	          j        ||j        ¬¦  «        | _        t          ||¬¦  «        | _	        t	          j        ||j        ¬¦  «        | _
        |j        r7t          |d|¬¦  «        | _        t	          j        ||j        ¬¦  «        | _        t          ||¦  «        | _        d S )Né   ©Úeps)rJ   rV   T)rJ   rU   rV   )rH   rI   rK   Ún_innerr   Ú	LayerNormÚlayer_norm_epsilonÚln_1rF   ÚattnÚln_2Úadd_cross_attentionÚcrossattentionÚln_cross_attnr¢   Úmlp)ra   rJ   rV   rK   Ú	inner_dimrb   s        €rB   rI   zGPT2Block.__init__÷   sÑ   ø€ Ý‰Œ×ÒÑÔÐØÔ(ˆØ&,¤nÐ&@�F”N�NÀaÈ+Áoˆ	å”L °&Ô2KÐLÑLÔLˆŒ	Ý!¨¸9ÐEÑEÔEˆŒ	Ý”L °&Ô2KÐLÑLÔLˆŒ	àÔ%ð 	ZÝ"/°vÐRVÐbkÐ"lÑ"lÔ"lˆDÔÝ!#¤¨k¸vÔ?XÐ!YÑ!YÔ!YˆDÔå˜9 fÑ-Ô-ˆŒˆˆrD   Fru   rv   r=   rw   rx   Ú	use_cacherz   c                 ó†  — |}|                       |¦  «        } | j        |f|||dœ|¤Ž\  }	}
|	|z   }|�\t          | d¦  «        st          d| › d�¦  «        ‚|}|                      |¦  «        }|                      |||||¬¦  «        \  }}
||z   }|}|                      |¦  «        }|                      |¦  «        }||z   }|S )N)rv   r=   rº   r¶   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)rv   r=   rw   rx   )r²   r³   r‚   rQ   r·   r¶   r´   r¸   )ra   ru   rv   r=   rw   rx   rº   r?   ÚresidualrA   rp   Úcross_attn_outputÚfeed_forward_hidden_statess                rB   r•   zGPT2Block.forward  s1  € ð !ˆØŸ	š	 -Ñ0Ô0ˆØ"˜œØð
à+Ø)Øð	
ð 
ð
 ð
ð 
‰ˆ�Qð $ hÑ.ˆà Ð,å˜4Ð!1Ñ2Ô2ð Ý ðZ¸dð Zð Zð Zñô ð ð %ˆHØ ×.Ò.¨}Ñ=Ô=ˆMØ#'×#6Ò#6ØØ /Ø-Ø&;Ø'=ð $7ñ $ô $Ñ Ð˜qð %Ð'8Ñ8ˆMà ˆØŸ	š	 -Ñ0Ô0ˆØ%)§X¢X¨mÑ%<Ô%<Ð"à Ð#=Ñ=ˆàÐrD   r–   r—   )r˜   r™   rš   rI   r›   r1   rœ   r   r�   rž   r•   rŸ   r    s   @rB   rª   rª   ö   sÙ   ø€ € € € € ð.ð .ð .ð .ð .ð .ð$ )-Ø37Ø59Ø;?Ø!&ð/ð /à˜UÔ.Ô/°$Ñ6ð/ð  ™ð/ð Ô)¨DÑ0ð	/ð
  %œ|¨dÑ2ð/ð !&Ô 1°DÑ 8ð/ð ˜$‘;ð/ð 
Œð/ð /ð /ð /ð /ð /ð /ð /rD   rª   c                   ód   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej        dej        dz  dej        fd„Z	ˆ xZ
S )
ÚGPT2SequenceSummaryaÌ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`GPT2Config`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    rJ   c                 óV  •— t          ¦   «                              ¦   «          t          |dd¦  «        | _        | j        dk    rt          ‚t          j        ¦   «         | _        t          |d¦  «        rW|j	        rPt          |d¦  «        r|j
        r|j        dk    r|j        }n|j        }t          j        |j        |¦  «        | _        t          |dd ¦  «        }|rt          |¦  «        nt          j        ¦   «         | _        t          j        ¦   «         | _        t          |d¦  «        r)|j        dk    rt          j        |j        ¦  «        | _        t          j        ¦   «         | _        t          |d	¦  «        r+|j        dk    r"t          j        |j        ¦  «        | _        d S d S d S )
NÚsummary_typeÚlastr³   Úsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)rH   rI   ÚgetattrrÂ   ÚNotImplementedErrorr   ÚIdentityÚsummaryr‚   rÄ   rÅ   Ú
num_labelsrK   ÚLinearr   Ú
activationÚfirst_dropoutrÇ   r[   Úlast_dropoutrÈ   )ra   rJ   Únum_classesÚactivation_stringrb   s       €rB   rI   zGPT2SequenceSummary.__init__S  sœ  ø€ Ý‰Œ×ÒÑÔÐå# F¨N¸FÑCÔCˆÔØÔ Ò&Ð&õ &Ð%å”{‘}”}ˆŒÝ�6Ð-Ñ.Ô.ð 	F°6Ô3Jð 	FÝ�vÐ7Ñ8Ô8ð 1¸VÔ=Zð 1Ð_eÔ_pÐstÒ_tÐ_tØ$Ô/��à$Ô0�Ýœ9 VÔ%7¸ÑEÔEˆDŒLå# FÐ,@À$ÑGÔGÐØIZÐ$m¥NÐ3DÑ$EÔ$EÐ$EÕ`bÔ`kÑ`mÔ`mˆŒåœ[™]œ]ˆÔÝ�6Ð2Ñ3Ô3ð 	J¸Ô8TÐWXÒ8XÐ8XÝ!#¤¨FÔ,HÑ!IÔ!IˆDÔåœK™MœMˆÔÝ�6Ð1Ñ2Ô2ð 	H°vÔ7RÐUVÒ7VÐ7VÝ "¤
¨6Ô+FÑ GÔ GˆDÔÐÐð	Hð 	HÐ7VÐ7VrD   Nru   Ú	cls_indexrz   c                 ó:  — | j         dk    r|dd…df         }�n-| j         dk    r|dd…df         }�n| j         dk    r|                     d¬¦  «        }nò| j         d	k    rÕ|€=t          j        |d
dd…dd…f         |j        d         dz
  t          j        ¬¦  «        }nl|                     d¦  «                             d¦  «        }|                     d|                     ¦   «         dz
  z  | 	                    d¦  «        fz   ¦  «        }| 
                    d|¦  «                             d¦  «        }n| j         dk    rt          ‚|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        rÃ   Nr(   Úfirstr   Úmeanr$   r+   rÔ   .r*   )r7   ©r(   r³   )rÂ   r×   r1   Ú	full_liker‡   ÚlongÚ	unsqueezeÚexpandr,   r0   ÚgatherÚsqueezerÊ   rÐ   rÌ   rÏ   rÑ   )ra   ru   rÔ   Úoutputs       rB   r•   zGPT2SequenceSummary.forwardp  sª  € ð Ô Ò&Ð&Ø" 1 1 1 b 5Ô)ˆF‰FØÔ 'Ò)Ð)Ø" 1 1 1 a 4Ô(ˆF‰FØÔ &Ò(Ð(Ø"×'Ò'¨AÐ'Ñ.Ô.ˆFˆFØÔ +Ò-Ð-ØÐ Ý!œOØ! # r¨ r¨1¨1¨1 *Ô-Ø!Ô'¨Ô+¨aÑ/Ýœ*ðñ ô �	�	ð &×/Ò/°Ñ3Ô3×=Ò=¸bÑAÔA�	Ø%×,Ò,¨U°i·m²m±o´oÈÑ6IÑ-JÈm×N`ÒN`ÐacÑNdÔNdÐMfÑ-fÑgÔg�	à"×)Ò)¨"¨iÑ8Ô8×@Ò@ÀÑDÔDˆFˆFØÔ &Ò(Ð(Ý%Ð%à×#Ò# FÑ+Ô+ˆØ—’˜fÑ%Ô%ˆØ—’ Ñ(Ô(ˆØ×"Ò" 6Ñ*Ô*ˆàˆrD   r–   )r˜   r™   rš   Ú__doc__r%   rI   r1   rœ   Ú
LongTensorr•   rŸ   r    s   @rB   rÀ   rÀ   9  s›   ø€ € € € € ðð ð2H˜zð Hð Hð Hð Hð Hð Hð< VZð)ð )Ø"Ô.ð)Ø;@Ô;KÈdÑ;Rð)à	Ô	ð)ð )ð )ð )ð )ð )ð )ð )rD   rÀ   c                   ó¼   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZe eedd¬¦  «         eed	d¬¦  «        d
œZddgZ ej        ¦   «         ˆ fd„¦   «         Zˆ xZS )ÚGPT2PreTrainedModelrJ   ÚtransformerTrª   rv   z.attnr$   )Ú
layer_nameÚindexz.crossattention)ru   Ú
attentionsÚcross_attentionsz	attn.biaszcrossattention.biasc           
      óô  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rFt	          j        |j        d| j        j        ¬¦  «         |j	        �t	          j
        |j	        ¦  «         t          |t          ¦  «        rc|                     ¦   «         D ]P\  }}|dk    rCt	          j        |d| j        j        t          j        d| j        j        z  ¦  «        z  ¬¦  «         ŒOdS dS )zInitialize the weights.r&   )r×   ÚstdNzc_proj.weightr/   )rH   Ú_init_weightsr}   r   ÚinitÚnormal_ÚweightrJ   Úinitializer_rangeÚbiasÚzeros_r   Únamed_parametersÚmathÚsqrtÚn_layer)ra   r9   Únamer-   rb   s       €rB   rë   z!GPT2PreTrainedModel._init_weights°  sû   ø€ õ 	‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�fÑ%Ô%ð 	)ÝŒL˜œ¨S°d´kÔ6SÐTÑTÔTÐTØŒ{Ð&Ý”˜FœKÑ(Ô(Ð(õ �f�oÑ.Ô.ð 	vØ!×2Ò2Ñ4Ô4ð vð v‘��aØ˜?Ò*Ð*å”L ¨°$´+Ô2OÕRVÔR[Ð\]Ð`dÔ`kÔ`sÑ\sÑRtÔRtÑ2tÐuÑuÔuÐuøð		vð 	vðvð vrD   )r˜   r™   rš   r%   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_attention_backendÚ_can_compile_fullgraphrª   r"   rF   Ú_can_record_outputsÚ"_keys_to_ignore_on_load_unexpectedr1   Úno_gradrë   rŸ   r    s   @rB   rã   rã   œ  sÝ   ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ€NØ"&ÐØ!Ðà"Ø$�n ]¸wÈaÐPÑPÔPØ*˜N¨=ÐEVÐ^_Ð`Ñ`Ô`ðð Ðð +6Ð7LÐ)MÐ&à€U„]�_„_ðvð vð vð vñ „_ðvð vð vð vð vrD   rã   z^
    Base class for outputs of models predicting if two sentences are consecutive or not.
    )Úcustom_introc                   óô   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	ej        dz  ed<   dZ
ej        dz  ed<   dZedz  ed<   dZeej                 dz  ed<   dZeej                 dz  ed	<   dS )
ÚGPT2DoubleHeadsModelOutputa\  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    mc_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `mc_labels` is provided):
        Multiple choice classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    mc_logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    NÚlossÚmc_lossÚlogitsÚ	mc_logitsrv   ru   rç   )r˜   r™   rš   rà   r  r1   rœ   r÷   r  r  r	  rv   r   ru   r›   rç   © rD   rB   r  r  Æ  sÌ   € € € € € € ðð ð  &*€Dˆ%Ô
˜dÑ
"Ð)Ð)Ñ)Ø(,€GˆUÔ Ñ%Ð,Ð,Ñ,Ø'+€FˆEÔ Ñ$Ð+Ð+Ñ+Ø*.€IˆuÔ  4Ñ'Ð.Ð.Ñ.Ø$(€O�U˜T‘\Ð(Ð(Ñ(Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6rD   r  c                   ó  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 	 	 	 	 	 	 	 dde	j
        dz  dedz  de	j        dz  de	j
        dz  d	e	j
        dz  d
e	j        dz  de	j        dz  de	j        dz  dedz  defd„¦   «         ¦   «         ¦   «         Zˆ xZS )Ú	GPT2Modelc                 ó6  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        t	          j        ‰j        | j        ¦  «        | _        t	          j        ‰j        | j        ¦  «        | _	        t	          j
        ‰j        ¦  «        | _        t	          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t	          j        | j        ‰j        ¬¦  «        | _        d| _        ‰j        | _        |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rV   )rª   )Ú.0ÚirJ   s     €rB   ú
<listcomp>z&GPT2Model.__init__.<locals>.<listcomp>ñ  s&   ø€ ÐhÐhÐhÀ1¥	¨&¸AÐ >Ñ >Ô >ÐhÐhÐhrD   r­   F)rH   rI   rK   rL   r   Ú	EmbeddingÚ
vocab_sizeÚwteÚmax_position_embeddingsÚwper[   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚnum_hidden_layersÚhr°   r±   Úln_fÚgradient_checkpointingrŠ   Ú	post_init©ra   rJ   rb   s    `€rB   rI   zGPT2Model.__init__è  sá   øø€ Ý‰Œ×Ò˜Ñ Ô Ð àÔ+ˆŒå”< Ô 1°4´>ÑBÔBˆŒÝ”< Ô >ÀÄÑOÔOˆŒå”J˜vÔ0Ñ1Ô1ˆŒ	Ý”ÐhÐhÐhÐhÍÈfÔNfÑHgÔHgÐhÑhÔhÑiÔiˆŒÝ”L ¤°VÔ5NÐOÑOÔOˆŒ	à&+ˆÔ#Ø$*Ô$?ˆÔ!ð 	�ŠÑÔÐÐÐrD   c                 ó   — | j         S r–   ©r  )ra   s    rB   Úget_input_embeddingszGPT2Model.get_input_embeddingsú  s	   € ØŒxˆrD   c                 ó   — || _         d S r–   r"  )ra   Únew_embeddingss     rB   Úset_input_embeddingszGPT2Model.set_input_embeddingsý  s   € Ø!ˆŒˆˆrD   NÚ	input_idsrv   r=   Útoken_type_idsÚposition_idsÚinputs_embedsrw   rx   rº   rz   c
                 ó*  — |
                      dd¦  «         |
                      dd¦  «         |�|�t          d¦  «        ‚|�T|                      ||¦  «         |                     ¦   «         }|                     d|d         ¦  «        }|j        d         }n;|�*|                     ¦   «         dd…         }|j        d         }nt          d¦  «        ‚|�|                     d|d         ¦  «        }|	r[|€t          | j        ¬¦  «        }| j        j        r8t          |t          ¦  «        s#t          |t          | j        ¬¦  «        ¦  «        }|€|                      |¦  «        }|€V|�|                     ¦   «         nd}t          j        |j        d	         |j        ¬
¦  «        |z   }|                     d¦  «        }|                      |¦  «        }||                     |j        ¦  «        z   }|�!|j        dk     r|                     |d¦  «        }t)          | j        ||||¬¦  «        }d}|�t+          | j        |||¬¦  «        }|�|                      |¦  «        }||z   }|                      |¦  «        }d|d	d…         z   |                     d¦  «        fz   }t/          | j        ¦  «        D ]%\  }} ||| j        r| j        s|nd||f||	|dœ|
¤Ž}Œ&|                      |¦  «        }|                     |¦  «        }|	r|nd}t9          ||¬¦  «        S )á¾  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        ry   NÚoutput_hidden_stateszDYou cannot specify both input_ids and inputs_embeds at the same timer(   r   z5You have to specify either input_ids or inputs_embeds)rJ   r$   ©rd   r¬   )rJ   r*  r=   rv   r)  )rJ   r*  r=   rw   rØ   )rx   rº   r)  )Úlast_hidden_staterv   )ÚpoprQ   Ú%warn_if_padding_and_no_attention_maskr0   rˆ   r‡   r   rJ   rµ   r}   r   r  Úget_seq_lengthr1   Úarangerd   rÛ   r  ÚtoÚndimr   r   r  Ú	enumerater  r  r.   r  r   )ra   r'  rv   r=   r(  r)  r*  rw   rx   rº   r?   Úinput_shapeÚ
batch_sizeÚpast_seen_tokensÚposition_embedsru   Úcausal_maskÚtoken_type_embedsÚoutput_shaper  Úblocks                        rB   r•   zGPT2Model.forward   s“  € ð< 	�
Š
Ð&¨Ñ-Ô-Ð-Ø�
Š
Ð)¨4Ñ0Ô0Ð0àÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKØ!Ÿš r¨;°r¬?Ñ;Ô;ˆIØ"œ¨Ô+ˆJˆJØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKØ&Ô,¨QÔ/ˆJˆJåÐTÑUÔUÐUàÐ%Ø+×0Ò0°°[À´_ÑEÔEˆNð ð 	iØÐ&Ý".°d´kÐ"BÑ"BÔ"B�àŒ{Ô.ð iµzÀ/ÕSfÑ7gÔ7gð iÝ"5°oÅ|Ð[_Ô[fÐGgÑGgÔGgÑ"hÔ"h�àÐ Ø ŸHšH YÑ/Ô/ˆMàÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLàŸ(š( <Ñ0Ô0ˆØ%¨×(:Ò(:¸=Ô;OÑ(PÔ(PÑPˆð Ð%¨.Ô*=ÀÒ*AÐ*AØ+×0Ò0°¸RÑ@Ô@ˆNå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð "&ÐØ Ð,Ý%>Ø”{Ø+Ø5Ø&;ð	&ñ &ô &Ð"ð Ð%Ø $§¢¨Ñ 8Ô 8ÐØ)Ð,=Ñ=ˆMàŸ	š	 -Ñ0Ô0ˆà˜{¨1¨2¨2œÑ.°-×2DÒ2DÀRÑ2HÔ2HÐ1JÑJˆå! $¤&Ñ)Ô)ð 
	ð 
	‰HˆAˆuØ!˜EØØ(,Ô(CÐ`ÈÌÐ`��Ð\`ØØ%ð		ð
 (>Ø#Ø)ð	ð 	ð ð	ð 	ˆMˆMð Ÿ	š	 -Ñ0Ô0ˆà%×*Ò*¨<Ñ8Ô8ˆà-6Ð@˜/˜/¸DˆÝ8Ø+Ø+ð
ñ 
ô 
ð 	
rD   )	NNNNNNNNN)r˜   r™   rš   rI   r#  r&  r!   r#   r   r1   rá   r   rœ   r�   rž   r   r•   rŸ   r    s   @rB   r  r  æ  sa  ø€ € € € € ðð ð ð ð ð$ð ð ð"ð "ð "ð  ØØð .2Ø(,Ø37Ø26Ø04Ø26Ø59Ø;?Ø!%ðr
ð r
àÔ# dÑ*ðr
ð  ™ðr
ð Ô)¨DÑ0ð	r
ð
 Ô(¨4Ñ/ðr
ð Ô&¨Ñ-ðr
ð Ô(¨4Ñ/ðr
ð  %œ|¨dÑ2ðr
ð !&Ô 1°DÑ 8ðr
ð ˜$‘;ðr
ð 
3ðr
ð r
ð r
ñ „^ñ „_ñ  Ôðr
ð r
ð r
ð r
ð r
rD   r  z‡
    The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                   ó2  ‡ — e Zd ZddiZˆ fd„Zee	 	 	 	 	 	 	 	 	 	 	 ddej        dz  de	dz  dej
        dz  d	ej        dz  d
ej        dz  dej
        dz  dej        dz  dej
        dz  dej        dz  dedz  deej        z  defd„¦   «         ¦   «         Zˆ xZS )ÚGPT2LMHeadModelúlm_head.weightútransformer.wte.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NF©rð   )
rH   rI   r  rä   r   rÎ   Ún_embdr  Úlm_headr  r   s     €rB   rI   zGPT2LMHeadModel.__init__�  s`   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆÔÝ”y ¤°Ô0AÈÐNÑNÔNˆŒð 	�ŠÑÔÐÐÐrD   Nr   r'  rv   r=   r(  r)  r*  rw   rx   Úlabelsrº   Úlogits_to_keeprz   c                 ób  —  | j         |f||||||||
dœ|¤Ž}|j        }t          |t          ¦  «        rt	          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|	� | j        ||	fd| j        j        i|¤Ž}t          |||j
        |j        |j        |j        ¬¦  «        S )ai  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        )rv   r=   r(  r)  r*  rw   rx   rº   Nr  )r  r  rv   ru   rç   rè   )rä   r/  r}   ÚintÚslicerG  Úloss_functionrJ   r  r   rv   ru   rç   rè   )ra   r'  rv   r=   r(  r)  r*  rw   rx   rH  rº   rI  r?   Útransformer_outputsru   Úslice_indicesr  r  s                     rB   r•   zGPT2LMHeadModel.forward‰  s   € ðF JZÈÔIYØðJ
à+Ø)Ø)Ø%Ø'Ø"7Ø#9ØðJ
ð J
ð ðJ
ð J
Ðð ,Ô=ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐà%�4Ô%ØØðð ð  œ;Ô1ðð ð	ð ˆDõ 1ØØØ/Ô?Ø-Ô;Ø*Ô5Ø0ÔAð
ñ 
ô 
ð 	
rD   )NNNNNNNNNNr   )r˜   r™   rš   Ú_tied_weights_keysrI   r   r   r1   rá   r   rœ   r�   rž   rK  r   r•   rŸ   r    s   @rB   r@  r@  x  sr  ø€ € € € € ð +Ð,DÐEÐðð ð ð ð ð Øð .2Ø(,Ø37Ø26Ø04Ø26Ø59Ø;?Ø*.Ø!%Ø-.ðC
ð C
àÔ# dÑ*ðC
ð  ™ðC
ð Ô)¨DÑ0ð	C
ð
 Ô(¨4Ñ/ðC
ð Ô&¨Ñ-ðC
ð Ô(¨4Ñ/ðC
ð  %œ|¨dÑ2ðC
ð !&Ô 1°DÑ 8ðC
ð Ô  4Ñ'ðC
ð ˜$‘;ðC
ð ˜eœlÑ*ðC
ð 
+ðC
ð C
ð C
ñ „^ñ ÔðC
ð C
ð C
ð C
ð C
rD   r@  a  
        The GPT2 Model transformer with a language modeling and a multiple-choice classification head on top e.g. for
    RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the
    input embeddings, the classification head takes as input the input of a specified classification token index in the
    input sequence).
    c                   ó  ‡ — e Zd ZddiZˆ fd„Zee	 	 	 	 	 	 	 	 	 	 ddej        dz  de	dz  dej
        dz  dej        dz  d	ej        dz  d
ej
        dz  dej        dz  dej        dz  dej        dz  dedz  defd„¦   «         ¦   «         Zˆ xZS )ÚGPT2DoubleHeadsModelrA  rB  c                 ó  •— t          ¦   «                              |¦  «         d|_        t          |¦  «        | _        t          j        |j        |j        d¬¦  «        | _	        t          |¦  «        | _        |                      ¦   «          d S )Nr$   FrE  )rH   rI   rÍ   r  rä   r   rÎ   rF  r  rG  rÀ   Úmultiple_choice_headr  r   s     €rB   rI   zGPT2DoubleHeadsModel.__init__Ü  sx   ø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆÔÝ$ VÑ,Ô,ˆÔÝ”y ¤°Ô0AÈÐNÑNÔNˆŒÝ$7¸Ñ$?Ô$?ˆÔ!ð 	�ŠÑÔÐÐÐrD   Nr'  rv   r=   r(  r)  r*  Úmc_token_idsrH  Ú	mc_labelsrº   rz   c           
      ó  —  | j         |f||||||
dœ|¤Ž}|j        }|                      |¦  «        }|                      ||¦  «                             d¦  «        }d}|	�Tt          ¦   «         } ||                     d|                     d¦  «        ¦  «        |	                     d¦  «        ¦  «        }d}|�­|                     |j	        ¦  «        }|ddd…dd…f          
                    ¦   «         }|ddd…f          
                    ¦   «         }t          ¦   «         } ||                     d|                     d¦  «        ¦  «        |                     d¦  «        ¦  «        }t          |||||j        |j        |j        ¬¦  «        S )af  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        mc_token_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
            Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
            1]`.
        labels (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids`. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to
            `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`
        mc_labels (`torch.LongTensor` of shape `(batch_size)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoTokenizer, GPT2DoubleHeadsModel

        >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
        >>> model = GPT2DoubleHeadsModel.from_pretrained("openai-community/gpt2")

        >>> # Add a [CLS] to the vocabulary (we should train it also!)
        >>> num_added_tokens = tokenizer.add_special_tokens({"cls_token": "[CLS]"})
        >>> # Update the model embeddings with the new vocabulary size
        >>> embedding_layer = model.resize_token_embeddings(len(tokenizer))

        >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"]
        >>> encoded_choices = [tokenizer.encode(s) for s in choices]
        >>> cls_token_location = [tokens.index(tokenizer.cls_token_id) for tokens in encoded_choices]

        >>> input_ids = torch.tensor(encoded_choices).unsqueeze(0)  # Batch size: 1, number of choices: 2
        >>> mc_token_ids = torch.tensor([cls_token_location])  # Batch size: 1

        >>> outputs = model(input_ids, mc_token_ids=mc_token_ids)
        >>> lm_logits = outputs.logits
        >>> mc_logits = outputs.mc_logits
        ```©rv   r=   r(  r)  r*  rº   r(   N.r$   )r  r  r  r	  rv   ru   rç   )rä   r/  rG  rT  rÞ   r   rˆ   r0   r4  rd   rŒ   r  rv   ru   rç   )ra   r'  rv   r=   r(  r)  r*  rU  rH  rV  rº   r?   rN  ru   Ú	lm_logitsr	  r  Úloss_fctÚlm_lossÚshift_logitsÚshift_labelss                        rB   r•   zGPT2DoubleHeadsModel.forwardæ  sµ  € ðB JZÈÔIYØð	J
à+Ø)Ø)Ø%Ø'Øð	J
ð 	J
ð ð	J
ð 	J
Ðð ,Ô=ˆà—L’L Ñ/Ô/ˆ	Ø×-Ò-¨m¸\ÑJÔJ×RÒRÐSUÑVÔVˆ	àˆØÐ Ý'Ñ)Ô)ˆHØ�h˜yŸ~š~¨b°)·.².ÀÑ2DÔ2DÑEÔEÀyÇ~Â~ÐVXÑGYÔGYÑZÔZˆGØˆØÐØ—Y’Y˜yÔ/Ñ0Ô0ˆFØ$ S¨#¨2¨#¨q¨q¨q [Ô1×<Ò<Ñ>Ô>ˆLØ! # q r r 'œ?×5Ò5Ñ7Ô7ˆLÝ'Ñ)Ô)ˆHØ�h˜|×0Ò0°°\×5FÒ5FÀrÑ5JÔ5JÑKÔKÈ\×M^ÒM^Ð_aÑMbÔMbÑcÔcˆGå)ØØØØØ/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
rD   )
NNNNNNNNNN)r˜   r™   rš   rP  rI   r   r   r1   rá   r   rœ   rž   r  r•   rŸ   r    s   @rB   rR  rR  Ñ  s^  ø€ € € € € ð +Ð,DÐEÐðð ð ð ð ð Øð .2Ø(,Ø37Ø26Ø04Ø26Ø04Ø*.Ø-1Ø!%ðc
ð c
àÔ# dÑ*ðc
ð  ™ðc
ð Ô)¨DÑ0ð	c
ð
 Ô(¨4Ñ/ðc
ð Ô&¨Ñ-ðc
ð Ô(¨4Ñ/ðc
ð Ô&¨Ñ-ðc
ð Ô  4Ñ'ðc
ð Ô# dÑ*ðc
ð ˜$‘;ðc
ð 
$ðc
ð c
ð c
ñ „^ñ Ôðc
ð c
ð c
ð c
ð c
rD   rR  aÔ  
    The GPT2 Model transformer with a sequence classification head on top (linear layer).

    [`GPT2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   óè   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dedz  dej	        dz  dej        dz  dej        dz  dej	        dz  d	ej        dz  d
e
dz  defd„¦   «         ¦   «         Zˆ xZS )ÚGPT2ForSequenceClassificationc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S rD  )
rH   rI   rÍ   r  rä   r   rÎ   rF  Úscorer  r   s     €rB   rI   z&GPT2ForSequenceClassification.__init__]  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ$ VÑ,Ô,ˆÔÝ”Y˜vœ}¨d¬oÀEÐJÑJÔJˆŒ
ð 	�ŠÑÔÐÐÐrD   Nr'  rv   r=   r(  r)  r*  rH  rº   rz   c	           
      ó¬  —  | j         |f||||||dœ|	¤Ž}
|
j        }|                      |¦  «        }|�|j        dd…         \  }}n|j        dd…         \  }}| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k                         |j        t          j
        ¦  «        }t          j        |j        d         |j        t          j
        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j        ||j        ¬	¦  «        |f         }d}|��Z| j        j        €f| j        dk    rd
| j        _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        d
k    rWt-          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j        j        dk    rGt1          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt5          ¦   «         } |||¦  «        }t7          |||
j        |
j        |
j        ¬¦  «        S )aB  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rX  Nr/   r$   z=Cannot handle batch sizes > 1 if no padding token is defined.r(   )rd   r7   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r.  Ú
regressionÚsingle_label_classificationÚmulti_label_classification)r  r  rv   ru   rç   )rä   r/  ra  r‡   rJ   Úpad_token_idrQ   r4  rd   r1   Úint32r3  ÚargmaxÚloggerÚwarning_oncerb   r˜   Úproblem_typerÍ   r7   rÚ   rK  r   rÞ   r   rˆ   r   r   rv   ru   rç   )ra   r'  rv   r=   r(  r)  r*  rH  rº   r?   rN  ru   r  r8  Úsequence_lengthÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr  rZ  s                        rB   r•   z%GPT2ForSequenceClassification.forwardf  s  € ð@ JZÈÔIYØð	J
à+Ø)Ø)Ø%Ø'Øð	J
ð 	J
ð ð	J
ð 	J
Ðð ,Ô=ˆØ—’˜MÑ*Ô*ˆàÐ Ø*3¬/¸"¸1¸"Ô*=Ñ'ˆJ˜˜à*7Ô*=¸b¸q¸bÔ*AÑ'ˆJ˜àŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 M×$9Ò$9Ñ$;Ô$;¸V¿^º^Ñ=MÔ=MÑNÔN�D�Dà#˜8 M°6Ñ:Ô:�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x × 2Ò 2°2°t´Ñ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨vÑ6Ô6�Ý/ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
rD   ©NNNNNNNN)r˜   r™   rš   rI   r   r   r1   rá   r   rœ   rž   r   r•   rŸ   r    s   @rB   r_  r_  N  s$  ø€ € € € € ðð ð ð ð ð Øð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%ð^
ð ^
àÔ# dÑ*ð^
ð  ™ð^
ð Ô)¨DÑ0ð	^
ð
 Ô(¨4Ñ/ð^
ð Ô&¨Ñ-ð^
ð Ô(¨4Ñ/ð^
ð Ô  4Ñ'ð^
ð ˜$‘;ð^
ð 
*ð^
ð ^
ð ^
ñ „^ñ Ôð^
ð ^
ð ^
ð ^
ð ^
rD   r_  c                   óè   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dedz  dej	        dz  dej        dz  dej        dz  dej	        dz  d	ej        dz  d
e
dz  defd„¦   «         ¦   «         Zˆ xZS )ÚGPT2ForTokenClassificationc                 ó¬  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |d¦  «        r|j        �|j        }n!t          |d¦  «        r|j        �|j        }nd}t          j	        |¦  «        | _
        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )NÚclassifier_dropoutÚhidden_dropoutgš™™™™™¹?)rH   rI   rÍ   r  rä   r‚   ru  rv  r   r[   r8   rÎ   rK   Ú
classifierr  )ra   rJ   ru  rb   s      €rB   rI   z#GPT2ForTokenClassification.__init__Ë  sÌ   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå$ VÑ,Ô,ˆÔÝ�6Ð/Ñ0Ô0ð 	%°VÔ5NÐ5ZØ!'Ô!:ÐÐÝ�VÐ-Ñ.Ô.ð 	%°6Ô3HÐ3TØ!'Ô!6ÐÐà!$ÐÝ”zÐ"4Ñ5Ô5ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐrD   Nr'  rv   r=   r(  r)  r*  rH  rº   rz   c	           
      ó�  —  | j         |f||||||dœ|	¤Ž}
|
j        }|                      |¦  «        }|                      |¦  «        }d}|�`|                     |j        ¦  «        }t          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }t          |||
j
        |
j        ¬¦  «        S )aR  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
            `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input
            sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rX  Nr(   )r  r  ru   rç   )rä   r/  r8   rw  r4  rd   r   rˆ   rÍ   r   ru   rç   )ra   r'  rv   r=   r(  r)  r*  rH  rº   r?   rN  ru   r  r  rZ  s                  rB   r•   z"GPT2ForTokenClassification.forwardÜ  só   € ð@ JZÈÔIYØð	J
à+Ø)Ø)Ø%Ø'Øð	J
ð 	J
ð ð	J
ð 	J
Ðð ,Ô=ˆØŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÐØ—Y’Y˜vœ}Ñ-Ô-ˆFÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDå$ØØØ-Ô;Ø*Ô5ð	
ñ 
ô 
ð 	
rD   rq  )r˜   r™   rš   rI   r   r   r1   rá   r   rœ   rž   r   r•   rŸ   r    s   @rB   rs  rs  É  s  ø€ € € € € ðð ð ð ð ð" Øð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%ð8
ð 8
àÔ# dÑ*ð8
ð  ™ð8
ð Ô)¨DÑ0ð	8
ð
 Ô(¨4Ñ/ð8
ð Ô&¨Ñ-ð8
ð Ô(¨4Ñ/ð8
ð Ô  4Ñ'ð8
ð ˜$‘;ð8
ð 
ð8
ð 8
ð 8
ñ „^ñ Ôð8
ð 8
ð 8
ð 8
ð 8
rD   rs  c                   óæ   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	fd„¦   «         ¦   «         Z
ˆ xZS )ÚGPT2ForQuestionAnsweringc                 óð   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        d¦  «        | _        |  	                    ¦   «          d S )Nr/   )
rH   rI   rÍ   r  rä   r   rÎ   rK   Ú
qa_outputsr  r   s     €rB   rI   z!GPT2ForQuestionAnswering.__init__  sc   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ$ VÑ,Ô,ˆÔÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐrD   Nr'  r=   r(  r)  r*  Ústart_positionsÚend_positionsrz   c                 ó¦  —  | j         |f||||dœ|¤Ž}	|	j        }
|                      |
¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d}|��|��t          |                     ¦   «         ¦  «        dk    r-|                     d¦  «                             |j	        ¦  «        }t          |                     ¦   «         ¦  «        dk    r-|                     d¦  «                             |j	        ¦  «        }|                     d¦  «        }| 
                    d|¦  «        }| 
                    d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }t          ||||	j        |	j        ¬	¦  «        S )
r,  )r=   r(  r)  r*  r$   r(   r+   Nr   )Úignore_indexr/   )r  Ústart_logitsÚ
end_logitsru   rç   )rä   r/  r|  r†   rÞ   rŒ   Úlenr0   r4  rd   Úclampr   r   ru   rç   )ra   r'  r=   r(  r)  r*  r}  r~  r?   ÚoutputsÚsequence_outputr  r�  r‚  Ú
total_lossÚignored_indexrZ  Ú
start_lossÚend_losss                      rB   r•   z GPT2ForQuestionAnswering.forward$  sï  € ð6 >N¸TÔ=MØð>
à)Ø)Ø%Ø'ð>
ð >
ð ð>
ð >
ˆð "Ô3ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÑ&¨=Ñ+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=×"@Ò"@ÀÔATÑ"UÔ"U�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9× <Ò <¸ZÔ=NÑ OÔ O�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
rD   )NNNNNNN)r˜   r™   rš   rI   r   r   r1   rá   rœ   r   r•   rŸ   r    s   @rB   rz  rz    s  ø€ € € € € ðð ð ð ð ð Øð .2Ø37Ø26Ø04Ø26Ø37Ø15ð@
ð @
àÔ# dÑ*ð@
ð Ô)¨DÑ0ð@
ð Ô(¨4Ñ/ð	@
ð
 Ô&¨Ñ-ð@
ð Ô(¨4Ñ/ð@
ð Ô)¨DÑ0ð@
ð Ô'¨$Ñ.ð@
ð 
&ð@
ð @
ð @
ñ „^ñ Ôð@
ð @
ð @
ð @
ð @
rD   rz  )rR  rz  r_  rs  r@  r  rã   )Nr&   )Grà   ró   Úcollections.abcr   Údataclassesr   r1   r   Útorch.nnr   r   r   Ú r
   rì   Úactivationsr   r   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_gpt2r%   Ú
get_loggerr˜   ri  rC   ÚModulerF   r¢   rª   rÀ   rã   r  r  r@  rR  r_  rs  rz  Ú__all__r
  rD   rB   ú<module>rž     sS  ðð "Ð !à €€€Ø $Ð $Ð $Ð $Ð $Ð $Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø JÐ JÐ JÐ JÐ JÐ JÐ JÐ JØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ #Ð #Ð #Ð #Ð #Ð #ðð ð ð ð ð ð ð ð ð ð ð ð HÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EØ *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€ð%ð %ð %ð %ð*W)ð W)ð W)ð W)ð W)�B”Iñ W)ô W)ð W)ðtð ð ð ð ˆbŒiñ ô ð ð"?ð ?ð ?ð ?ð ?Ð*ñ ?ô ?ð ?ðF`ð `ð `ð `ð `˜"œ)ñ `ô `ð `ðF ð&vð &vð &vð &vð &v˜/ñ &vô &vñ „ð&vðR €ððñ ô ð
 ð7ð 7ð 7ð 7ð 7 ñ 7ô 7ñ „ñô ð7ð4 ðN
ð N
ð N
ð N
ð N
Ð#ñ N
ô N
ñ „ðN
ðb €ððñ ô ðP
ð P
ð P
ð P
ð P
Ð)¨?ñ P
ô P
ñô ðP
ðf €ððñ ô ðr
ð r
ð r
ð r
ð r
Ð.°ñ r
ô r
ñô ðr
ðj €ððñ ô ðj
ð j
ð j
ð j
ð j
Ð$7ñ j
ô j
ñô ðj
ðZ ðL
ð L
ð L
ð L
ð L
Ð!4ñ L
ô L
ñ „ðL
ð^ ðL
ð L
ð L
ð L
ð L
Ð2ñ L
ô L
ñ „ðL
ð^ð ð €€€rD   