§
    ‚Štj[�  ã                   ó:  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$  e¦   «         rddlm%Z%  e"j&        e'¦  «        Z(de)de)dej*        fd„Z+d„ Z,dej*        dej*        fd„Z-dej*        dej*        dej*        dej*        fd„Z. G d„ dej/        ¦  «        Z0 G d „ d!e0¦  «        Z1e0e1d"œZ2 G d#„ d$ej/        ¦  «        Z3 G d%„ d&e¦  «        Z4e! G d'„ d(e¦  «        ¦   «         Z5e! G d)„ d*e5¦  «        ¦   «         Z6 e!d+¬,¦  «         G d-„ d.e5e¦  «        ¦   «         Z7 e!d/¬,¦  «         G d0„ d1e5¦  «        ¦   «         Z8e! G d2„ d3e5¦  «        ¦   «         Z9g d4¢Z:dS )5zPyTorch GPT-J model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú!flash_attn_supports_top_left_maskÚis_flash_attn_available)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPast)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú
GPTJConfig)Ú_flash_attention_forwardÚnum_posÚdimÚreturnc                 ó€  — ddt          j        d|dt           j        ¬¦  «        |z  z  z  }t          j        dt          j        | t           j        ¬¦  «                             ¦   «         |¦  «                             ¦   «         }t          j        t          j        |¦  «        t          j        |¦  «        fd¬¦  «        S )	Ng      ð?i'  r   é   )Údtypezi , j -> i jr   ©r   )ÚtorchÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r   r   Úinv_freqÚsinusoid_inps       úd/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gptj/modeling_gptj.pyÚcreate_sinusoidal_positionsr-   /   s–   € Ø�e¥¤¨Q°°Q½e¼kÐ JÑ JÔ JÈSÑ PÑQÑR€HÝ”< µ´¸WÍEÌKÐ0XÑ0XÔ0X×0^Ò0^Ñ0`Ô0`ÐbjÑkÔk×qÒqÑsÔs€LÝŒ9•e”i Ñ-Ô-­u¬y¸Ñ/FÔ/FÐGÈQÐOÑOÔOÐOó    c                 óv   — |                       |j        ¦  «                             |j        d         dd¦  «        S ©Nr   r   )ÚtoÚdeviceÚrepeatÚshape)Úembed_positionsÚposition_idss     r,   Úget_embed_positionsr7   5   s5   € Ø×Ò˜lÔ1Ñ2Ô2×9Ò9¸,Ô:LÈQÔ:OÐQRÐTUÑVÔVÐVr.   Úxc                 ó²   — | d d …d d …d d …d d d…f         }| d d …d d …d d …dd d…f         }t          j        | |fd¬¦  «        } |                      d¦  «        S )Nr   r   éÿÿÿÿr!   éþÿÿÿ)r"   ÚstackÚflatten)r8   Úx1Úx2s      r,   Úrotate_every_twor@   9   ss   € Ø	
ˆ1ˆ1ˆ1ˆaˆaˆa����C�C�a�Cˆ<Œ€BØ	
ˆ1ˆ1ˆ1ˆaˆaˆa����A�D�q�Dˆ=Ô	€BÝŒ�b�S˜"�I 2Ð&Ñ&Ô&€AØ�9Š9�R‰=Œ=Ðr.   Útensorr(   r)   c                 óÊ   — t          j        |d d …d d …d d d …f         dd¦  «        }t          j        |d d …d d …d d d …f         dd¦  «        }| |z  t          | ¦  «        |z  z   S )Nr   r   )r"   Úrepeat_interleaver@   )rA   r(   r)   s      r,   Úapply_rotary_pos_embrD   @   sy   € Ý
Ô
! # a a a¨¨¨¨D°!°!°! mÔ"4°a¸Ñ
;Ô
;€CÝ
Ô
! # a a a¨¨¨¨D°!°!°! mÔ"4°a¸Ñ
;Ô
;€CØ�S‰LÕ-¨fÑ5Ô5¸Ñ;Ñ<Ð<r.   c                   ó0  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z	 dd„Zd„ Z	 	 	 	 	 ddej	        d	e
dz  d
ej	        dz  dej        dz  dedz  dedz  deej        eej                 f         eej        eej                 eej        df         f         z  dz  fd„Zˆ xZS )ÚGPTJAttentionNc                 ó(  •— t          ¦   «                              ¦   «          || _        |j        | _        t          j        |j        ¦  «        | _        t          j        |j	        ¦  «        | _
        d| _        || _        |€(t                               d| j        j        › d�¦  «         |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t+          d| j        › d| j        › d�¦  «        ‚t-          j        | j        ¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        t          j        | j        | j        d¬¦  «        | _        |j        | _        | j        p| j        | _        |                       d	tC          | j        | j        ¦  «        d¬
¦  «         d S )NTzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).F©Úbiasr5   )Ú
persistent)"ÚsuperÚ__init__ÚconfigÚmax_position_embeddingsÚmax_positionsr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	is_causalÚ	layer_idxÚloggerÚwarning_onceÚ	__class__Ú__name__Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚhead_dimÚ
ValueErrorÚmathÚsqrtÚ
scale_attnÚLinearÚk_projÚv_projÚq_projÚout_projÚ
rotary_dimÚpos_embd_dimÚregister_bufferr-   )ÚselfrM   rV   rY   s      €r,   rL   zGPTJAttention.__init__G   sý  ø€ Ý‰Œ×ÒÑÔÐØˆŒØ#Ô;ˆÔåœJ vÔ'8Ñ9Ô9ˆÔÝœZ¨Ô(:Ñ;Ô;ˆÔàˆŒØ"ˆŒØÐÝ×Òð, ¤Ô!8ð ,ð ,ð ,ñô ð ð  Ô+ˆŒØ#)Ô#=ˆÔ Øœ¨$Ô*BÑBˆŒØŒ=˜4Ô3Ñ3°t´~ÒEÐEÝðHÐX\ÔXfð Hð HØ+/Ô+CðHð Hð Hñô ð õ œ) D¤MÑ2Ô2ˆŒå”i ¤°´ÀUÐKÑKÔKˆŒÝ”i ¤°´ÀUÐKÑKÔKˆŒÝ”i ¤°´ÀUÐKÑKÔKˆŒÝœ	 $¤.°$´.ÀuÐMÑMÔMˆŒØ Ô+ˆŒØ œOÐ=¨t¬~ˆÔØ×ÒØÕ:¸4Ô;MÈtÔO`ÑaÔaÐnsð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r.   c                 ó€  — |                      ¦   «         dd…         ||fz   }|                     |¦  «        }|r|S t          |j        ¦  «        dk    r|                     ddddd¦  «        S t          |j        ¦  «        dk    r|                     dddd¦  «        S t          d	t          |j        ¦  «        › �¦  «        ‚)
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        Nr:   é   r   r   r   r   é   ú3Input tensor rank should be one of [4, 5], but is: )ÚsizeÚviewÚlenr4   Úpermuter_   )rk   rA   r]   Úattn_head_sizeÚrotaryÚ	new_shapes         r,   Ú_split_headszGPTJAttention._split_headsl   sÃ   € ð —K’K‘M”M # 2 #Ô&Ð*=¸~Ð)NÑNˆ	Ø—’˜YÑ'Ô'ˆØð 	ØˆMÝˆvŒ|ÑÔ Ò!Ð!Ø—>’> ! Q¨¨1¨aÑ0Ô0Ð0Ý�”ÑÔ !Ò#Ð#Ø—>’> ! Q¨¨1Ñ-Ô-Ð-åÐfÕSVÐW]ÔWcÑSdÔSdÐfÐfÑgÔgÐgr.   c                 óÈ  — t          |j        ¦  «        dk    r,|                     ddddd¦  «                             ¦   «         }ngt          |j        ¦  «        dk    r+|                     dddd¦  «                             ¦   «         }n$t	          dt          |j        ¦  «        › �¦  «        ‚|                     ¦   «         dd	…         ||z  fz   }|                     |¦  «        S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        rm   r   r   r   r   rn   ro   Nr;   )rr   r4   rs   Ú
contiguousr_   rp   rq   )rk   rA   r]   rt   rv   s        r,   Ú_merge_headszGPTJAttention._merge_heads{   sÔ   € õ ˆvŒ|ÑÔ Ò!Ð!Ø—^’^ A q¨!¨Q°Ñ2Ô2×=Ò=Ñ?Ô?ˆFˆFÝ�”ÑÔ !Ò#Ð#Ø—^’^ A q¨!¨QÑ/Ô/×:Ò:Ñ<Ô<ˆFˆFåÐfÕSVÐW]ÔWcÑSdÔSdÐfÐfÑgÔgÐgØ—K’K‘M”M # 2 #Ô&Ð*=ÀÑ*NÐ)PÑPˆ	Ø�{Š{˜9Ñ%Ô%Ð%r.   c                 óÄ  — |                      t          j        ¦  «        }|                      t          j        ¦  «        }t          j        ||                     dd¦  «        ¦  «        }|| j        z  }|�||z   }t          j                             |d¬¦  «        }|                      |j	        ¦  «        }|  
                    |¦  «        }t          j        ||¦  «        }||fS )Nr:   r;   r!   )r1   r"   Úfloat32ÚmatmulÚ	transposerb   r   Ú
functionalÚsoftmaxr    rR   )rk   ÚqueryÚkeyÚvalueÚattention_maskÚattn_weightsÚattn_outputs          r,   Ú_attnzGPTJAttention._attnˆ   sÁ   € ð —’�œÑ'Ô'ˆØ�fŠf•U”]Ñ#Ô#ˆå”| E¨3¯=ª=¸¸RÑ+@Ô+@ÑAÔAˆØ# d¤oÑ5ˆàÐ%Ø'¨.Ñ8ˆLå”}×,Ò,¨\¸rÐ,ÑBÔBˆØ#—’ u¤{Ñ3Ô3ˆØ×(Ò(¨Ñ6Ô6ˆå”l <°Ñ7Ô7ˆà˜LÐ(Ð(r.   c                 ó¶   — | j         }|j        |j        k    r!|                     |j        ¦  «        }|| _         |                     |j        d         dd¦  «        S r0   )r5   r2   r1   r3   r4   )rk   r6   r5   s      r,   Ú_get_embed_positionsz"GPTJAttention._get_embed_positions¡   sY   € ØÔ.ˆØÔ! \Ô%8Ò8Ð8Ø-×0Ò0°Ô1DÑEÔEˆOØ#2ˆDÔ Ø×%Ò% lÔ&8¸Ô&;¸QÀÑBÔBÐBr.   FÚhidden_statesÚ
layer_pastr„   r6   Ú	use_cacheÚoutput_attentionsr   .c                 óÂ  — |                       |¦  «        }|                      |¦  «        }	|                      |¦  «        }
|                      || j        | j        d¦  «        }|                      |	| j        | j        d¦  «        }	|                      |
| j        | j        d¦  «        }
|                      |¦  «        }|                     d¦  «                             dd|j	        d         ¦  «        }t          j        |d|¦  «                             |	j        ¦  «        }t          j        ||j	        d         dz  d¬¦  «        \  }}| j        �·|	d d …d d …d d …d | j        …f         }|	d d …d d …d d …| j        d …f         }|d d …d d …d d …d | j        …f         }|d d …d d …d d …| j        d …f         }t!          |||¦  «        }t!          |||¦  «        }t          j        ||gd¬¦  «        }	t          j        ||gd¬¦  «        }n"t!          |	||¦  «        }	t!          |||¦  «        }|	                     dddd¦  «        }	|                     dddd¦  «        }|�|                     |	|
| j        ¦  «        \  }	}
|                      ||	|
|¦  «        \  }}|                      || j        | j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||fS )	NTFr:   r   r   r!   r   r   )rf   rd   re   rw   r]   r^   r‰   Ú	unsqueezer3   r4   r"   Úgatherr1   r    Úsplitrh   rD   r'   rs   ÚupdaterV   r‡   rz   rg   rT   )rk   rŠ   r‹   r„   r6   rŒ   r�   Úkwargsr�   r‚   rƒ   r5   Úrepeated_position_idsÚsincosr(   r)   Úk_rotÚk_passÚq_rotÚq_passr†   r…   s                         r,   ÚforwardzGPTJAttention.forward¨   só  € ð —’˜MÑ*Ô*ˆØ�kŠk˜-Ñ(Ô(ˆØ—’˜MÑ*Ô*ˆà×!Ò! %¨Ô)AÀ4Ä=ÐRVÑWÔWˆØ×Ò  TÔ%=¸t¼}ÈdÑSÔSˆØ×!Ò! %¨Ô)AÀ4Ä=ÐRWÑXÔXˆà×3Ò3°LÑAÔAˆà ,× 6Ò 6°rÑ :Ô :× AÒ AÀ!ÀQÈÔH]Ð^`ÔHaÑ bÔ bÐÝ”˜o¨qÐ2GÑHÔH×KÒKÈCÌIÑVÔVˆÝ”;˜v v¤|°BÔ'7¸1Ñ'<À"ÐEÑEÔE‰ˆˆSàŒ?Ð&Ø˜˜˜˜1˜1˜1˜a˜a˜aÐ!2 4¤?Ð!2Ð2Ô3ˆEØ˜˜˜˜A˜A˜A˜q˜q˜q $¤/Ð"3Ð"3Ð3Ô4ˆFà˜!˜!˜!˜Q˜Q˜Q   Ð#4 T¤_Ð#4Ð4Ô5ˆEØ˜1˜1˜1˜a˜a˜a    D¤OÐ$5Ð$5Ð5Ô6ˆFå(¨°°SÑ9Ô9ˆEÝ(¨°°SÑ9Ô9ˆEå”)˜U F˜O°Ð4Ñ4Ô4ˆCÝ”I˜u f˜o°2Ð6Ñ6Ô6ˆEˆEå& s¨C°Ñ5Ô5ˆCÝ(¨°°SÑ9Ô9ˆEà�kŠk˜!˜Q  1Ñ%Ô%ˆØ—’˜a  A qÑ)Ô)ˆàÐ!Ø#×*Ò*¨3°°t´~ÑFÔF‰JˆC�ð %)§J¢J¨u°c¸5À.Ñ$QÔ$QÑ!ˆ�\à×'Ò'¨°TÔ5MÈtÌ}Ñ]Ô]ˆØ—m’m KÑ0Ô0ˆØ×(Ò(¨Ñ5Ô5ˆà˜LÐ(Ð(r.   ©N©NNNFF)rZ   Ú
__module__Ú__qualname__rL   rw   rz   r‡   r‰   r"   ÚFloatTensorr
   Ú
LongTensorÚboolÚtupleÚTensorrš   Ú__classcell__©rY   s   @r,   rF   rF   F   s\  ø€ € € € € ð#
ð #
ð #
ð #
ð #
ð #
ðJhð hð hð&ð &ð &ð$ ð)ð )ð )ð )ð2Cð Cð Cð $(Ø37Ø04Ø!&Ø).ð9)ð 9)àÔ(ð9)ð ˜D‘Lð9)ð Ô)¨DÑ0ð	9)ð
 Ô&¨Ñ-ð9)ð ˜$‘;ð9)ð   $™;ð9)ð 	ˆeŒl˜E %¤,Ô/Ð/Ô0Ø
�”˜e E¤LÔ1°5¸¼ÀsÐ9JÔ3KÐKÔ
Lñ	Mà
ñ	ð9)ð 9)ð 9)ð 9)ð 9)ð 9)ð 9)ð 9)r.   rF   c                   ó  ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej        dedz  dej        dz  dej        dz  d	e	dz  d
e	dz  de
ej        e
ej                 f         e
ej        e
ej                 e
ej        df         f         z  dz  fd„Zˆ xZS )ÚGPTJFlashAttention2aD  
    GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 ó`   •—  t          ¦   «         j        |i |¤Ž t          ¦   «         | _        d S r›   )rK   rL   r   Ú_flash_attn_uses_top_left_mask)rk   Úargsr“   rY   s      €r,   rL   zGPTJFlashAttention2.__init__ë   s6   ø€ Ø�‰ŒÔ˜$Ð) &Ð)Ð)Ð)õ
 /PÑ.QÔ.QˆÔ+Ð+Ð+r.   NFrŠ   r‹   r„   r6   rŒ   r�   r   .c           
      ó4	  — |                       |¦  «        }|                      |¦  «        }	|                      |¦  «        }
|                      || j        | j        d¦  «        }|                      |	| j        | j        d¦  «        }	|                      |
| j        | j        d¦  «        }
|                      |¦  «        }|                     d¦  «                             dd|j	        d         ¦  «        }t          j        |d|¦  «                             |	j        ¦  «        }t          j        ||j	        d         dz  d¬¦  «        \  }}| j        �·|	d d …d d …d d …d | j        …f         }|	d d …d d …d d …| j        d …f         }|d d …d d …d d …d | j        …f         }|d d …d d …d d …| j        d …f         }t!          |||¦  «        }t!          |||¦  «        }t          j        ||gd¬¦  «        }	t          j        ||gd¬¦  «        }n"t!          |	||¦  «        }	t!          |||¦  «        }|	                     dddd¦  «        }	|                     dddd¦  «        }|�|                     |	|
| j        ¦  «        \  }	}
|	                     dddd¦  «                             ¦   «         }	|                     dddd¦  «                             ¦   «         }|
                     dddd¦  «                             ¦   «         }
|j        }|j        j        d	k    r|j        j        nd
}|t          j        k    r¹t          j        |¦  «        rt          j        |¦  «        }n3t7          | j        d¦  «        r| j        j        }n| j         j        j        }t<                               d|› d�¦  «         |                     |¦  «        }|	                     |¦  «        }	|
                     |¦  «        }
| j         r| j        j!        nd}|j	        d         }tE          ||	|
|||| j#        | j$        ¬¦  «        }| %                    |j	        d         |j	        d         |j	        d         |j	        d         z  ¦  «        }|  &                    |¦  «        }|  '                    |¦  «        }||fS )NTFr:   r   r   r!   r   r   ÚmpsÚcpuÚ_is_quantizedz¾The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in ú.g        )ÚdropoutrU   Úuse_top_left_mask)(rf   rd   re   rw   r]   r^   r‰   r�   r3   r4   r"   r�   r1   r    r‘   rh   rD   r'   rs   r’   rV   ry   r2   Útyper|   Úis_autocast_enabledÚget_autocast_dtypeÚhasattrrM   ÚweightrW   rX   ÚtrainingrQ   r   rU   r©   Úreshaperg   rT   )rk   rŠ   r‹   r„   r6   rŒ   r�   r“   r�   r‚   rƒ   r5   r”   r•   r(   r)   r–   r—   r˜   r™   Úinput_dtypeÚdevice_typeÚtarget_dtypeÚattention_dropoutÚquery_lengthr…   r†   s                              r,   rš   zGPTJFlashAttention2.forwardó   s®  € ð —’˜MÑ*Ô*ˆØ�kŠk˜-Ñ(Ô(ˆØ—’˜MÑ*Ô*ˆà×!Ò! %¨Ô)AÀ4Ä=ÐRVÑWÔWˆØ×Ò  TÔ%=¸t¼}ÈdÑSÔSˆØ×!Ò! %¨Ô)AÀ4Ä=ÐRWÑXÔXˆà×3Ò3°LÑAÔAˆà ,× 6Ò 6°rÑ :Ô :× AÒ AÀ!ÀQÈÔH]Ð^`ÔHaÑ bÔ bÐÝ”˜o¨qÐ2GÑHÔH×KÒKÈCÌIÑVÔVˆÝ”;˜v v¤|°BÔ'7¸1Ñ'<À"ÐEÑEÔE‰ˆˆSàŒ?Ð&Ø˜˜˜˜1˜1˜1˜a˜a˜aÐ!2 4¤?Ð!2Ð2Ô3ˆEØ˜˜˜˜A˜A˜A˜q˜q˜q $¤/Ð"3Ð"3Ð3Ô4ˆFà˜!˜!˜!˜Q˜Q˜Q   Ð#4 T¤_Ð#4Ð4Ô5ˆEØ˜1˜1˜1˜a˜a˜a    D¤OÐ$5Ð$5Ð5Ô6ˆFå(¨°°SÑ9Ô9ˆEÝ(¨°°SÑ9Ô9ˆEå”)˜U F˜O°Ð4Ñ4Ô4ˆCÝ”I˜u f˜o°2Ð6Ñ6Ô6ˆEˆEå& s¨C°Ñ5Ô5ˆCÝ(¨°°SÑ9Ô9ˆEð
 �kŠk˜!˜Q  1Ñ%Ô%ˆØ—’˜a  A qÑ)Ô)ˆð Ð!Ø#×*Ò*¨3°°t´~ÑFÔF‰JˆC�ð �kŠk˜!˜Q  1Ñ%Ô%×0Ò0Ñ2Ô2ˆØ—’˜a  A qÑ)Ô)×4Ò4Ñ6Ô6ˆØ—’˜a  A qÑ)Ô)×4Ò4Ñ6Ô6ˆð ”kˆØ+0¬<Ô+<ÀÒ+EÐ+E�e”lÔ'Ð'È5ˆØ�%œ-Ò'Ð'ÝÔ(¨Ñ5Ô5ð 8Ý$Ô7¸ÑDÔD��å˜œ oÑ6Ô6ð 8Ø#œ{Ô0��à#œ{Ô1Ô7�å×Òð$à ð$ð $ð $ñô ð ð —H’H˜\Ñ*Ô*ˆEØ—&’&˜Ñ&Ô&ˆCØ—H’H˜\Ñ*Ô*ˆEà6:´mÐL˜DœKÔ2Ð2ÈÐà”{ 1”~ˆõ 0ØØØØØØ%Ø”nØ"ÔAð	
ñ 	
ô 	
ˆð #×*Ò*ØÔ˜qÔ! <Ô#5°aÔ#8¸,Ô:LÈQÔ:OÐR^ÔRdÐefÔRgÑ:gñ
ô 
ˆð —m’m KÑ0Ô0ˆØ×(Ò(¨Ñ5Ô5ˆØ˜LÐ(Ð(r.   rœ   )rZ   r�   rž   Ú__doc__rL   r"   rŸ   r
   r    r¡   r¢   r£   rš   r¤   r¥   s   @r,   r§   r§   ä   s)  ø€ € € € € ðð ðRð Rð Rð Rð Rð $(Ø37Ø04Ø!&Ø).ðo)ð o)àÔ(ðo)ð ˜D‘Lðo)ð Ô)¨DÑ0ð	o)ð
 Ô&¨Ñ-ðo)ð ˜$‘;ðo)ð   $™;ðo)ð 	ˆeŒl˜E %¤,Ô/Ð/Ô0Ø
�”˜e E¤LÔ1°5¸¼ÀsÐ9JÔ3KÐKÔ
Lñ	Mà
ñ	ðo)ð o)ð o)ð o)ð o)ð o)ð o)ð o)r.   r§   )ÚeagerÚflash_attention_2c                   óH   ‡ — e Zd Zˆ fd„Zdej        dz  dej        fd„Zˆ xZS )ÚGPTJMLPc                 ó(  •— t          ¦   «                              ¦   «          |j        }t          j        ||¦  «        | _        t          j        ||¦  «        | _        t          |j                 | _	        t          j
        |j        ¦  «        | _        d S r›   )rK   rL   Ún_embdr   rc   Úfc_inÚfc_outr	   Úactivation_functionÚactrP   rS   r°   )rk   Úintermediate_sizerM   r\   rY   s       €r,   rL   zGPTJMLP.__init__l  so   ø€ Ý‰Œ×ÒÑÔÐØ”Mˆ	å”Y˜yÐ*;Ñ<Ô<ˆŒ
Ý”iÐ 1°9Ñ=Ô=ˆŒå˜&Ô4Ô5ˆŒÝ”z &Ô"4Ñ5Ô5ˆŒˆˆr.   rŠ   Nr   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r›   )rÅ   rÈ   rÆ   r°   )rk   rŠ   s     r,   rš   zGPTJMLP.forwardv  sL   € ØŸ
š
 =Ñ1Ô1ˆØŸš Ñ/Ô/ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐr.   )rZ   r�   rž   rL   r"   rŸ   rš   r¤   r¥   s   @r,   rÂ   rÂ   k  se   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð UÔ%6¸Ñ%=ð À%ÔBSð ð ð ð ð ð ð ð r.   rÂ   c                   óè   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej        dz  dedz  dej        dz  dej        dz  dedz  d	edz  d
e	ej
                 e	ej
        e	ej        df         f         z  dz  fd„Zˆ xZS )Ú	GPTJBlockNc                 ó,  •— t          ¦   «                              ¦   «          |j        �|j        n	d|j        z  }t	          j        |j        |j        ¬¦  «        | _        t          |j	                 ||¦  «        | _
        t          ||¦  «        | _        d S )Nrn   ©Úeps)rK   rL   Ún_innerrÄ   r   Ú	LayerNormÚlayer_norm_epsilonÚln_1ÚGPTJ_ATTENTION_CLASSESÚ_attn_implementationÚattnrÂ   Úmlp)rk   rM   rV   Ú	inner_dimrY   s       €r,   rL   zGPTJBlock.__init__  s|   ø€ Ý‰Œ×ÒÑÔÐØ&,¤nÐ&@�F”N�NÀaÈ&Ì-ÑFWˆ	Ý”L ¤°FÔ4MÐNÑNÔNˆŒ	Ý*¨6Ô+FÔGÈÐPYÑZÔZˆŒ	Ý˜9 fÑ-Ô-ˆŒˆˆr.   FrŠ   r‹   r„   r6   rŒ   r�   r   .c                 ó®   — |}|                       |¦  «        }|                      ||||||¬¦  «        \  }	}
|                      |¦  «        }|	|z   |z   }||
fS )N)rŠ   r‹   r„   r6   rŒ   r�   )rÓ   rÖ   r×   )rk   rŠ   r‹   r„   r6   rŒ   r�   r“   ÚresidualÚattn_outputsr…   Úfeed_forward_hidden_statess               r,   rš   zGPTJBlock.forward†  sx   € ð !ˆØŸ	š	 -Ñ0Ô0ˆØ%)§Y¢YØ'Ø!Ø)Ø%ØØ/ð &/ñ &
ô &
Ñ"ˆ�lð &*§X¢X¨mÑ%<Ô%<Ð"Ø$Ð'AÑAÀHÑLˆà˜lÐ*Ð*r.   r›   rœ   )rZ   r�   rž   rL   r"   rŸ   r
   r    r¡   r¢   r£   rš   r¤   r¥   s   @r,   rÌ   rÌ   ~  sú   ø€ € € € € ð.ð .ð .ð .ð .ð .ð $(Ø37Ø04Ø!&Ø).ð+ð +àÔ(¨4Ñ/ð+ð ˜D‘Lð+ð Ô)¨DÑ0ð	+ð
 Ô&¨Ñ-ð+ð ˜$‘;ð+ð   $™;ð+ð 
ˆuŒ|Ô	˜u U¤\°5¸Ô9JÈCÐ9OÔ3PÐ%PÔQÑ	QÐTXÑ	Xð+ð +ð +ð +ð +ð +ð +ð +r.   rÌ   c                   óF   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
ˆ fd„Zˆ xZS )ÚGPTJPreTrainedModelrM   ÚtransformerTrÌ   Úpast_key_valuesc                 óÜ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r4t	          j        |j        t          |j        |j	        ¦  «        ¦  «         d S d S r›   )
rK   Ú_init_weightsÚ
isinstancerF   ÚinitÚcopy_r5   r-   rO   ri   )rk   ÚmodulerY   s     €r,   râ   z!GPTJPreTrainedModel._init_weightsª  sg   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�f�mÑ,Ô,ð 	wÝŒJ�vÔ-Õ/JÈ6ÔK_ÐagÔatÑ/uÔ/uÑvÔvÐvÐvÐvð	wð 	wr.   )rZ   r�   rž   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_can_compile_fullgraphrâ   r¤   r¥   s   @r,   rÞ   rÞ      sx   ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ!Ððwð wð wð wð wð wð wð wð wr.   rÞ   c                   óø   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  de	dz  dej
        dz  dej        dz  d	ej        dz  d
ej
        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )Ú	GPTJModelc                 óî  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        | j        ¦  «        | _        t          j        ‰j	        ¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        | j        ‰j        ¬¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rV   )rÌ   )Ú.0ÚirM   s     €r,   ú
<listcomp>z&GPTJModel.__init__.<locals>.<listcomp>¹  s&   ø€ Ð^Ð^Ð^À1¥	¨&¸AÐ >Ñ >Ô >Ð^Ð^Ð^r.   rÎ   F)rK   rL   rÄ   r\   Ú
vocab_sizer   Ú	EmbeddingÚwterP   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerÚhrÑ   rÒ   Úln_fÚgradient_checkpointingÚ	post_init©rk   rM   rY   s    `€r,   rL   zGPTJModel.__init__²  sÆ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð àœˆŒØ Ô+ˆŒÝ”< Ô 1°4´>ÑBÔBˆŒÝ”J˜vÔ0Ñ1Ô1ˆŒ	Ý”Ð^Ð^Ð^Ð^ÍÈfÌnÑH]ÔH]Ð^Ñ^Ô^Ñ_Ô_ˆŒÝ”L ¤°VÔ5NÐOÑOÔOˆŒ	à&+ˆÔ#ð 	�ŠÑÔÐÐÐr.   c                 ó   — | j         S r›   ©r÷   )rk   s    r,   Úget_input_embeddingszGPTJModel.get_input_embeddingsÁ  s	   € ØŒxˆr.   c                 ó   — || _         d S r›   r  )rk   Únew_embeddingss     r,   Úset_input_embeddingszGPTJModel.set_input_embeddingsÄ  s   € Ø!ˆŒˆˆr.   NÚ	input_idsrà   r„   Útoken_type_idsr6   Úinputs_embedsrŒ   r�   Úoutput_hidden_statesÚreturn_dictr   c           	      ó„  — |�|n| j         j        }|	�|	n| j         j        }	|�|n| j         j        }|
�|
n| j         j        }
|du |duz  rt          d¦  «        ‚| j        r%| j        r|rt           	                    d¦  «         d}|€|  
                    |¦  «        }|r|€t          | j         ¬¦  «        }|j        d         }|€K|�|                     ¦   «         nd}t          j        ||j        ¬¦  «        |z   }|                     d¦  «        }t%          | j         ||||¬	¦  «        }|}|�0|                     d
|¦  «        }|  
                    |¦  «        }||z   }|                      |¦  «        }d
||                     d
¦  «        f}|rdnd}|	rdnd}t-          | j        ¦  «        D ]4\  }}|	r||fz   } |||||||¬¦  «        }|d         }|r||d         fz   }Œ5|                      |¦  «        }|                     |¦  «        }|	r||fz   }|
st3          d„ ||||fD ¦   «         ¦  «        S t5          ||||¬¦  «        S )á�  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rM   r   r   ©r2   )rM   r
  r„   rà   r6   r:   © )r‹   r„   r6   rŒ   r�   c              3   ó   K  — | ]}|®|V — Œ	d S r›   r  )rò   Úvs     r,   ú	<genexpr>z$GPTJModel.forward.<locals>.<genexpr>&  s1   è è € ð ð ØÐghÐgt�ÐgtÐgtÐgtÐgtðð r.   )Úlast_hidden_staterà   rŠ   Ú
attentions)rM   r�   r  rŒ   r  r_   rÿ   r·   rW   rX   r÷   r   r4   Úget_seq_lengthr"   r#   r2   r�   r   rq   rù   rp   Ú	enumeraterý   rþ   r¢   r   )rk   r  rà   r„   r	  r6   r
  rŒ   r�   r  r  r“   Ú
seq_lengthÚpast_key_values_lengthÚcausal_maskrŠ   Útoken_type_embedsÚoutput_shapeÚall_self_attentionsÚall_hidden_statesró   ÚblockÚoutputss                          r,   rš   zGPTJModel.forwardÇ  s*  € ð* 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ Ø ŸHšH YÑ/Ô/ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOà"Ô(¨Ô+ˆ
ØÐØIXÐId _×%CÒ%CÑ%EÔ%EÐ%EÐjkÐ"Ý œ<¨
¸=Ô;OÐPÑPÔPÐSiÑiˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆàÐ%Ø+×0Ò0°°ZÑ@Ô@ˆNØ $§¢¨Ñ 8Ô 8ÐØ)Ð,=Ñ=ˆMàŸ	š	 -Ñ0Ô0ˆØ˜J¨×(:Ò(:¸2Ñ(>Ô(>Ð?ˆà$5Ð?˜b˜b¸4ÐØ"6Ð@˜B˜B¸DÐÝ! $¤&Ñ)Ô)ð 	Jð 	J‰HˆAˆuØ#ð IØ$5¸Ð8HÑ$HÐ!à�eØØ*Ø*Ø)Ø#Ø"3ðñ ô ˆGð $ AœJˆMØ ð JØ&9¸WÀQ¼Z¸MÑ&IÐ#øàŸ	š	 -Ñ0Ô0ˆà%×*Ò*¨<Ñ8Ô8ˆàð 	EØ 1°]Ð4DÑ DÐàð 	Ýð ð Ø)¨?Ð<MÐObÐcðñ ô ñ ô ð õ 'Ø+Ø+Ø+Ø*ð	
ñ 
ô 
ð 	
r.   ©
NNNNNNNNNN)rZ   r�   rž   rL   r  r  r   r"   r    r
   rŸ   r¡   r¢   r   rš   r¤   r¥   s   @r,   rï   rï   °  s]  ø€ € € € € ðð ð ð ð ðð ð ð"ð "ð "ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø!%Ø)-Ø,0Ø#'ðg
ð g
àÔ# dÑ*ðg
ð  ™ðg
ð Ô)¨DÑ0ð	g
ð
 Ô(¨4Ñ/ðg
ð Ô&¨Ñ-ðg
ð Ô(¨4Ñ/ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð 
Ð(Ñ	(ðg
ð g
ð g
ñ „^ðg
ð g
ð g
ð g
ð g
r.   rï   zK
    The GPT-J Model transformer with a language modeling head on top.
    )Úcustom_introc                   ó   ‡ — e Zd ZddiZˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dedz  dej	        dz  d	ej        dz  d
ej        dz  dej	        dz  dej        dz  de
dz  de
dz  de
dz  de
dz  deej        z  deez  fd„¦   «         Zˆ xZS )ÚGPTJForCausalLMzlm_head.weightztransformer.wte.weightc                 óâ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r›   )
rK   rL   rï   rß   r   rc   rÄ   rõ   Úlm_headr   r  s     €r,   rL   zGPTJForCausalLM.__init__:  s[   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý$ VÑ,Ô,ˆÔÝ”y ¤°Ô0AÑBÔBˆŒð 	�ŠÑÔÐÐÐr.   Nr   r  rà   r„   r	  r6   r
  ÚlabelsrŒ   r�   r  r  Úlogits_to_keepr   c                 ó¼  — |�|n| j         j        }|                      ||||||||	|
|¬¦
  «
        }|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j         j        dœ|¤Ž}|s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        |j        ¬¦  «        S )aG  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N©	rà   r„   r	  r6   r
  rŒ   r�   r  r  r   )Úlogitsr'  rõ   r   ©Úlossr+  rà   rŠ   r  r  )rM   r  rß   rã   ÚintÚslicer&  Úloss_functionrõ   r   rà   rŠ   r  )rk   r  rà   r„   r	  r6   r
  r'  rŒ   r�   r  r  r(  r“   Útransformer_outputsrŠ   Úslice_indicesr+  r-  Úoutputs                       r,   rš   zGPTJForCausalLM.forwardB  s?  € ð6 &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ñ 
ô 
Ðð ,¨AÔ.ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�YÐ!4°Q°R°RÔ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå%ØØØ/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r.   )NNNNNNNNNNNr   )rZ   r�   rž   Ú_tied_weights_keysrL   r   r"   r    r
   rŸ   r¡   r.  r£   r¢   r   rš   r¤   r¥   s   @r,   r$  r$  2  sb  ø€ € € € € ð +Ð,DÐEÐðð ð ð ð ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ð<
ð <
àÔ# dÑ*ð<
ð  ™ð<
ð Ô)¨DÑ0ð	<
ð
 Ô(¨4Ñ/ð<
ð Ô&¨Ñ-ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð   $™;ð<
ð # T™kð<
ð ˜D‘[ð<
ð ˜eœlÑ*ð<
ð 
Ð'Ñ	'ð<
ð <
ð <
ñ „^ð<
ð <
ð <
ð <
ð <
r.   r$  aã  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dedz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
e	dz  de	dz  de	dz  de	dz  de
ez  fd„¦   «         Zˆ xZS )ÚGPTJForSequenceClassificationc                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S )NFrH   )
rK   rL   Ú
num_labelsrï   rß   r   rc   rÄ   Úscorer   r  s     €r,   rL   z&GPTJForSequenceClassification.__init__‘  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ$ VÑ,Ô,ˆÔÝ”Y˜vœ}¨d¬oÀEÐJÑJÔJˆŒ
ð 	�ŠÑÔÐÐÐr.   Nr  rà   r„   r	  r6   r
  r'  rŒ   r�   r  r  r   c                 ó6  — |�|n| j         j        }|                      ||||||||	|
|¬¦
  «
        }|d         }|                      |¦  «        }|�|j        d         }n|j        d         }| j         j        €|dk    rt          d¦  «        ‚| j         j        €d}n¨|�}|| j         j        k                         |j        t          j
        ¦  «        }t          j        |j        d         |j        t          j
        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d�¦  «         |t          j        ||j        ¬	¦  «        |f         }d}|��t|                     |j        ¦  «        }| j         j        €f| j        dk    rd
| j         _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j         _        nd| j         _        | j         j        d
k    rWt-          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt1          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt5          ¦   «         } |||¦  «        }|s|f|dd…         z   }|�|f|z   n|S t7          |||j        |j        |j        ¬¦  «        S )a!  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr*  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r:   )r2   r    zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr,  )rM   r  rß   r9  r4   Úpad_token_idr_   r1   r2   r"   Úint32r#   ÚargmaxrW   rX   rY   rZ   Úproblem_typer8  r    Úlongr.  r   Úsqueezer   rq   r   r   rà   rŠ   r  )rk   r  rà   r„   r	  r6   r
  r'  rŒ   r�   r  r  r“   r1  rŠ   r+  Ú
batch_sizeÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr-  Úloss_fctr3  s                           r,   rš   z%GPTJForSequenceClassification.forwardš  sP  € ð4 &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ñ 
ô 
Ðð ,¨AÔ.ˆØ—’˜MÑ*Ô*ˆàÐ Ø"œ¨Ô+ˆJˆJà&Ô,¨QÔ/ˆJàŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÑØ—Y’Y˜}Ô3Ñ4Ô4ˆFØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 M×$9Ò$9Ñ$;Ô$;¸V¿^º^Ñ=MÔ=MÑNÔN�D�Dà#˜8 M°6Ñ:Ô:�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x × 2Ò 2°2°t´Ñ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨vÑ6Ô6�Øð 	FØ#Ð%Ð(;¸A¸B¸BÔ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå/ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r.   )NNNNNNNNNNN)rZ   r�   rž   rL   r   r"   r    r
   rŸ   r¡   r¢   r   rš   r¤   r¥   s   @r,   r6  r6  ‚  sU  ø€ € € € € ðð ð ð ð ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%Ø)-Ø,0Ø#'ðb
ð b
àÔ# dÑ*ðb
ð  ™ðb
ð Ô)¨DÑ0ð	b
ð
 Ô(¨4Ñ/ðb
ð Ô&¨Ñ-ðb
ð Ô(¨4Ñ/ðb
ð Ô  4Ñ'ðb
ð ˜$‘;ðb
ð   $™;ðb
ð # T™kðb
ð ˜D‘[ðb
ð 
Ð1Ñ	1ðb
ð b
ð b
ñ „^ðb
ð b
ð b
ð b
ð b
r.   r6  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚGPTJForQuestionAnsweringc                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r›   )
rK   rL   r8  rï   rß   r   rc   r[   Ú
qa_outputsr   r  s     €r,   rL   z!GPTJForQuestionAnswering.__init__  sf   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ$ VÑ,Ô,ˆÔÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr.   Nr  r„   r	  r6   r
  Ústart_positionsÚend_positionsr�   r  r  r   c           
      ó  — |
�|
n| j         j        }
|                      |||||||	|
¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «                             ¦   «         }|                     d¦  «                             ¦   «         }d}|��|��t          |                     ¦   «         ¦  «        dk    r-|                     d¦  «         	                    |j
        ¦  «        }t          |                     ¦   «         ¦  «        dk    r-|                     d¦  «         	                    |j
        ¦  «        }|                     d¦  «        }|                     d|¦  «        }|                     d|¦  «        }t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|
s||f|dd…         z   }|�|f|z   n|S t          ||||j        |j        ¬	¦  «        S )
r  N)r„   r	  r6   r
  r�   r  r  r   r   r:   r!   )Úignore_indexr   )r-  Ústart_logitsÚ
end_logitsrŠ   r  )rM   r  rß   rM  r‘   rC  ry   rr   rp   r1   r2   Úclampr   r   rŠ   r  )rk   r  r„   r	  r6   r
  rN  rO  r�   r  r  r“   r   Úsequence_outputr+  rR  rS  Ú
total_lossÚignored_indexrI  Ú
start_lossÚend_lossr3  s                          r,   rš   z GPTJForQuestionAnswering.forward  s4  € ð* &1Ð%<�k�kÀ$Ä+ÔBYˆà×"Ò"ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð #ñ 	
ô 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/×:Ò:Ñ<Ô<ˆØ×'Ò'¨Ñ+Ô+×6Ò6Ñ8Ô8ˆ
àˆ
ØÑ&¨=Ñ+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=×"@Ò"@ÀÔATÑ"UÔ"U�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9× <Ò <¸ZÔ=NÑ OÔ O�à(×-Ò-¨aÑ0Ô0ˆMØ-×3Ò3°A°}ÑEÔEˆOØ)×/Ò/°°=ÑAÔAˆMå'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r.   r!  )rZ   r�   rž   rL   r   r"   r    rŸ   r¡   r¢   r   rš   r¤   r¥   s   @r,   rK  rK     sG  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ðC
ð C
àÔ# dÑ*ðC
ð Ô)¨DÑ0ðC
ð Ô(¨4Ñ/ð	C
ð
 Ô&¨Ñ-ðC
ð Ô(¨4Ñ/ðC
ð Ô)¨DÑ0ðC
ð Ô'¨$Ñ.ðC
ð   $™;ðC
ð # T™kðC
ð ˜D‘[ðC
ð 
Ð-Ñ	-ðC
ð C
ð C
ñ „^ðC
ð C
ð C
ð C
ð C
r.   rK  )r$  rK  r6  rï   rÞ   );r¾   r`   r"   r   Útorch.nnr   r   r   Ú r   rä   Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_gptjr   r   Ú
get_loggerrZ   rW   r.  r£   r-   r7   r@   rD   ÚModulerF   r§   rÔ   rÂ   rÌ   rÞ   rï   r$  r6  rK  Ú__all__r  r.   r,   ú<module>ri     s�  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø hÐ hÐ hÐ hÐ hÐ hÐ hÐ hØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø *Ð *Ð *Ð *Ð *Ð *ð ÐÑÔð KØJÐJÐJÐJÐJÐJð 
ˆÔ	˜HÑ	%Ô	%€ðP¨ð P°3ð P¸5¼<ð Pð Pð Pð PðWð Wð Wð˜œð ¨¬ð ð ð ð ð= ¤ð =°E´Lð =ÀuÄ|ð =ÐX]ÔXdð =ð =ð =ð =ð[)ð [)ð [)ð [)ð [)�B”Iñ [)ô [)ð [)ð|~)ð ~)ð ~)ð ~)ð ~)˜-ñ ~)ô ~)ð ~)ðD Ø,ðð Ð ðð ð ð ð ˆbŒiñ ô ð ð&+ð +ð +ð +ð +Ð*ñ +ô +ð +ðD ðwð wð wð wð w˜/ñ wô wñ „ðwð ð~
ð ~
ð ~
ð ~
ð ~
Ð#ñ ~
ô ~
ñ „ð~
ðB €ððñ ô ð
H
ð H
ð H
ð H
ð H
Ð)¨?ñ H
ô H
ñô ð
H
ðV €ððñ ô ðm
ð m
ð m
ð m
ð m
Ð$7ñ m
ô m
ñô ðm
ð` ðN
ð N
ð N
ð N
ð N
Ð2ñ N
ô N
ñ „ðN
ðbð ð €€€r.   