§
    ‚ŠtjÚO  ã                   óV  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZ  ej        e¦  «        Zdededej         fd„Z!dej         dej         fd„Z"dej         dej         dej         dej         fd„Z# G d„ dej$        ¦  «        Z% G d„ dej$        ¦  «        Z& G d„ de¦  «        Z'e G d „ d!e¦  «        ¦   «         Z(e G d"„ d#e(¦  «        ¦   «         Z) ed$¬%¦  «         G d&„ d'e(e¦  «        ¦   «         Z*g d(¢Z+dS ))zPyTorch CodeGen model.é    N)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚCodeGenConfigÚnum_posÚdimÚreturnc                 ó€  — ddt          j        d|dt           j        ¬¦  «        |z  z  z  }t          j        dt          j        | t           j        ¬¦  «                             ¦   «         |¦  «                             ¦   «         }t          j        t          j        |¦  «        t          j        |¦  «        fd¬¦  «        S )	Ng      ð?i'  r   é   )Údtypezi , j -> i jr   ©r   )ÚtorchÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r   r   Úinv_freqÚsinusoid_inps       új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/codegen/modeling_codegen.pyÚcreate_sinusoidal_positionsr%   (   s–   € Ø�e¥¤¨Q°°Q½e¼kÐ JÑ JÔ JÈSÑ PÑQÑR€HÝ”< µ´¸WÍEÌKÐ0XÑ0XÔ0X×0^Ò0^Ñ0`Ô0`ÐbjÑkÔk×qÒqÑsÔs€LÝŒ9•e”i Ñ-Ô-­u¬y¸Ñ/FÔ/FÐGÈQÐOÑOÔOÐOó    Úxc                 ó²   — | d d …d d …d d …d d d…f         }| d d …d d …d d …dd d…f         }t          j        | |fd¬¦  «        } |                      d¦  «        S )Nr   r   éÿÿÿÿr   éþÿÿÿ)r   ÚstackÚflatten)r'   Úx1Úx2s      r$   Úrotate_every_twor/   /   ss   € Ø	
ˆ1ˆ1ˆ1ˆaˆaˆa����C�C�a�Cˆ<Œ€BØ	
ˆ1ˆ1ˆ1ˆaˆaˆa����A�D�q�Dˆ=Ô	€BÝŒ�b�S˜"�I 2Ð&Ñ&Ô&€AØ�9Š9�R‰=Œ=Ðr&   Útensorr    r!   c                 óÊ   — t          j        |d d …d d …d d d …f         dd¦  «        }t          j        |d d …d d …d d d …f         dd¦  «        }| |z  t          | ¦  «        |z  z   S )Nr   r   )r   Úrepeat_interleaver/   )r0   r    r!   s      r$   Úapply_rotary_pos_embr3   7   sy   € Ý
Ô
! # a a a¨¨¨¨D°!°!°! mÔ"4°a¸Ñ
;Ô
;€CÝ
Ô
! # a a a¨¨¨¨D°!°!°! mÔ"4°a¸Ñ
;Ô
;€CØ�S‰LÕ-¨fÑ5Ô5¸Ñ;Ñ<Ð<r&   c                   ó0  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z	 dd„Z	 	 	 	 	 ddej        dz  de	dz  d	ej        dz  d
ej
        dz  dedz  dedz  deej        eej                 f         eej        eej                 eej        df         f         z  dz  fd„Zˆ xZS )ÚCodeGenAttentionNc                 óz  •— t          ¦   «                              ¦   «          |j        | _        t	          j        |j        ¦  «        | _        t	          j        |j        ¦  «        | _	        || _
        |€(t                               d| j        j        › d�¦  «         |j        | _        |j        | _        | j        | j        z  | _        | j        | j        z  | j        k    r t'          d| j        › d| j        › d�¦  «        ‚t)          j        | j        ¦  «        | _        t	          j        | j        | j        dz  d¬¦  «        | _        t	          j        | j        | j        d¬¦  «        | _        |j        | _        | j        p| j        | _        |                      d	t;          | j        | j        ¦  «        d¬
¦  «         d S )NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r   F)ÚbiasÚembed_positions)Ú
persistent)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_positionsr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	layer_idxÚloggerÚwarning_onceÚ	__class__Ú__name__Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚhead_dimÚ
ValueErrorÚmathÚsqrtÚ
scale_attnÚLinearÚqkv_projÚout_projÚ
rotary_dimÚpos_embd_dimÚregister_bufferr%   )ÚselfÚconfigrC   rF   s      €r$   r;   zCodeGenAttention.__init__>   s¼  ø€ Ý‰Œ×ÒÑÔÐà#Ô;ˆÔÝœJ vÔ'8Ñ9Ô9ˆÔÝœZ¨Ô(:Ñ;Ô;ˆÔØ"ˆŒØÐÝ×Òð, ¤Ô!8ð ,ð ,ð ,ñô ð ð  Ô+ˆŒØ#)Ô#=ˆÔ Øœ¨$Ô*BÑBˆŒØŒ=˜4Ô3Ñ3°t´~ÒEÐEÝðHÐX\ÔXfð Hð HØ+/Ô+CðHð Hð Hñô ð õ œ) D¤MÑ2Ô2ˆŒÝœ	 $¤.°$´.À1Ñ2DÈ5ÐQÑQÔQˆŒåœ	 $¤.°$´.ÀuÐMÑMÔMˆŒØ Ô+ˆŒØ œOÐ=¨t¬~ˆÔØ×ÒØÕ:¸4Ô;MÈtÔO`ÑaÔaÐnsð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r&   c                 óÄ   — |                      |j        d d…         ||z  |fz   ¦  «        }|                      |j        d d…         dz   |j        dd …         z   ¦  «        }|S )Nr)   r*   )r)   )ÚreshapeÚshape)rV   r'   Ún_headÚdim_headÚmp_numÚreshapeds         r$   Ú_split_headszCodeGenAttention._split_heads^   sc   € Ø—9’9˜QœW S b Sœ\¨V°vÑ-=¸xÐ,HÑHÑIÔIˆØ×#Ò# A¤G¨C¨R¨C¤L°5Ñ$8¸8¼>È"È#È#Ô;NÑ$NÑOÔOˆØˆr&   c                 óÈ  — t          |j        ¦  «        dk    r,|                     ddddd¦  «                             ¦   «         }ngt          |j        ¦  «        dk    r+|                     dddd¦  «                             ¦   «         }n$t	          dt          |j        ¦  «        › �¦  «        ‚|                     ¦   «         dd	…         ||z  fz   }|                     |¦  «        S )
zM
        Merges attn_head_size dim and num_attn_heads dim into n_ctx
        é   r   r   r   r   é   z3Input tensor rank should be one of [4, 5], but is: Nr*   )ÚlenrZ   ÚpermuteÚ
contiguousrL   ÚsizeÚview)rV   r0   rJ   Úattn_head_sizeÚ	new_shapes        r$   Ú_merge_headszCodeGenAttention._merge_headsc   sÔ   € õ ˆvŒ|ÑÔ Ò!Ð!Ø—^’^ A q¨!¨Q°Ñ2Ô2×=Ò=Ñ?Ô?ˆFˆFÝ�”ÑÔ !Ò#Ð#Ø—^’^ A q¨!¨QÑ/Ô/×:Ò:Ñ<Ô<ˆFˆFåÐfÕSVÐW]ÔWcÑSdÔSdÐfÐfÑgÔgÐgØ—K’K‘M”M # 2 #Ô&Ð*=ÀÑ*NÐ)PÑPˆ	Ø�{Š{˜9Ñ%Ô%Ð%r&   c                 ó¾  — |                      t          j        ¦  «        }|                      t          j        ¦  «        }t          j        ||                     dd¦  «        ¦  «        }|�||z   }|| j        z  } t          j        d¬¦  «        |¦  «        }|                      |j        ¦  «        }|  	                    |¦  «        }t          j        ||¦  «        }||fS )Nr)   r*   r   )
Útor   Úfloat32ÚmatmulÚ	transposerO   r   ÚSoftmaxr   r@   )rV   ÚqueryÚkeyÚvalueÚattention_maskÚattn_weightsÚattn_outputs          r$   Ú_attnzCodeGenAttention._attnp   sÃ   € ð —’�œÑ'Ô'ˆØ�fŠf•U”]Ñ#Ô#ˆå”| E¨3¯=ª=¸¸RÑ+@Ô+@ÑAÔAˆàÐ%Ø'¨.Ñ8ˆLà# d¤oÑ5ˆØ)•r”z bÐ)Ñ)Ô)¨,Ñ7Ô7ˆØ#—’ u¤{Ñ3Ô3ˆØ×(Ò(¨Ñ6Ô6ˆå”l <°Ñ7Ô7ˆà˜LÐ(Ð(r&   FÚhidden_statesÚ
layer_pastrt   Úposition_idsÚ	use_cacheÚoutput_attentionsr   .c                 ó  — |                       |¦  «        }d}|                     |j        d d…         |dfz   ¦  «        }	| j        | j        z  |z  }
t          j        |	|
d¬¦  «        \  }}}|                      || j        | j        |¬¦  «        }|                      || j        | j        |¬¦  «        }|                      || j        | j        |¬¦  «        }|                     dddd¦  «        }| j	        }|j
        |j
        k    r!|                     |j
        ¦  «        }|| _	        ||         }t          j        ||j        d         dz  d¬¦  «        \  }}| j        �·|d d …d d …d d …d | j        …f         }|d d …d d …d d …| j        d …f         }|d d …d d …d d …d | j        …f         }|d d …d d …d d …| j        d …f         }t          |||¦  «        }t          |||¦  «        }t          j        ||gd¬¦  «        }t          j        ||gd¬¦  «        }n"t          |||¦  «        }t          |||¦  «        }|                     dddd¦  «        }|                     dddd¦  «        }|�7|                     |                     |j        ¦  «        || j        ¦  «        \  }}|                      ||||¦  «        \  }}|                      || j        | j        ¦  «        }|                      |¦  «        }|                      |¦  «        }||fS )	Nrb   r)   r   )r]   r   r   r   r   )rQ   rY   rZ   rK   rJ   r   Úsplitr_   rd   r8   Údevicerl   rS   r3   r   Úupdater   rC   rw   rj   rR   rB   )rV   rx   ry   rt   rz   r{   r|   Úqkvr]   Ú	qkv_splitÚ	local_dimrq   rs   rr   r8   Úsincosr    r!   Úk_rotÚk_passÚq_rotÚq_passrv   ru   s                           r$   ÚforwardzCodeGenAttention.forward‰   s8  € ð �mŠm˜MÑ*Ô*ˆàˆØ—K’K ¤	¨#¨2¨#¤°&¸"°Ñ =Ñ>Ô>ˆ	à”M DÔ$<Ñ<ÀÑFˆ	Ý!œK¨	°9À"ÐEÑEÔEÑˆˆu�cØ×!Ò! %¨Ô)AÀ4Ä=ÐY_Ð!Ñ`Ô`ˆØ×Ò  TÔ%=¸t¼}ÐU[ÐÑ\Ô\ˆà×!Ò! %¨Ô)AÀ4Ä=ÐY_Ð!Ñ`Ô`ˆØ—’˜a  A qÑ)Ô)ˆàÔ.ˆØÔ! \Ô%8Ò8Ð8Ø-×0Ò0°Ô1DÑEÔEˆOØ#2ˆDÔ à  Ô.ˆÝ”;˜v v¤|°BÔ'7¸1Ñ'<À"ÐEÑEÔE‰ˆˆSàŒ?Ð&Ø˜˜˜˜1˜1˜1˜a˜a˜aÐ!2 4¤?Ð!2Ð2Ô3ˆEØ˜˜˜˜A˜A˜A˜q˜q˜q $¤/Ð"3Ð"3Ð3Ô4ˆFà˜!˜!˜!˜Q˜Q˜Q   Ð#4 T¤_Ð#4Ð4Ô5ˆEØ˜1˜1˜1˜a˜a˜a    D¤OÐ$5Ð$5Ð5Ô6ˆFå(¨°°SÑ9Ô9ˆEÝ(¨°°SÑ9Ô9ˆEå”)˜U F˜O°Ð4Ñ4Ô4ˆCÝ”I˜u f˜o°2Ð6Ñ6Ô6ˆEˆEå& s¨C°Ñ5Ô5ˆCÝ(¨°°SÑ9Ô9ˆEà�kŠk˜!˜Q  1Ñ%Ô%ˆØ—’˜a  A qÑ)Ô)ˆð Ð!Ø#×*Ò*¨3¯6ª6°-Ô2EÑ+FÔ+FÈÈtÌ~Ñ^Ô^‰JˆC�ð %)§J¢J¨u°c¸5À.Ñ$QÔ$QÑ!ˆ�\à×'Ò'¨°TÔ5MÈtÌ}Ñ]Ô]ˆØ—m’m KÑ0Ô0ˆØ×(Ò(¨Ñ5Ô5ˆØ˜LÐ(Ð(r&   ©N©NNNFF)rG   Ú
__module__Ú__qualname__r;   r_   rj   rw   r   ÚFloatTensorr   Ú
LongTensorÚboolÚtupleÚTensorr‰   Ú__classcell__©rF   s   @r$   r5   r5   =   s\  ø€ € € € € ð
ð 
ð 
ð 
ð 
ð 
ð@ð ð ð
&ð &ð &ð$ ð)ð )ð )ð )ð8 $(Ø37Ø04Ø!&Ø).ð@)ð @)àÔ(¨4Ñ/ð@)ð ˜D‘Lð@)ð Ô)¨DÑ0ð	@)ð
 Ô&¨Ñ-ð@)ð ˜$‘;ð@)ð   $™;ð@)ð 	ˆeŒl˜E %¤,Ô/Ð/Ô0Ø
�”˜e E¤LÔ1°5¸¼ÀsÐ9JÔ3KÐKÔ
Lñ	Mà
ñ	ð@)ð @)ð @)ð @)ð @)ð @)ð @)ð @)r&   r5   c                   óH   ‡ — e Zd Zˆ fd„Zdej        dz  dej        fd„Zˆ xZS )Ú
CodeGenMLPc                 ó(  •— t          ¦   «                              ¦   «          |j        }t          j        ||¦  «        | _        t          j        ||¦  «        | _        t          |j                 | _	        t          j
        |j        ¦  «        | _        d S rŠ   )r:   r;   Ún_embdr   rP   Úfc_inÚfc_outr   Úactivation_functionÚactr>   rA   Údropout)rV   Úintermediate_sizerW   rI   rF   s       €r$   r;   zCodeGenMLP.__init__Î   so   ø€ Ý‰Œ×ÒÑÔÐØ”Mˆ	å”Y˜yÐ*;Ñ<Ô<ˆŒ
Ý”iÐ 1°9Ñ=Ô=ˆŒå˜&Ô4Ô5ˆŒÝ”z &Ô"4Ñ5Ô5ˆŒˆˆr&   rx   Nr   c                 ó®   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }|S rŠ   )r™   rœ   rš   r�   )rV   rx   s     r$   r‰   zCodeGenMLP.forwardØ   sL   € ØŸ
š
 =Ñ1Ô1ˆØŸš Ñ/Ô/ˆØŸš MÑ2Ô2ˆØŸš ]Ñ3Ô3ˆØÐr&   )rG   rŒ   r�   r;   r   rŽ   r‰   r“   r”   s   @r$   r–   r–   Í   se   ø€ € € € € ð6ð 6ð 6ð 6ð 6ð UÔ%6¸Ñ%=ð À%ÔBSð ð ð ð ð ð ð ð r&   r–   c                   óè   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej        dz  dedz  dej        dz  dej        dz  dedz  d	edz  d
e	ej
                 e	ej
        e	ej        df         f         z  dz  fd„Zˆ xZS )ÚCodeGenBlockNc                 ó  •— t          ¦   «                              ¦   «          |j        �|j        n	d|j        z  }t	          j        |j        |j        ¬¦  «        | _        t          ||¦  «        | _	        t          ||¦  «        | _        d S )Nrb   ©Úeps)r:   r;   Ún_innerr˜   r   Ú	LayerNormÚlayer_norm_epsilonÚln_1r5   Úattnr–   Úmlp)rV   rW   rC   Ú	inner_dimrF   s       €r$   r;   zCodeGenBlock.__init__ã   ss   ø€ Ý‰Œ×ÒÑÔÐØ&,¤nÐ&@�F”N�NÀaÈ&Ì-ÑFWˆ	Ý”L ¤°FÔ4MÐNÑNÔNˆŒ	Ý$ V¨YÑ7Ô7ˆŒ	Ý˜i¨Ñ0Ô0ˆŒˆˆr&   Frx   ry   rt   rz   r{   r|   r   .c                 ó®   — |}|                       |¦  «        }|                      ||||||¬¦  «        \  }	}
|                      |¦  «        }|	|z   |z   }||
fS )N)rx   ry   rt   rz   r{   r|   )r¨   r©   rª   )rV   rx   ry   rt   rz   r{   r|   ÚkwargsÚresidualÚattn_outputsru   Úfeed_forward_hidden_statess               r$   r‰   zCodeGenBlock.forwardê   sx   € ð !ˆØŸ	š	 -Ñ0Ô0ˆØ%)§Y¢YØ'Ø!Ø)Ø%ØØ/ð &/ñ &
ô &
Ñ"ˆ�lð &*§X¢X¨mÑ%<Ô%<Ð"Ø$Ð'AÑAÀHÑLˆà˜lÐ*Ð*r&   rŠ   r‹   )rG   rŒ   r�   r;   r   rŽ   r   r�   r�   r‘   r’   r‰   r“   r”   s   @r$   r¡   r¡   á   sú   ø€ € € € € ð1ð 1ð 1ð 1ð 1ð 1ð $(Ø37Ø04Ø!&Ø).ð+ð +àÔ(¨4Ñ/ð+ð ˜D‘Lð+ð Ô)¨DÑ0ð	+ð
 Ô&¨Ñ-ð+ð ˜$‘;ð+ð   $™;ð+ð 
ˆuŒ|Ô	˜u U¤\°5¸Ô9JÈCÐ9OÔ3PÐ%PÔQÑ	QÐTXÑ	Xð+ð +ð +ð +ð +ð +ð +ð +r&   r¡   c                   óB   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	ˆ fd„Z
ˆ xZS )ÚCodeGenPreTrainedModelrW   ÚtransformerTr¡   Úpast_key_valuesc                 óÜ   •— t          ¦   «                              |¦  «         t          |t          ¦  «        r4t	          j        |j        t          |j        |j	        ¦  «        ¦  «         d S d S rŠ   )
r:   Ú_init_weightsÚ
isinstancer5   ÚinitÚcopy_r8   r%   r=   rT   )rV   ÚmodulerF   s     €r$   r¶   z$CodeGenPreTrainedModel._init_weights  sh   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ð 	wÝŒJ�vÔ-Õ/JÈ6ÔK_ÐagÔatÑ/uÔ/uÑvÔvÐvÐvÐvð	wð 	wr&   )rG   rŒ   r�   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_can_compile_fullgraphr¶   r“   r”   s   @r$   r²   r²     ss   ø€ € € € € € àÐÐÑØ%ÐØ&*Ð#Ø'Ð(ÐØ#4Ð"5ÐØ!Ððwð wð wð wð wð wð wð wð wr&   r²   c                   óø   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  de	dz  dej
        dz  dej        dz  d	ej        dz  d
ej
        dz  dedz  dedz  dedz  dedz  deez  fd„¦   «         Zˆ xZS )ÚCodeGenModelc                 ó<  •‡— t          ¦   «                              ‰¦  «         ‰j        | _        ‰j        | _        t          j        ‰j        | j        ¦  «        | _        t          j        ‰j	        ¦  «        | _
        t          j        ˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        t          j        | j        ‰j        ¬¦  «        | _        t%          ‰j        ‰j        ‰j        z  ¦  «        | _        d| _        |                      ¦   «          d S )Nc                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))rC   )r¡   )Ú.0ÚirW   s     €r$   ú
<listcomp>z)CodeGenModel.__init__.<locals>.<listcomp>  s&   ø€ ÐaÐaÐaÀa¥¨V¸qÐ AÑ AÔ AÐaÐaÐar&   r£   F)r:   r;   r˜   rI   Ú
vocab_sizer   Ú	EmbeddingÚwter>   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerÚhr¦   r§   Úln_fÚminrS   Ún_ctxrJ   Úgradient_checkpointingÚ	post_init©rV   rW   rF   s    `€r$   r;   zCodeGenModel.__init__  sæ   øø€ Ý‰Œ×Ò˜Ñ Ô Ð àœˆŒØ Ô+ˆŒÝ”< Ô 1°4´>ÑBÔBˆŒÝ”J˜vÔ0Ñ1Ô1ˆŒ	Ý”ÐaÐaÐaÐaÍ5ÐQWÔQ_ÑK`ÔK`ÐaÑaÔaÑbÔbˆŒÝ”L ¤°VÔ5NÐOÑOÔOˆŒ	Ý˜fÔ/°´ÀÔA[Ñ1[Ñ\Ô\ˆŒà&+ˆÔ#ð 	�ŠÑÔÐÐÐr&   c                 ó   — | j         S rŠ   ©rÊ   )rV   s    r$   Úget_input_embeddingsz!CodeGenModel.get_input_embeddings%  s	   € ØŒxˆr&   c                 ó   — || _         d S rŠ   rØ   )rV   Únew_embeddingss     r$   Úset_input_embeddingsz!CodeGenModel.set_input_embeddings(  s   € Ø!ˆŒˆˆr&   NÚ	input_idsr´   rt   Útoken_type_idsrz   Úinputs_embedsr{   r|   Úoutput_hidden_statesÚreturn_dictr   c           	      óš  — |�|n| j         j        }|	�|	n| j         j        }	|�|n| j         j        }|
�|
n| j         j        }
|du |duz  rt          d¦  «        ‚| j        r%| j        r|rt           	                    d¦  «         d}|€|  
                    |¦  «        }|r|€t          | j         ¬¦  «        }|j        d         }|€V|�|                     ¦   «         nd}t          j        |j        d         |j        ¬¦  «        |z   }|                     d¦  «        }t%          | j         ||||¬	¦  «        }|}|�0|                     d
|¦  «        }|  
                    |¦  «        }||z   }|                      |¦  «        }d
||                     d
¦  «        f}|rdnd}|	rdnd}t-          | j        ¦  «        D ]4\  }}|	r||fz   } |||||||¬¦  «        }|d         }|r||d         fz   }Œ5|                      |¦  «        }|                     |¦  «        }|	r||fz   }|
st3          d„ ||||fD ¦   «         ¦  «        S t5          ||||¬¦  «        S )a�  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rW   r   r   )r   )rW   rß   rt   r´   rz   r)   © )ry   rt   rz   r{   r|   c              3   ó   K  — | ]}|®|V — Œ	d S rŠ   rã   )rÅ   Úvs     r$   ú	<genexpr>z'CodeGenModel.forward.<locals>.<genexpr>Š  s1   è è € ð ð ØÐghÐgt�ÐgtÐgtÐgtÐgtðð r&   )Úlast_hidden_stater´   rx   Ú
attentions)rW   r|   rà   r{   rá   rL   rÔ   ÚtrainingrD   rE   rÊ   r   rZ   Úget_seq_lengthr   r   r   Ú	unsqueezer
   rg   rÌ   rf   Ú	enumeraterÐ   rÑ   r‘   r   )rV   rÝ   r´   rt   rÞ   rz   rß   r{   r|   rà   rá   r­   Ú
seq_lengthÚpast_seen_tokensÚcausal_maskrx   Útoken_type_embedsÚoutput_shapeÚall_self_attentionsÚall_hidden_statesrÆ   ÚblockÚoutputss                          r$   r‰   zCodeGenModel.forward+  s2  € ð* 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð "+Ð!6�I�I¸D¼KÔ<Qˆ	Ø%0Ð%<�k�kÀ$Ä+ÔBYˆà˜Ð -°tÐ";Ñ<ð 	[ÝÐYÑZÔZÐZàÔ&ð 	"¨4¬=ð 	"Øð "Ý×#Ò#Øpñô ð ð "�	àÐ Ø ŸHšH YÑ/Ô/ˆMàð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOà"Ô(¨Ô+ˆ
ØÐØCRÐC^˜×=Ò=Ñ?Ô?Ð?ÐdeÐÝ œ<¨Ô(;¸AÔ(>À}ÔG[Ð\Ñ\Ô\Ð_oÑoˆLØ'×1Ò1°!Ñ4Ô4ˆLå(Ø”;Ø'Ø)Ø+Ø%ð
ñ 
ô 
ˆð &ˆàÐ%Ø+×0Ò0°°ZÑ@Ô@ˆNØ $§¢¨Ñ 8Ô 8ÐØ)Ð,=Ñ=ˆMàŸ	š	 -Ñ0Ô0ˆØ˜J¨×(:Ò(:¸2Ñ(>Ô(>Ð?ˆà$5Ð?˜b˜b¸4ÐØ"6Ð@˜B˜B¸DÐÝ! $¤&Ñ)Ô)ð 	Jð 	J‰HˆAˆuØ#ð IØ$5¸Ð8HÑ$HÐ!à�eØØ*Ø*Ø)Ø#Ø"3ðñ ô ˆGð $ AœJˆMØ ð JØ&9¸WÀQ¼Z¸MÑ&IÐ#øàŸ	š	 -Ñ0Ô0ˆà%×*Ò*¨<Ñ8Ô8ˆàð 	EØ 1°]Ð4DÑ DÐàð 	Ýð ð Ø)¨?Ð<MÐObÐcðñ ô ñ ô ð õ 'Ø+Ø+Ø+Ø*ð	
ñ 
ô 
ð 	
r&   )
NNNNNNNNNN)rG   rŒ   r�   r;   rÙ   rÜ   r   r   r�   r   rŽ   r�   r‘   r   r‰   r“   r”   s   @r$   rÂ   rÂ     s]  ø€ € € € € ðð ð ð ð ð ð ð ð"ð "ð "ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø!%Ø)-Ø,0Ø#'ðg
ð g
àÔ# dÑ*ðg
ð  ™ðg
ð Ô)¨DÑ0ð	g
ð
 Ô(¨4Ñ/ðg
ð Ô&¨Ñ-ðg
ð Ô(¨4Ñ/ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð 
Ð(Ñ	(ðg
ð g
ð g
ñ „^ðg
ð g
ð g
ð g
ð g
r&   rÂ   zM
    The CodeGen Model transformer with a language modeling head on top.
    )Úcustom_introc                   ó   ‡ — e Zd ZddiZˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej        dz  dedz  dej	        dz  d	ej        dz  d
ej        dz  dej	        dz  dej        dz  de
dz  de
dz  de
dz  de
dz  deej        z  deez  fd„¦   «         Zˆ xZS )ÚCodeGenForCausalLMzlm_head.weightztransformer.wte.weightc                 óâ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S rŠ   )
r:   r;   rÂ   r³   r   rP   r˜   rÈ   Úlm_headrÕ   rÖ   s     €r$   r;   zCodeGenForCausalLM.__init__ž  s[   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ý'¨Ñ/Ô/ˆÔÝ”y ¤°Ô0AÑBÔBˆŒð 	�ŠÑÔÐÐÐr&   Nr   rÝ   r´   rt   rÞ   rz   rß   Úlabelsr{   r|   rà   rá   Úlogits_to_keepr   c                 ó¼  — |�|n| j         j        }|                      ||||||||	|
|¬¦
  «
        }|d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j         j        dœ|¤Ž}|s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        |j        ¬¦  «        S )aG  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)	r´   rt   rÞ   rz   rß   r{   r|   rà   rá   r   )Úlogitsrû   rÈ   r   )Úlossrþ   r´   rx   rè   rã   )rW   rá   r³   r·   ÚintÚslicerú   Úloss_functionrÈ   r   r´   rx   rè   )rV   rÝ   r´   rt   rÞ   rz   rß   rû   r{   r|   rà   rá   rü   r­   Útransformer_outputsrx   Úslice_indicesrþ   rÿ   Úoutputs                       r$   r‰   zCodeGenForCausalLM.forward¦  s?  € ð6 &1Ð%<�k�kÀ$Ä+ÔBYˆà"×.Ò.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ñ 
ô 
Ðð ,¨AÔ.ˆå8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜m¨A¨A¨A¨}¸a¸a¸aÐ,?Ô@ÑAÔAˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDàð 	FØ�YÐ!4°Q°R°RÔ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå%ØØØ/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r&   )NNNNNNNNNNNr   )rG   rŒ   r�   Ú_tied_weights_keysr;   r   r   r�   r   rŽ   r�   r   r’   r‘   r   r‰   r“   r”   s   @r$   rø   rø   –  sb  ø€ € € € € ð +Ð,DÐEÐðð ð ð ð ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ð<
ð <
àÔ# dÑ*ð<
ð  ™ð<
ð Ô)¨DÑ0ð	<
ð
 Ô(¨4Ñ/ð<
ð Ô&¨Ñ-ð<
ð Ô(¨4Ñ/ð<
ð Ô  4Ñ'ð<
ð ˜$‘;ð<
ð   $™;ð<
ð # T™kð<
ð ˜D‘[ð<
ð ˜eœlÑ*ð<
ð 
Ð'Ñ	'ð<
ð <
ð <
ñ „^ð<
ð <
ð <
ð <
ð <
r&   rø   )rø   rÂ   r²   ),Ú__doc__rM   r   r   Ú r   r¸   Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_codegenr   Ú
get_loggerrG   rD   r   r’   r%   r/   r3   ÚModuler5   r–   r¡   r²   rÂ   rø   Ú__all__rã   r&   r$   ú<module>r     s'  ðð Ð à €€€à €€€Ø Ð Ð Ð Ð Ð à &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9Ø OÐ OÐ OÐ OÐ OÐ OÐ OÐ OØ -Ð -Ð -Ð -Ð -Ð -ðð ð ð ð ð ð ð ð 1Ð 0Ð 0Ð 0Ð 0Ð 0ð 
ˆÔ	˜HÑ	%Ô	%€ðP¨ð P°3ð P¸5¼<ð Pð Pð Pð Pð˜œð ¨¬ð ð ð ð ð= ¤ð =°E´Lð =ÀuÄ|ð =ÐX]ÔXdð =ð =ð =ð =ðL)ð L)ð L)ð L)ð L)�r”yñ L)ô L)ð L)ð`ð ð ð ð �”ñ ô ð ð( +ð  +ð  +ð  +ð  +Ð-ñ  +ô  +ð  +ðF ðwð wð wð wð w˜_ñ wô wñ „ðwð ð
ð 
ð 
ð 
ð 
Ð)ñ 
ô 
ñ „ð
ðD €ððñ ô ð
H
ð H
ð H
ð H
ð H
Ð/°ñ H
ô H
ñô ð
H
ðV KÐ
JÐ
J€€€r&   