§
    ‚ŠtjXn  ã                   ó6  — d dl Z d dlmZ d dlZd dlmZ d dlmZmZmZ ddl	m
Z
 ddlmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+  e%j,        e-¦  «        Z. G d„ dej/        ¦  «        Z0 G d„ dej/        ¦  «        Z1	 	 d3dej2        dej3        dej3        dej3        dej3        dz  de4dz  de4de e"         fd „Z5 G d!„ d"ej2        ¦  «        Z6 G d#„ d$e¦  «        Z7e# G d%„ d&e¦  «        ¦   «         Z8e# G d'„ d(e8¦  «        ¦   «         Z9 e#d)¬*¦  «         G d+„ d,e8e¦  «        ¦   «         Z:e# G d-„ d.e8¦  «        ¦   «         Z; e#d/¬*¦  «         G d0„ d1e8¦  «        ¦   «         Z<g d2¢Z=dS )4é    N)ÚCallable)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚBioGptConfigc                   ód   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej        ded	ej        dz  fˆ fd
„Zˆ xZ	S )Ú BioGptLearnedPositionalEmbeddingzN
    This module learns positional embeddings up to a fixed maximum size.
    Únum_embeddingsÚembedding_dimc                 ój   •— d| _         t          ¦   «                              || j         z   |¦  «         d S )Né   )ÚoffsetÚsuperÚ__init__)Úselfr!   r"   Ú	__class__s      €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/biogpt/modeling_biogpt.pyr'   z)BioGptLearnedPositionalEmbedding.__init__8   s3   ø€ ð ˆŒÝ‰Œ×Ò˜¨$¬+Ñ5°}ÑEÔEÐEÐEÐEó    r   NÚattention_maskÚpast_key_values_lengthÚposition_idsc                 óÖ   •— |€>t          j        |d¬¦  «        }||z  dz
                       ¦   «         }|dd…|d…f         }t          ¦   «                              || j        z   ¦  «        S )z3`input_ids_shape` is expected to be [bsz x seqlen].Nr   ©Údim)ÚtorchÚcumsumÚlongr&   Úforwardr%   )r(   r,   r-   r.   r)   s       €r*   r5   z(BioGptLearnedPositionalEmbedding.forward>   sq   ø€ ð ÐÝ œ<¨¸AÐ>Ñ>Ô>ˆLØ(¨>Ñ9¸AÑ=×CÒCÑEÔEˆLà'¨¨¨Ð+AÐ+BÐ+BÐ(BÔCˆLå‰wŒw�Š˜|¨d¬kÑ9Ñ:Ô:Ð:r+   )r   N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr'   r2   Ú
LongTensorr5   Ú__classcell__©r)   s   @r*   r    r    3   s¯   ø€ € € € € ðð ðF sð F¸3ð Fð Fð Fð Fð Fð Fð '(Ø04ð	;ð ;àÔ(ð;ð !$ð;ð Ô&¨Ñ-ð	;ð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r+   r    c            
       óV   ‡ — e Zd ZdZddededededz  fˆ fd„Zd	ej        fˆ fd
„Z	ˆ xZ
S )ÚBioGptScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    ç      ð?r!   r"   Úpadding_idxÚembed_scaleNc                 ó\   •— t          ¦   «                              |||¦  «         || _        d S ©N)r&   r'   rB   )r(   r!   r"   rA   rB   r)   s        €r*   r'   z"BioGptScaledWordEmbedding.__init__T   s-   ø€ Ý‰Œ×Ò˜¨¸ÑDÔDÐDØ&ˆÔÐÐr+   Ú	input_idsc                 óV   •— t          ¦   «                              |¦  «        | j        z  S rD   )r&   r5   rB   )r(   rE   r)   s     €r*   r5   z!BioGptScaledWordEmbedding.forwardX   s!   ø€ Ý‰wŒw�Š˜yÑ)Ô)¨DÔ,<Ñ<Ð<r+   )r@   )r6   r7   r8   r9   r:   Úfloatr'   r2   ÚTensorr5   r<   r=   s   @r*   r?   r?   O   s–   ø€ € € € € ðð ð'ð ' sð '¸3ð 'ÈSð 'Ð_dÐgkÑ_kð 'ð 'ð 'ð 'ð 'ð 'ð= ¤ð =ð =ð =ð =ð =ð =ð =ð =ð =ð =r+   r?   ç        ÚmoduleÚqueryÚkeyÚvaluer,   ÚscalingÚdropoutÚkwargsc                 ó®  — |€|                      d¦  «        dz  }t          j        ||                     dd¦  «        ¦  «        |z  }|�||z   }t          j                             |d¬¦  «        }t          j                             ||| j        ¬¦  «        }t          j        ||¦  «        }	|	                     dd¦  «         	                    ¦   «         }	|	|fS )Néÿÿÿÿç      à¿r$   r   r0   ©ÚpÚtrainingr   )
Úsizer2   ÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxrO   rV   Ú
contiguous)
rJ   rK   rL   rM   r,   rN   rO   rP   Úattn_weightsÚattn_outputs
             r*   Úeager_attention_forwardr`   \   sÈ   € ð €Ø—*’*˜R‘.”. DÑ(ˆõ ”<  s§}¢}°Q¸Ñ':Ô':Ñ;Ô;¸gÑE€LàÐ!Ø# nÑ4ˆå”=×(Ò(¨¸2Ð(Ñ>Ô>€LÝ”=×(Ò(¨¸È6Ì?Ð(Ñ[Ô[€Lå”,˜|¨UÑ3Ô3€KØ×'Ò'¨¨1Ñ-Ô-×8Ò8Ñ:Ô:€Kà˜Ð$Ð$r+   c                   óì   ‡ — e Zd ZdZ	 	 	 	 	 	 ddededed	ed
edededz  dedz  fˆ fd„Z	 	 	 dde	j
        de	j
        dz  dedz  de	j
        dz  dee         dee	j
        e	j
        dz  f         fd„Zˆ xZS )ÚBioGptAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrI   FTNÚ	embed_dimÚ	num_headsrO   Ú
is_decoderÚbiasÚ	is_causalÚconfigÚ	layer_idxc	                 óz  •— t          ¦   «                              ¦   «          || _        || _        || _        ||z  | _        || _        | j        |z  | j        k    rt          d| j        › d|› d�¦  «        ‚| j        dz  | _        || _	        || _
        || _        |€/| j	        r(t                               d| j        j        › d�¦  «         t!          j        |||¬¦  «        | _        t!          j        |||¬¦  «        | _        t!          j        |||¬¦  «        | _        t!          j        |||¬¦  «        | _        d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rS   zInstantiating a decoder z¸ without passing `layer_idx` is not recommended and will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.©rf   )r&   r'   rc   rd   rO   Úhead_dimrh   Ú
ValueErrorrN   re   rg   ri   ÚloggerÚwarning_oncer)   r6   rZ   ÚLinearÚk_projÚv_projÚq_projÚout_proj)
r(   rc   rd   rO   re   rf   rg   rh   ri   r)   s
            €r*   r'   zBioGptAttention.__init__{   sY  ø€ õ 	‰Œ×ÒÑÔÐØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒàŒM˜IÑ%¨$¬.Ò8Ð8Ýð3ÈdÌnð 3ð 3Ø%.ð3ð 3ð 3ñô ð ð ”} dÑ*ˆŒØ$ˆŒØ"ˆŒØ"ˆŒØÐ ¤ÐÝ×Òð,¨4¬>Ô+Bð ,ð ,ð ,ñô ð õ ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝ”i 	¨9¸4Ð@Ñ@Ô@ˆŒÝœ	 )¨Y¸TÐBÑBÔBˆŒˆˆr+   Úhidden_statesÚkey_value_statesÚpast_key_valuesr,   rP   Úreturnc                 óŽ  — |du}|j         dd…         }g |¢d‘| j        ‘R }|                      |¦  «                             |¦  «                             dd¦  «        }	d}
|�Ht          |t          ¦  «        r1|j                             | j	        ¦  «        }
|r|j
        }n
|j        }n|}|r|n|}|r3|�1|
r/|j        | j	                 j        }|j        | j	                 j        }nÞ|                      |¦  «        }|                      |¦  «        }g |j         dd…         ¢d‘| j        ‘R }|                     |¦  «                             dd¦  «        }|                     |¦  «                             dd¦  «        }|�E|                     ||| j	        ¦  «        \  }}|r$t          |t          ¦  «        rd|j        | j	        <   t%          j        | j        j        t,          ¦  «        } || |	|||f| j        sdn| j        | j        dœ|¤Ž\  }} |j        g |¢d‘R Ž                      ¦   «         }|                      |¦  «        }||fS )	z#Input shape: Batch x Time x ChannelNrR   r   r$   FTrI   )rO   rN   )Úshaperl   rs   ÚviewrY   Ú
isinstancer   Ú
is_updatedÚgetri   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrq   rr   Úupdater   Úget_interfacerh   Ú_attn_implementationr`   rV   rO   rN   Úreshaper]   rt   )r(   ru   rv   rw   r,   rP   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesr}   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚattention_interfacer_   r^   s                      r*   r5   zBioGptAttention.forward¢   s¨  € ð .°TÐ9Ðð $Ô)¨#¨2¨#Ô.ˆà8˜Ð8 bÐ8¨$¬-Ð8Ð8ˆð —{’{ =Ñ1Ô1×6Ò6°|ÑDÔD×NÒNÈqÐRSÑTÔTˆàˆ
ØÐ&Ý˜/Õ+>Ñ?Ô?ð 7Ø,Ô7×;Ò;¸D¼NÑKÔK�
Ø%ð Pà+:Ô+PÐ(Ð(à+:Ô+OÐ(Ð(à'6Ð$à-?ÐRÐ)Ð)À]ˆØð 	F /Ð"=À*Ð"=à-Ô4°T´^ÔDÔIˆJØ/Ô6°t´~ÔFÔMˆLˆLàŸš ^Ñ4Ô4ˆJØŸ;š; ~Ñ6Ô6ˆLØF˜Ô-¨c¨r¨cÔ2ÐF°BÐF¸¼ÐFÐFˆHØ#Ÿš¨Ñ2Ô2×<Ò<¸QÀÑBÔBˆJØ'×,Ò,¨XÑ6Ô6×@Ò@ÀÀAÑFÔFˆLàÐ*Ø+?×+FÒ+FÀzÐS_ÐaeÔaoÑ+pÔ+pÑ(�
˜Là%ð F­*°_ÕFYÑ*ZÔ*Zð FØAE�OÔ.¨t¬~Ñ>å(?Ô(MØŒKÔ,Õ.Eñ)
ô )
Ðð %8Ð$7ØØØØØð	%
ð  $œ}Ð>�C�C°$´,Ø”Lð	%
ð 	%
ð ð	%
ð 	%
Ñ!ˆ�\ð *�kÔ)Ð;¨;Ð;¸Ð;Ð;Ð;×FÒFÑHÔHˆØ—m’m KÑ0Ô0ˆà˜LÐ(Ð(r+   )rI   FTFNN)NNN)r6   r7   r8   r9   r:   rG   Úboolr   r'   r2   rH   r	   r   r   Útupler5   r<   r=   s   @r*   rb   rb   x   s]  ø€ € € € € ØGÐGð Ø ØØØ&*Ø $ð%Cð %Càð%Cð ð%Cð ð	%Cð
 ð%Cð ð%Cð ð%Cð ˜tÑ#ð%Cð ˜‘:ð%Cð %Cð %Cð %Cð %Cð %CðT 15Ø(,Ø.2ðH)ð H)à”|ðH)ð  œ,¨Ñ-ðH)ð  ™ð	H)ð
 œ tÑ+ðH)ð Ð-Ô.ðH)ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ðH)ð H)ð H)ð H)ð H)ð H)ð H)ð H)r+   rb   c                   óª   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 ddej        dej        dz  dedz  d	e	dz  d
ej
        dz  dee         dej        fd„Zˆ xZS )ÚBioGptDecoderLayerNrh   ri   c           	      ó  •— t          ¦   «                              ¦   «          |j        | _        t	          | j        |j        |j        dd||¬¦  «        | _        |j        | _	        t          |j                 | _        |j        | _        t          j        | j        ¦  «        | _        t          j        | j        |j        ¦  «        | _        t          j        |j        | j        ¦  «        | _        t          j        | j        ¦  «        | _        d S )NT)rc   rd   rO   re   rg   rh   ri   )r&   r'   Úhidden_sizerc   rb   Únum_attention_headsÚattention_probs_dropout_probÚ	self_attnÚhidden_dropout_probrO   r   Ú
hidden_actÚactivation_fnÚactivation_dropoutrZ   Ú	LayerNormÚself_attn_layer_normrp   Úintermediate_sizeÚfc1Úfc2Úfinal_layer_norm)r(   rh   ri   r)   s      €r*   r'   zBioGptDecoderLayer.__init__î   sÛ   ø€ Ý‰Œ×ÒÑÔÐØÔ+ˆŒå(Ø”nØÔ0ØÔ7ØØØØð
ñ 
ô 
ˆŒð Ô1ˆŒÝ# FÔ$5Ô6ˆÔØ"(Ô";ˆÔå$&¤L°´Ñ$@Ô$@ˆÔ!å”9˜Tœ^¨VÔ-EÑFÔFˆŒÝ”9˜VÔ5°t´~ÑFÔFˆŒÝ "¤¨T¬^Ñ <Ô <ˆÔÐÐr+   Tru   r,   rw   Ú	use_cacher.   rP   rx   c                 ó&  — |}|                       |¦  «        } | j        d||||dœ|¤Ž\  }}t          j                             || j        | j        ¬¦  «        }||z   }|}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        }t          j                             || j	        | j        ¬¦  «        }|  
                    |¦  «        }t          j                             || j        | j        ¬¦  «        }||z   }|S )a‘  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            past_key_values (`Cache`): cached past key and value projection states
        )ru   rw   r,   r.   rT   © )r    rš   rZ   r[   rO   rV   r¤   r¢   r�   rž   r£   )	r(   ru   r,   rw   r¥   r.   rP   ÚresidualÚ_s	            r*   r5   zBioGptDecoderLayer.forward  s(  € ð  !ˆà×1Ò1°-Ñ@Ô@ˆð *˜4œ>ð 
Ø'Ø+Ø)Ø%ð	
ð 
ð
 ð
ð 
Ñˆ�qõ œ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØ  =Ñ0ˆð !ˆØ×-Ò-¨mÑ<Ô<ˆØŸš Ñ/Ô/ˆØ×*Ò*¨=Ñ9Ô9ˆÝœ×-Ò-¨m¸tÔ?VÐaeÔanÐ-ÑoÔoˆØŸš Ñ/Ô/ˆÝœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆØ  =Ñ0ˆàÐr+   rD   )NNTN)r6   r7   r8   r   r:   r'   r2   rH   r	   r’   r;   r   r   r5   r<   r=   s   @r*   r•   r•   í   sà   ø€ € € € € ð=ð =˜|ð =¸¸d¹
ð =ð =ð =ð =ð =ð =ð4 /3Ø(,Ø!%Ø04ð)ð )à”|ð)ð œ tÑ+ð)ð  ™ð	)ð
 ˜$‘;ð)ð Ô&¨Ñ-ð)ð Ð+Ô,ð)ð 
Œð)ð )ð )ð )ð )ð )ð )ð )r+   r•   c                   ó<   — e Zd ZU eed<   dZdZdZdZdZ	dZ
eedœZdS )ÚBioGptPreTrainedModelrh   ÚbiogptT)ru   Ú
attentionsN)r6   r7   r8   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphr•   rb   Ú_can_record_outputsr§   r+   r*   r«   r«   1  sT   € € € € € € àÐÐÑØ ÐØ&*Ð#ØÐØ€NØÐØ!Ðà+Ø%ðð ÐÐÐr+   r«   c                   óè   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej	        dz  dej
        dz  dej
        dz  dedz  dedz  d	ej	        dz  d
ee         deez  fd„¦   «         ¦   «         ¦   «         Zˆ xZS )ÚBioGptModelrh   c                 ój  •‡— t          ¦   «                              ‰¦  «         ‰| _        ‰j        | _        ‰j        | _        ‰j        | _        ‰j        | _	        ‰j
        rt          j        ‰j        ¦  «        nd}t          ‰j        | j        | j	        |¬¦  «        | _        t!          ‰j        | j        ¦  «        | _        t'          j        ˆfd„t+          ‰j        ¦  «        D ¦   «         ¦  «        | _        t'          j        | j        ¦  «        | _        d| _        |                      ¦   «          d S )Nr@   )rB   c                 ó2   •— g | ]}t          ‰|¬ ¦  «        ‘ŒS ))ri   )r•   )Ú.0Úirh   s     €r*   ú
<listcomp>z(BioGptModel.__init__.<locals>.<listcomp>P  s(   ø€ Ð$vÐ$vÐ$vÐQRÕ%7¸È!Ð%LÑ%LÔ%LÐ$vÐ$vÐ$vr+   F)r&   r'   rh   Ú	layerdropr›   rO   r—   rc   Úpad_token_idrA   Úscale_embeddingÚmathÚsqrtr?   Ú
vocab_sizeÚembed_tokensr    Úmax_position_embeddingsÚembed_positionsrZ   Ú
ModuleListÚrangeÚnum_hidden_layersr�   rŸ   Ú
layer_normÚgradient_checkpointingÚ	post_init)r(   rh   rB   r)   s    ` €r*   r'   zBioGptModel.__init__B  s  øø€ Ý‰Œ×Ò˜Ñ Ô Ð ØˆŒØÔ)ˆŒØÔ1ˆŒØÔ+ˆŒØ!Ô.ˆÔØ7=Ô7MÐV•d”i Ô 2Ñ3Ô3Ð3ÐSVˆå5ØÔ˜tœ~¨tÔ/?È[ð
ñ 
ô 
ˆÔõ  @ÀÔ@^Ð`dÔ`nÑoÔoˆÔå”mÐ$vÐ$vÐ$vÐ$vÕV[Ð\bÔ\tÑVuÔVuÐ$vÑ$vÔ$vÑwÔwˆŒÝœ, t¤~Ñ6Ô6ˆŒà&+ˆÔ#à�ŠÑÔÐÐÐr+   NrE   r,   Úinputs_embedsrw   r¥   r.   rP   rx   c           	      ó`  — |d u |d uz  rt          d¦  «        ‚|€|                      |¦  «        }|r|€t          | j        ¬¦  «        }|                     ¦   «         d d…         \  }}	|�|                     ¦   «         nd}
|€!|
|	z   }t          j        |||j        ¬¦  «        }|}t          | j        |||¬¦  «        }|€3t          j
        |	|j        ¬¦  «        |
z   }|                     d¦  «        }|                      ||
|¬¦  «        }||z   }t          j                             || j        | j        ¬¦  «        }t#          | j        ¦  «        D ]:\  }}| j        r t          j        g ¦  «        }|| j        k     rŒ, ||f||||d	œ|¤Ž}Œ;|                      |¦  «        }t-          ||¬
¦  «        S )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time)rh   rR   r   ©Údevice)rh   rÌ   r,   rw   )r.   rT   )r,   rw   r¥   r.   )Úlast_hidden_staterw   )rm   rÃ   r
   rh   rW   Úget_seq_lengthr2   ÚonesrÏ   r   ÚarangeÚ	unsqueezerÅ   rZ   r[   rO   rV   Ú	enumerater�   Úrandr½   rÉ   r   )r(   rE   r,   rÌ   rw   r¥   r.   rP   Ú
batch_sizeÚ
seq_lengthr-   Úmask_seq_lengthÚself_attn_cacheÚcausal_maskÚ	positionsru   ÚidxÚdecoder_layerÚdropout_probabilitys                      r*   r5   zBioGptModel.forwardW  s  € ð ˜Ð -°tÐ";Ñ<ð 	uÝÐsÑtÔtÐtàÐ Ø ×-Ò-¨iÑ8Ô8ˆMð ð 	?˜Ð0Ý*°$´+Ð>Ñ>Ô>ˆOà!.×!3Ò!3Ñ!5Ô!5°c°r°cÔ!:Ñˆ
�JØETÐE` ×!?Ò!?Ñ!AÔ!AÐ!AÐfgÐàÐ!à4°zÑAˆOÝ"œZ¨
°OÈMÔL`ÐaÑaÔaˆNà)ˆå(Ø”;Ø'Ø)Ø+ð	
ñ 
ô 
ˆð ÐÝ œ<¨
¸=Ô;OÐPÑPÔPÐSiÑiˆLØ'×1Ò1°!Ñ4Ô4ˆLà×(Ò(¨Ð9OÐ^jÐ(ÑkÔkˆ	Ø%¨	Ñ1ˆÝœ×-Ò-¨m¸t¼|ÐVZÔVcÐ-ÑdÔdˆå"+¨D¬KÑ"8Ô"8ð 	ð 	ÑˆC�ØŒ}ð Ý&+¤j°¡n¤nÐ#Ø&¨¬Ò7Ð7Øà)˜MØðà*Ø /Ø#Ø)ðð ð ðð ˆMˆMð Ÿš¨Ñ6Ô6ˆå8Ø+Ø+ð
ñ 
ô 
ð 	
r+   )NNNNNN)r6   r7   r8   r   r'   r   r   r   r2   r;   ÚFloatTensorr	   r’   r   r   r“   r   r5   r<   r=   s   @r*   r·   r·   @  s%  ø€ € € € € ð˜|ð ð ð ð ð ð ð*  ØØð .2Ø37Ø26Ø(,Ø!%Ø04ðB
ð B
àÔ# dÑ*ðB
ð Ô)¨DÑ0ðB
ð Ô(¨4Ñ/ð	B
ð
  ™ðB
ð ˜$‘;ðB
ð Ô&¨Ñ-ðB
ð Ð+Ô,ðB
ð 
Ð:Ñ	:ðB
ð B
ð B
ñ „^ñ „_ñ  ÔðB
ð B
ð B
ð B
ð B
r+   r·   zR
    BioGPT Model with a `language modeling` head on top for CLM fine-tuning.
    )Úcustom_introc                   ó  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 dde	j
        dz  d	e	j        dz  d
e	j        dz  dedz  de	j
        dz  dedz  de	j
        dz  dee	j        z  dee         deez  fd„¦   «         ¦   «         Zˆ xZS )ÚBioGptForCausalLMzoutput_projection.weightzbiogpt.embed_tokens.weightc                 óæ   •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        |  	                    ¦   «          d S ©NFrk   )
r&   r'   r·   r¬   rZ   rp   r—   rÂ   Úoutput_projectionrË   ©r(   rh   r)   s     €r*   r'   zBioGptForCausalLM.__init__§  sb   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å! &Ñ)Ô)ˆŒÝ!#¤¨6Ô+=¸vÔ?PÐW\Ð!]Ñ!]Ô!]ˆÔð 	�ŠÑÔÐÐÐr+   c                 ó   — | j         S rD   ©ræ   ©r(   s    r*   Úget_output_embeddingsz'BioGptForCausalLM.get_output_embeddings°  s   € ØÔ%Ð%r+   c                 ó   — || _         d S rD   ré   )r(   Únew_embeddingss     r*   Úset_output_embeddingsz'BioGptForCausalLM.set_output_embeddings³  s   € Ø!/ˆÔÐÐr+   Nr   rE   r,   rÌ   rw   Úlabelsr¥   r.   Úlogits_to_keeprP   rx   c	           	      ó^  —  | j         |f|||||dœ|	¤Ž}
|
d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }d}|� | j        d||| j        j        dœ|	¤Ž}t          |||
j	        |
j
        |
j        |
j        ¬¦  «        S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        )r,   rÌ   rw   r¥   r.   r   N)Úlogitsrï   rÂ   )Úlossrò   rw   ru   r­   Úcross_attentionsr§   )r¬   r|   r:   Úsliceræ   Úloss_functionrh   rÂ   r   rw   ru   r­   rô   )r(   rE   r,   rÌ   rw   rï   r¥   r.   rð   rP   Úoutputsru   Úslice_indicesrò   ró   s                  r*   r5   zBioGptForCausalLM.forward¶  sý   € ð( �$”+Øð
à)Ø'Ø+ØØ%ð
ð 
ð ð
ð 
ˆð   œ
ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ×'Ò'¨°a°a°a¸ÈÈÈÐ6IÔ(JÑKÔKˆàˆØÐØ%�4Ô%Ðp¨V¸FÈtÌ{ÔOeÐpÐpÐioÐpÐpˆDå0ØØØ#Ô3Ø!Ô/ØÔ)Ø$Ô5ð
ñ 
ô 
ð 	
r+   ©NNNNNNNr   )r6   r7   r8   Ú_tied_weights_keysr'   rë   rî   r   r   r2   r;   rà   r	   r’   r:   rH   r   r   r“   r   r5   r<   r=   s   @r*   rã   rã   Ÿ  sS  ø€ € € € € ð 5Ð6RÐSÐðð ð ð ð ð&ð &ð &ð0ð 0ð 0ð Øð .2Ø37Ø26Ø(,Ø*.Ø!%Ø04Ø-.ð+
ð +
àÔ# dÑ*ð+
ð Ô)¨DÑ0ð+
ð Ô(¨4Ñ/ð	+
ð
  ™ð+
ð Ô  4Ñ'ð+
ð ˜$‘;ð+
ð Ô&¨Ñ-ð+
ð ˜eœlÑ*ð+
ð Ð+Ô,ð+
ð 
Ð2Ñ	2ð+
ð +
ð +
ñ „^ñ Ôð+
ð +
ð +
ð +
ð +
r+   rã   c                   óî   ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  de	dz  dej        dz  dej        dz  d	e
dz  d
ej        dz  deez  fd„¦   «         ¦   «         Zˆ xZS )ÚBioGptForTokenClassificationc                 óx  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          |d¦  «        r|j        �|j        }n|j        }t          j	        |¦  «        | _
        t          j        |j        |j        ¦  «        | _        |                      ¦   «          d S )NÚclassifier_dropout)r&   r'   Ú
num_labelsr·   r¬   Úhasattrrþ   r›   rZ   ÚDropoutrO   rp   r—   Ú
classifierrË   )r(   rh   rþ   r)   s      €r*   r'   z%BioGptForTokenClassification.__init__è  s¥   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå! &Ñ)Ô)ˆŒÝ�6Ð/Ñ0Ô0ð 	<°VÔ5NÐ5ZØ!'Ô!:ÐÐà!'Ô!;ÐÝ”zÐ"4Ñ5Ô5ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒà�ŠÑÔÐÐÐr+   NrE   Útoken_type_idsr,   rw   rÌ   rï   r¥   r.   rx   c	           	      óˆ  —  | j         |f|||||dœ|	¤Ž}
|
d         }|                      |¦  «        }|                      |¦  «        }d}|�Üt          ¦   «         }|�”|                     d¦  «        dk    }|                     d| j        ¦  «        }t          j        ||                     d¦  «        t          j        |j	        ¦  «         
                    |¦  «        ¦  «        } |||¦  «        }n8 ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }t          |||
j        |
j        ¬¦  «        S )á�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ©rw   r,   rÌ   r¥   r.   r   NrR   r   )ró   rò   ru   r­   )r¬   rO   r  r   r{   rÿ   r2   ÚwhereÚtensorÚignore_indexÚtype_asr   ru   r­   )r(   rE   r  r,   rw   rÌ   rï   r¥   r.   rP   Útransformer_outputsru   rò   ró   Úloss_fctÚactive_lossÚactive_logitsÚactive_labelss                     r*   r5   z$BioGptForTokenClassification.forwardö  s\  € ð( *˜dœkØð
à+Ø)Ø'ØØ%ð
ð 
ð ð
ð 
Ðð ,¨AÔ.ˆØŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÐÝ'Ñ)Ô)ˆHØÐ)Ø,×1Ò1°"Ñ5Ô5¸Ò:�Ø &§¢¨B°´Ñ @Ô @�Ý %¤Ø §¢¨R¡¤µ%´,¸xÔ?TÑ2UÔ2U×2]Ò2]Ð^dÑ2eÔ2eñ!ô !�ð  �x ¨}Ñ=Ô=��à�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR�å$ØØØ-Ô;Ø*Ô5ð	
ñ 
ô 
ð 	
r+   )NNNNNNNN)r6   r7   r8   r'   r   r   r2   r;   rà   r	   r’   r“   r   r5   r<   r=   s   @r*   rü   rü   æ  s  ø€ € € € € ðð ð ð ð ð Øð .2Ø26Ø37Ø(,Ø26Ø*.Ø!%Ø04ð2
ð 2
àÔ# dÑ*ð2
ð Ô(¨4Ñ/ð2
ð Ô)¨DÑ0ð	2
ð
  ™ð2
ð Ô(¨4Ñ/ð2
ð Ô  4Ñ'ð2
ð ˜$‘;ð2
ð Ô&¨Ñ-ð2
ð 
Ð&Ñ	&ð2
ð 2
ð 2
ñ „^ñ Ôð2
ð 2
ð 2
ð 2
ð 2
r+   rü   aÛ  
    The BioGpt Model transformer with a sequence classification head on top (linear layer).

    [`BioGptForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it is required to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej        dz  dej	        dz  de
dz  dej	        dz  d	ej        dz  d
edz  dej        dz  deej        z  deez  fd„¦   «         ¦   «         Zd„ Zd„ Zˆ xZS )ÚBioGptForSequenceClassificationrh   c                 óþ   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        | j        d¬¦  «        | _        |  	                    ¦   «          d S rå   )
r&   r'   rÿ   r·   r¬   rZ   rp   r—   ÚscorerË   rç   s     €r*   r'   z(BioGptForSequenceClassification.__init__<  si   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒÝ! &Ñ)Ô)ˆŒÝ”Y˜vÔ1°4´?ÈÐOÑOÔOˆŒ
ð 	�ŠÑÔÐÐÐr+   Nr   rE   r,   rw   rÌ   rï   r¥   r.   rð   rx   c	           	      ó  —  | j         |f|||||dœ|	¤Ž}
|
d         }t          |t          ¦  «        rt          | d¦  «        n|}|                      |dd…|dd…f         ¦  «        }|�|j        dd…         \  }}n|j        dd…         \  }}| j        j        €|dk    rt          d¦  «        ‚| j        j        €d}n¨|�}|| j        j        k     	                    |j
        t          j        ¦  «        }t          j        |j        d         |j
        t          j        ¬¦  «        }||z                       d¦  «        }n)d}t                               | j        j        › d	�¦  «         |t          j        ||j
        ¬
¦  «        |f         }d}|��Z| j        j        €f| j        dk    rd| j        _        nN| j        dk    r7|j        t          j        k    s|j        t          j        k    rd| j        _        nd| j        _        | j        j        dk    rWt/          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j        j        dk    rGt3          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j        j        dk    rt7          ¦   «         } |||¦  «        }t9          |||
j        |
j        |
j        ¬¦  «        S )r  r  r   Nr$   r   z=Cannot handle batch sizes > 1 if no padding token is defined.rR   )rÏ   ÚdtypezŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rÎ   Ú
regressionÚsingle_label_classificationÚmulti_label_classification)ró   rò   rw   ru   r­   ) r¬   r|   r:   rõ   r  rz   rh   r¾   rm   ÚtorÏ   r2   Úint32rÓ   Úargmaxrn   ro   r)   r6   Úproblem_typerÿ   r  r4   r   Úsqueezer   r{   r   r   rw   ru   r­   )r(   rE   r,   rw   rÌ   rï   r¥   r.   rð   rP   r  ru   rø   rò   r×   Úsequence_lengthÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsró   r  s                         r*   r5   z'BioGptForSequenceClassification.forwardE  s@  € ð( *˜dœkØð
à+Ø)Ø'ØØ%ð
ð 
ð ð
ð 
Ðð ,¨AÔ.ˆÝ8BÀ>ÕSVÑ8WÔ8WÐk�˜~˜o¨tÑ4Ô4Ð4Ð]kˆØ—’˜M¨!¨!¨!¨]¸A¸A¸AÐ*=Ô>Ñ?Ô?ˆàÐ Ø*3¬/¸"¸1¸"Ô*=Ñ'ˆJ˜˜à*7Ô*=¸b¸q¸bÔ*AÑ'ˆJ˜àŒ;Ô#Ð+°
¸a²°ÝÐ\Ñ]Ô]Ð]ØŒ;Ô#Ð+Ø!#ÐÐØÐ"à%¨¬Ô)AÒA×EÒEÀfÄmÕUZÔU`ÑaÔaˆLÝ!œL¨¬¸Ô)<ÀVÄ]ÕZ_ÔZeÐfÑfÔfˆMØ"/°,Ñ">×!FÒ!FÀrÑ!JÔ!JÐÐà!#ÐÝ×ÒØ”>Ô*ð Zð Zð Zñô ð ð
 �uœ|¨J¸v¼}ÐMÑMÔMÐOaÐaÔbˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 M×$9Ò$9Ñ$;Ô$;¸V¿^º^Ñ=MÔ=MÑNÔN�D�Dà#˜8 M°6Ñ:Ô:�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x × 2Ò 2°2°t´Ñ GÔ GÈÏÊÐUWÉÌÑYÔY��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨vÑ6Ô6�å/ØØ Ø/Ô?Ø-Ô;Ø*Ô5ð
ñ 
ô 
ð 	
r+   c                 ó   — | j         j        S rD   ©r¬   rÃ   rê   s    r*   Úget_input_embeddingsz4BioGptForSequenceClassification.get_input_embeddingsœ  s   € ØŒ{Ô'Ð'r+   c                 ó   — || j         _        d S rD   r$  )r(   rM   s     r*   Úset_input_embeddingsz4BioGptForSequenceClassification.set_input_embeddingsŸ  s   € Ø#(ˆŒÔ Ð Ð r+   rù   )r6   r7   r8   r   r'   r   r   r2   r;   rà   r	   r’   r:   rH   r“   r   r5   r%  r'  r<   r=   s   @r*   r  r  -  sP  ø€ € € € € ð˜|ð ð ð ð ð ð ð Øð .2Ø37Ø(,Ø26Ø*.Ø!%Ø04Ø-.ðS
ð S
àÔ# dÑ*ðS
ð Ô)¨DÑ0ðS
ð  ™ð	S
ð
 Ô(¨4Ñ/ðS
ð Ô  4Ñ'ðS
ð ˜$‘;ðS
ð Ô&¨Ñ-ðS
ð ˜eœlÑ*ðS
ð 
Ð1Ñ	1ðS
ð S
ð S
ñ „^ñ ÔðS
ðj(ð (ð (ð)ð )ð )ð )ð )ð )ð )r+   r  )rã   rü   r  r·   r«   )NrI   )>rÀ   Úcollections.abcr   r2   Útorch.nnrZ   r   r   r   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_biogptr   Ú
get_loggerr6   rn   Ú	Embeddingr    r?   ÚModulerH   rG   r`   rb   r•   r«   r·   rã   rü   r  Ú__all__r§   r+   r*   ú<module>r;     sh  ðð* €€€Ø $Ð $Ð $Ð $Ð $Ð $à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà !Ð !Ð !Ð !Ð !Ð !Ø CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CÐ CØ )Ð )Ð )Ð )Ð )Ð )Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð GÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RØ 7Ð 7Ð 7Ð 7Ð 7Ð 7Ø 5Ð 5Ð 5Ð 5Ð 5Ð 5Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð;ð ;ð ;ð ;ð ; r¤|ñ ;ô ;ð ;ð8
=ð 
=ð 
=ð 
=ð 
= ¤ñ 
=ô 
=ð 
=ð& !Øð%ð %ØŒIð%àŒ<ð%ð 
Œð%ð Œ<ð	%ð
 ”L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ô(ð%ð %ð %ð %ð8r)ð r)ð r)ð r)ð r)�b”iñ r)ô r)ð r)ðjAð Að Að Að AÐ3ñ Aô Að AðH ðð ð ð ð ˜Oñ ô ñ „ðð ð[
ð [
ð [
ð [
ð [
Ð'ñ [
ô [
ñ „ð[
ð| €ððñ ô ð
?
ð ?
ð ?
ð ?
ð ?
Ð-¨ñ ?
ô ?
ñô ð
?
ðD ðC
ð C
ð C
ð C
ð C
Ð#8ñ C
ô C
ñ „ðC
ðL €ððñ ô ðe)ð e)ð e)ð e)ð e)Ð&;ñ e)ô e)ñô ðe)ðPð ð €€€r+   