§
    ‚ŠtjÛå  ã                   óx  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ ddlm Z m!Z! ddl"m#Z#  e!j$        e%¦  «        Z&g d¢Z' e d¬¦  «        e G d„ de¦  «        ¦   «         ¦   «         Z( G d„ dej)        ¦  «        Z* G d„ dej)        ¦  «        Z+ G d„ dej)        ¦  «        Z, G d„ dej)        ¦  «        Z- G d„ dej)        ¦  «        Z. G d „ d!ej)        ¦  «        Z/ G d"„ d#ej)        ¦  «        Z0 G d$„ d%ej)        ¦  «        Z1 G d&„ d'e¦  «        Z2 G d(„ d)ej)        ¦  «        Z3 G d*„ d+ej)        ¦  «        Z4 G d,„ d-ej)        ¦  «        Z5 G d.„ d/ej)        ¦  «        Z6 G d0„ d1ej)        ¦  «        Z7e  G d2„ d3e¦  «        ¦   «         Z8e  G d4„ d5e8¦  «        ¦   «         Z9 e d6¬¦  «         G d7„ d8e8¦  «        ¦   «         Z:e  G d9„ d:e8¦  «        ¦   «         Z;e  G d;„ d<e8¦  «        ¦   «         Z<e  G d=„ d>e8¦  «        ¦   «         Z=g d?¢Z>dS )@zPyTorch CANINE model.é    N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚModelOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚloggingé   )ÚCanineConfig)é   é+   é;   é=   éI   éa   ég   éq   é‰   é•   é�   é­   éµ   éÁ   éÓ   éß   a  
    Output type of [`CanineModel`]. Based on [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly
    different `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
    Transformer encoders.
    )Úcustom_introc                   ó¤   — e Zd ZU dZdZej        dz  ed<   dZej        dz  ed<   dZ	e
ej                 dz  ed<   dZe
ej                 dz  ed<   dS )ÚCanineModelOutputWithPoolinga§  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Sequence of hidden-states at the output of the last layer of the model (i.e. the output of the final
        shallow Transformer encoder).
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Hidden-state of the first token of the sequence (classification token) at the last layer of the deep
        Transformer encoder, further processed by a Linear layer and a Tanh activation function. The Linear layer
        weights are trained from the next sentence prediction (classification) objective during pretraining.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of each
        encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length //
        config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of each layer plus the
        initial input to each Transformer encoder. The hidden states of the shallow encoders have length
        `sequence_length`, but the hidden states of the deep encoder have length `sequence_length` //
        `config.downsampling_rate`.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape `(batch_size,
        num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length //
        config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions weights after the
        attention softmax, used to compute the weighted average in the self-attention heads.
    NÚlast_hidden_stateÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   ÚtorchÚFloatTensorÚ__annotations__r-   r.   Útupler/   © ó    úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/canine/modeling_canine.pyr+   r+   1   s‰   € € € € € € ðð ð, 37Ð�uÔ(¨4Ñ/Ð6Ð6Ñ6Ø.2€M�5Ô$ tÑ+Ð2Ð2Ñ2Ø59€M�5˜Ô*Ô+¨dÑ2Ð9Ð9Ñ9Ø26€J��eÔ'Ô(¨4Ñ/Ð6Ð6Ñ6Ð6Ð6r9   r+   c                   ó¶   ‡ — e Zd ZdZˆ fd„Zdedefd„Zdededefd„Z	 	 	 	 dd	ej	        dz  d
ej	        dz  dej	        dz  dej
        dz  dej
        f
d„Zˆ xZS )ÚCanineEmbeddingsz<Construct the character, position and token_type embeddings.c           	      ó˜  •— t          ¦   «                              ¦   «          || _        |j        |j        z  }t          |j        ¦  «        D ]0}d|› �}t          | |t          j        |j	        |¦  «        ¦  «         Œ1t          j        |j	        |j        ¦  «        | _
        t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        |                      dt'          j        |j        ¦  «                             d¦  «        d¬¦  «         d S )NÚHashBucketCodepointEmbedder_©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistent)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚnum_hash_functionsÚrangeÚsetattrr   Ú	EmbeddingÚnum_hash_bucketsÚchar_position_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr4   ÚarangeÚmax_position_embeddingsÚexpand)ÚselfrG   Úshard_embedding_sizeÚiÚnameÚ	__class__s        €r:   rF   zCanineEmbeddings.__init__Y   s2  ø€ Ý‰Œ×ÒÑÔÐàˆŒð  &Ô1°VÔ5NÑNÐÝ�vÔ0Ñ1Ô1ð 	]ð 	]ˆAØ5°!Ð5Ð5ˆDÝ�D˜$¥¤¨VÔ-DÐFZÑ [Ô [Ñ\Ô\Ð\Ð\Ý(*¬°VÔ5LÈfÔN`Ñ(aÔ(aˆÔ%Ý%'¤\°&Ô2HÈ&ÔJ\Ñ%]Ô%]ˆÔ"åœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒð 	×ÒØ�EœL¨Ô)GÑHÔH×OÒOÐPWÑXÔXÐejð 	ñ 	
ô 	
ð 	
ð 	
ð 	
r9   Ú
num_hashesÚnum_bucketsc                 óê   — |t          t          ¦  «        k    r$t          dt          t          ¦  «        › �¦  «        ‚t          d|…         }g }|D ]"}|dz   |z  |z  }|                     |¦  «         Œ#|S )a—  
        Converts ids to hash bucket ids via multiple hashing.

        Args:
            input_ids: The codepoints or other IDs to be hashed.
            num_hashes: The number of hash functions to use.
            num_buckets: The number of hash buckets (i.e. embeddings in each table).

        Returns:
            A list of tensors, each of which is the hash bucket IDs from one hash function.
        z`num_hashes` must be <= Nr   )ÚlenÚ_PRIMESÚ
ValueErrorÚappend)rZ   Ú	input_idsr_   r`   ÚprimesÚresult_tensorsÚprimeÚhasheds           r:   Ú_hash_bucket_tensorsz%CanineEmbeddings._hash_bucket_tensorsn   s„   € ð ��G™œÒ$Ð$ÝÐF½½G¹¼ÐFÐFÑGÔGÐGå˜˜*˜Ô%ˆàˆØð 	*ð 	*ˆEØ  1‘}¨Ñ-°Ñ<ˆFØ×!Ò! &Ñ)Ô)Ð)Ð)ØÐr9   Úembedding_sizec                 ó0  — ||z  dk    rt          d|› d|› d�¦  «        ‚|                      |||¬¦  «        }g }t          |¦  «        D ]8\  }}d|› �}	 t          | |	¦  «        |¦  «        }
|                     |
¦  «         Œ9t          j        |d¬¦  «        S )	zDConverts IDs (e.g. codepoints) into embeddings via multiple hashing.r   zExpected `embedding_size` (z) % `num_hashes` (z) == 0)r_   r`   r>   rC   ©Údim)rd   rk   Ú	enumerateÚgetattrre   r4   Úcat)rZ   rf   rl   r_   r`   Úhash_bucket_tensorsÚembedding_shardsr\   Úhash_bucket_idsr]   Úshard_embeddingss              r:   Ú_embed_hash_bucketsz$CanineEmbeddings._embed_hash_buckets…   sÉ   € à˜JÑ&¨!Ò+Ð+ÝÐo¸>ÐoÐoÐ]gÐoÐoÐoÑpÔpÐpà"×7Ò7¸	ÈjÐfqÐ7ÑrÔrÐØÐÝ"+Ð,?Ñ"@Ô"@ð 	6ð 	6ÑˆAˆØ5°!Ð5Ð5ˆDØ2�w t¨TÑ2Ô2°?ÑCÔCÐØ×#Ò#Ð$4Ñ5Ô5Ð5Ð5åŒyÐ)¨rÐ2Ñ2Ô2Ð2r9   Nrf   Útoken_type_idsrA   Úinputs_embedsÚreturnc                 ó,  — |�|                      ¦   «         }n|                      ¦   «         d d…         }|d         }|€| j        d d …d |…f         }|€+t          j        |t          j        | j        j        ¬¦  «        }|€6|                      || j        j        | j        j	        | j        j
        ¦  «        }|                      |¦  «        }||z   }|                      |¦  «        }	||	z  }|                      |¦  «        }|                      |¦  «        }|S )NrC   r   ©ÚdtypeÚdevice)ÚsizerA   r4   ÚzerosÚlongr~   rw   rG   rH   rI   rM   rP   rN   rQ   rU   )
rZ   rf   rx   rA   ry   Úinput_shapeÚ
seq_lengthrP   Ú
embeddingsÚposition_embeddingss
             r:   ÚforwardzCanineEmbeddings.forward“   s  € ð Ð Ø#Ÿ.š.Ñ*Ô*ˆKˆKà'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKà  ”^ˆ
àÐØÔ,¨Q¨Q¨Q°°°¨^Ô<ˆLàÐ!Ý"œ[¨½E¼JÈtÔO`ÔOgÐhÑhÔhˆNàÐ Ø ×4Ò4Ø˜4œ;Ô2°D´KÔ4RÐTXÔT_ÔTpñô ˆMð !%× :Ò :¸>Ñ JÔ JÐØ"Ð%:Ñ:ˆ
à"×;Ò;¸LÑIÔIÐØÐ)Ñ)ˆ
à—^’^ JÑ/Ô/ˆ
Ø—\’\ *Ñ-Ô-ˆ
ØÐr9   )NNNN)r0   r1   r2   r3   rF   Úintrk   rw   r4   Ú
LongTensorr5   r†   Ú__classcell__©r^   s   @r:   r<   r<   V   s  ø€ € € € € ØFÐFð
ð 
ð 
ð 
ð 
ð*¸#ð ÈCð ð ð ð ð.3¸Sð 3Ècð 3Ð`cð 3ð 3ð 3ð 3ð  .2Ø26Ø04Ø26ð!ð !àÔ# dÑ*ð!ð Ô(¨4Ñ/ð!ð Ô&¨Ñ-ð	!ð
 Ô(¨4Ñ/ð!ð 
Ô	ð!ð !ð !ð !ð !ð !ð !ð !r9   r<   c                   óF   ‡ — e Zd ZdZˆ fd„Zdej        dej        fd„Zˆ xZS )ÚCharactersToMoleculeszeConvert character sequence to initial molecule sequence (i.e. downsample) using strided convolutions.c                 ó"  •— t          ¦   «                              ¦   «          t          j        |j        |j        |j        |j        ¬¦  «        | _        t          |j                 | _	        t          j
        |j        |j        ¬¦  «        | _
        d S )N©Úin_channelsÚout_channelsÚkernel_sizeÚstrider?   )rE   rF   r   ÚConv1drH   Údownsampling_rateÚconvr
   Ú
hidden_actÚ
activationrQ   rR   ©rZ   rG   r^   s     €r:   rF   zCharactersToMolecules.__init__º   sz   ø€ Ý‰Œ×ÒÑÔÐå”IØÔ*ØÔ+ØÔ0ØÔ+ð	
ñ 
ô 
ˆŒ	õ ! Ô!2Ô3ˆŒåœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr9   Úchar_encodingrz   c                 óP  — |d d …dd…d d …f         }t          j        |dd¦  «        }|                      |¦  «        }t          j        |dd¦  «        }|                      |¦  «        }|d d …dd…d d …f         }t          j        ||gd¬¦  «        }|                      |¦  «        }|S )Nr   r   é   rC   rn   )r4   Ú	transposer•   r—   rr   rQ   )rZ   r™   Úcls_encodingÚdownsampledÚdownsampled_truncatedÚresults         r:   r†   zCharactersToMolecules.forwardÇ   s¹   € à$ Q Q Q¨¨!¨¨Q¨Q¨Q YÔ/ˆõ œ¨°q¸!Ñ<Ô<ˆØ—i’i Ñ.Ô.ˆÝ”o k°1°aÑ8Ô8ˆØ—o’o kÑ2Ô2ˆð !,¨A¨A¨A¨q°¨t°Q°Q°Q¨JÔ 7Ðõ ”˜LÐ*?Ð@ÀaÐHÑHÔHˆà—’ Ñ'Ô'ˆàˆr9   ©	r0   r1   r2   r3   rF   r4   ÚTensorr†   r‰   rŠ   s   @r:   rŒ   rŒ   ·   si   ø€ € € € € ØoÐoðUð Uð Uð Uð Uð U¤\ð °e´lð ð ð ð ð ð ð ð r9   rŒ   c                   ó^   ‡ — e Zd ZdZˆ fd„Z	 ddej        dej        dz  dej        fd„Zˆ xZS )	ÚConvProjectionz�
    Project representations from hidden_size*2 back to hidden_size across a window of w = config.upsampling_kernel_size
    characters.
    c                 óh  •— t          ¦   «                              ¦   «          || _        t          j        |j        dz  |j        |j        d¬¦  «        | _        t          |j	                 | _
        t          j        |j        |j        ¬¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr›   r   rŽ   r?   )rE   rF   rG   r   r“   rH   Úupsampling_kernel_sizer•   r
   r–   r—   rQ   rR   rS   rT   rU   r˜   s     €r:   rF   zConvProjection.__init__é   s—   ø€ Ý‰Œ×ÒÑÔÐØˆŒÝ”IØÔ*¨QÑ.ØÔ+ØÔ5Øð	
ñ 
ô 
ˆŒ	õ ! Ô!2Ô3ˆŒÝœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr9   NÚinputsÚfinal_seq_char_positionsrz   c                 ó¢  — t          j        |dd¦  «        }| j        j        dz
  }|dz  }||z
  }t	          j        ||fd¦  «        }|                       ||¦  «        ¦  «        }t          j        |dd¦  «        }|                      |¦  «        }|                      |¦  «        }|  	                    |¦  «        }|}|�t          d¦  «        ‚|}	|	S )Nr   r›   r   z,CanineForMaskedLM is currently not supported)r4   rœ   rG   r¦   r   ÚConstantPad1dr•   r—   rQ   rU   ÚNotImplementedError)
rZ   r§   r¨   Ú	pad_totalÚpad_begÚpad_endÚpadr    Úfinal_char_seqÚ	query_seqs
             r:   r†   zConvProjection.forwardö   sØ   € õ ” ¨¨AÑ.Ô.ˆð
 ”KÔ6¸Ñ:ˆ	Ø˜q‘.ˆØ˜gÑ%ˆåÔ ¨Ð1°1Ñ5Ô5ˆà—’˜3˜3˜v™;œ;Ñ'Ô'ˆÝ” ¨¨AÑ.Ô.ˆØ—’ Ñ(Ô(ˆØ—’ Ñ'Ô'ˆØ—’˜fÑ%Ô%ˆØˆà#Ð/õ
 &Ð&TÑUÔUÐUà&ˆIàÐr9   ©Nr¡   rŠ   s   @r:   r¤   r¤   ã   s‡   ø€ € € € € ðð ð
>ð >ð >ð >ð >ð  9=ð"ð "à”ð"ð #(¤,°Ñ"5ð"ð 
Œð	"ð "ð "ð "ð "ð "ð "ð "r9   r¤   c                   ó”   ‡ — e Zd Zˆ fd„Z	 	 d
dej        dej        dej        dz  dedz  deej        ej        dz  f         f
d	„Z	ˆ xZ
S )ÚCanineSelfAttentionc                 ód  •— t          ¦   «                              ¦   «          |j        |j        z  dk    r0t	          |d¦  «        s t          d|j        › d|j        › d�¦  «        ‚|j        | _        t          |j        |j        z  ¦  «        | _        | j        | j        z  | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j
        |j        | j        ¦  «        | _        t          j        |j        ¦  «        | _        d S )Nr   rl   zThe hidden size (z6) is not a multiple of the number of attention heads (ú))rE   rF   rH   Únum_attention_headsÚhasattrrd   r‡   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluerS   Úattention_probs_dropout_probrU   r˜   s     €r:   rF   zCanineSelfAttention.__init__  s  ø€ Ý‰Œ×ÒÑÔÐØÔ Ô :Ñ:¸aÒ?Ð?ÍÐPVÐXhÑHiÔHiÐ?Ýð8 FÔ$6ð 8ð 8Ø Ô4ð8ð 8ð 8ñô ð ð
 $*Ô#=ˆÔ Ý#& vÔ'9¸FÔ<VÑ'VÑ#WÔ#WˆÔ Ø!Ô5¸Ô8PÑPˆÔå”Y˜vÔ1°4Ô3EÑFÔFˆŒ
Ý”9˜VÔ/°Ô1CÑDÔDˆŒÝ”Y˜vÔ1°4Ô3EÑFÔFˆŒ
å”z &Ô"EÑFÔFˆŒˆˆr9   NFÚfrom_tensorÚ	to_tensorÚattention_maskÚoutput_attentionsrz   c                 ó¢  — |j         \  }}}|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }	|                      |¦  «                             |d| j        | j        ¦  «                             dd¦  «        }
t          j	        |
|                     dd¦  «        ¦  «        }|t          j        | j        ¦  «        z  }|�t|j        dk    rLt          j        |d¬¦  «        }d|                     ¦   «         z
  t          j        |j        ¦  «        j        z  }||                     |j        ¦  «        z   }t&          j                             |d¬¦  «        }|                      |¦  «        }t          j	        ||	¦  «        }|                     dddd¦  «                             ¦   «         }|                     ¦   «         d d…         | j        fz   } |j        |Ž }|r||fn|f}|S )	NrC   r   r›   éþÿÿÿr   rn   g      ð?r   )Úshaper½   Úviewr·   r¹   rœ   r¾   r¼   r4   ÚmatmulÚmathÚsqrtÚndimÚ	unsqueezeÚfloatÚfinfor}   ÚminÚtor   Ú
functionalÚsoftmaxrU   ÚpermuteÚ
contiguousr   rº   )rZ   rÀ   rÁ   rÂ   rÃ   Ú
batch_sizerƒ   Ú_Ú	key_layerÚvalue_layerÚquery_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                   r:   r†   zCanineSelfAttention.forward.  s.  € ð %0Ô$5Ñ!ˆ
�J ð �HŠH�YÑÔßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �JŠJ�yÑ!Ô!ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	ð �JŠJ�{Ñ#Ô#ßŠT�*˜b $Ô":¸DÔ<TÑUÔUßŠY�q˜!‰_Œ_ð 	õ !œ<¨°Y×5HÒ5HÈÈRÑ5PÔ5PÑQÔQÐà+­d¬i¸Ô8PÑ.QÔ.QÑQÐØÐ%ØÔ" aÒ'Ð'å!&¤°ÀQÐ!GÑ!GÔ!G�ð #&¨×(<Ò(<Ñ(>Ô(>Ñ">Å%Ä+ÐN^ÔNdÑBeÔBeÔBiÑ!i�à/°.×2CÒ2CÐDTÔDZÑ2[Ô2[Ñ[Ðõ œ-×/Ò/Ð0@ÀbÐ/ÑIÔIˆð Ÿ,š, Ñ7Ô7ˆåœ _°kÑBÔBˆà%×-Ò-¨a°°A°qÑ9Ô9×DÒDÑFÔFˆØ"/×"4Ò"4Ñ"6Ô"6°s¸°sÔ";¸tÔ?QÐ>SÑ"SÐØ*˜Ô*Ð,CÐDˆà6GÐ]�= /Ð2Ð2ÈmÐM]ˆàˆr9   ©NF)r0   r1   r2   rF   r4   r¢   r5   Úboolr7   r†   r‰   rŠ   s   @r:   r´   r´     s±   ø€ € € € € ðGð Gð Gð Gð Gð, 48Ø).ð;ð ;à”\ð;ð ”<ð;ð Ô)¨DÑ0ð	;ð
   $™;ð;ð 
ˆuŒ|˜Uœ\¨DÑ0Ð0Ô	1ð;ð ;ð ;ð ;ð ;ð ;ð ;ð ;r9   r´   c                   óv   ‡ — e Zd Zˆ fd„Zdeej                 dej        deej        ej        f         fd„Zˆ xZS )ÚCanineSelfOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j        |j	        ¦  «        | _
        d S ©Nr?   )rE   rF   r   r»   rH   ÚdenserQ   rR   rS   rT   rU   r˜   s     €r:   rF   zCanineSelfOutput.__init__m  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr9   r.   Úinput_tensorrz   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r²   ©rå   rU   rQ   ©rZ   r.   ræ   s      r:   r†   zCanineSelfOutput.forwards  sB   € ð Ÿ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr9   ©	r0   r1   r2   rF   r7   r4   r5   r†   r‰   rŠ   s   @r:   râ   râ   l  s   ø€ € € € € ð>ð >ð >ð >ð >ðØ" 5Ô#4Ô5ðØEJÔEVðà	ˆuÔ  %Ô"3Ð3Ô	4ðð ð ð ð ð ð ð r9   râ   c                   óÀ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddededededed	efˆ fd
„Z	 	 ddeej	                 dej	        dz  dedz  deej	        ej	        dz  f         fd„Z
ˆ xZS )ÚCanineAttentionav  
    Additional arguments related to local attention:

        - **local** (`bool`, *optional*, defaults to `False`) -- Whether to apply local attention.
        - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks be able to
          attend
        to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all** (`bool`,
        *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to attend to all
        positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*, defaults to 128) -- The
        width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride** (`int`, *optional*, defaults to
        128) -- The number of elements to skip when moving to the next block in `from_tensor`. -
        **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each block-wise chunk in
        *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The number of elements to
        skip when moving to the next block in `to_tensor`.
    Fé€   Úalways_attend_to_first_positionÚfirst_position_attends_to_allÚattend_from_chunk_widthÚattend_from_chunk_strideÚattend_to_chunk_widthÚattend_to_chunk_stridec	                 óN  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          |¦  «        | _        || _        ||k     rt          d¦  «        ‚||k     rt          d¦  «        ‚|| _        || _	        || _
        || _        || _        || _        d S )Nze`attend_from_chunk_width` < `attend_from_chunk_stride` would cause sequence positions to get skipped.z``attend_to_chunk_width` < `attend_to_chunk_stride`would cause sequence positions to get skipped.)rE   rF   r´   rZ   râ   ÚoutputÚlocalrd   rî   rï   rð   rñ   rò   ró   ©
rZ   rG   rö   rî   rï   rð   rñ   rò   ró   r^   s
            €r:   rF   zCanineAttention.__init__�  s¿   ø€ õ 	‰Œ×ÒÑÔÐÝ'¨Ñ/Ô/ˆŒ	Ý& vÑ.Ô.ˆŒð ˆŒ
Ø"Ð%=Ò=Ð=ÝØwñô ð ð !Ð#9Ò9Ð9ÝØrñô ð ð 0OˆÔ,Ø-JˆÔ*Ø'>ˆÔ$Ø(@ˆÔ%Ø%:ˆÔ"Ø&<ˆÔ#Ð#Ð#r9   Nr.   rÂ   rÃ   rz   c                 ó4  — | j         s"|                      ||||¦  «        }|d         }�n.|j        d         x}}|x}}	g }
| j        r|
                     d¦  «         d}nd}t          ||| j        ¦  «        D ]1}t          ||| j        z   ¦  «        }|
                     ||f¦  «         Œ2g }| j        r|                     d|f¦  «         t          d|| j	        ¦  «        D ]1}t          ||| j
        z   ¦  «        }|                     ||f¦  «         Œ2t          |
¦  «        t          |¦  «        k    rt          d|
› d|
› d�¦  «        ‚g }g }t          |
|¦  «        D ]ç\  \  }}\  }}|d d …||…d d …f         }|	d d …||…d d …f         }|d d …||…||…f         }| j        rR|d d …||…dd…f         }t          j        ||gd¬¦  «        }|	d d …dd…d d …f         }t          j        ||gd¬¦  «        }|                      ||||¦  «        }|                     |d         ¦  «         |r|                     |d         ¦  «         Œèt          j        |d¬¦  «        }|                      ||¦  «        }|f}| j         s||dd …         z   }n|t%          |¦  «        z   }|S )	Nr   r   )r   r   z/Expected to have same number of `from_chunks` (z) and `to_chunks` (z). Check strides.r›   rn   )rö   rZ   rÆ   rï   re   rJ   rñ   rÏ   rð   ró   rò   rb   rd   Úziprî   r4   rr   rõ   r7   )rZ   r.   rÂ   rÃ   Úself_outputsÚattention_outputÚfrom_seq_lengthÚto_seq_lengthrÀ   rÁ   Úfrom_chunksÚ
from_startÚchunk_startÚ	chunk_endÚ	to_chunksÚattention_output_chunksÚattention_probs_chunksÚfrom_endÚto_startÚto_endÚfrom_tensor_chunkÚto_tensor_chunkÚattention_mask_chunkÚcls_attention_maskÚcls_positionÚattention_outputs_chunkrÞ   s                              r:   r†   zCanineAttention.forward­  s…  € ð Œzð 9	IØŸ9š9 ]°MÀ>ÐSdÑeÔeˆLØ+¨AœÐÑà.;Ô.AÀ!Ô.DÐDˆO˜mØ&3Ð3ˆK˜)ð ˆKØÔ1ð Ø×"Ò" 6Ñ*Ô*Ð*ð �
�
à�
Ý$ Z°À$ÔB_Ñ`Ô`ð =ð =�Ý °¸tÔ?[Ñ1[Ñ\Ô\�	Ø×"Ò" K°Ð#;Ñ<Ô<Ð<Ð<ð ˆIØÔ1ð 5Ø× Ò  ! ]Ð!3Ñ4Ô4Ð4Ý$ Q¨°tÔ7RÑSÔSð ;ð ;�Ý ¨{¸TÔ=WÑ/WÑXÔX�	Ø× Ò  +¨yÐ!9Ñ:Ô:Ð:Ð:å�;ÑÔ¥3 y¡>¤>Ò1Ð1Ý ðCÀkð Cð CØ$/ðCð Cð Cñô ð ð ')Ð#Ø%'Ð"Ý>AÀ+ÈyÑ>YÔ>Yð Nð NÑ:Ñ&�˜XÑ(:¨°6Ø$/°°°°:¸hÐ3FÈÈÈÐ0IÔ$JÐ!Ø"+¨A¨A¨A¨x¸¨ÀÀÀÐ,AÔ"B�ð (6°a°a°a¸ÀHÐ9LÈhÐW]ÈoÐ6]Ô'^Ð$ØÔ7ð XØ)7¸¸¸¸:ÀhÐ;NÐPQÐRSÐPSÐ8SÔ)TÐ&Ý+0¬9Ð6HÐJ^Ð5_ÐefÐ+gÑ+gÔ+gÐ(à#,¨Q¨Q¨Q°°!°°Q°Q°Q¨YÔ#7�LÝ&+¤i°¸Ð0OÐUVÐ&WÑ&WÔ&W�Oà*.¯)ª)Ø% Ð8LÐN_ñ+ô +Ð'ð (×.Ò.Ð/FÀqÔ/IÑJÔJÐJØ$ð NØ*×1Ò1Ð2IÈ!Ô2LÑMÔMÐMøå$œyÐ)@ÀaÐHÑHÔHÐàŸ;š;Ð'7¸ÑGÔGÐØ#Ð%ˆØŒzð 	>Ø ¨Q¨R¨RÔ 0Ñ0ˆGˆGà¥Ð&<Ñ =Ô =Ñ=ˆGØˆr9   ©FFFrí   rí   rí   rí   rß   )r0   r1   r2   r3   rà   r‡   rF   r7   r4   r5   r†   r‰   rŠ   s   @r:   rì   rì   |  s  ø€ € € € € ðð ð& Ø05Ø.3Ø'*Ø(+Ø%(Ø&)ð=ð =ð *.ð	=ð
 (,ð=ð "%ð=ð #&ð=ð  #ð=ð !$ð=ð =ð =ð =ð =ð =ðF 48Ø).ð	Gð Gà˜UÔ.Ô/ðGð Ô)¨DÑ0ðGð   $™;ð	Gð
 
ˆuÔ  %Ô"3°dÑ":Ð:Ô	;ðGð Gð Gð Gð Gð Gð Gð Gr9   rì   c                   óB   ‡ — e Zd Zˆ fd„Zdej        dej        fd„Zˆ xZS )ÚCanineIntermediatec                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _        d S |j        | _        d S r²   )rE   rF   r   r»   rH   Úintermediate_sizerå   Ú
isinstancer–   Ústrr
   Úintermediate_act_fnr˜   s     €r:   rF   zCanineIntermediate.__init__ø  sn   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3KÑLÔLˆŒ
Ý�fÔ'­Ñ-Ô-ð 	9Ý'-¨fÔ.?Ô'@ˆDÔ$Ð$Ð$à'-Ô'8ˆDÔ$Ð$Ð$r9   r.   rz   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r²   )rå   r  ©rZ   r.   s     r:   r†   zCanineIntermediate.forward   s,   € ØŸ
š
 =Ñ1Ô1ˆØ×0Ò0°Ñ?Ô?ˆØÐr9   )r0   r1   r2   rF   r4   r5   r†   r‰   rŠ   s   @r:   r  r  ÷  s`   ø€ € € € € ð9ð 9ð 9ð 9ð 9ð UÔ%6ð ¸5Ô;Lð ð ð ð ð ð ð ð r9   r  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                 dej        dej        fd„Zˆ xZS )ÚCanineOutputc                 ó  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        |j        |j        ¬¦  «        | _        t          j	        |j
        ¦  «        | _        d S rä   )rE   rF   r   r»   r  rH   rå   rQ   rR   rS   rT   rU   r˜   s     €r:   rF   zCanineOutput.__init__  sf   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ7¸Ô9KÑLÔLˆŒ
Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒÝ”z &Ô"<Ñ=Ô=ˆŒˆˆr9   r.   ræ   rz   c                 óŠ   — |                       |¦  «        }|                      |¦  «        }|                      ||z   ¦  «        }|S r²   rè   ré   s      r:   r†   zCanineOutput.forward  s@   € ØŸ
š
 =Ñ1Ô1ˆØŸš ]Ñ3Ô3ˆØŸš }°|Ñ'CÑDÔDˆØÐr9   rê   rŠ   s   @r:   r  r    sp   ø€ € € € € ð>ð >ð >ð >ð >ð U¨5Ô+<Ô%=ð ÈUÔM^ð ÐchÔctð ð ð ð ð ð ð ð r9   r  c                   ó˜   ‡ — e Zd Zˆ fd„Z	 	 d
deej                 dej        dz  dedz  deej        ej        dz  f         fd„Zd	„ Z	ˆ xZ
S )ÚCanineLayerc	           
      óô   •— t          ¦   «                              ¦   «          |j        | _        d| _        t	          ||||||||¦  «        | _        t          |¦  «        | _        t          |¦  «        | _	        d S ©Nr   )
rE   rF   Úchunk_size_feed_forwardÚseq_len_dimrì   Ú	attentionr  Úintermediater  rõ   r÷   s
            €r:   rF   zCanineLayer.__init__  s|   ø€ õ 	‰Œ×ÒÑÔÐØ'-Ô'EˆÔ$ØˆÔÝ(ØØØ+Ø)Ø#Ø$Ø!Ø"ñ	
ô 	
ˆŒõ /¨vÑ6Ô6ˆÔÝ" 6Ñ*Ô*ˆŒˆˆr9   NFr.   rÂ   rÃ   rz   c                 ó¨   — |                       |||¬¦  «        }|d         }|dd …         }t          | j        | j        | j        |¦  «        }|f|z   }|S )N)rÃ   r   r   )r"  r   Úfeed_forward_chunkr   r!  )rZ   r.   rÂ   rÃ   Úself_attention_outputsrû   rÞ   Úlayer_outputs           r:   r†   zCanineLayer.forward0  sv   € ð "&§¢ØØØ/ð "0ñ "
ô "
Ðð
 2°!Ô4Ðà(¨¨¨Ô,ˆå0ØÔ# TÔ%AÀ4ÔCSÐUeñ
ô 
ˆð  �/ GÑ+ˆàˆr9   c                 ó\   — |                       |¦  «        }|                      ||¦  «        }|S r²   )r#  rõ   )rZ   rû   Úintermediate_outputr'  s       r:   r%  zCanineLayer.feed_forward_chunkF  s2   € Ø"×/Ò/Ð0@ÑAÔAÐØ—{’{Ð#6Ð8HÑIÔIˆØÐr9   rß   )r0   r1   r2   rF   r7   r4   r5   rà   r†   r%  r‰   rŠ   s   @r:   r  r    s·   ø€ € € € € ð+ð +ð +ð +ð +ð< 48Ø).ð	ð à˜UÔ.Ô/ðð Ô)¨DÑ0ðð   $™;ð	ð
 
ˆuÔ  %Ô"3°dÑ":Ð:Ô	;ðð ð ð ð,ð ð ð ð ð ð r9   r  c                   ó–   ‡ — e Zd Z	 	 	 	 	 	 	 dˆ fd„	Z	 	 	 	 ddeej                 dej        dz  dedz  d	edz  d
edz  deez  fd„Z	ˆ xZ
S )ÚCanineEncoderFrí   c	           
      óð   •‡‡‡‡‡‡‡‡— t          ¦   «                              ¦   «          ‰| _        t          j        ˆˆˆˆˆˆˆˆfd„t          ‰j        ¦  «        D ¦   «         ¦  «        | _        d| _        d S )Nc                 ó<   •— g | ]}t          ‰‰	‰‰‰‰‰‰¦  «        ‘ŒS r8   )r  )
Ú.0rÖ   rî   rñ   rð   ró   rò   rG   rï   rö   s
     €€€€€€€€r:   ú
<listcomp>z*CanineEncoder.__init__.<locals>.<listcomp>[  sM   ø€ ð ð ð ð õ ØØØ3Ø1Ø+Ø,Ø)Ø*ñ	ô 	ðð ð r9   F)	rE   rF   rG   r   Ú
ModuleListrJ   Únum_hidden_layersÚlayerÚgradient_checkpointingr÷   s
    ````````€r:   rF   zCanineEncoder.__init__M  s£   øøøøøøøøø€ õ 	‰Œ×ÒÑÔÐØˆŒÝ”]ðð ð ð ð ð ð ð ð ð ð õ ˜vÔ7Ñ8Ô8ðñ ô ñ
ô 
ˆŒ
ð ',ˆÔ#Ð#Ð#r9   NTr.   rÂ   rÃ   Úoutput_hidden_statesÚreturn_dictrz   c                 ó  — |rdnd }|rdnd }t          | j        ¦  «        D ]0\  }}	|r||fz   } |	|||¦  «        }
|
d         }|r||
d         fz   }Œ1|r||fz   }|st          d„ |||fD ¦   «         ¦  «        S t          |||¬¦  «        S )Nr8   r   r   c              3   ó   K  — | ]}|®|V — Œ	d S r²   r8   ©r.  Úvs     r:   ú	<genexpr>z(CanineEncoder.forward.<locals>.<genexpr>„  s(   è è € ÐmÐm˜qÐ_`Ð_l˜Ð_lÐ_lÐ_lÐ_lÐmÐmr9   )r,   r.   r/   )rp   r2  r7   r   )rZ   r.   rÂ   rÃ   r4  r5  Úall_hidden_statesÚall_self_attentionsr\   Úlayer_moduleÚlayer_outputss              r:   r†   zCanineEncoder.forwardk  sù   € ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4Ðå(¨¬Ñ4Ô4ð 	Pð 	P‰OˆAˆ|Ø#ð IØ$5¸Ð8HÑ$HÐ!à(˜L¨¸ÐHYÑZÔZˆMà)¨!Ô,ˆMØ ð PØ&9¸]È1Ô=MÐ<OÑ&OÐ#øàð 	EØ 1°]Ð4DÑ DÐàð 	nÝÐmÐm ]Ð4EÐGZÐ$[ÐmÑmÔmÑmÔmÐmÝØ+Ø+Ø*ð
ñ 
ô 
ð 	
r9   r  )NFFT)r0   r1   r2   rF   r7   r4   r5   rà   r   r†   r‰   rŠ   s   @r:   r+  r+  L  s×   ø€ € € € € ð Ø(-Ø&+Ø #Ø!$Ø!Ø"ð,ð ,ð ,ð ,ð ,ð ,ðB 48Ø).Ø,1Ø#'ð
ð 
à˜UÔ.Ô/ð
ð Ô)¨DÑ0ð
ð   $™;ð	
ð
 # T™kð
ð ˜D‘[ð
ð 
�Ñ	 ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r9   r+  c                   óN   ‡ — e Zd Zˆ fd„Zdeej                 dej        fd„Zˆ xZS )ÚCaninePoolerc                 óÀ   •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          j        ¦   «         | _        d S r²   )rE   rF   r   r»   rH   rå   ÚTanhr—   r˜   s     €r:   rF   zCaninePooler.__init__�  sC   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ýœ'™)œ)ˆŒˆˆr9   r.   rz   c                 ór   — |d d …df         }|                       |¦  «        }|                      |¦  «        }|S )Nr   )rå   r—   )rZ   r.   Úfirst_token_tensorÚpooled_outputs       r:   r†   zCaninePooler.forward’  s@   € ð +¨1¨1¨1¨a¨4Ô0ÐØŸ
š
Ð#5Ñ6Ô6ˆØŸš¨Ñ6Ô6ˆØÐr9   rê   rŠ   s   @r:   r@  r@  Œ  se   ø€ € € € € ð$ð $ð $ð $ð $ð
 U¨5Ô+<Ô%=ð À%ÔBSð ð ð ð ð ð ð ð r9   r@  c                   óN   ‡ — e Zd Zˆ fd„Zdeej                 dej        fd„Zˆ xZS )ÚCaninePredictionHeadTransformc                 óV  •— t          ¦   «                              ¦   «          t          j        |j        |j        ¦  «        | _        t          |j        t          ¦  «        rt          |j                 | _
        n|j        | _
        t          j        |j        |j        ¬¦  «        | _        d S rä   )rE   rF   r   r»   rH   rå   r  r–   r  r
   Útransform_act_fnrQ   rR   r˜   s     €r:   rF   z&CaninePredictionHeadTransform.__init__œ  s…   ø€ Ý‰Œ×ÒÑÔÐÝ”Y˜vÔ1°6Ô3EÑFÔFˆŒ
Ý�fÔ'­Ñ-Ô-ð 	6Ý$*¨6Ô+<Ô$=ˆDÔ!Ð!à$*Ô$5ˆDÔ!Ýœ fÔ&8¸fÔ>SÐTÑTÔTˆŒˆˆr9   r.   rz   c                 ó„   — |                       |¦  «        }|                      |¦  «        }|                      |¦  «        }|S r²   )rå   rI  rQ   r  s     r:   r†   z%CaninePredictionHeadTransform.forward¥  s=   € ØŸ
š
 =Ñ1Ô1ˆØ×-Ò-¨mÑ<Ô<ˆØŸš }Ñ5Ô5ˆØÐr9   rê   rŠ   s   @r:   rG  rG  ›  sj   ø€ € € € € ðUð Uð Uð Uð Uð U¨5Ô+<Ô%=ð À%ÔBSð ð ð ð ð ð ð ð r9   rG  c                   óN   ‡ — e Zd Zˆ fd„Zdeej                 dej        fd„Zˆ xZS )ÚCanineLMPredictionHeadc                 ó  •— t          ¦   «                              ¦   «          t          |¦  «        | _        t	          j        |j        |j        d¬¦  «        | _        t	          j	        t          j        |j        ¦  «        ¦  «        | _        d S )NT)Úbias)rE   rF   rG  Ú	transformr   r»   rH   Ú
vocab_sizeÚdecoderÚ	Parameterr4   r€   rN  r˜   s     €r:   rF   zCanineLMPredictionHead.__init__­  sj   ø€ Ý‰Œ×ÒÑÔÐÝ6°vÑ>Ô>ˆŒõ ”y Ô!3°VÔ5FÈTÐRÑRÔRˆŒå”L¥¤¨VÔ->Ñ!?Ô!?Ñ@Ô@ˆŒ	ˆ	ˆ	r9   r.   rz   c                 óZ   — |                       |¦  «        }|                      |¦  «        }|S r²   )rO  rQ  r  s     r:   r†   zCanineLMPredictionHead.forward¹  s*   € ØŸš }Ñ5Ô5ˆØŸš ]Ñ3Ô3ˆØÐr9   rê   rŠ   s   @r:   rL  rL  ¬  sj   ø€ € € € € ðAð Að Að Að Að U¨5Ô+<Ô%=ð À%ÔBSð ð ð ð ð ð ð ð r9   rL  c                   óZ   ‡ — e Zd Zˆ fd„Zdeej                 deej                 fd„Zˆ xZS )ÚCanineOnlyMLMHeadc                 óp   •— t          ¦   «                              ¦   «          t          |¦  «        | _        d S r²   )rE   rF   rL  Úpredictionsr˜   s     €r:   rF   zCanineOnlyMLMHead.__init__À  s/   ø€ Ý‰Œ×ÒÑÔÐÝ1°&Ñ9Ô9ˆÔÐÐr9   Úsequence_outputrz   c                 ó0   — |                       |¦  «        }|S r²   )rW  )rZ   rX  Úprediction_scoress      r:   r†   zCanineOnlyMLMHead.forwardÄ  s   € ð !×,Ò,¨_Ñ=Ô=ÐØ Ð r9   )	r0   r1   r2   rF   r7   r4   r¢   r†   r‰   rŠ   s   @r:   rU  rU  ¿  sl   ø€ € € € € ð:ð :ð :ð :ð :ð!à˜uœ|Ô,ð!ð 
ˆuŒ|Ô	ð!ð !ð !ð !ð !ð !ð !ð !r9   rU  c                   ó2   ‡ — e Zd ZU eed<   dZdZˆ fd„Zˆ xZS )ÚCaninePreTrainedModelrG   ÚcanineTc                 ó  •— t          ¦   «                              |¦  «         t          |t          ¦  «        rQt	          j        |j        t          j        |j        j	        d         ¦  «         
                    d¦  «        ¦  «         d S d S )NrC   rB   )rE   Ú_init_weightsr  r<   ÚinitÚcopy_rA   r4   rW   rÆ   rY   )rZ   Úmoduler^   s     €r:   r_  z#CaninePreTrainedModel._init_weightsÒ  s{   ø€ Ý‰Œ×Ò˜fÑ%Ô%Ð%Ý�fÕ.Ñ/Ô/ð 	iÝŒJ�vÔ*­E¬L¸Ô9LÔ9RÐSUÔ9VÑ,WÔ,W×,^Ò,^Ð_fÑ,gÔ,gÑhÔhÐhÐhÐhð	ið 	ir9   )	r0   r1   r2   r   r6   Úbase_model_prefixÚsupports_gradient_checkpointingr_  r‰   rŠ   s   @r:   r\  r\  Ì  s[   ø€ € € € € € àÐÐÑØ ÐØ&*Ð#ðið ið ið ið ið ið ið ið ir9   r\  c                   ó  ‡ — e Zd Zdˆ fd„	Zd„ Zdej        defd„Zdej        ded	ej        fd
„Z	e
	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dedz  dedz  dedz  d	eez  fd„¦   «         Zˆ xZS )ÚCanineModelTc           
      ó  •— t          ¦   «                              |¦  «         || _        t          j        |¦  «        }d|_        t          |¦  «        | _        t          |ddd|j	        |j	        |j	        |j	        ¬¦  «        | _
        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        t          |¦  «        | _        |rt#          |¦  «        nd| _        |                      ¦   «          dS )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        r   TF)rö   rî   rï   rð   rñ   rò   ró   N)rE   rF   rG   ÚcopyÚdeepcopyr1  r<   Úchar_embeddingsr+  Úlocal_transformer_strideÚinitial_char_encoderrŒ   Úchars_to_moleculesÚencoderr¤   Ú
projectionÚfinal_char_encoderr@  ÚpoolerÚ	post_init)rZ   rG   Úadd_pooling_layerÚshallow_configr^   s       €r:   rF   zCanineModel.__init__Ú  s÷   ø€ õ
 	‰Œ×Ò˜Ñ Ô Ð ØˆŒÝœ vÑ.Ô.ˆØ+,ˆÔ(å/°Ñ7Ô7ˆÔå$1ØØØ,1Ø*/Ø$*Ô$CØ%+Ô%DØ"(Ô"AØ#)Ô#Bð	%
ñ 	%
ô 	%
ˆÔ!õ #8¸Ñ"?Ô"?ˆÔå$ VÑ,Ô,ˆŒÝ(¨Ñ0Ô0ˆŒå"/°Ñ"?Ô"?ˆÔà.?ÐI•l 6Ñ*Ô*Ð*ÀTˆŒð 	�ŠÑÔÐÐÐr9   c                 ó  — |j         d         |j         d         }}|j         d         }t          j        ||d|f¦  «                             ¦   «         }t          j        ||dft          j        |j        ¬¦  «        }||z  }|S )aP  
        Create 3D attention mask from a 2D tensor mask.

        Args:
            from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...].
            to_mask: int32 Tensor of shape [batch_size, to_seq_length].

        Returns:
            float Tensor of shape [batch_size, from_seq_length, to_seq_length].
        r   r   )r   r}   r~   )rÆ   r4   ÚreshaperÍ   ÚonesÚfloat32r~   )rZ   rÀ   Úto_maskrÕ   rü   rý   Úbroadcast_onesÚmasks           r:   Ú)_create_3d_attention_mask_from_input_maskz5CanineModel._create_3d_attention_mask_from_input_maskü  s…   € ð '2Ô&7¸Ô&:¸KÔ<MÈaÔ<P�Oˆ
àœ aÔ(ˆå”- ¨*°a¸Ð)GÑHÔH×NÒNÑPÔPˆõ
 œ¨*°oÀqÐ)IÕQVÔQ^ÐgnÔguÐvÑvÔvˆð  Ñ'ˆàˆr9   Úchar_attention_maskr”   c                 óè   — |j         \  }}t          j        ||d|f¦  «        }t          j                             ||¬¦  «        |                     ¦   «         ¦  «        }|                     d¬¦  «        S )z[Downsample 2D character attention mask to 2D molecule attention mask using MaxPool1d layer.r   )r‘   r’   rn   )rÆ   r4   rv  r   Ú	MaxPool1drÍ   Úsqueeze)rZ   r}  r”   rÕ   Úchar_seq_lenÚpoolable_char_maskÚpooled_molecule_masks          r:   Ú_downsample_attention_maskz&CanineModel._downsample_attention_mask  s}   € ð $7Ô#<Ñ ˆ
�LÝ"œ]Ð+>ÀÈQÐP\Ð@]Ñ^Ô^Ðõ  %œx×1Ò1Ð>OÐXiÐ1ÑjÔjØ×$Ò$Ñ&Ô&ñ 
ô  
Ðð
 $×+Ò+°Ð+Ñ2Ô2Ð2r9   Ú	moleculesÚchar_seq_lengthrz   c                 óú   — | j         j        }|dd…dd…dd…f         }t          j        ||d¬¦  «        }|dd…dd…dd…f         }||z  }t          j        |||z   d¬¦  «        }t          j        ||gd¬¦  «        S )zDRepeats molecules to make them the same length as the char sequence.Nr   rÅ   )Úrepeatsro   rC   rn   )rG   r”   r4   Úrepeat_interleaverr   )	rZ   r…  r†  ÚrateÚmolecules_without_extra_clsÚrepeatedÚlast_moleculeÚremainder_lengthÚremainder_repeateds	            r:   Ú_repeat_moleculeszCanineModel._repeat_molecules&  s±   € ð Œ{Ô,ˆà&/°°°°1°2°2°q°q°q°Ô&9Ð#åÔ*Ð+FÐPTÐZ\Ð]Ñ]Ô]ˆð " ! ! ! R S S¨!¨!¨! )Ô,ˆØ*¨TÑ1ÐÝ"Ô4Øà$ tÑ+Øð	
ñ 
ô 
Ðõ Œy˜(Ð$6Ð7¸RÐ@Ñ@Ô@Ð@r9   Nrf   rÂ   rx   rA   ry   rÃ   r4  r5  c	                 óì  — |�|n| j         j        }|�|n| j         j        }|rdnd }
|rdnd }|�|n| j         j        }|�|�t	          d¦  «        ‚|�+|                      ||¦  «         |                     ¦   «         }n.|�|                     ¦   «         d d…         }nt	          d¦  «        ‚|\  }}|�|j        n|j        }|€t          j	        ||f|¬¦  «        }|€!t          j
        |t          j        |¬¦  «        }|                      || j         j        ¬¦  «        }|                      ||||¬¦  «        }|                      |�|n||¦  «        }|                      ||||¬	¦  «        }|j        }|                      |¦  «        }t'          | j         |d d …d
d…d d …f         |¬¦  «        }|                      |||||¬¦  «        }|d
         }| j        �|                      |¦  «        nd }|                      ||d         ¬¦  «        }t          j        ||gd¬¦  «        }|                      |¦  «        }t'          | j         ||¬¦  «        }|                      ||||¬	¦  «        }|j        }|r&|r|j        n|d         }|
|j        z   |z   |j        z   }
|r&|r|j        n|d         }||j        z   |z   |j        z   }|s$||f}|t9          d„ |
|fD ¦   «         ¦  «        z  }|S t;          |||
|¬¦  «        S )Nr8   zDYou cannot specify both input_ids and inputs_embeds at the same timerC   z5You have to specify either input_ids or inputs_embeds)r~   r|   )r”   )rf   rA   rx   ry   )rÂ   rÃ   r4  r   r   )rG   ry   rÂ   )rÂ   rÃ   r4  r5  )r†  rn   c              3   ó   K  — | ]}|®|V — Œ	d S r²   r8   r8  s     r:   r:  z&CanineModel.forward.<locals>.<genexpr>Ô  s(   è è € ÐaÐa !ÐSTÐS`˜AÐS`ÐS`ÐS`ÐS`ÐaÐar9   )r,   r-   r.   r/   )rG   rÃ   r4  r5  rd   Ú%warn_if_padding_and_no_attention_maskr   r~   r4   rw  r€   r�   r„  r”   rj  r|  rl  r,   rm  r   rn  rq  r�  rr   ro  rp  r.   r/   r7   r+   ) rZ   rf   rÂ   rx   rA   ry   rÃ   r4  r5  Úkwargsr;  r<  r‚   rÕ   rƒ   r~   Úmolecule_attention_maskÚinput_char_embeddingsr}  Úinit_chars_encoder_outputsÚinput_char_encodingÚinit_molecule_encodingÚencoder_outputsÚmolecule_sequence_outputrE  Úrepeated_moleculesÚconcatrX  Úfinal_chars_encoder_outputsÚdeep_encoder_hidden_statesÚdeep_encoder_self_attentionsrõ   s                                    r:   r†   zCanineModel.forward?  s)  € ð 2CÐ1NÐ-Ð-ÐTXÔT_ÔTqÐà$8Ð$DÐ Ð È$Ì+ÔJjð 	ð #7Ð@˜B˜B¸DÐØ$5Ð?˜b˜b¸4ÐØ%0Ð%<�k�kÀ$Ä+ÔBYˆàÐ  ]Ð%>ÝÐcÑdÔdÐdØÐ"Ø×6Ò6°yÀ.ÑQÔQÐQØ#Ÿ.š.Ñ*Ô*ˆKˆKØÐ&Ø'×,Ò,Ñ.Ô.¨s°¨sÔ3ˆKˆKåÐTÑUÔUÐUà!,Ñˆ
�JØ%.Ð%:�Ô!Ð!ÀÔ@TˆàÐ!Ý"œZ¨*°jÐ)AÈ6ÐRÑRÔRˆNØÐ!Ý"œ[¨½E¼JÈvÐVÑVÔVˆNð #'×"AÒ"AØ¨d¬kÔ.Kð #Bñ #
ô #
Ðð
 !%× 4Ò 4ØØ%Ø)Ø'ð	 !5ñ !
ô !
Ðð #×LÒLØ"Ð.ˆIˆI°MÀ>ñ
ô 
Ðð &*×%>Ò%>Ø!Ø.Ø/Ø!5ð	 &?ñ &
ô &
Ð"ð 9ÔJÐð  "&×!8Ò!8Ð9LÑ!MÔ!MÐå";Ø”;Ø0°°°°A°a°C¸¸¸°Ô;Ø2ð#
ñ #
ô #
Ðð Ÿ,š,Ø"Ø2Ø/Ø!5Ø#ð 'ñ 
ô 
ˆð $3°1Ô#5Ð ØAEÄÐAX˜ŸšÐ$<Ñ=Ô=Ð=Ð^bˆð "×3Ò3Ð4LÐ^iÐjlÔ^mÐ3ÑnÔnÐõ ”Ð/Ð1CÐDÈ"ÐMÑMÔMˆð Ÿ/š/¨&Ñ1Ô1ˆå2Ø”;Ø)Ø)ð
ñ 
ô 
ˆð '+×&=Ò&=ØØ)Ø/Ø!5ð	 '>ñ '
ô '
Ð#ð 6ÔGˆàð 	ØJUÐ)m¨Ô)FÐ)FÐ[jÐklÔ[mÐ&à!Ø,Ô:ñ;à,ñ-ð .Ô;ñ<ð ð ð 	ØITÐ+m¨?Ô+EÐ+EÐZiÐjlÔZmÐ(à#Ø,Ô7ñ8à.ñ/ð .Ô8ñ9ð  ð ð 	Ø% }Ð5ˆFØ•eÐaÐaÐ(9Ð;NÐ'OÐaÑaÔaÑaÔaÑaˆFØˆMå+Ø-Ø'Ø+Ø*ð	
ñ 
ô 
ð 	
r9   )T)NNNNNNNN)r0   r1   r2   rF   r|  r4   r¢   r‡   r„  r�  r   rˆ   r5   rà   r7   r+   r†   r‰   rŠ   s   @r:   rf  rf  Ø  s‹  ø€ € € € € ð ð  ð  ð  ð  ð  ðDð ð ð63¸e¼lð 3Ð_bð 3ð 3ð 3ð 3ðA¨5¬<ð AÈ#ð AÐRWÔR^ð Að Að Að Að2 ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ð\
ð \
àÔ# dÑ*ð\
ð Ô)¨DÑ0ð\
ð Ô(¨4Ñ/ð	\
ð
 Ô&¨Ñ-ð\
ð Ô(¨4Ñ/ð\
ð   $™;ð\
ð # T™kð\
ð ˜D‘[ð\
ð 
Ð-Ñ	-ð\
ð \
ð \
ñ „^ð\
ð \
ð \
ð \
ð \
r9   rf  zž
    CANINE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚCanineForSequenceClassificationc                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r²   ©rE   rF   Ú
num_labelsrf  r]  r   rS   rT   rU   r»   rH   Ú
classifierrr  r˜   s     €r:   rF   z(CanineForSequenceClassification.__init__æ  óy   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå! &Ñ)Ô)ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr9   Nrf   rÂ   rx   rA   ry   ÚlabelsrÃ   r4  r5  rz   c
           
      óì  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|��Z| j         j        €f| j        dk    rd| j         _        nN| j        dk    r7|j        t          j	        k    s|j        t          j
        k    rd| j         _        nd| j         _        | j         j        dk    rWt          ¦   «         }| j        dk    r1 ||                     ¦   «         |                     ¦   «         ¦  «        }nŽ |||¦  «        }n�| j         j        dk    rGt          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }n*| j         j        dk    rt          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t!          |||j        |j        ¬	¦  «        S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©rÂ   rx   rA   ry   rÃ   r4  r5  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrC   r›   ©ÚlossÚlogitsr.   r/   )rG   r5  r]  rU   r¦  Úproblem_typer¥  r}   r4   r�   r‡   r   r€  r   rÇ   r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r¨  rÃ   r4  r5  r”  rÞ   rE  r°  r¯  Úloss_fctrõ   s                    r:   r†   z'CanineForSequenceClassification.forwardñ  s  € ð( &1Ð%<�k�kÀ$Ä+ÔBYˆà—+’+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆàˆØÑØŒ{Ô'Ð/Ø”? aÒ'Ð'Ø/;�D”KÔ,Ð,Ø”_ qÒ(Ð(¨f¬l½e¼jÒ.HÐ.HÈFÌLÕ\aÔ\eÒLeÐLeØ/L�D”KÔ,Ð,à/K�D”KÔ,àŒ{Ô'¨<Ò7Ð7Ý"™9œ9�Ø”? aÒ'Ð'Ø#˜8 F§N¢NÑ$4Ô$4°f·n²nÑ6FÔ6FÑGÔG�D�Dà#˜8 F¨FÑ3Ô3�D�DØ”Ô)Ð-JÒJÐJÝ+Ñ-Ô-�Ø�x §¢¨B°´Ñ @Ô @À&Ç+Â+ÈbÁ/Ä/ÑRÔR��Ø”Ô)Ð-IÒIÐIÝ,Ñ.Ô.�Ø�x ¨Ñ/Ô/�Øð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå'ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r9   ©	NNNNNNNNN)r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r¢  r¢  ß  s1  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðD
ð D
àÔ# dÑ*ðD
ð Ô)¨DÑ0ðD
ð Ô(¨4Ñ/ð	D
ð
 Ô&¨Ñ-ðD
ð Ô(¨4Ñ/ðD
ð Ô  4Ñ'ðD
ð   $™;ðD
ð # T™kðD
ð ˜D‘[ðD
ð 
Ð)Ñ	)ðD
ð D
ð D
ñ „^ðD
ð D
ð D
ð D
ð D
r9   r¢  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚCanineForMultipleChoicec                 ó  •— t          ¦   «                              |¦  «         t          |¦  «        | _        t	          j        |j        ¦  «        | _        t	          j        |j	        d¦  «        | _
        |                      ¦   «          d S r  )rE   rF   rf  r]  r   rS   rT   rU   r»   rH   r¦  rr  r˜   s     €r:   rF   z CanineForMultipleChoice.__init__;  sl   ø€ Ý‰Œ×Ò˜Ñ Ô Ð å! &Ñ)Ô)ˆŒÝ”z &Ô"<Ñ=Ô=ˆŒÝœ) FÔ$6¸Ñ:Ô:ˆŒð 	�ŠÑÔÐÐÐr9   Nrf   rÂ   rx   rA   ry   r¨  rÃ   r4  r5  rz   c
           
      ó¸  — |	�|	n| j         j        }	|�|j        d         n|j        d         }|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�)|                     d|                     d¦  «        ¦  «        nd}|�=|                     d|                     d¦  «        |                     d¦  «        ¦  «        nd}|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }|                     d|¦  «        }d}|�t          ¦   «         } |||¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j
        |j        ¬¦  «        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rC   rÅ   rª  r›   r®  )rG   r5  rÆ   rÇ   r   r]  rU   r¦  r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r¨  rÃ   r4  r5  r”  Únum_choicesrÞ   rE  r°  Úreshaped_logitsr¯  r²  rõ   s                      r:   r†   zCanineForMultipleChoice.forwardE  s(  € ðX &1Ð%<�k�kÀ$Ä+ÔBYˆØ,5Ð,A�i”o aÔ(Ð(À}ÔGZÐ[\ÔG]ˆà>GÐ>S�I—N’N 2 y§~¢~°bÑ'9Ô'9Ñ:Ô:Ð:ÐY]ˆ	ØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØM[ÐMg˜×,Ò,¨R°×1DÒ1DÀRÑ1HÔ1HÑIÔIÐIÐmqˆØGSÐG_�|×(Ò(¨¨\×->Ò->¸rÑ-BÔ-BÑCÔCÐCÐeiˆð Ð(ð ×Ò˜r =×#5Ò#5°bÑ#9Ô#9¸=×;MÒ;MÈbÑ;QÔ;QÑRÔRÐRàð 	ð —+’+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð   œ
ˆàŸš ]Ñ3Ô3ˆØ—’ Ñ/Ô/ˆØ Ÿ+š+ b¨+Ñ6Ô6ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜O¨VÑ4Ô4ˆDàð 	FØ%Ð'¨'°!°"°"¬+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå(ØØ"Ø!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r9   r³  )r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   rµ  rµ  9  s1  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðW
ð W
àÔ# dÑ*ðW
ð Ô)¨DÑ0ðW
ð Ô(¨4Ñ/ð	W
ð
 Ô&¨Ñ-ðW
ð Ô(¨4Ñ/ðW
ð Ô  4Ñ'ðW
ð   $™;ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*ðW
ð W
ð W
ñ „^ðW
ð W
ð W
ð W
ð W
r9   rµ  c                   óê   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	edz  d
edz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚCanineForTokenClassificationc                 ó6  •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        ¦  «        | _        t          j	        |j
        |j        ¦  «        | _        |                      ¦   «          d S r²   r¤  r˜   s     €r:   rF   z%CanineForTokenClassification.__init__¢  r§  r9   Nrf   rÂ   rx   rA   ry   r¨  rÃ   r4  r5  rz   c
           
      óÂ  — |	�|	n| j         j        }	|                      ||||||||	¬¦  «        }|d         }|                      |¦  «        }|                      |¦  «        }d}|�Ft          ¦   «         } ||                     d| j        ¦  «        |                     d¦  «        ¦  «        }|	s|f|dd…         z   }|�|f|z   n|S t          |||j	        |j
        ¬¦  «        S )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, CanineForTokenClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/canine-s")
        >>> model = CanineForTokenClassification.from_pretrained("google/canine-s")

        >>> inputs = tokenizer(
        ...     "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt"
        ... )

        >>> with torch.no_grad():
        ...     logits = model(**inputs).logits

        >>> predicted_token_class_ids = logits.argmax(-1)

        >>> # Note that tokens are classified rather then input words which means that
        >>> # there might be more predicted token classes than words.
        >>> # Multiple token classes might account for the same word
        >>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
        >>> predicted_tokens_classes  # doctest: +SKIP
        ```

        ```python
        >>> labels = predicted_token_class_ids
        >>> loss = model(**inputs, labels=labels).loss
        >>> round(loss.item(), 2)  # doctest: +SKIP
        ```Nrª  r   rC   r›   r®  )rG   r5  r]  rU   r¦  r   rÇ   r¥  r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   r¨  rÃ   r4  r5  r”  rÞ   rX  r°  r¯  r²  rõ   s                    r:   r†   z$CanineForTokenClassification.forward­  s  € ð` &1Ð%<�k�kÀ$Ä+ÔBYˆà—+’+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆàŸ,š, Ñ7Ô7ˆØ—’ Ñ1Ô1ˆàˆØÐÝ'Ñ)Ô)ˆHØ�8˜FŸKšK¨¨D¬OÑ<Ô<¸f¿kºkÈ"¹o¼oÑNÔNˆDàð 	FØ�Y ¨¨¨¤Ñ,ˆFØ)-Ð)9�T�G˜fÑ$Ð$¸vÐEå$ØØØ!Ô/ØÔ)ð	
ñ 
ô 
ð 	
r9   r³  )r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r»  r»     s1  ø€ € € € € ð	ð 	ð 	ð 	ð 	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ðO
ð O
àÔ# dÑ*ðO
ð Ô)¨DÑ0ðO
ð Ô(¨4Ñ/ð	O
ð
 Ô&¨Ñ-ðO
ð Ô(¨4Ñ/ðO
ð Ô  4Ñ'ðO
ð   $™;ðO
ð # T™kðO
ð ˜D‘[ðO
ð 
Ð&Ñ	&ðO
ð O
ð O
ñ „^ðO
ð O
ð O
ð O
ð O
r9   r»  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej        dz  dej        dz  dej        dz  dej        dz  dej        dz  dej        dz  d	ej        dz  d
edz  dedz  dedz  de	e
z  fd„¦   «         Zˆ xZS )ÚCanineForQuestionAnsweringc                 óú   •— t          ¦   «                              |¦  «         |j        | _        t          |¦  «        | _        t          j        |j        |j        ¦  «        | _        |  	                    ¦   «          d S r²   )
rE   rF   r¥  rf  r]  r   r»   rH   Ú
qa_outputsrr  r˜   s     €r:   rF   z#CanineForQuestionAnswering.__init__  se   ø€ Ý‰Œ×Ò˜Ñ Ô Ð Ø Ô+ˆŒå! &Ñ)Ô)ˆŒÝœ) FÔ$6¸Ô8IÑJÔJˆŒð 	�ŠÑÔÐÐÐr9   Nrf   rÂ   rx   rA   ry   Ústart_positionsÚend_positionsrÃ   r4  r5  rz   c           
      óf  — |
�|
n| j         j        }
|                      |||||||	|
¬¦  «        }|d         }|                      |¦  «        }|                     dd¬¦  «        \  }}|                     d¦  «        }|                     d¦  «        }d }|�ç|�åt          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }t          |                     ¦   «         ¦  «        dk    r|                     d¦  «        }|                     d¦  «        }|                     d|¦  «         |                     d|¦  «         t          |¬¦  «        } |||¦  «        } |||¦  «        }||z   dz  }|
s||f|dd …         z   }|�|f|z   n|S t          ||||j        |j        ¬¦  «        S )	Nrª  r   r   rC   rn   )Úignore_indexr›   )r¯  Ústart_logitsÚ
end_logitsr.   r/   )rG   r5  r]  rÁ  Úsplitr€  rb   r   Úclamp_r   r   r.   r/   )rZ   rf   rÂ   rx   rA   ry   rÂ  rÃ  rÃ   r4  r5  r”  rÞ   rX  r°  rÆ  rÇ  Ú
total_lossÚignored_indexr²  Ú
start_lossÚend_lossrõ   s                          r:   r†   z"CanineForQuestionAnswering.forward  sú  € ð &1Ð%<�k�kÀ$Ä+ÔBYˆà—+’+ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ñ 	
ô 	
ˆð " !œ*ˆà—’ Ñ1Ô1ˆØ#)§<¢<°°r <Ñ#:Ô#:Ñ ˆ�jØ#×+Ò+¨BÑ/Ô/ˆØ×'Ò'¨Ñ+Ô+ˆ
àˆ
ØÐ&¨=Ð+Då�?×'Ò'Ñ)Ô)Ñ*Ô*¨QÒ.Ð.Ø"1×"9Ò"9¸"Ñ"=Ô"=�Ý�=×%Ò%Ñ'Ô'Ñ(Ô(¨1Ò,Ð,Ø -× 5Ò 5°bÑ 9Ô 9�à(×-Ò-¨aÑ0Ô0ˆMØ×"Ò" 1 mÑ4Ô4Ð4Ø× Ò   MÑ2Ô2Ð2å'°]ÐCÑCÔCˆHØ!˜ ,°Ñ@Ô@ˆJØ�x 
¨MÑ:Ô:ˆHØ$ xÑ/°1Ñ4ˆJàð 	RØ" JÐ/°'¸!¸"¸"´+Ñ=ˆFØ/9Ð/E�Z�M FÑ*Ð*È6ÐQå+ØØ%Ø!Ø!Ô/ØÔ)ð
ñ 
ô 
ð 	
r9   )
NNNNNNNNNN)r0   r1   r2   rF   r   r4   rˆ   r5   rà   r7   r   r†   r‰   rŠ   s   @r:   r¿  r¿     s3  ø€ € € € € ðð ð ð ð ð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ð=
ð =
àÔ# dÑ*ð=
ð Ô)¨DÑ0ð=
ð Ô(¨4Ñ/ð	=
ð
 Ô&¨Ñ-ð=
ð Ô(¨4Ñ/ð=
ð Ô)¨DÑ0ð=
ð Ô'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
Ð-Ñ	-ð=
ð =
ð =
ñ „^ð=
ð =
ð =
ð =
ð =
r9   r¿  )rµ  r¿  r¢  r»  r  rf  r\  )?r3   rh  rÉ   Údataclassesr   r4   r   Útorch.nnr   r   r   Ú r	   r`  Úactivationsr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   Úconfiguration_caniner   Ú
get_loggerr0   Úloggerrc   r+   ÚModuler<   rŒ   r¤   r´   râ   rì   r  r  r  r+  r@  rG  rL  rU  r\  rf  r¢  rµ  r»  r¿  Ú__all__r8   r9   r:   ú<module>rÝ     s  ðð Ð à €€€Ø €€€Ø !Ð !Ð !Ð !Ð !Ð !à €€€Ø Ð Ð Ð Ð Ð Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ Aà &Ð &Ð &Ð &Ð &Ð &Ø !Ð !Ð !Ð !Ð !Ð !Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð .Ð -Ð -Ð -Ð -Ð -Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø .Ð .Ð .Ð .Ð .Ð .ð 
ˆÔ	˜HÑ	%Ô	%€ð UÐ
TÐ
T€ð €ððñ ô ð ð7ð 7ð 7ð 7ð 7 ;ñ 7ô 7ñ „ñô ð7ð:^ð ^ð ^ð ^ð ^�r”yñ ^ô ^ð ^ðB)ð )ð )ð )ð )˜BœIñ )ô )ð )ðX5ð 5ð 5ð 5ð 5�R”Yñ 5ô 5ð 5ðpNð Nð Nð Nð N˜"œ)ñ Nô Nð Nðbð ð ð ð �r”yñ ô ð ð xð xð xð xð x�b”iñ xô xð xðvð ð ð ð ˜œñ ô ð ðð ð ð ð �2”9ñ ô ð ð5ð 5ð 5ð 5ð 5Ð,ñ 5ô 5ð 5ðp=
ð =
ð =
ð =
ð =
�B”Iñ =
ô =
ð =
ð@ð ð ð ð �2”9ñ ô ð ðð ð ð ð  B¤Iñ ô ð ð"ð ð ð ð ˜RœYñ ô ð ð&
!ð 
!ð 
!ð 
!ð 
!˜œ	ñ 
!ô 
!ð 
!ð ðið ið ið ið i˜Oñ iô iñ „ðið ðC
ð C
ð C
ð C
ð C
Ð'ñ C
ô C
ñ „ðC
ðL €ððñ ô ðQ
ð Q
ð Q
ð Q
ð Q
Ð&;ñ Q
ô Q
ñô ðQ
ðh ðc
ð c
ð c
ð c
ð c
Ð3ñ c
ô c
ñ „ðc
ðL ð\
ð \
ð \
ð \
ð \
Ð#8ñ \
ô \
ñ „ð\
ð~ ðI
ð I
ð I
ð I
ð I
Ð!6ñ I
ô I
ñ „ðI
ðXð ð €€€r9   