§
    ‚ŠtjR=  ã                   óè   — d dl mZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ  e	j        e¦  «        ZdZd	d
dœZg d¢g d¢dœZddddddddœZ ed¬¦  «         G d„ de¦  «        ¦   «         ZdgZdS )é    )ÚAnyé   )ÚBatchEncoding)Ú
AddedToken)ÚSentencePieceBackend)Úlogging)Úrequiresu   â–�zsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)Ú__java__Ú
__python__Ú	__en_XX__)r   r   r   Ú__javascript__Ú__php__Ú__ruby__Ú__go__)ÚbaseÚmultir   r   r   r   r   r   r   )ÚjavaÚpythonÚen_XXÚ
javascriptÚphpÚrubyÚgo)Úsentencepiece)Úbackendsc                   ó°  ‡ — e Zd ZU dZeZddgZg Zee	         e
d<   g Zee	         e
d<   	 	 	 	 	 	 	 	 	 	 	 	 	 d)deeef         dz  fˆ fd„Zed„ ¦   «         Zd„ Zedefd„¦   «         Zej        deddfd„¦   «         Zdededz  dedz  fd„Zd„ Zd„ Z	 	 	 d*dee         dedee         dz  dedef
ˆ fd „Zd!„ Zd"„ Zd+d#„Zd$eddfd%„Zd$edefd&„Zd,ˆ fd(„	Zˆ xZ S )-ÚPLBartTokenizeraÕ  
    Construct an PLBART tokenizer.

    Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
    [SentencePiece](https://github.com/google/sentencepiece).

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        src_lang (`str`, *optional*):
            A string representing the source language.
        tgt_lang (`str`, *optional*):
            A string representing the target language.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The start of sequence token.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The cls token, which is a special token used as the first token for all tasks.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token(`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masking tasks. This
            is only used in the `"base"` tokenizer type. For `"multi"` tokenizer, masking is never done for the
            downstream tasks.
        language_codes (`str`, *optional*, defaults to `"base"`):
            What language codes to use. Should be one of `"base"` or `"multi"`.
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:
            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
              - `nbest_size = {0,1}`: No sampling is performed.
              - `nbest_size > 1`: samples from the nbest_size results.
              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.
            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
              BPE-dropout.

    Examples:

    ```python
    >>> from transformers import PLBartTokenizer

    >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-python-en_XX", src_lang="python", tgt_lang="en_XX")
    >>> example_python_phrase = "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])"
    >>> expected_translation_english = "Returns the maximum value of a b c."
    >>> inputs = tokenizer(example_python_phrase, text_target=expected_translation_english, return_tensors="pt")
    ```Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensú<s>ú</s>ú<unk>ú<pad>ú<mask>r   NTÚsp_model_kwargsc                 óŽ  •‡ ‡— t          |t          ¦  «        rt          |dd¬¦  «        n|}|€i n|‰ _        ‰                      |
¦  «        }
‰                      |¦  «        }|	‰ _        t          ‰ j                 }|‰ _        i ‰ _        i ‰ _	        dddddœ‰ _
        d	„ ‰ j
                             ¦   «         D ¦   «         ‰ _        d‰ _        t          |¦  «        Š|�!‰                     ˆfd
„|D ¦   «         ¦  «          t!          ¦   «         j        d%i d|“d|“d|“d|“d|“d|“d|“d|“d|
“d|“d‰“d‰ j        “d|“d|	“dd“dd“|¤Ž t%          ‰ j        ¦  «        ‰ _        ˆ fd„t+          |¦  «        D ¦   «         ‰ _        d„ ‰ j                             ¦   «         D ¦   «         ‰ _	        dddddœ‰ _
        ‰ j        dk    r9t%          ‰ j        ¦  «        t%          ‰ j        ¦  «        z   ‰ j        z   ‰ j
        d <   ‰ j
                             ‰ j        ¦  «         d!„ ‰ j
                             ¦   «         D ¦   «         ‰ _        h d"£}|                     t          ‰ j                 ¦  «         d}|D ]<}‰ j                             |d ¦  «        }|�‰ j                             |d ¦  «         d}Œ=|r(‰                      ¦   «          ‰                      ¦   «          d}‰ j                             ¦   «         D ]-\  }}|‰ j        v rŒt          |dddd¬#¦  «        ‰ j        |<   d}Œ.|r(‰                      ¦   «          ‰                      ¦   «          ‰ j        dk    r-|
‰ _        ‰ j        �‰ j        ‰ j                 n‰ j        ‰ _        n"|
�|
nd$‰ _        ‰ j        ‰ j                 ‰ _        |‰ _        ‰                      ‰ j        ¦  «         d S )&NTF)ÚlstripÚrstripr   é   é   r   )r$   r'   r%   r&   c                 ó   — i | ]\  }}||“Œ	S © r0   ©Ú.0ÚkÚvs      úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/plbart/tokenization_plbart.pyú
<dictcomp>z,PLBartTokenizer.__init__.<locals>.<dictcomp>š   ó   € Ð%ZÐ%ZÐ%Z©t¨q°! a¨Ð%ZÐ%ZÐ%Zó    c                 ó   •— g | ]}|‰v¯|‘Œ	S r0   r0   )r2   ÚtÚ_additional_special_tokenss     €r5   ú
<listcomp>z,PLBartTokenizer.__init__.<locals>.<listcomp>    s$   ø€ Ð]Ð]Ð]�q¸ÐB\Ð9\Ð9\�Ð9\Ð9\Ð9\r8   r
   Ú	bos_tokenÚ	eos_tokenÚ	unk_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensr)   Úclean_up_tokenization_spacesÚlanguage_codesÚspecial_tokens_patternÚprefix_suffixÚtoken_type_ids_patternÚ	all_zerosc                 ó<   •— i | ]\  }}|‰j         |z   ‰j        z   “ŒS r0   )Úsp_model_sizeÚfairseq_offset)r2   ÚiÚcodeÚselfs      €r5   r6   z,PLBartTokenizer.__init__.<locals>.<dictcomp>¹   s<   ø€ ð  
ð  
ð  
ÙCJÀ1ÀdˆD�$Ô$ qÑ(¨4Ô+>Ñ>ð 
ð  
ð  
r8   c                 ó   — i | ]\  }}||“Œ	S r0   r0   r1   s      r5   r6   z,PLBartTokenizer.__init__.<locals>.<dictcomp>¼   s   € ÐNÐNÐN©¨¨A  1ÐNÐNÐNr8   r   r(   c                 ó   — i | ]\  }}||“Œ	S r0   r0   r1   s      r5   r6   z,PLBartTokenizer.__init__.<locals>.<dictcomp>Ã   r7   r8   >   r$   r%   r'   r&   r(   )ÚspecialÚ
normalizedr+   r,   r   r0   ) Ú
isinstanceÚstrr   r)   Ú!_convert_lang_code_special_formatrH   ÚFAIRSEQ_LANGUAGE_CODESr
   Úlang_code_to_idÚid_to_lang_codeÚfairseq_tokens_to_idsÚitemsÚfairseq_ids_to_tokensrO   ÚlistÚextendÚsuperÚ__init__ÚlenÚsp_modelrN   Ú	enumerateÚupdateÚ_added_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_trieÚ_update_total_vocab_sizeÚ	_src_langÚcur_lang_code_idrE   Úset_src_lang_special_tokens)rR   r
   r=   r>   r@   rA   r?   rB   rC   rH   rD   rE   r)   rF   rG   ÚkwargsÚfairseq_language_codesÚreserved_tokensÚremovedÚtokenÚidxÚsyncedr;   Ú	__class__s   `                     @€r5   rc   zPLBartTokenizer.__init__t   sÍ  øøø€ õ& KUÐU_ÕadÑJeÔJeÐu•Z 
°4ÀÐFÑFÔFÐFÐkuˆ
à%4Ð%<˜r˜rÀ/ˆÔØ×9Ò9¸(ÑCÔCˆØ×9Ò9¸(ÑCÔCˆØ,ˆÔÝ!7¸Ô8KÔ!LÐð %ˆŒØ!ˆÔØ!ˆÔØ-.¸ÀAÐPQÐ%RÐ%RˆÔ"Ø%ZÐ%Z°tÔ7Q×7WÒ7WÑ7YÔ7YÐ%ZÑ%ZÔ%ZˆÔ"ØˆÔÝ%)Ð*@Ñ%AÔ%AÐ"à$Ð0Ø&×-Ò-Ø]Ð]Ð]Ð]Ð5Ð]Ñ]Ô]ñô ð ð 	�‰ŒÔð 	
ð 	
ð 	
Ø!�zð	
à�ið	
ð  �ið	
ð  �ið		
ð
  �ið	
ð  �ið	
ð  �ið	
ð "�zð	
ð �Xð	
ð �Xð	
ð 'AÐ&@ð	
ð !Ô0Ð0ð	
ð *FÐ)Eð	
ð *˜>ð	
ð $3 ?ð	
ð  $/ ;Øð#	
ð 	
ð 	
õ* ! ¤Ñ/Ô/ˆÔð 
ð  
ð  
ð  
ÝNWÐXnÑNoÔNoð 
ñ  
ô  
ˆÔð  OÐN°Ô1E×1KÒ1KÑ1MÔ1MÐNÑNÔNˆÔØ-.¸ÀAÐPQÐ%RÐ%RˆÔ"àÔ &Ò(Ð(Ý36°t´}Ñ3EÔ3EÍÈDÔL`ÑHaÔHaÑ3aÐdhÔdwÑ3wˆDÔ& xÑ0àÔ"×)Ò)¨$Ô*>Ñ?Ô?Ð?Ø%ZÐ%Z°tÔ7Q×7WÒ7WÑ7YÔ7YÐ%ZÑ%ZÔ%ZˆÔ"ØEÐEÐEˆØ×ÒÕ5°dÔ6IÔJÑKÔKÐKàˆØ$ð 	ð 	ˆEØÔ,×0Ò0°¸Ñ=Ô=ˆCØˆØÔ*×.Ò.¨s°DÑ9Ô9Ð9Ø�øØð 	,Ø×ÒÑÔÐØ×)Ò)Ñ+Ô+Ð+àˆØÔ4×:Ò:Ñ<Ô<ð 	ð 	‰JˆE�3Ø�dÔ0Ð0Ð0ØÝ.8Ø˜t°¸eÈEð/ñ /ô /ˆDÔ& sÑ+ð ˆFˆFØð 	,Ø×ÒÑÔÐØ×)Ò)Ñ+Ô+Ð+àÔ &Ò(Ð(Ø%ˆDŒNà8<¼Ð8R�Ô$ T¤^Ô4Ð4ÐX\ÔXfð Ô!Ð!ð *2Ð)=˜X˜XÀ;ˆDŒNØ$(Ô$8¸¼Ô$HˆDÔ!à ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r8   c                 óü   — t          t          | di ¦  «        ¦  «        }t          | dd¦  «        }t          | d¦  «        rt          | j        ¦  «        nd}t          | dd¦  «        dk    r||z   |z   dz   S ||z   |z   S )Nr[   rO   r-   re   r   rH   r   )rd   ÚgetattrÚhasattrre   )rR   Úlang_code_countrO   Ú
base_vocabs       r5   Ú
vocab_sizezPLBartTokenizer.vocab_sizeé   sŽ   € å�g dÐ,=¸rÑBÔBÑCÔCˆÝ  Ð'7¸Ñ;Ô;ˆÝ+2°4¸Ñ+DÔ+DÐK•S˜œÑ'Ô'Ð'È!ˆ
Ý�4Ð)¨6Ñ2Ô2°fÒ<Ð<Ø Ñ/°.Ñ@À1ÑDÐDØ˜OÑ+¨nÑ<Ð<r8   c                 óp  ‡— | j                              ¦   «         Št          | j                             ¦   «         ¦  «        D ]<}| j                             |¦  «        }|dk    r| j        n	|| j        z   }|‰vr|‰|<   Œ=‰                     ˆfd„| j	         
                    ¦   «         D ¦   «         ¦  «         ‰S )z,Override to use fairseq vocabulary structurer   c                 ó$   •— i | ]\  }}|‰v¯	||“ŒS r0   r0   )r2   rt   ru   Úvocabs      €r5   r6   z-PLBartTokenizer.get_vocab.<locals>.<dictcomp>ü   s+   ø€ ÐlÐlÐl¡Z U¨CÐY^ÐfkÐYkÐYk�e˜SÐYkÐYkÐYkr8   )r]   ÚcopyÚrangere   Úget_piece_sizeÚ	IdToPieceÚunk_token_idrO   rg   rh   r^   )rR   rP   Úsp_tokenÚvocab_idr€   s       @r5   Ú	get_vocabzPLBartTokenizer.get_vocabò   s¾   ø€ àÔ*×/Ò/Ñ1Ô1ˆÝ�t”}×3Ò3Ñ5Ô5Ñ6Ô6ð 	+ð 	+ˆAØ”}×.Ò.¨qÑ1Ô1ˆHà,-°ªF¨F�tÔ(Ð(¸¸TÔ=PÑ9PˆHØ˜uÐ$Ð$Ø"*��h‘øà�ŠÐlÐlÐlÐl°4Ô3M×3SÒ3SÑ3UÔ3UÐlÑlÔlÑmÔmÐmØˆr8   Úreturnc                 ó   — | j         S ©N)rm   ©rR   s    r5   rD   zPLBartTokenizer.src_langÿ   s
   € àŒ~Ðr8   Únew_src_langc                 ór   — |                       |¦  «        }|| _        |                      | j        ¦  «         d S r‹   )rY   rm   ro   )rR   r�   s     r5   rD   zPLBartTokenizer.src_lang  s8   € à×=Ò=¸lÑKÔKˆØ%ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r8   Úreturn_tensorsrD   rE   c                 óê   — |�|€t          d¦  «        ‚|                      |¦  «        | _        |                      |¦  «        | _         | |fd|dœ|¤Ž}|                      | j        ¦  «        }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionNzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensr�   Úforced_bos_token_id)Ú
ValueErrorrY   rD   rE   Úconvert_tokens_to_ids)rR   Ú
raw_inputsr�   rD   rE   Úextra_kwargsÚinputsÚtgt_lang_ids           r5   Ú_build_translation_inputsz)PLBartTokenizer._build_translation_inputs	  sŒ   € ð Ð˜xÐ/ÝÐ`ÑaÔaÐaØ×>Ò>¸xÑHÔHˆŒØ×>Ò>¸xÑHÔHˆŒØ��jÐi°TÈ.ÐiÐiÐ\hÐiÐiˆØ×0Ò0°´Ñ?Ô?ˆØ(3ˆÐ$Ñ%Øˆr8   c                 óˆ   — || j         v r| j         |         S | j                             |¦  «        }|r
|| j        z   n| j        S )z0Converts a token (str) in an id using the vocab.)r]   re   Ú	PieceToIdrO   r…   )rR   rt   Úspm_ids      r5   Ú_convert_token_to_idz$PLBartTokenizer._convert_token_to_id  sP   € à�DÔ.Ð.Ð.ØÔ-¨eÔ4Ð4Ø”×(Ò(¨Ñ/Ô/ˆð 06ÐLˆv˜Ô+Ñ+Ð+¸4Ô;LÐLr8   c                 ór   — || j         v r| j         |         S | j                             || j        z
  ¦  «        S )z=Converts an index (integer) in a token (str) using the vocab.)r_   re   r„   rO   )rR   Úindexs     r5   Ú_convert_id_to_tokenz$PLBartTokenizer._convert_id_to_token  s<   € à�DÔ.Ð.Ð.ØÔ-¨eÔ4Ð4ØŒ}×&Ò& u¨tÔ/BÑ'BÑCÔCÐCr8   r   r   Ú	src_textsÚ	tgt_textsc                 ó¢   •— |                       |¦  «        | _        |                       |¦  «        | _         t          ¦   «         j        ||fi |¤ŽS r‹   )rY   rD   rE   rb   Úprepare_seq2seq_batch)rR   r¡   rD   r¢   rE   rp   rw   s         €r5   r¤   z%PLBartTokenizer.prepare_seq2seq_batch%  sP   ø€ ð ×>Ò>¸xÑHÔHˆŒØ×>Ò>¸xÑHÔHˆŒØ,�u‰wŒwÔ,¨Y¸	ÐLÐLÀVÐLÐLÐLr8   c                 ó6   — |                       | j        ¦  «        S r‹   )ro   rD   rŒ   s    r5   Ú_switch_to_input_modez%PLBartTokenizer._switch_to_input_mode1  ó   € Ø×/Ò/°´Ñ>Ô>Ð>r8   c                 ó6   — |                       | j        ¦  «        S r‹   )Úset_tgt_lang_special_tokensrE   rŒ   s    r5   Ú_switch_to_target_modez&PLBartTokenizer._switch_to_target_mode4  r§   r8   c                 ó¼   — |                       |¦  «        }|�| j        |         nd| _        g | _        | j        �| j        | j        g| _        dS | j        g| _        dS )z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].N©rY   r[   Úcur_lang_coder"   Úeos_token_idr#   )rR   rD   s     r5   ro   z+PLBartTokenizer.set_src_lang_special_tokens7  sn   € à×9Ò9¸(ÑCÔCˆØ?GÐ?S˜TÔ1°(Ô;Ð;ÐY]ˆÔØˆÔØÔÐ)Ø"&Ô"3°TÔ5GÐ!HˆDÔÐÐà"&Ô"3Ð!4ˆDÔÐÐr8   Úlangc                 ó¼   — |                       |¦  «        }|�| j        |         nd| _        g | _        | j        �| j        | j        g| _        dS | j        g| _        dS )zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].Nr¬   ©rR   r¯   s     r5   r©   z+PLBartTokenizer.set_tgt_lang_special_tokensA  sn   € à×5Ò5°dÑ;Ô;ˆà;?Ð;K˜TÔ1°$Ô7Ð7ÐQUˆÔØˆÔØÔÐ)Ø"&Ô"3°TÔ5GÐ!HˆDÔÐÐà"&Ô"3Ð!4ˆDÔÐÐr8   c                 ó<   — t                                ||¦  «        }|S )z;Convert Language Codes to format tokenizer uses if required)ÚFAIRSEQ_LANGUAGE_CODES_MAPÚgetr±   s     r5   rY   z1PLBartTokenizer._convert_lang_code_special_formatL  s   € å)×-Ò-¨d°DÑ9Ô9ˆØˆr8   Fc                 óH   •—  t          ¦   «         j        d||| j        dœ|¤ŽS )zOOverride to use self.clean_up_tokenization_spaces as default for batched input.)Ú	token_idsÚskip_special_tokensrG   r0   )rb   ÚdecoderG   )rR   r¶   r·   rG   rp   rw   s        €r5   r¸   zPLBartTokenizer.decodeQ  s=   ø€ à�u‰wŒwŒ~ð 
ØØ 3Ø)-Ô)Jð
ð 
ð ð	
ð 
ð 	
r8   )r$   r%   r%   r$   r&   r'   r(   r   NNNNT)r   Nr   )r‰   N)FN)!Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr"   r`   ÚintÚ__annotations__r#   ÚdictrX   r   rc   Úpropertyr}   rˆ   rD   Úsetterr™   r�   r    r   r¤   r¦   rª   ro   r©   rY   r¸   Ú__classcell__)rw   s   @r5   r   r   /   s¹  ø€ € € € € € ð;ð ;ðz *ÐØ$Ð&6Ð7Ðà!€M�4˜”9Ð!Ð!Ñ!Ø!€M�4˜”9Ð!Ð!Ñ!ð
 ØØØØØØØØØØ15Ø"&Ø%)ðs9ð s9ð ˜c 3˜hœ¨$Ñ.ðs9ð s9ð s9ð s9ð s9ð s9ðj ð=ð =ñ „Xð=ðð ð ð ð˜#ð ð ð ñ „Xðð „_ð9 Sð 9¨Tð 9ð 9ð 9ñ „_ð9ð
Ø*-ðØ9<¸t¹ðØORÐUYÉzðð ð ð ðMð Mð MðDð Dð Dð  Ø&*Ø ð
Mð 
Mà˜”9ð
Mð ð
Mð ˜”9˜tÑ#ð	
Mð
 ð
Mð 
ð
Mð 
Mð 
Mð 
Mð 
Mð 
Mð?ð ?ð ?ð?ð ?ð ?ð5ð 5ð 5ð 5ð	5°ð 	5¸ð 	5ð 	5ð 	5ð 	5ð°cð ¸cð ð ð ð ð

ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r8   r   N)Útypingr   Útokenization_pythonr   Útokenization_utils_baser   Ú tokenization_utils_sentencepiecer   Úutilsr   Úutils.import_utilsr	   Ú
get_loggerr¹   ÚloggerÚSPIECE_UNDERLINEr½   rZ   r³   r   Ú__all__r0   r8   r5   ú<module>rÐ      s>  ðð Ð Ð Ð Ð Ð à 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø DÐ DÐ DÐ DÐ DÐ DØ Ð Ð Ð Ð Ð Ø *Ð *Ð *Ð *Ð *Ð *ð 
ˆÔ	˜HÑ	%Ô	%€àÐ à#<ÐP`ÐaÐaÐ ð 4Ð3Ð3ØgÐgÐgðð Ð ð ØØØ"ØØØ
ðð Ð ð 
€Ð%Ð&Ñ&Ô&ðh
ð h
ð h
ð h
ð h
Ð*ñ h
ô h
ñ 'Ô&ðh
ðV	 Ð
€€€r8   