§
    ‚Štj!  ã                   ó¢   — d dl mZmZmZmZ d dlmZ ddlmZ ddl	m
Z
 ddlmZ  ej        e¦  «        Zddd	œZg d
¢Z G d„ de
¦  «        ZdgZdS )é    )Ú	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚUnigramé   )Ú
AddedToken)ÚTokenizersBackend)Úloggingzsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CNc                   ó"  ‡ — e Zd ZU dZeZddgZeZg Z	e
e         ed<   g Ze
e         ed<   	 	 	 	 	 	 	 	 	 	 	 ddeez  e
z  dz  fˆ fd„Zedefd„¦   «         Zej        deddfd„¦   «         Zdededz  dedz  fd„Zd„ Zd„ Zdd„Zdeddfd„Zˆ xZS )ÚMBartTokenizeruC  
    Construct an MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Examples:

    ```python
    >>> from transformers import MBartTokenizer

    >>> tokenizer = MBartTokenizer.from_pretrained(
    ...     "facebook/mbart-large-en-ro", src_lang="en_XX", tgt_lang="ro_RO"
    ... )
    >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
    >>> expected_translation_romanian = "Åžeful ONU declarÄƒ cÄƒ nu existÄƒ o soluÅ£ie militarÄƒ Ã®n Siria"
    >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_romanian, return_tensors="pt")
    ```Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensNú<s>ú</s>ú<unk>ú<pad>ú<mask>Úvocabc                 óP  •‡ ‡— t          |t          ¦  «        rt          |dd¬¦  «        n|}t                               ¦   «         Š|�!‰                     ˆfd„|D ¦   «         ¦  «         |€�t          |¦  «        dft          |¦  «        dft          |¦  «        dft          |¦  «        dfg}|dgz  }t          D ]}|                     |df¦  «         Œ|                     t          |¦  «        df¦  «         |‰ _        t          t          ‰ j        dd¬¦  «        ¦  «        ‰ _
        d ‰ j
        _        t          j        t          j        ¦   «         t          j        d	d
d¬¦  «        g¦  «        ‰ j
        _        t#          j        d	d
d¬¦  «        ‰ j
        _         t'          ¦   «         j        d||||||||	|
‰dœ
|¤Ž ˆ fd„t          D ¦   «         ‰ _        d‰ _        dddddœ‰ _        ‰ j                             ‰ j        ¦  «         ‰                      t          |¦  «        ¦  «        ‰ j        d<   d„ ‰ j                             ¦   «         D ¦   «         ‰ _        |	�|	nd‰ _        ‰                      ‰ j        ¦  «        ‰ _        |
‰ _        ‰                      ‰ j        ¦  «         d S )NTF)ÚlstripÚrstripc                 ó   •— g | ]}|‰v¯|‘Œ	S © r7   )Ú.0ÚtÚ_additional_special_tokenss     €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/mbart/tokenization_mbart.pyú
<listcomp>z+MBartTokenizer.__init__.<locals>.<listcomp>T   s$   ø€ Ð]Ð]Ð]�q¸ÐB\Ð9\Ð9\�Ð9\Ð9\Ð9\ó    g        )õ   â–�g       Àr   )Úunk_idÚbyte_fallbackr>   Úalways)ÚreplacementÚprepend_schemeÚsplit)
Ú	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ	unk_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensc                 ó<   •— i | ]}|‰                      |¦  «        “ŒS r7   )Úconvert_tokens_to_ids)r8   Ú	lang_codeÚselfs     €r;   ú
<dictcomp>z+MBartTokenizer.__init__.<locals>.<dictcomp>   s6   ø€ ð  
ð  
ð  
ØAJˆI�t×1Ò1°)Ñ<Ô<ð 
ð  
ð  
r=   é   r   é   )r-   r0   r.   r/   r1   c                 ó   — i | ]\  }}||“Œ	S r7   r7   )r8   ÚkÚvs      r;   rS   z+MBartTokenizer.__init__.<locals>.<dictcomp>�   s   € Ð%ZÐ%ZÐ%Z©t¨q°! a¨Ð%ZÐ%ZÐ%Zr=   r   r7   ) Ú
isinstanceÚstrr	   ÚFAIRSEQ_LANGUAGE_CODESÚcopyÚextendÚappendÚ_vocabr   r   Ú
_tokenizerÚ
normalizerr   ÚSequenceÚWhitespaceSplitÚ	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__Úlang_code_to_idÚfairseq_offsetÚfairseq_tokens_to_idsÚupdaterP   ÚitemsÚfairseq_ids_to_tokensÚ	_src_langÚcur_lang_coderM   Úset_src_lang_special_tokens)rR   r2   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   ÚkwargsrQ   r:   Ú	__class__s   `             @€r;   rh   zMBartTokenizer.__init__@   sò  øøø€ õ KUÐU_ÕadÑJeÔJeÐu•Z 
°4ÀÐFÑFÔFÐFÐkuˆ
å%;×%@Ò%@Ñ%BÔ%BÐ"Ø$Ð0Ø&×-Ò-Ø]Ð]Ð]Ð]Ð5Ð]Ñ]Ô]ñô ð ð ˆ=å�Y‘” Ð%Ý�Y‘” Ð%Ý�Y‘” Ð%Ý�Y‘” Ð%ð	ˆEð �m�_Ñ$ˆEÝ3ð /ð /�	Ø—’˜i¨Ð-Ñ.Ô.Ð.Ð.Ø�LŠL�#˜j™/œ/¨3Ð/Ñ0Ô0Ð0àˆŒÝ#¥G¨D¬KÀÐQVÐ$WÑ$WÔ$WÑXÔXˆŒà%)ˆŒÔ"å(6Ô(?åÔ.Ñ0Ô0ÝÔ(°UÈ8Ð[_Ð`Ñ`Ô`ðñ)
ô )
ˆŒÔ%õ #+Ô"4ÀÐW_ÐgkÐ"lÑ"lÔ"lˆŒÔà�‰ŒÔð 	
ØØØØØØØ!ØØØ&@ð	
ð 	
ð ð	
ð 	
ð 	
ð 
ð  
ð  
ð  
ÝNdð 
ñ  
ô  
ˆÔð  ˆÔð ØØØð	&
ð &
ˆÔ"ð 	Ô"×)Ò)¨$Ô*>Ñ?Ô?Ð?Ø/3×/IÒ/IÍ#ÈjÉ/Ì/Ñ/ZÔ/ZˆÔ" 8Ñ,Ø%ZÐ%Z°tÔ7Q×7WÒ7WÑ7YÔ7YÐ%ZÑ%ZÔ%ZˆÔ"à%-Ð%9˜˜¸wˆŒØ!×7Ò7¸¼ÑGÔGˆÔØ ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r=   Úreturnc                 ó   — | j         S ©N)ro   ©rR   s    r;   rL   zMBartTokenizer.src_lang”   s
   € àŒ~Ðr=   Únew_src_langc                 óH   — || _         |                      | j         ¦  «         d S rv   )ro   rq   )rR   rx   s     r;   rL   zMBartTokenizer.src_lang˜   s%   € à%ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r=   Úreturn_tensorsrL   rM   c                 ó†   — |�|€t          d¦  «        ‚|| _         | |fd|dœ|¤Ž}|                      |¦  «        }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionNzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensrz   Úforced_bos_token_id)Ú
ValueErrorrL   rP   )rR   Ú
raw_inputsrz   rL   rM   Úextra_kwargsÚinputsÚtgt_lang_ids           r;   Ú_build_translation_inputsz(MBartTokenizer._build_translation_inputs�   sg   € ð Ð˜xÐ/ÝÐ`ÑaÔaÐaØ ˆŒØ��jÐi°TÈ.ÐiÐiÐ\hÐiÐiˆØ×0Ò0°Ñ:Ô:ˆØ(3ˆÐ$Ñ%Øˆr=   c                 ó6   — |                       | j        ¦  «        S rv   )rq   rL   rw   s    r;   Ú_switch_to_input_modez$MBartTokenizer._switch_to_input_mode©   s   € Ø×/Ò/°´Ñ>Ô>Ð>r=   c                 ó\   — | j         €| j        | _         |                      | j         ¦  «        S rv   )rM   ro   Úset_tgt_lang_special_tokensrw   s    r;   Ú_switch_to_target_modez%MBartTokenizer._switch_to_target_mode¬   s)   € ØŒ=Ð Ø œNˆDŒMØ×/Ò/°´Ñ>Ô>Ð>r=   c                 óŒ  — |                       |¦  «        | _        g | _        | j        | j        g| _        |                      | j        ¦  «        }|                      | j        ¦  «        }t          j        |dgz   |z   |ddgz   |z   t          t          ||z   | j        | j        z   ¦  «        ¦  «        ¬¦  «        | j
        _        dS )z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].ú$Aú$B©ÚsingleÚpairÚspecial_tokensN©rP   rp   r+   Úeos_token_idr,   Úconvert_ids_to_tokensr   ÚTemplateProcessingÚlistÚzipr`   Úpost_processor)rR   rL   Úprefix_tokens_strÚsuffix_tokens_strs       r;   rq   z*MBartTokenizer.set_src_lang_special_tokens±   sÍ   € à!×7Ò7¸ÑAÔAˆÔØˆÔØ"Ô/°Ô1CÐDˆÔà ×6Ò6°tÔ7IÑJÔJÐØ ×6Ò6°tÔ7IÑJÔJÐå)3Ô)FØ$¨ vÑ-Ð0AÑAØ" d¨D \Ñ1Ð4EÑEÝ¥Ð$5Ð8IÑ$IÈ4ÔK]Ð`dÔ`rÑKrÑ sÔ sÑtÔtð*
ñ *
ô *
ˆŒÔ&Ð&Ð&r=   Úlangc                 óŒ  — |                       |¦  «        | _        g | _        | j        | j        g| _        |                      | j        ¦  «        }|                      | j        ¦  «        }t          j        |dgz   |z   |ddgz   |z   t          t          ||z   | j        | j        z   ¦  «        ¦  «        ¬¦  «        | j
        _        dS )zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].rŠ   r‹   rŒ   Nr�   )rR   r™   r—   r˜   s       r;   r‡   z*MBartTokenizer.set_tgt_lang_special_tokensÀ   sÍ   € à!×7Ò7¸Ñ=Ô=ˆÔØˆÔØ"Ô/°Ô1CÐDˆÔà ×6Ò6°tÔ7IÑJÔJÐØ ×6Ò6°tÔ7IÑJÔJÐå)3Ô)FØ$¨ vÑ-Ð0AÑAØ" d¨D \Ñ1Ð4EÑEÝ¥Ð$5Ð8IÑ$IÈ4ÔK]Ð`dÔ`rÑKrÑ sÔ sÑtÔtð*
ñ *
ô *
ˆŒÔ&Ð&Ð&r=   )Nr-   r.   r.   r-   r/   r0   r1   NNN)rt   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr+   r”   ÚintÚ__annotations__r,   rZ   Údictrh   ÚpropertyrL   Úsetterrƒ   r…   rˆ   rq   r‡   Ú__classcell__)rs   s   @r;   r(   r(   !   s±  ø€ € € € € € ðð ð. *ÐØ$Ð&6Ð7ÐØ€Eà!€M�4˜”9Ð!Ð!Ñ!Ø!€M�4˜”9Ð!Ð!Ñ!ð +/ØØØØØØØØØØ"&ðR9ð R9à�T‰z˜DÑ  4Ñ'ðR9ð R9ð R9ð R9ð R9ð R9ðh ð˜#ð ð ð ñ „Xðð „_ð9 Sð 9¨Tð 9ð 9ð 9ñ „_ð9ð
Ø*-ð
Ø9<¸t¹ð
ØORÐUYÉzð
ð 
ð 
ð 
ð?ð ?ð ?ð?ð ?ð ?ð

ð 
ð 
ð 
ð
°ð 
¸ð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r=   r(   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_pythonr	   Útokenization_utils_tokenizersr
   Úutilsr   Ú
get_loggerr›   ÚloggerrŸ   r[   r(   Ú__all__r7   r=   r;   ú<module>r±      sð   ðð  GÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FØ %Ð %Ð %Ð %Ð %Ð %à -Ð -Ð -Ð -Ð -Ð -Ø >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð $=ÐP`ÐaÐaÐ ð {ð  {ð  {Ð ðl
ð l
ð l
ð l
ð l
Ð&ñ l
ô l
ð l
ð^ Ð
€€€r=   