§
    ‚Štj`9  ã                   ó®   — d dl mZmZmZmZmZmZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZ  ej        e¦  «        Zddd	œZg d
¢Z G d„ de¦  «        ZdgZdS )é    )ÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEé   )Ú
AddedTokenÚBatchEncoding)ÚTokenizersBackend)Úloggingzsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)ÊÚace_ArabÚace_LatnÚacm_ArabÚacq_ArabÚaeb_ArabÚafr_LatnÚajp_ArabÚaka_LatnÚamh_EthiÚapc_ArabÚarb_ArabÚars_ArabÚary_ArabÚarz_ArabÚasm_BengÚast_LatnÚawa_DevaÚayr_LatnÚazb_ArabÚazj_LatnÚbak_CyrlÚbam_LatnÚban_LatnÚbel_CyrlÚbem_LatnÚben_BengÚbho_DevaÚbjn_ArabÚbjn_LatnÚbod_TibtÚbos_LatnÚbug_LatnÚbul_CyrlÚcat_LatnÚceb_LatnÚces_LatnÚcjk_LatnÚckb_ArabÚcrh_LatnÚcym_LatnÚdan_LatnÚdeu_LatnÚdik_LatnÚdyu_LatnÚdzo_TibtÚell_GrekÚeng_LatnÚepo_LatnÚest_LatnÚeus_LatnÚewe_LatnÚfao_LatnÚpes_ArabÚfij_LatnÚfin_LatnÚfon_LatnÚfra_LatnÚfur_LatnÚfuv_LatnÚgla_LatnÚgle_LatnÚglg_LatnÚgrn_LatnÚguj_GujrÚhat_LatnÚhau_LatnÚheb_HebrÚhin_DevaÚhne_DevaÚhrv_LatnÚhun_LatnÚhye_ArmnÚibo_LatnÚilo_LatnÚind_LatnÚisl_LatnÚita_LatnÚjav_LatnÚjpn_JpanÚkab_LatnÚkac_LatnÚkam_LatnÚkan_KndaÚkas_ArabÚkas_DevaÚkat_GeorÚknc_ArabÚknc_LatnÚkaz_CyrlÚkbp_LatnÚkea_LatnÚkhm_KhmrÚkik_LatnÚkin_LatnÚkir_CyrlÚkmb_LatnÚkon_LatnÚkor_HangÚkmr_LatnÚlao_LaooÚlvs_LatnÚlij_LatnÚlim_LatnÚlin_LatnÚlit_LatnÚlmo_LatnÚltg_LatnÚltz_LatnÚlua_LatnÚlug_LatnÚluo_LatnÚlus_LatnÚmag_DevaÚmai_DevaÚmal_MlymÚmar_DevaÚmin_LatnÚmkd_CyrlÚplt_LatnÚmlt_LatnÚmni_BengÚkhk_CyrlÚmos_LatnÚmri_LatnÚzsm_LatnÚmya_MymrÚnld_LatnÚnno_LatnÚnob_LatnÚnpi_DevaÚnso_LatnÚnus_LatnÚnya_LatnÚoci_LatnÚgaz_LatnÚory_OryaÚpag_LatnÚpan_GuruÚpap_LatnÚpol_LatnÚpor_LatnÚprs_ArabÚpbt_ArabÚquy_LatnÚron_LatnÚrun_LatnÚrus_CyrlÚsag_LatnÚsan_DevaÚsat_BengÚscn_LatnÚshn_MymrÚsin_SinhÚslk_LatnÚslv_LatnÚsmo_LatnÚsna_LatnÚsnd_ArabÚsom_LatnÚsot_LatnÚspa_LatnÚals_LatnÚsrd_LatnÚsrp_CyrlÚssw_LatnÚsun_LatnÚswe_LatnÚswh_LatnÚszl_LatnÚtam_TamlÚtat_CyrlÚtel_TeluÚtgk_CyrlÚtgl_LatnÚtha_ThaiÚtir_EthiÚtaq_LatnÚtaq_TfngÚtpi_LatnÚtsn_LatnÚtso_LatnÚtuk_LatnÚtum_LatnÚtur_LatnÚtwi_LatnÚtzm_TfngÚuig_ArabÚukr_CyrlÚumb_LatnÚurd_ArabÚuzn_LatnÚvec_LatnÚvie_LatnÚwar_LatnÚwol_LatnÚxho_LatnÚydd_HebrÚyor_LatnÚyue_HantÚzho_HansÚzho_HantÚzul_Latnc                   óÌ  ‡ — e Zd ZU dZeZddgZeZg Z	e
e         ed<   g Ze
e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d)deeeef         z  dz  dee
e         z  dz  dedz  fˆ fd„Zedefd„¦   «         Zej        deddfd„¦   «         Zdededz  dedz  fd„Z	 	 	 	 	 	 	 	 d*de
e         dede
e         dz  dededz  d edz  d!ededz  d"edefd#„Zd$„ Zd%„ Zd+d&„Zd'eddfd(„Zˆ xZS ),ÚNllbTokenizera	  
    Construct an NLLB tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Examples:

    ```python
    >>> from transformers import NllbTokenizer

    >>> tokenizer = NllbTokenizer.from_pretrained(
    ...     "facebook/nllb-200-distilled-600M", src_lang="eng_Latn", tgt_lang="fra_Latn"
    ... )
    >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
    >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
    >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="pt")
    ```

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values.
        src_lang (`str`, *optional*):
            The language to use as source language for translation.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        legacy_behaviour (`bool`, *optional*, defaults to `False`):
            Whether to use legacy behaviour (suffix pattern) or new behaviour (prefix pattern).
    Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensNú<s>ú</s>ú<unk>ú<pad>ú<mask>FÚvocabÚmergesÚ_spm_precompiled_charsmapc                 ó$  •— |�|}n	|€t           }t          |	t          ¦  «        rt          |	ddd¬¦  «        n|	}	|| _        |€>t          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        di}|| _        |pg | _        t          t          | j        | j        d t          |¦  «        dd¬¦  «        ¦  «        | _	        |�Rt          j        t          j        |¦  «        t          j        t          d	¦  «        d
¦  «        g¦  «        | j	        _        t!          j        ddd¬¦  «        | j	        _        t'          j        ddd¬¦  «        | j	        _         t+          ¦   «         j        d|||||||
||	||dœ|¤Ž d| _        dddddœ| _        d„ | j                             ¦   «         D ¦   «         | _        |
�|
nd| _        |                      | j        ¦  «        | _        || _        |                      | j        ¦  «         d S )NT)Ú
normalizedÚlstripÚspecialr   é   é   r
   F)ræ   rç   ÚdropoutÚ	unk_tokenÚfuse_unkÚbyte_fallbackz {2,}ú u   â–�Úalways)ÚreplacementÚprepend_schemeÚsplit)Ú	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenrð   Ú	pad_tokenÚsrc_langÚtgt_langÚ
mask_tokenÚextra_special_tokensÚlegacy_behaviour)rá   rä   râ   rã   c                 ó   — i | ]\  }}||“Œ	S © r  )Ú.0ÚkÚvs      úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/nllb/tokenization_nllb.pyú
<dictcomp>z*NllbTokenizer.__init__.<locals>.<dictcomp>±   s   € Ð%ZÐ%ZÐ%Z©t¨q°! a¨Ð%ZÐ%ZÐ%Zó    r?   r  ) ÚFAIRSEQ_LANGUAGE_CODESÚ
isinstanceÚstrr   r  Ú_vocabÚ_mergesr   r	   Ú
_tokenizerr   ÚSequenceÚPrecompiledÚReplacer   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__Úfairseq_offsetÚfairseq_tokens_to_idsÚitemsÚfairseq_ids_to_tokensÚ	_src_langÚconvert_tokens_to_idsÚcur_lang_coderþ   Úset_src_lang_special_tokens)Úselfræ   rç   rø   rù   rú   rû   rð   rü   rÿ   rý   rþ   rè   Úadditional_special_tokensr   r  ÚkwargsÚ	__class__s                    €r  r  zNllbTokenizer.__init__Y   sV  ø€ ð*  Ð+Ø(<Ð%Ð%Ø&Ð.Ý(>Ð%õ ˜*¥cÑ*Ô*ð�J�z¨d¸4ÈÐNÑNÔNÐNàð 	ð
 !1ˆÔàˆ=å�I‘” Ý�I‘” Ý�I‘” Ý�I‘” ð	ˆEð ˆŒØ�| ˆŒå#ÝØ”kØ”|ØÝ˜i™.œ.ØØ#ðñ ô ñ	
ô 	
ˆŒð %Ð0Ý)4Ô)=åÔ+Ð,EÑFÔFÝÔ'­¨h©¬¸Ñ=Ô=ðñ*ô *ˆDŒOÔ&õ )7Ô(@ÈUÐckÐswÐ(xÑ(xÔ(xˆŒÔ%Ý"*Ô"4ÀÐW_ÐgkÐ"lÑ"lÔ"lˆŒÔà�‰ŒÔð 	
ØØØØØØØØØ!Ø!:Ø-ð	
ð 	
ð ð	
ð 	
ð 	
ð   ˆÔàØØØð	&
ð &
ˆÔ"ð &[Ð%Z°tÔ7Q×7WÒ7WÑ7YÔ7YÐ%ZÑ%ZÔ%ZˆÔ"à%-Ð%9˜˜¸zˆŒØ!×7Ò7¸¼ÑGÔGˆÔØ ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r	  Úreturnc                 ó   — | j         S ©N)r  ©r!  s    r  rý   zNllbTokenizer.src_lang¸   s
   € àŒ~Ðr	  Únew_src_langc                 óH   — || _         |                      | j         ¦  «         d S r'  )r  r   )r!  r)  s     r  rý   zNllbTokenizer.src_lang¼   s%   € à%ˆŒØ×(Ò(¨¬Ñ8Ô8Ð8Ð8Ð8r	  Úreturn_tensorsrý   rþ   c                 ó†   — |�|€t          d¦  «        ‚|| _         | |fd|dœ|¤Ž}|                      |¦  «        }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionNzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensr+  Úforced_bos_token_id)Ú
ValueErrorrý   r  )r!  Ú
raw_inputsr+  rý   rþ   Úextra_kwargsÚinputsÚtgt_lang_ids           r  Ú_build_translation_inputsz'NllbTokenizer._build_translation_inputsÁ   sg   € ð Ð˜xÐ/ÝÐ`ÑaÔaÐaØ ˆŒØ��jÐi°TÈ.ÐiÐiÐ\hÐiÐiˆØ×0Ò0°Ñ:Ô:ˆØ(3ˆÐ$Ñ%Øˆr	  r?   rI   ÚlongestTÚ	src_textsÚ	tgt_textsÚ
max_lengthÚmax_target_lengthÚpaddingÚ
truncationc
           	      óæ   — || _         || _        |€| j        } | |fd||||	dœ|
¤Ž}|€|S |€|}|                      ¦   «           | |fd||||	dœ|
¤Ž}|d         |d<   |                      ¦   «          |S )NT)r-  r+  r8  r:  r;  )r-  r+  r:  r8  r;  rÝ   Úlabels)rý   rþ   Úmodel_max_lengthÚ_switch_to_target_modeÚ_switch_to_input_mode)r!  r6  rý   r7  rþ   r8  r9  r:  r+  r;  r#  Úmodel_inputsr=  s                r  Úprepare_seq2seq_batchz#NllbTokenizer.prepare_seq2seq_batchÍ   sé   € ð !ˆŒØ ˆŒàÐØÔ.ˆJà�tØð
à#Ø)Ø!ØØ!ð
ð 
ð ð
ð 
ˆð ÐØÐð Ð$Ø *Ðð 	×#Ò#Ñ%Ô%Ð%Ø�Øð
à#Ø)ØØ(Ø!ð
ð 
ð ð
ð 
ˆð "(¨Ô!4ˆ�XÑð 	×"Ò"Ñ$Ô$Ð$àÐr	  c                 ó6   — |                       | j        ¦  «        S r'  )r   rý   r(  s    r  r@  z#NllbTokenizer._switch_to_input_mode  s   € Ø×/Ò/°´Ñ>Ô>Ð>r	  c                 ó\   — | j         €| j        | _         |                      | j         ¦  «        S r'  )rþ   r  Úset_tgt_lang_special_tokensr(  s    r  r?  z$NllbTokenizer._switch_to_target_mode  s)   € ØŒ=Ð Ø œNˆDŒMØ×/Ò/°´Ñ>Ô>Ð>r	  c                 óÔ  — |                       |¦  «        | _        |}| j        rcg | _        | j        | j        g| _        t          j        d| j        |gdd| j        |g| j        | j        f|| j        fg¬¦  «        | j	        _
        dS | j        g| _        | j        g| _        t          j        |d| j        g|dd| j        g| j        | j        f|| j        fg¬¦  «        | j	        _
        dS )zÄReset the special tokens to the source lang setting.
        - In legacy mode: No prefix and suffix=[eos, src_lang_code].
        - In default mode: Prefix=[src_lang_code], suffix = [eos]
        ú$Aú$B©ÚsingleÚpairÚspecial_tokensN©r  r  r  rß   Úeos_token_idrà   r   ÚTemplateProcessingrù   r  Úpost_processor)r!  rý   Úlang_code_tokens      r  r   z)NllbTokenizer.set_src_lang_special_tokens  s  € ð
 "×7Ò7¸ÑAÔAˆÔØ"ˆàÔ ð 	Ø!#ˆDÔØ"&Ô"3°TÔ5GÐ!HˆDÔÝ-7Ô-JØ˜dœn¨oÐ>Ø˜D $¤.°/ÐBØ!%¤°Ô1BÐ CÀoÐW[ÔWiÐEjÐkð.ñ .ô .ˆDŒOÔ*Ð*Ð*ð #'Ô"4Ð!5ˆDÔØ"&Ô"3Ð!4ˆDÔÝ-7Ô-JØ'¨¨t¬~Ð>Ø% t¨T°4´>ÐBØ!%¤°Ô1BÐ CÀoÐW[ÔWiÐEjÐkð.ñ .ô .ˆDŒOÔ*Ð*Ð*r	  Úlangc                 óÔ  — |                       |¦  «        | _        |}| j        rcg | _        | j        | j        g| _        t          j        d| j        |gdd| j        |g| j        | j        f|| j        fg¬¦  «        | j	        _
        dS | j        g| _        | j        g| _        t          j        |d| j        g|dd| j        g| j        | j        f|| j        fg¬¦  «        | j	        _
        dS )zÄReset the special tokens to the target lang setting.
        - In legacy mode: No prefix and suffix=[eos, tgt_lang_code].
        - In default mode: Prefix=[tgt_lang_code], suffix = [eos]
        rG  rH  rI  NrM  )r!  rR  rQ  s      r  rE  z)NllbTokenizer.set_tgt_lang_special_tokens$  s  € ð
 "×7Ò7¸Ñ=Ô=ˆÔØˆàÔ ð 	Ø!#ˆDÔØ"&Ô"3°TÔ5GÐ!HˆDÔÝ-7Ô-JØ˜dœn¨oÐ>Ø˜D $¤.°/ÐBØ!%¤°Ô1BÐ CÀoÐW[ÔWiÐEjÐkð.ñ .ô .ˆDŒOÔ*Ð*Ð*ð #'Ô"4Ð!5ˆDÔØ"&Ô"3Ð!4ˆDÔÝ-7Ô-JØ'¨¨t¬~Ð>Ø% t¨T°4´>ÐBØ!%¤°Ô1BÐ CÀoÐW[ÔWiÐEjÐkð.ñ .ô .ˆDŒOÔ*Ð*Ð*r	  )NNrá   râ   râ   rá   rã   rä   rå   NNNNNF)r?   NrI   NNr5  NT)r%  N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   Úmodelrß   ÚlistÚintÚ__annotations__rà   r  Údictr  Úpropertyrý   Úsetterr4  Úboolr   rB  r@  r?  r   rE  Ú__classcell__)r$  s   @r  rÜ   rÜ   !   s˜  ø€ € € € € € ð.ð .ð` *ÐØ$Ð&6Ð7ÐØ€Eà!€M�4˜”9Ð!Ð!Ñ!Ø!€M�4˜”9Ð!Ð!Ñ!ð .2Ø)-ØØØØØØØØØØ04Ø"&Ø!Øð!]9ð ]9à�T˜#˜s˜(”^Ñ# dÑ*ð]9ð �d˜3”i‘ $Ñ&ð]9ð $'¨¡:ð]9ð ]9ð ]9ð ]9ð ]9ð ]9ð~ ð˜#ð ð ð ñ „Xðð „_ð9 Sð 9¨Tð 9ð 9ð 9ñ „_ð9ð
Ø*-ð
Ø9<¸t¹ð
ØORÐUYÉzð
ð 
ð 
ð 
ð #Ø&*Ø"Ø!%Ø(,Ø Ø%)Øð4ð 4à˜”9ð4ð ð4ð ˜”9˜tÑ#ð	4ð
 ð4ð ˜$‘Jð4ð  ™:ð4ð ð4ð ˜d™
ð4ð ð4ð 
ð4ð 4ð 4ð 4ðl?ð ?ð ?ð?ð ?ð ?ð
ð ð ð ð2°ð ¸ð ð ð ð ð ð ð ð r	  rÜ   N)Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_pythonr   r   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrT  ÚloggerrX  r
  rÜ   Ú__all__r  r	  r  ú<module>rl     s  ðð  [Ð ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZØ !Ð !Ð !Ð !Ð !Ð !à <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð $=ÐP`ÐaÐaÐ ð R&ð  R&ð  R&Ð ðZð Zð Zð Zð ZÐ%ñ Zô Zð Zðz Ð
€€€r	  