§
    ‚ŠtjåE  ã                   ó¢   — d Z ddlZddlZddlZddlZddlmZ ddlmZ  ej	        e
¦  «        Zdddd	œZd
„ Zd„ Zd„ Z G d„ de¦  «        ZdgZdS )zTokenization classes for FSMT.é    Né   )ÚPreTrainedTokenizer)Úloggingzvocab-src.jsonzvocab-tgt.jsonz
merges.txt)Úsrc_vocab_fileÚtgt_vocab_fileÚmerges_filec                 ó~   — t          ¦   «         }| d         }| dd…         D ]}|                     ||f¦  «         |}Œ|S )zƒ
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    r   é   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/fsmt/tokenization_fsmt.pyÚ	get_pairsr   "   sP   € õ
 ‰EŒE€EØ�Q”€IØ�Q�R�R”ð ð ˆØ�	Š	�9˜dÐ#Ñ$Ô$Ð$Øˆ	ˆ	Ø€Ló    c                 ó6  — |                       dd¦  «        } t          j        dd| ¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       d	d
¦  «        } |                       dd
¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       dd¦  «        } |                       d d!¦  «        } |                       d"d#¦  «        } |                       d$d%¦  «        } |                       d&d'¦  «        } |                       d(d)¦  «        } |                       d*d+¦  «        } |                       d,d-¦  «        } t          j        d.d| ¦  «        } |                       d/d0¦  «        } |                       d1d2¦  «        } |                       d3d4¦  «        } |                       d5d6¦  «        } |                       d7d8¦  «        } |                       d9d:¦  «        } |                       d;d<¦  «        } |                       d=d>¦  «        } |                       d?d@¦  «        } | S )Azz
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
    u   ï¼Œú,u   ã€‚\s*z. u   ã€�u   â€�ú"u   â€œu   âˆ¶ú:u   ï¼šu   ï¼Ÿú?u   ã€Šu   ã€‹u   ï¼‰ú)u   ï¼�ú!u   ï¼ˆú(u   ï¼›ú;u   ï¼‘Ú1u   ã€�u   ã€Œu   ï¼�Ú0u   ï¼“Ú3u   ï¼’Ú2u   ï¼•Ú5u   ï¼–Ú6u   ï¼™Ú9u   ï¼—Ú7u   ï¼˜Ú8u   ï¼”Ú4u   ï¼Ž\s*u   ï½žú~u   â€™ú'u   â€¦z...u   â”�ú-u   ã€ˆú<u   ã€‰ú>u   ã€�ú[u   ã€‘ú]u   ï¼…ú%)ÚreplaceÚreÚsub)Útexts    r   Úreplace_unicode_punctr3   /   sµ  € ð �<Š<˜˜sÑ#Ô#€DÝŒ6�)˜T 4Ñ(Ô(€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DÝŒ6�)˜T 4Ñ(Ô(€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜uÑ%Ô%€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ�<Š<˜˜sÑ#Ô#€DØ€Kr   c                 ó¸   — g }| D ]A}t          j        |¦  «        }|                     d¦  «        rŒ,|                     |¦  «         ŒBd                     |¦  «        S )zw
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl
    ÚCÚ )ÚunicodedataÚcategoryÚ
startswithÚappendÚjoin)r2   Úoutputr   Úcats       r   Úremove_non_printing_charr>   Z   sd   € ð €FØð ð ˆÝÔ" 4Ñ(Ô(ˆØ�>Š>˜#ÑÔð 	ØØ�Š�dÑÔÐÐØ�7Š7�6‰?Œ?Ðr   c            
       óª  ‡ — e Zd ZdZeZddgZ	 	 	 	 	 	 	 	 	 d&ˆ fd
„	Zdee	e
f         fd„Zede
fd„¦   «         Zd„ Zd„ Zd„ Zd„ Zed„ ¦   «         Zed„ ¦   «         Zd„ Zd„ Zd„ Zd'd„Zd„ Zd„ Zd„ Z	 d(dee
         dee
         dz  dee
         fd„Z	 d)dee
         dee
         dz  dedee
         fˆ fd „Zd(d!e	d"e	dz  dee	         fd#„Z d$„ Z!d%„ Z"ˆ xZ#S )*ÚFSMTTokenizera	  
    Construct an FAIRSEQ Transformer tokenizer. Based on Byte-Pair Encoding. The tokenization process is the following:

    - Moses preprocessing and tokenization.
    - Normalizing all inputs text.
    - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
      "__classify__") to a vocabulary.
    - The argument `langs` defines a pair of languages.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        langs (`List[str]`, *optional*):
            A list of two languages to translate from and to, for instance `["en", "ru"]`.
        src_vocab_file (`str`, *optional*):
            File containing the vocabulary for the source language.
        tgt_vocab_file (`st`, *optional*):
            File containing the vocabulary for the target language.
        merges_file (`str`, *optional*):
            File containing the merges.
        do_lower_case (`bool`, *optional*, defaults to `False`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.

    Ú	input_idsÚattention_maskNFú<unk>ú<s>ú</s>ú<pad>c
                 ó  •— 	 dd l }n# t          $ r t          d¦  «        ‚w xY w|| _        || _        || _        || _        || _        i | _        i | _        i | _	        |r#t          |¦  «        dk    r|\  | _        | _        nt          d|› d�¦  «        ‚t          |d¬¦  «        5 }t          j        |¦  «        | _        d d d ¦  «         n# 1 swxY w Y   t          |d¬¦  «        5 }t          j        |¦  «        }d„ |                     ¦   «         D ¦   «         | _        d d d ¦  «         n# 1 swxY w Y   t          |d¬¦  «        5 }|                     ¦   «                              d	¦  «        d d
…         }d d d ¦  «         n# 1 swxY w Y   d„ |D ¦   «         }t-          t/          |t1          t          |¦  «        ¦  «        ¦  «        ¦  «        | _        i | _         t7          ¦   «         j        d|||||||||	dœ	|
¤Ž d S )Nr   únYou need to install sacremoses to use XLMTokenizer. See https://pypi.org/project/sacremoses/ for installation.é   zFarg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got zw. Usually that means that tokenizer can't find a mapping for the given model path in  and other maps of this tokenizer.úutf-8©Úencodingc                 ó   — i | ]\  }}||“Œ	S © rN   ©Ú.0ÚkÚvs      r   ú
<dictcomp>z*FSMTTokenizer.__init__.<locals>.<dictcomp>Ð   s   € Ð?Ð?Ð?¡T Q¨˜A˜qÐ?Ð?Ð?r   ú
éÿÿÿÿc                 ó`   — g | ]+}t          |                     ¦   «         d d…         ¦  «        ‘Œ,S )NrI   )ÚtupleÚsplit)rP   Úmerges     r   ú
<listcomp>z*FSMTTokenizer.__init__.<locals>.<listcomp>Ó   s1   € Ð?Ð?Ð?¨u•%˜Ÿš™œ b q bÔ)Ñ*Ô*Ð?Ð?Ð?r   )	Úlangsr   r   r   Údo_lower_caseÚ	unk_tokenÚ	bos_tokenÚ	sep_tokenÚ	pad_tokenrN   )Ú
sacremosesÚImportErrorÚsmr   r   r   r\   Úcache_moses_punct_normalizerÚcache_moses_tokenizerÚcache_moses_detokenizerÚlenÚsrc_langÚtgt_langÚ
ValueErrorÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadrX   ÚdictÚzipÚrangeÚ	bpe_ranksÚcacheÚsuperÚ__init__)Úselfr[   r   r   r   r\   r]   r^   r_   r`   Úkwargsra   Úsrc_vocab_handleÚtgt_vocab_handleÚ	tgt_vocabÚmerges_handleÚmergesÚ	__class__s                    €r   rx   zFSMTTokenizer.__init__¡   s  ø€ ð	ØÐÐÐÐøÝð 	ð 	ð 	ÝðMñô ð ð	øøøð ˆŒà,ˆÔØ,ˆÔØ&ˆÔØ*ˆÔð -/ˆÔ)à%'ˆÔ"Ø')ˆÔ$àð 	•S˜‘Z”Z 1’_�_Ø+0Ñ(ˆDŒM˜4œ=˜=åð8ÐY^ð 8ð 8ð 8ñô ð õ �.¨7Ð3Ñ3Ô3ð 	7Ð7GÝœ9Ð%5Ñ6Ô6ˆDŒLð	7ð 	7ð 	7ñ 	7ô 	7ð 	7ð 	7ð 	7ð 	7ð 	7ð 	7øøøð 	7ð 	7ð 	7ð 	7å�.¨7Ð3Ñ3Ô3ð 	@Ð7GÝœ	Ð"2Ñ3Ô3ˆIØ?Ð?¨Y¯_ª_Ñ->Ô->Ð?Ñ?Ô?ˆDŒLð	@ð 	@ð 	@ñ 	@ô 	@ð 	@ð 	@ð 	@ð 	@ð 	@ð 	@øøøð 	@ð 	@ð 	@ð 	@õ �+¨Ð0Ñ0Ô0ð 	;°MØ"×'Ò'Ñ)Ô)×/Ò/°Ñ5Ô5°c°r°cÔ:ˆFð	;ð 	;ð 	;ñ 	;ô 	;ð 	;ð 	;ð 	;ð 	;ð 	;ð 	;øøøð 	;ð 	;ð 	;ð 	;à?Ð?¸Ð?Ñ?Ô?ˆÝ�c &­%µ°F±´Ñ*<Ô*<Ñ=Ô=Ñ>Ô>ˆŒØˆŒ
Ø�‰ŒÔð 	
ØØ)Ø)Ø#Ø'ØØØØð	
ð 	
ð ð	
ð 	
ð 	
ð 	
ð 	
s>   ƒ ˆ"Â&CÃCÃCÃ(8D,Ä,D0Ä3D0Å0FÆFÆFÚreturnc                 ó*   — |                       ¦   «         S ©N)Úget_src_vocab©ry   s    r   Ú	get_vocabzFSMTTokenizer.get_vocabä   s   € Ø×!Ò!Ñ#Ô#Ð#r   c                 ó   — | j         S rƒ   )Úsrc_vocab_sizer…   s    r   Ú
vocab_sizezFSMTTokenizer.vocab_sizeè   s   € àÔ"Ð"r   c                 óž   — || j         vr%| j                             |¬¦  «        }|| j         |<   | j         |                              |¦  «        S ©N©Úlang)rd   rc   ÚMosesPunctNormalizerÚ	normalize)ry   r2   r�   Úpunct_normalizers       r   Úmoses_punct_normzFSMTTokenizer.moses_punct_normì   sR   € Ø�tÔ8Ð8Ð8Ø#œw×;Ò;ÀÐ;ÑFÔFÐØ6FˆDÔ-¨dÑ3ØÔ0°Ô6×@Ò@ÀÑFÔFÐFr   c                 ó¦   — || j         vr%| j                             |¬¦  «        }|| j         |<   | j         |                              |ddd¬¦  «        S )NrŒ   TF)Úaggressive_dash_splitsÚ
return_strÚescape)re   rc   ÚMosesTokenizerÚtokenize)ry   r2   r�   Úmoses_tokenizers       r   Úmoses_tokenizezFSMTTokenizer.moses_tokenizeò   sc   € Ø�tÔ1Ð1Ð1Ø"œg×4Ò4¸$Ð4Ñ?Ô?ˆOØ/>ˆDÔ& tÑ,ØÔ)¨$Ô/×8Ò8Ø¨¸%Èð 9ñ 
ô 
ð 	
r   c                 óž   — || j         vr%| j                             |¬¦  «        }|| j         |<   | j         |                              |¦  «        S r‹   )rf   rc   ÚMosesDetokenizerÚ
detokenize)ry   Útokensr�   Úmoses_detokenizers       r   Úmoses_detokenizezFSMTTokenizer.moses_detokenizeú   sR   € Ø�tÔ3Ð3Ð3Ø $¤× 8Ò 8¸dÐ 8Ñ CÔ CÐØ1BˆDÔ(¨Ñ.ØÔ+¨DÔ1×<Ò<¸VÑDÔDÐDr   c                 ón   — t          |¦  «        }|                      ||¦  «        }t          |¦  «        }|S rƒ   )r3   r‘   r>   )ry   r2   r�   s      r   Úmoses_pipelinezFSMTTokenizer.moses_pipeline   s6   € Ý$ TÑ*Ô*ˆØ×$Ò$ T¨4Ñ0Ô0ˆÝ'¨Ñ-Ô-ˆØˆr   c                 ó*   — t          | j        ¦  «        S rƒ   )rg   rn   r…   s    r   rˆ   zFSMTTokenizer.src_vocab_size  ó   € å�4”<Ñ Ô Ð r   c                 ó*   — t          | j        ¦  «        S rƒ   )rg   rp   r…   s    r   Útgt_vocab_sizezFSMTTokenizer.tgt_vocab_size
  r£   r   c                 ó0   — t          | j        fi | j        ¤ŽS rƒ   )rr   rn   Úadded_tokens_encoderr…   s    r   r„   zFSMTTokenizer.get_src_vocab  ó   € Ý�D”LÐ>Ð> DÔ$=Ð>Ð>Ð>r   c                 ó0   — t          | j        fi | j        ¤ŽS rƒ   )rr   rp   Úadded_tokens_decoderr…   s    r   Úget_tgt_vocabzFSMTTokenizer.get_tgt_vocab  r¨   r   c                 ó¦  ‡ — t          |d d…         ¦  «        |d         dz   fz   }|‰ j        v r‰ j        |         S t          |¦  «        }|s|dz   S 	 t          |ˆ fd„¬¦  «        }|‰ j        vr�n8|\  }}g }d}|t          |¦  «        k     ræ	 |                     ||¦  «        }	|                     |||	…         ¦  «         |	}n-# t          $ r  |                     ||d …         ¦  «         Y n†w xY w||         |k    rC|t          |¦  «        dz
  k     r-||dz            |k    r| 	                    ||z   ¦  «         |dz  }n | 	                    ||         ¦  «         |dz  }|t          |¦  «        k     °æt          |¦  «        }|}t          |¦  «        dk    rnt          |¦  «        }�ŒWd	 
                    |¦  «        }|d
k    rd}|‰ j        |<   |S )NrU   ú</w>Tc                 óT   •— ‰j                              | t          d¦  «        ¦  «        S )NÚinf)ru   ÚgetÚfloat)Úpairry   s    €r   ú<lambda>z#FSMTTokenizer.bpe.<locals>.<lambda>  s    ø€ °´×1CÒ1CÀDÍ%ÐPUÉ,Ì,Ñ1WÔ1W€ r   ©Úkeyr   r
   rI   ú z
  </w>z
</w>)rW   rv   r   Úminru   rg   ÚindexÚextendrj   r:   r;   )
ry   Útokenr   r   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   `         r   ÚbpezFSMTTokenizer.bpe  s  ø€ Ý�U˜3˜B˜3”ZÑ Ô  E¨"¤I°Ñ$6Ð#8Ñ8ˆØ�D”JÐÐØ”:˜eÔ$Ð$Ý˜$‘”ˆàð 	"Ø˜6‘>Ð!ð	(Ý˜Ð$WÐ$WÐ$WÐ$WÐXÑXÔXˆFØ˜Tœ^Ð+Ð+ÙØ"‰MˆE�6ØˆHØˆAØ•c˜$‘i”i’-�-ðØŸ
š
 5¨!Ñ,Ô,�Að
 —O’O D¨¨1¨¤IÑ.Ô.Ð.Ø�A�Aøõ "ð ð ð Ø—O’O D¨¨¨¤HÑ-Ô-Ð-Ø�Eðøøøð ˜”7˜eÒ#Ð#¨­C°©I¬I¸©MÒ(9Ð(9¸dÀ1ÀqÁ5¼kÈVÒ>SÐ>SØ—O’O E¨F¡NÑ3Ô3Ð3Ø˜‘F�A�Aà—O’O D¨¤GÑ,Ô,Ð,Ø˜‘F�Að •c˜$‘i”i’-�-õ  ˜X‘”ˆHØˆDÝ�4‰yŒy˜AŠ~ˆ~Øå! $™œ�ñ9	(ð: �xŠx˜‰~Œ~ˆØ�:ÒÐØˆDØ ˆŒ
�5ÑØˆs   ÂC Ã'C/Ã.C/Úenc                 óv  — | j         }| j        r|                     ¦   «         }|r|                     ¦   «         }n.|                      ||¬¦  «        }|                      ||¬¦  «        }g }|D ]L}|rH|                     t          |                      |¦  «                             d¦  «        ¦  «        ¦  «         ŒM|S )av  
        Tokenize a string given language code using Moses.

        Details of tokenization:

            - [sacremoses](https://github.com/alvations/sacremoses): port of Moses
            - Install with `pip install sacremoses`

        Args:
            - lang: ISO language code (default = 'en') (string). Languages should belong of the model supported
              languages. However, we don't enforce it.
            - bypass_tokenizer: Allow users to preprocess and tokenize the sentences externally (default = False)
              (bool). If True, we only apply BPE.

        Returns:
            List of tokens.
        rŒ   r¶   )	rh   r\   ÚlowerrX   r¡   r™   r¹   ÚlistrÁ   )ry   r2   r�   Úbypass_tokenizerÚsplit_tokensrº   s         r   Ú	_tokenizezFSMTTokenizer._tokenize@  sÄ   € ð* Œ}ˆàÔð 	 Ø—:’:‘<”<ˆDàð 	8Ø—:’:‘<”<ˆDˆDà×&Ò& t°$Ð&Ñ7Ô7ˆDØ×&Ò& t°$Ð&Ñ7Ô7ˆDàˆØð 	Fð 	FˆEØð FØ×#Ò#¥D¨¯ª°%©¬×)>Ò)>¸sÑ)CÔ)CÑ$DÔ$DÑEÔEÐEøàÐr   c                 ór   — | j                              || j                              | j        ¦  «        ¦  «        S )z0Converts a token (str) in an id using the vocab.)rn   r°   r]   )ry   rº   s     r   Ú_convert_token_to_idz"FSMTTokenizer._convert_token_to_idg  s,   € àŒ|×Ò  t¤|×'7Ò'7¸¼Ñ'GÔ'GÑHÔHÐHr   c                 óB   — | j                              || j        ¦  «        S )z=Converts an index (integer) in a token (str) using the vocab.)rp   r°   r]   )ry   r¸   s     r   Ú_convert_id_to_tokenz"FSMTTokenizer._convert_id_to_tokenk  s   € àŒ|×Ò  t¤~Ñ6Ô6Ð6r   c                 ó¢   — d„ |D ¦   «         }d                      |¦  «                             ¦   «         }|                      || j        ¦  «        }|S )z:Converts a sequence of tokens (string) in a single string.c                 ób   — g | ],}|                      d d¦  «                              dd ¦  «        ‘Œ-S )r¶   r6   r­   )r/   )rP   Úts     r   rZ   z:FSMTTokenizer.convert_tokens_to_string.<locals>.<listcomp>s  s6   € ÐJÐJÐJ¸a�!—)’)˜C Ñ$Ô$×,Ò,¨V°SÑ9Ô9ÐJÐJÐJr   r6   )r;   rX   rŸ   ri   )ry   r�   r2   s      r   Úconvert_tokens_to_stringz&FSMTTokenizer.convert_tokens_to_stringo  sO   € ð KÐJÀ6ÐJÑJÔJˆØ—’˜‘”×&Ò&Ñ(Ô(ˆà×$Ò$ V¨T¬]Ñ;Ô;ˆØˆr   Útoken_ids_0Útoken_ids_1c                 ó6   — | j         g}|€||z   S ||z   |z   |z   S )až  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A FAIRSEQ Transformer sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )Úsep_token_id)ry   rÑ   rÒ   Úseps       r   Ú build_inputs_with_special_tokensz.FSMTTokenizer.build_inputs_with_special_tokensy  s7   € ð& Ô Ð!ˆð ÐØ Ñ$Ð$Ø˜SÑ  ;Ñ.°Ñ4Ð4r   Úalready_has_special_tokensc                 óà   •— |r$t          ¦   «                              ||d¬¦  «        S |�/dgt          |¦  «        z  dgz   dgt          |¦  «        z  z   dgz   S dgt          |¦  «        z  dgz   S )aÄ  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)rÑ   rÒ   r×   Nr   r
   )rw   Úget_special_tokens_maskrg   )ry   rÑ   rÒ   r×   r€   s       €r   rÙ   z%FSMTTokenizer.get_special_tokens_mask“  s’   ø€ ð& &ð 	Ý‘7”7×2Ò2Ø'°[Ð]að 3ñ ô ð ð Ð"Ø�C�#˜kÑ*Ô*Ñ*¨q¨cÑ1°a°S½3¸{Ñ;KÔ;KÑ5KÑLÐPQÈsÑRÐRØ�•c˜+Ñ&Ô&Ñ&¨1¨#Ñ-Ð-r   Úsave_directoryÚfilename_prefixc           	      óÜ  — t           j                             |¦  «        s t                               d|› d�¦  «         d S t           j                             ||r|dz   ndt          d         z   ¦  «        }t           j                             ||r|dz   ndt          d         z   ¦  «        }t           j                             ||r|dz   ndt          d         z   ¦  «        }t          |dd	¬
¦  «        5 }|                     t          j
        | j        ddd¬¦  «        dz   ¦  «         d d d ¦  «         n# 1 swxY w Y   t          |dd	¬
¦  «        5 }d„ | j                             ¦   «         D ¦   «         }|                     t          j
        |ddd¬¦  «        dz   ¦  «         d d d ¦  «         n# 1 swxY w Y   d}t          |dd	¬
¦  «        5 }	t          | j                             ¦   «         d„ ¬¦  «        D ][\  }
}||k    r t                               d|› d�¦  «         |}|	                     d                     |
¦  «        dz   ¦  «         |dz  }Œ\	 d d d ¦  «         n# 1 swxY w Y   |||fS )NzVocabulary path (z) should be a directoryr)   r6   r   r   r   ÚwrJ   rK   rI   TF)ÚindentÚ	sort_keysÚensure_asciirT   c                 ó   — i | ]\  }}||“Œ	S rN   rN   rO   s      r   rS   z1FSMTTokenizer.save_vocabulary.<locals>.<dictcomp>Â  s   € Ð?Ð?Ð?¡$ ! Q˜˜AÐ?Ð?Ð?r   r   c                 ó   — | d         S )Nr
   rN   )Úkvs    r   r³   z/FSMTTokenizer.save_vocabulary.<locals>.<lambda>Ç  s   € ÐY[Ð\]ÔY^€ r   r´   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r¶   r
   )ÚosÚpathÚisdirÚloggerÚerrorr;   ÚVOCAB_FILES_NAMESrk   Úwriterl   Údumpsrn   rp   ro   Úsortedru   Úwarning)ry   rÚ   rÛ   r   r   r   Úfr}   r¸   ÚwriterÚ
bpe_tokensÚtoken_indexs               r   Úsave_vocabularyzFSMTTokenizer.save_vocabulary¯  sp  € ÝŒw�}Š}˜^Ñ,Ô,ð 	Ý�LŠLÐT¨^ÐTÐTÐTÑUÔUÐUØˆFåœŸšØ°oÐM˜_¨sÑ2Ð2È2ÕQbÐcsÔQtÑtñ
ô 
ˆõ œŸšØ°oÐM˜_¨sÑ2Ð2È2ÕQbÐcsÔQtÑtñ
ô 
ˆõ ”g—l’lØ°oÐM˜_¨sÑ2Ð2È2ÕQbÐcpÔQqÑqñ
ô 
ˆõ �. #°Ð8Ñ8Ô8ð 	c¸AØ�GŠG•D”J˜tœ|°AÀÐTYÐZÑZÔZÐ]aÑaÑbÔbÐbð	cð 	cð 	cñ 	cô 	cð 	cð 	cð 	cð 	cð 	cð 	cøøøð 	cð 	cð 	cð 	cõ �. #°Ð8Ñ8Ô8ð 	`¸AØ?Ð?¨$¬,×*<Ò*<Ñ*>Ô*>Ð?Ñ?Ô?ˆIØ�GŠG•D”J˜y°¸dÐQVÐWÑWÔWÐZ^Ñ^Ñ_Ô_Ð_ð	`ð 	`ð 	`ñ 	`ô 	`ð 	`ð 	`ð 	`ð 	`ð 	`ð 	`øøøð 	`ð 	`ð 	`ð 	`ð ˆÝ�+˜s¨WÐ5Ñ5Ô5ð 		¸Ý+1°$´.×2FÒ2FÑ2HÔ2HÐN^ÐN^Ð+_Ñ+_Ô+_ð ð Ñ'�
˜KØ˜KÒ'Ð'Ý—N’NðM°ð Mð Mð Mñô ð ð (�EØ—’˜SŸXšX jÑ1Ô1°DÑ8Ñ9Ô9Ð9Ø˜‘
��ðð		ð 		ð 		ñ 		ô 		ð 		ð 		ð 		ð 		ð 		ð 		øøøð 		ð 		ð 		ð 		ð ˜~¨{Ð:Ð:s8   Ã14D1Ä1D5Ä8D5ÅAF,Æ,F0Æ3F0ÇBIÉI"É%I"c                 óB   — | j                              ¦   «         }d |d<   |S )Nrc   )Ú__dict__Úcopy)ry   Ústates     r   Ú__getstate__zFSMTTokenizer.__getstate__Ó  s#   € Ø”×"Ò"Ñ$Ô$ˆØˆˆd‰Øˆr   c                 óh   — || _         	 dd l}n# t          $ r t          d¦  «        ‚w xY w|| _        d S )Nr   rH   )rô   ra   rb   rc   )ry   Údra   s      r   Ú__setstate__zFSMTTokenizer.__setstate__Ø  s]   € ØˆŒð	ØÐÐÐÐøÝð 	ð 	ð 	ÝðMñô ð ð	øøøð ˆŒˆˆs   ‰ Ž()	NNNNFrC   rD   rE   rF   )rÂ   Frƒ   )NF)$Ú__name__Ú
__module__Ú__qualname__Ú__doc__ré   Úvocab_files_namesÚmodel_input_namesrx   rr   ÚstrÚintr†   Úpropertyr‰   r‘   r™   rŸ   r¡   rˆ   r¥   r„   r«   rÁ   rÈ   rÊ   rÌ   rÐ   rÅ   rÖ   ÚboolrÙ   rW   rò   r÷   rú   Ú__classcell__)r€   s   @r   r@   r@   p   s±  ø€ € € € € ð+ð +ðZ *ÐØ$Ð&6Ð7Ðð ØØØØØØØØð@
ð @
ð @
ð @
ð @
ð @
ðF$˜4  S œ>ð $ð $ð $ð $ð ð#˜Cð #ð #ð #ñ „Xð#ðGð Gð Gð
ð 
ð 
ðEð Eð Eðð ð ð ð!ð !ñ „Xð!ð ð!ð !ñ „Xð!ð?ð ?ð ?ð?ð ?ð ?ð*ð *ð *ðX%ð %ð %ð %ðNIð Ið Ið7ð 7ð 7ðð ð ð GKð5ð 5Ø œ9ð5Ø37¸´9¸tÑ3Cð5à	ˆcŒð5ð 5ð 5ð 5ð6 puð.ð .Ø œ9ð.Ø37¸´9¸tÑ3Cð.Øhlð.à	ˆcŒð.ð .ð .ð .ð .ð .ð8";ð ";¨cð ";ÀCÈ$ÁJð ";ÐZ_Ð`cÔZdð ";ð ";ð ";ð ";ðHð ð ð
ð ð ð ð ð ð r   r@   )rþ   rl   rä   r0   r7   Útokenization_pythonr   Úutilsr   Ú
get_loggerrû   rç   ré   r   r3   r>   r@   Ú__all__rN   r   r   ú<module>r
     sô   ðð %Ð $à €€€Ø 	€	€	€	Ø 	€	€	€	Ø Ð Ð Ð à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð 'Ø&Øðð Ð ð
ð 
ð 
ð(ð (ð (ðV
ð 
ð 
ð,sð sð sð sð sÐ'ñ sô sð sðl Ð
€€€r   