§
    ~ŠtjI  ã                   óŽ   — d dl mZmZmZmZmZmZ ddlmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZmZmZ ddlmZ  G d„ d	e¦  «        Zd
S )é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnioné   )Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚBertNormalizerÚ	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   óF  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 d"deeeeeef         f                  deeee	e
eef                  f                  d	eeef         d
edee         dedee         dedefˆ fd„Zededefd„¦   «         Zdddgdg ddfdeee	e         f         dedede	eeef                  dede	e         d
ee         defd„Zdddgdg dddfdeee         eee                  f         dedede	eeef                  dede	e         d
ee         ded ee         fd!„Zˆ xZS )#ÚCharBPETokenizeraà  Original BPE Tokenizer

    Represents the BPE algorithm, as introduced by Rico Sennrich
    (https://arxiv.org/abs/1508.07909)

    The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
    Sennrich subword-nmt implementation by the following options that you can deactivate:
        - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
            * removing any control characters and replacing all whitespaces by the classic one.
            * handle chinese chars by putting spaces around them.
            * strip all accents.
        - spitting on punctuation in addition to whitespaces (deactivate it with
          `split_on_whitespace_only=True`)
    Nú<unk>ú</w>FTÚvocabÚmergesÚ	unk_tokenÚsuffixÚdropoutÚ	lowercaseÚunicode_normalizerÚbert_normalizerÚsplit_on_whitespace_onlyc
           
      óV  •— |�1|�/t          t          |||t          |¦  «        |¬¦  «        ¦  «        }
n,t          t          t          |¦  «        ||¬¦  «        ¦  «        }
|
                     t          |¦  «        ¦  «        �#|
                     t          |¦  «        g¦  «         g }|r|t          |¦  «        gz  }|r|t          d¬¦  «        gz  }|r|t          ¦   «         gz  }t          |¦  «        dk    r5t          |¦  «        dk    rt          |¦  «        |
_
        n|d         |
_
        |	rt          j        ¦   «         |
_        nt          j        ¦   «         |
_        t          j        |¬¦  «        |
_        d|||||||	d	œ}t%          ¦   «                              |
|¦  «         d S )
N)r   r   Úend_of_word_suffix)r   r   r$   F)r   r   r   )r   r   )Úmodelr   r   r   r   r    r!   r"   )r   r   ÚstrÚtoken_to_idÚadd_special_tokensr   r   r   Úlenr   Ú
normalizerr   ÚWhitespaceSplitÚpre_tokenizerÚBertPreTokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r"   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/tokenizers/implementations/char_level_bpe.pyr1   zCharBPETokenizer.__init__   sÖ  ø€ ð Ð Ð!3Ý!ÝØØØ#Ý! )™nœnØ'-ðñ ô ñô ˆIˆIõ "¥#µ°I±´ÈÐdjÐ"kÑ"kÔ"kÑlÔlˆIà× Ò ¥ Y¡¤Ñ0Ô0Ð<Ø×(Ò(­#¨i©.¬.Ð)9Ñ:Ô:Ð:ð ˆàð 	MØÕ7Ð8JÑKÔKÐLÑLˆKàð 	=Ø�N°UÐ;Ñ;Ô;Ð<Ñ<ˆKàð 	)Ø�I™KœK˜=Ñ(ˆKõ ˆ{ÑÔ˜aÒÐÝ�;ÑÔ !Ò#Ð#Ý'/°Ñ'<Ô'<�	Ô$Ð$à'2°1¤~�	Ô$à#ð 	HÝ&4Ô&DÑ&FÔ&FˆIÔ#Ð#å&4Ô&EÑ&GÔ&GˆIÔ#å$Ô/°vÐ>Ñ>Ô>ˆ	Ôð Ø"ØØØ"Ø"4Ø.Ø(@ð	
ð 	
ˆ
õ 	‰Œ×Ò˜ JÑ/Ô/Ð/Ð/Ð/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t          j        | |¦  «        \  }}t          ||fi |¤ŽS )N)r   Ú	read_filer   )r9   r:   Úkwargsr   r   s        r7   Ú	from_filezCharBPETokenizer.from_file\   s/   € åœ n°oÑFÔF‰ˆˆvÝ  vÐ8Ð8°Ð8Ð8Ð8r8   i0u  r	   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc	           	      ó¤   — t          j        |||||||¬¦  «        }	t          |t          ¦  «        r|g}| j                             ||	¬¦  «         dS )z%Train the model using the given files©r@   rA   rB   rC   rD   r$   rE   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer&   Ú
_tokenizerÚtrain)
r2   r?   r@   rA   rB   rC   rD   r   rE   rH   s
             r7   rL   zCharBPETokenizer.traina   sl   € õ Ô%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ð
ñ 
ô 
ˆõ �e�SÑ!Ô!ð 	Ø�GˆEØŒ×Ò˜e¨WÐÑ5Ô5Ð5Ð5Ð5r8   ÚiteratorÚlengthc
           	      óv   — t          j        |||||||¬¦  «        }
| j                             ||
|	¬¦  «         dS )z(Train the model using the given iteratorrG   )rH   rN   N)r   rI   rK   Útrain_from_iterator)r2   rM   r@   rA   rB   rC   rD   r   rE   rN   rH   s              r7   rP   z$CharBPETokenizer.train_from_iterator{   sd   € õ Ô%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ð
ñ 
ô 
ˆð 	Œ×+Ò+ØØØð 	,ñ 	
ô 	
ð 	
ð 	
ð 	
r8   )	NNr   r   NFNTF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r&   r   Úintr   r   r
   ÚfloatÚboolr1   Ústaticmethodr>   rL   r   rP   Ú__classcell__)r6   s   @r7   r   r   	   s¥  ø€ € € € € ðð ð" 7;Ø>BØ,3ØØ#'ØØ,0Ø $Ø).ðA0ð A0à˜˜c 4¨¨S¨¤>Ð1Ô2Ô3ðA0ð ˜˜s D¨¨s°C¨x¬Ô$9Ð9Ô:Ô;ðA0ð ˜˜j˜Ô)ð	A0ð
 ðA0ð ˜%”ðA0ð ðA0ð % SœMðA0ð ðA0ð #'ðA0ð A0ð A0ð A0ð A0ð A0ðF ð9 #ð 9¸ð 9ð 9ð 9ñ „\ð9ð  ØØ8?°yØ"Ø&(Ø &Ø"ð6ð 6à�S˜$˜sœ)�^Ô$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ô3Ô4ð6ð ð6ð ˜sœ)ð6ð ˜”ð6ð ð6ð 6ð 6ð 6ð:  ØØ8?°yØ"Ø&(Ø &Ø"Ø $ð
ð 
à˜ œ x°¸´Ô'>Ð>Ô?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ô3Ô4ð
ð ð
ð ˜sœ)ð
ð ˜”ð
ð ð
ð ˜”ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r8   r   N)Útypingr   r   r   r   r   r   Ú r
   r   r   r   r   Úmodelsr   r4   r   r   r   r   Úbase_tokenizerr   r   © r8   r7   ú<module>r_      så   ðØ ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HÐ HØ Ð Ð Ð Ð Ð Ø ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZØ )Ð )Ð )Ð )Ð )Ð )ðM
ð M
ð M
ð M
ð M
�}ñ M
ô M
ð M
ð M
ð M
r8   