§
    ~Štj‰  ã                   ó‚   — d dl mZmZmZmZmZmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlmZ ddlmZ  G d„ de¦  «        Zd	S )
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚNFKCé   )ÚBaseTokenizerc                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 d deeeeeef         f                  deeee	e
eef                  f                  d	eeef         d
ededee         dee         fˆ fd„Zededefd„¦   «         Zdddgdg dfdeee	e         f         dedede	eeef                  dede	e         defd„Zdddgdg ddfdeee         eee                  f         dedede	eeef                  dede	e         dedee         fd„Zˆ xZS )!ÚSentencePieceBPETokenizerzrSentencePiece BPE Tokenizer

    Represents the BPE algorithm, with the pretokenization used by SentencePiece
    Nú<unk>õ   â–�TFÚvocabÚmergesÚ	unk_tokenÚreplacementÚadd_prefix_spaceÚdropoutÚfuse_unkc           	      ó  •— |�$|�"t          t          |||||¬¦  «        ¦  «        }nt          t          |||¬¦  «        ¦  «        }|                     t          |¦  «        ¦  «        �#|                     t          |¦  «        g¦  «         t          ¦   «         |_        |rdnd}	t          j        ||	¬¦  «        |_	        t          j        ||	¬¦  «        |_        d||||dœ}
t          ¦   «                              ||
¦  «         d S )N)r   r   r   ÚalwaysÚnever)r   Úprepend_schemeÚSentencePieceBPE)Úmodelr   r   r   r   )r
   r   Útoken_to_idÚstrÚadd_special_tokensr   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   Ú	tokenizerr    Ú
parametersÚ	__class__s              €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/tokenizers/implementations/sentencepiece_bpe.pyr+   z"SentencePieceBPETokenizer.__init__   s  ø€ ð Ð Ð!3Ý!¥# e¨V¸WÐPYÐdlÐ"mÑ"mÔ"mÑnÔnˆIˆIå!¥#¨gÀÐU]Ð"^Ñ"^Ô"^Ñ_Ô_ˆIà× Ò ¥ Y¡¤Ñ0Ô0Ð<Ø×(Ò(­#¨i©.¬.Ð)9Ñ:Ô:Ð:å#™vœvˆ	ÔØ%5ÐB˜˜¸7ˆÝ"0Ô":À{ÐcqÐ"rÑ"rÔ"rˆ	ÔÝ$Ô.¸;ÐWeÐfÑfÔfˆ	Ôð (Ø"Ø&Ø 0Øð
ð 
ˆ
õ 	‰Œ×Ò˜ JÑ/Ô/Ð/Ð/Ð/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t          j        | |¦  «        \  }}t          ||fi |¤ŽS )N)r   Ú	read_filer   )r2   r3   Úkwargsr   r   s        r0   Ú	from_filez#SentencePieceBPETokenizer.from_file1   s/   € åœ n°oÑFÔF‰ˆˆvÝ(¨°ÐAÐA¸&ÐAÐAÐAr1   i0u  é   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc                 ó¢   — t          j        ||||||¬¦  «        }t          |t          ¦  «        r|g}| j                             ||¬¦  «         dS )z%Train the model using the given files©r:   r;   r<   r=   r>   r?   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer$   Ú
_tokenizerÚtrain)	r,   r9   r:   r;   r<   r=   r>   r?   rB   s	            r0   rF   zSentencePieceBPETokenizer.train6   si   € õ Ô%Ø!Ø'Ø)Ø)Ø-Ø'ð
ñ 
ô 
ˆõ �e�SÑ!Ô!ð 	Ø�GˆEØŒ×Ò˜e¨WÐÑ5Ô5Ð5Ð5Ð5r1   ÚiteratorÚlengthc	                 ót   — t          j        ||||||¬¦  «        }	| j                             ||	|¬¦  «         dS )z(Train the model using the given iteratorrA   )rB   rH   N)r   rC   rE   Útrain_from_iterator)
r,   rG   r:   r;   r<   r=   r>   r?   rH   rB   s
             r0   rJ   z-SentencePieceBPETokenizer.train_from_iteratorN   sa   € õ Ô%Ø!Ø'Ø)Ø)Ø-Ø'ð
ñ 
ô 
ˆð 	Œ×+Ò+ØØØð 	,ñ 	
ô 	
ð 	
ð 	
ð 	
r1   )NNr   r   TNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r$   r   Úintr   r   r	   ÚboolÚfloatr+   Ústaticmethodr7   rF   r   rJ   Ú__classcell__)r/   s   @r0   r   r   
   s_  ø€ € € € € ðð ð 7;Ø>BØ,3Ø Ø!%Ø#'Ø#(ð0ð 0à˜˜c 4¨¨S¨¤>Ð1Ô2Ô3ð0ð ˜˜s D¨¨s°C¨x¬Ô$9Ð9Ô:Ô;ð0ð ˜˜j˜Ô)ð	0ð
 ð0ð ð0ð ˜%”ð0ð ˜4”.ð0ð 0ð 0ð 0ð 0ð 0ðB ðB #ð B¸ð Bð Bð Bñ „\ðBð  ØØ8?°yØ"Ø&(Ø"ð6ð 6à�S˜$˜sœ)�^Ô$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ô3Ô4ð6ð ð6ð ˜sœ)ð6ð ð6ð 6ð 6ð 6ð6  ØØ8?°yØ"Ø&(Ø"Ø $ð
ð 
à˜ œ x°¸´Ô'>Ð>Ô?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ô3Ô4ð
ð ð
ð ˜sœ)ð
ð ð
ð ˜”ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r1   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   Úbase_tokenizerr   r   © r1   r0   ú<module>rZ      sÓ   ðØ ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PÐ PØ !Ð !Ð !Ð !Ð !Ð !Ø 'Ð 'Ð 'Ð 'Ð 'Ð 'à )Ð )Ð )Ð )Ð )Ð )ð]
ð ]
ð ]
ð ]
ð ]
 ñ ]
ô ]
ð ]
ð ]
ð ]
r1   