§
    ~Štj°  ã                   óŽ   — d dl mZmZmZmZmZmZ d dlmZm	Z	m
Z
mZmZmZ d dlmZ d dlmZmZmZ ddlmZ  G d„ de¦  «        Zd	S )
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processorsÚtrainers)ÚBPE)Ú	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   óö  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddeeeeeef         f                  deeee	e
eef                  f                  dededee         d	ee         d
ee         dee         defˆ fd„Zededefd„¦   «         Zdddg fdeee	e         f         dededede	eeef                  f
d„Zdddg dfdeee         eee                  f         dededede	eeef                  dee         fd„Zˆ xZS )ÚByteLevelBPETokenizerzjByteLevelBPETokenizer

    Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
    NFÚvocabÚmergesÚadd_prefix_spaceÚ	lowercaseÚdropoutÚunicode_normalizerÚcontinuing_subword_prefixÚend_of_word_suffixÚtrim_offsetsc
           	      ój  •— |�(|�&t          t          ||||pd|pd¬¦  «        ¦  «        }
nt          t          ¦   «         ¦  «        }
g }|r|t          |¦  «        gz  }|r|t          ¦   «         gz  }t	          |¦  «        dk    r5t	          |¦  «        dk    rt          |¦  «        |
_        n|d         |
_        t          j        |¬¦  «        |
_	        t          j        ¦   «         |
_        t          j        |	¬¦  «        |
_        d|||||||	dœ}t          ¦   «                              |
|¦  «         d S )	NÚ )r   r   r   r   r   )r   )r   ÚByteLevelBPE)Úmodelr   r   r   r   r   r   r   )r
   r   r   r   Úlenr   Ú
normalizerr   Ú	ByteLevelÚpre_tokenizerr   Údecoderr   Úpost_processorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €úg/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/tokenizers/implementations/byte_level_bpe.pyr+   zByteLevelBPETokenizer.__init__   sk  ø€ ð Ð Ð!3Ý!ÝØØØ#Ø.GÐ.MÈ2Ø'9Ð'?¸Rðñ ô ñô ˆIˆIõ "¥#¡%¤%Ñ(Ô(ˆIð ˆàð 	MØÕ7Ð8JÑKÔKÐLÑLˆKàð 	)Ø�I™KœK˜=Ñ(ˆKõ ˆ{ÑÔ˜aÒÐÝ�;ÑÔ !Ò#Ð#Ý'/°Ñ'<Ô'<�	Ô$Ð$à'2°1¤~�	Ô$å"0Ô":ÐL\Ð"]Ñ"]Ô"]ˆ	ÔÝ$Ô.Ñ0Ô0ˆ	ÔÝ#-Ô#7À\Ð#RÑ#RÔ#Rˆ	Ô ð $Ø 0Ø"ØØ"4Ø)BØ"4Ø(ð	
ð 	
ˆ
õ 	‰Œ×Ò˜ JÑ/Ô/Ð/Ð/Ð/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t          j        | |¦  «        \  }}t          ||fi |¤ŽS )N)r   Ú	read_filer   )r3   r4   Úkwargsr   r   s        r1   Ú	from_filezByteLevelBPETokenizer.from_fileJ   s/   € åœ n°oÑFÔF‰ˆˆvÝ$ U¨FÐ=Ð=°fÐ=Ð=Ð=r2   i0u  é   TÚfilesÚ
vocab_sizeÚmin_frequencyÚshow_progressÚspecial_tokensc                 óØ   — t          j        ||||t          j                             ¦   «         ¬¦  «        }t          |t          ¦  «        r|g}| j                             ||¬¦  «         dS )z%Train the model using the given files©r;   r<   r=   r>   Úinitial_alphabet)ÚtrainerN)	r   Ú
BpeTrainerr   r&   ÚalphabetÚ
isinstanceÚstrÚ
_tokenizerÚtrain)r,   r:   r;   r<   r=   r>   rB   s          r1   rH   zByteLevelBPETokenizer.trainO   su   € õ Ô%Ø!Ø'Ø'Ø)Ý+Ô5×>Ò>Ñ@Ô@ð
ñ 
ô 
ˆõ �e�SÑ!Ô!ð 	Ø�GˆEØŒ×Ò˜e¨WÐÑ5Ô5Ð5Ð5Ð5r2   ÚiteratorÚlengthc                 óª   — t          j        ||||t          j                             ¦   «         ¬¦  «        }| j                             |||¬¦  «         dS )z(Train the model using the given iteratorr@   )rB   rJ   N)r   rC   r   r&   rD   rG   Útrain_from_iterator)r,   rI   r;   r<   r=   r>   rJ   rB   s           r1   rL   z)ByteLevelBPETokenizer.train_from_iteratord   sm   € õ Ô%Ø!Ø'Ø'Ø)Ý+Ô5×>Ò>Ñ@Ô@ð
ñ 
ô 
ˆð 	Œ×+Ò+ØØØð 	,ñ 	
ô 	
ð 	
ð 	
ð 	
r2   )	NNFFNNNNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rF   r   Úintr   r   ÚboolÚfloatr+   Ústaticmethodr8   r	   rH   r   rL   Ú__classcell__)r0   s   @r1   r   r   
   s2  ø€ € € € € ðð ð 7;Ø>BØ!&ØØ#'Ø,0Ø37Ø,0Ø"ð80ð 80à˜˜c 4¨¨S¨¤>Ð1Ô2Ô3ð80ð ˜˜s D¨¨s°C¨x¬Ô$9Ð9Ô:Ô;ð80ð ð	80ð
 ð80ð ˜%”ð80ð % SœMð80ð $,¨C¤=ð80ð % SœMð80ð ð80ð 80ð 80ð 80ð 80ð 80ðt ð> #ð >¸ð >ð >ð >ñ „\ð>ð  ØØ"Ø79ð6ð 6à�S˜$˜sœ)�^Ô$ð6ð ð6ð ð	6ð
 ð6ð ˜U 3¨
 ?Ô3Ô4ð6ð 6ð 6ð 6ð0  ØØ"Ø79Ø $ð
ð 
à˜ œ x°¸´Ô'>Ð>Ô?ð
ð ð
ð ð	
ð
 ð
ð ˜U 3¨
 ?Ô3Ô4ð
ð ˜”ð
ð 
ð 
ð 
ð 
ð 
ð 
ð 
r2   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   r   r   Úbase_tokenizerr   r   © r2   r1   ú<module>r\      så   ðØ ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?Ð ?à \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ø !Ð !Ð !Ð !Ð !Ð !Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ Sà )Ð )Ð )Ð )Ð )Ð )ðp
ð p
ð p
ð p
ð p
˜Mñ p
ô p
ð p
ð p
ð p
r2   