§
    ‚Štjû  ã                   ó¨   — d Z ddlmZmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej        e¦  «        Zddd	d
œZddiZdZ G d„ de¦  «        ZdgZdS )zTokenization classes for Qwen2.é    )Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filezqwen/qwen-tokenizeri €  zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                   óˆ   ‡ — e Zd ZeZddgZeZ	 	 	 	 	 	 	 ddee	ee
f         z  dz  deee         z  dz  deded	ef
ˆ fd
„Zˆ xZS )ÚQwen2TokenizerÚ	input_idsÚattention_maskNú<|endoftext|>ÚvocabÚmergesÚ	unk_tokenÚ	eos_tokenÚ	pad_tokenc                 ój  •— |�|nd| _         |�|nddi| _        |pg | _        t          t	          | j        | j        d d dddd¬¦  «        ¦  «        | _        t          j        ¦   «         | j        _        t          j
        ¦   «         | j        _        t          j        t          j        t          t           ¦  «        dd¬¦  «        t          j        | j         d¬¦  «        g¦  «        | j        _         t%          ¦   «         j        d|||||d	œ|¤Ž |                      d
„ | j        D ¦   «         ¦  «         d S )NFr   r   Ú )r   r   Údropoutr   Úcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkÚbyte_fallbackÚisolated)ÚbehaviorÚinvert)Úadd_prefix_spaceÚ	use_regex)r   Ú	bos_tokenr   r   r$   c                 ó0   — g | ]}t          |d ¬¦  «        ‘ŒS )T)Úspecial)r   )Ú.0Útokens     új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/qwen2/tokenization_qwen2.pyú
<listcomp>z+Qwen2Tokenizer.__init__.<locals>.<listcomp>b   s%   € Ð^Ð^Ð^¸U� E°4Ð8Ñ8Ô8Ð^Ð^Ð^ó    © )r$   Ú_vocabÚ_mergesr   r	   Ú
_tokenizerr   Ú	ByteLevelÚdecoderr   ÚNFCÚ
normalizerr   ÚSequenceÚSplitr   ÚPRETOKENIZE_REGEXÚpre_tokenizerÚsuperÚ__init__Ú
add_tokensÚall_special_tokens)
Úselfr   r   r   r&   r   r   r$   ÚkwargsÚ	__class__s
            €r+   r;   zQwen2Tokenizer.__init__)   s~  ø€ ð 5EÐ4PÐ 0Ð 0ÐV[ˆÔð Ð ð ˆEð   ðð 	Œð �| ˆŒÝ#ÝØ”kØ”|ØØØ*,Ø#%ØØ#ð	ñ 	ô 	ñ
ô 
ˆŒõ #+Ô"4Ñ"6Ô"6ˆŒÔÝ%0¤_Ñ%6Ô%6ˆŒÔ"Ý(6Ô(?åÔ$ÝÕ+Ñ,Ô,Ø'Ø ðñ ô õ
 Ô(Ø%)Ô%:Ø#ðñ ô ð
ñ)
ô )
ˆŒÔ%ð 	�‰ŒÔð 	
ØØØØØ-ð	
ð 	
ð ð	
ð 	
ð 	
ð 	�ŠÐ^Ð^ÀdÔF]Ð^Ñ^Ô^Ñ_Ô_Ð_Ð_Ð_r-   )NNr   Nr   r   N)Ú__name__Ú
__module__Ú__qualname__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   ÚmodelÚstrÚdictÚintÚlistr;   Ú__classcell__)r@   s   @r+   r   r   $   sÏ   ø€ € € € € Ø)ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø(ØØ(Ø(Øð9`ð 9`à�T˜#˜s˜(”^Ñ# dÑ*ð9`ð �d˜3”i‘ $Ñ&ð9`ð ð	9`ð ð9`ð ð9`ð 9`ð 9`ð 9`ð 9`ð 9`ð 9`ð 9`ð 9`ð 9`r-   r   N)Ú__doc__Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrA   ÚloggerrD   ÚMAX_MODEL_INPUT_SIZESr8   r   Ú__all__r.   r-   r+   ú<module>rV      s÷   ðð &Ð %à ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZØ !Ð !Ð !Ð !Ð !Ð !à >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð ØØ&ðð Ð ð /°Ð6Ð ð JÐ ð>`ð >`ð >`ð >`ð >`Ð&ñ >`ô >`ð >`ðB Ð
€€€r-   