§
    ‚Štj9  ã                   óÈ   — d Z ddlZddlZddlmZmZ  e¦   «         rddlZddlmZ ddl	m
Z
  e
j        e¦  «        ZddiZd	„ Z G d
„ d¦  «        Z G d„ de¦  «        ZdgZdS )z Tokenization classes for CPMAnt.é    N)Úis_rjieba_availableÚrequires_backendsé   )ÚPreTrainedTokenizer)ÚloggingÚ
vocab_filez	vocab.txtc                 ó  — t          j        ¦   «         }t          | dd¬¦  «        5 }|                     ¦   «         }ddd¦  «         n# 1 swxY w Y   t	          |¦  «        D ]\  }}|                     d¦  «        }|||<   Œ |S )z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingNú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens         úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/cpmant/tokenization_cpmant.pyÚ
load_vocabr   "   sÄ   € åÔ#Ñ%Ô%€EÝ	ˆj˜#¨Ð	0Ñ	0Ô	0ð $°FØ×!Ò!Ñ#Ô#ˆð$ð $ð $ñ $ô $ð $ð $ð $ð $ð $ð $øøøð $ð $ð $ð $å! &Ñ)Ô)ð ð ‰ˆˆuØ—’˜TÑ"Ô"ˆØˆˆe‰ˆØ€Ls   ¦AÁAÁAc                   ó   — e Zd Zdd„Zd„ ZdS )ÚWordpieceTokenizerú<unk>éÈ   c                 ó0   — || _         || _        || _        d S ©N)r   Ú	unk_tokenÚmax_input_chars_per_word)Úselfr   r"   r#   s       r   Ú__init__zWordpieceTokenizer.__init__.   s   € ØˆŒ
Ø"ˆŒØ(@ˆÔ%Ð%Ð%ó    c                 óÀ  — t          |¦  «        }t          |¦  «        | j        k    r| j        gS d}g }|t          |¦  «        k     r—t          |¦  «        }d }||k     r4d                     |||…         ¦  «        }|| j        v r|}n|dz  }||k     °4|€ |                     | j        ¦  «         |dz  }n|                     |¦  «         |}|t          |¦  «        k     °—|S )Nr   Ú é   )ÚlistÚlenr#   r"   Újoinr   Úappend)r$   r   ÚcharsÚstartÚ
sub_tokensÚendÚ
cur_substrÚsubstrs           r   ÚtokenizezWordpieceTokenizer.tokenize3   s   € Ý�U‘”ˆÝˆu‰:Œ:˜Ô5Ò5Ð5Ø”NÐ#Ð#àˆØˆ
Ø•c˜%‘j”jÒ Ð Ý�e‘*”*ˆCØˆJØ˜#’+�+ØŸš  u¨S yÔ!1Ñ2Ô2�Ø˜TœZÐ'Ð'Ø!'�JØØ�q‘�ð ˜#’+�+ð Ð!Ø×!Ò! $¤.Ñ1Ô1Ð1Ø˜‘
��à×!Ò! *Ñ-Ô-Ð-Ø�ð •c˜%‘j”jÒ Ð ð  Ðr&   N)r   r   )Ú__name__Ú
__module__Ú__qualname__r%   r4   © r&   r   r   r   -   s;   € € € € € ðAð Að Að Að
ð ð ð ð r&   r   c                   ó  ‡ — e Zd ZdZeZddgZdZ	 	 	 	 	 	 	 	 	 d ˆ fd„	Ze	d„ ¦   «         Z
e	d„ ¦   «         Ze	d„ ¦   «         Ze	defd„¦   «         Zd„ Zd„ Zˆ fd„Zd„ Zdee         defd„Zd„ Zd„ Zd!dededz  dee         fd„Zˆ xZS )"ÚCpmAntTokenizera²  
    Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        bod_token (`str`, *optional*, defaults to `"<d>"`):
            The beginning of document token.
        eod_token (`str`, *optional*, defaults to `"</d>"`):
            The end of document token.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        line_token (`str`, *optional*, defaults to `"</n>"`):
            The line token.
        space_token (`str`, *optional*, defaults to `"</_>"`):
            The space token.
    Ú	input_idsÚattention_maskFú<d>ú</d>ú<s>ú</s>ú<pad>r   ú</n>ú</_>Úleftc                 óÚ  •— t          | dg¦  «         || _        || _        t          |¦  «        | _        | j        |	         | j        d<   | j        |         | j        d<   | j        |	= | j        |= t          j        t          | j                             ¦   «         d„ ¬¦  «        ¦  «        | _        d„ | j                             ¦   «         D ¦   «         | _	        t          | j        |¬¦  «        | _         t          ¦   «         j        d||||||||	|
dd	d
dœ|¤Ž |	|fD ]:}| j                             |d ¦  «        }|�| j                             |d ¦  «         Œ;|                      ¦   «          d S )NÚrjiebaú r   c                 ó   — | d         S ©Nr)   r8   ©Úxs    r   ú<lambda>z*CpmAntTokenizer.__init__.<locals>.<lambda>‚   ó   € ÐZ[Ð\]ÔZ^€ r&   ©Úkeyc                 ó   — i | ]\  }}||“Œ	S r8   r8   )Ú.0ÚkÚvs      r   ú
<dictcomp>z,CpmAntTokenizer.__init__.<locals>.<dictcomp>ƒ   s   € Ð>Ð>Ð>¡  A˜˜1Ð>Ð>Ð>r&   )r   r"   Ú	all_zerosTÚbos)Ú	bod_tokenÚ	eod_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenr"   Ú
line_tokenÚspace_tokenÚpadding_sideÚtoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_patternr8   )r   rW   rX   r   Úencoderr   r   ÚsortedÚitemsÚdecoderr   Úwordpiece_tokenizerÚsuperr%   Úadded_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_total_vocab_size)r$   r   rW   rX   rY   rZ   r[   r"   r\   r]   r^   ÚkwargsÚspecial_tokenÚtoken_idÚ	__class__s                 €r   r%   zCpmAntTokenizer.__init__j   s“  ø€ õ 	˜$  
Ñ+Ô+Ð+Ø"ˆŒØ"ˆŒÝ! *Ñ-Ô-ˆŒØ œL¨Ô5ˆŒ�SÑØ!œ\¨*Ô5ˆŒ�TÑàŒL˜Ð%ØŒL˜Ð$å"Ô.­v°d´l×6HÒ6HÑ6JÔ6JÐP^ÐP^Ð/_Ñ/_Ô/_Ñ`Ô`ˆŒØ>Ð>¨¬×);Ò);Ñ)=Ô)=Ð>Ñ>Ô>ˆŒå#5¸D¼LÐT]Ð#^Ñ#^Ô#^ˆÔ à�‰ŒÔð 	
ØØØØØØØ!Ø#Ø%Ø#.Ø26Ø#(ð	
ð 	
ð ð	
ð 	
ð 	
ð *¨:Ð6ð 	?ð 	?ˆMØÔ0×4Ò4°]ÀDÑIÔIˆHØÐ#ØÔ*×.Ò.¨x¸Ñ>Ô>Ð>øØ×%Ò%Ñ'Ô'Ð'Ð'Ð'r&   c                 ó&   — | j         | j                 S r!   )rb   rW   ©r$   s    r   Úbod_token_idzCpmAntTokenizer.bod_token_idœ   ó   € àŒ|˜DœNÔ+Ð+r&   c                 ó&   — | j         | j                 S r!   )rb   rX   rq   s    r   Úeod_token_idzCpmAntTokenizer.eod_token_id    rs   r&   c                 ó   — | j         d         S )Nr   ©rb   rq   s    r   Ú
newline_idzCpmAntTokenizer.newline_id¤   s   € àŒ|˜DÔ!Ð!r&   Úreturnc                 ó*   — t          | j        ¦  «        S r!   )r+   rb   rq   s    r   Ú
vocab_sizezCpmAntTokenizer.vocab_size¨   s   € å�4”<Ñ Ô Ð r&   c                 ó0   — t          | j        fi | j        ¤ŽS r!   )Údictrb   rh   rq   s    r   Ú	get_vocabzCpmAntTokenizer.get_vocab¬   s   € Ý�D”LÐ>Ð> DÔ$=Ð>Ð>Ð>r&   c                 ó”   — g }t          j        |d¦  «        D ]/}|                     | j                             |¦  «        ¦  «         Œ0|S )zTokenize a string.F)rF   ÚcutÚextendrf   r4   )r$   ÚtextÚoutput_tokensrK   s       r   Ú	_tokenizezCpmAntTokenizer._tokenize¯   sR   € àˆÝ”˜D %Ñ(Ô(ð 	Gð 	GˆAØ× Ò  Ô!9×!BÒ!BÀ1Ñ!EÔ!EÑFÔFÐFÐFØÐr&   c                 ón   •‡ — d„ |D ¦   «         }ˆ fd„|D ¦   «         } t          ¦   «         j        |fi |¤ŽS )zDecode ids into a string.c                 ó   — g | ]
}|d k    ¯|‘ŒS )r   r8   )rQ   Úis     r   ú
<listcomp>z+CpmAntTokenizer._decode.<locals>.<listcomp>¸   s   € Ð4Ð4Ð4˜1¨Q°!ªV¨V�Q¨V¨V¨Vr&   c                 óV   •— g | ]%}|‰j         k    ¯|‰j        k    ¯|‰j        k    ¯#|‘Œ&S r8   )Úpad_token_idÚeos_token_idÚbos_token_id)rQ   rK   r$   s     €r   rˆ   z+CpmAntTokenizer._decode.<locals>.<listcomp>¹   sI   ø€ ð 
ð 
ð 
Ø A¨Ô):Ò$:Ð$:¸qÀDÔDUÒ?UÐ?UÐZ[Ð_cÔ_pÒZpÐZpˆAÐZpÐZpÐZpr&   )rg   Ú_decode)r$   Ú	token_idsrl   ro   s   `  €r   r�   zCpmAntTokenizer._decode¶   s_   øø€ à4Ð4 	Ð4Ñ4Ô4ˆ	ð
ð 
ð 
ð 
Ø ð
ñ 
ô 
ˆ	ð �u‰wŒwŒ˜yÐ3Ð3¨FÐ3Ð3Ð3r&   c                 ó   — || j         v S r!   rw   ©r$   r   s     r   ÚcheckzCpmAntTokenizer.check¾   s   € Ø˜œÐ$Ð$r&   r   c                 ó,   — d                      |¦  «        S )Nr(   )r,   )r$   r   s     r   Úconvert_tokens_to_stringz(CpmAntTokenizer.convert_tokens_to_stringÁ   s   € Ø�wŠw�v‰ŒÐr&   c                 ór   — | j                              || j                              | j        ¦  «        ¦  «        S )z0Converts a token (str) in an id using the vocab.)rb   Úgetr"   r�   s     r   Ú_convert_token_to_idz$CpmAntTokenizer._convert_token_to_idÄ   s,   € àŒ|×Ò  t¤|×'7Ò'7¸¼Ñ'GÔ'GÑHÔHÐHr&   c                 óB   — | j                              || j        ¦  «        S )z=Converts an index (integer) in a token (str) using the vocab.)re   r•   r"   )r$   r   s     r   Ú_convert_id_to_tokenz$CpmAntTokenizer._convert_id_to_tokenÈ   s   € àŒ|×Ò  t¤~Ñ6Ô6Ð6r&   NÚsave_directoryÚfilename_prefixc                 ó   — t           j                             |¦  «        r6t           j                             ||r|dz   ndt          d         z   ¦  «        }n|r|dz   nd|z   }d}d| j        v r| j        d         | j        d<   | j        d= d| j        v r| j        d         | j        d<   | j        d= t          j        t          | j         	                    ¦   «         d	„ ¬
¦  «        ¦  «        | _        t          |dd¬¦  «        5 }| j         	                    ¦   «         D ]H\  }}||k    r t                               d|› d�¦  «         |}|                     |dz   ¦  «         |dz  }ŒI	 d d d ¦  «         n# 1 swxY w Y   |fS )Nú-r(   r   r   rG   rC   r   rB   c                 ó   — | d         S rI   r8   rJ   s    r   rL   z1CpmAntTokenizer.save_vocabulary.<locals>.<lambda>Ú   rM   r&   rN   Úwr   r   zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r)   )ÚosÚpathÚisdirr,   ÚVOCAB_FILES_NAMESrb   r   r   rc   rd   r   ÚloggerÚwarningÚwrite)r$   r™   rš   r   r   Úwriterr   Útoken_indexs           r   Úsave_vocabularyzCpmAntTokenizer.save_vocabularyÌ   sñ  € ÝŒ7�=Š=˜Ñ(Ô(ð 	]ÝœŸšØ¸/Ð!Q °3Ñ!6Ð!6ÈrÕUfÐgsÔUtÑ tñô ˆJˆJð 4CÐJ˜/¨CÑ/Ð/ÈÈnÑ\ˆJØˆØ�$”,ÐÐØ#'¤<°Ô#4ˆDŒL˜Ñ Ø”˜SÐ!Ø�4”<ÐÐØ#'¤<°Ô#5ˆDŒL˜Ñ Ø”˜TÐ"Ý"Ô.­v°d´l×6HÒ6HÑ6JÔ6JÐP^ÐP^Ð/_Ñ/_Ô/_Ñ`Ô`ˆŒÝ�*˜c¨GÐ4Ñ4Ô4ð 		¸Ø&*¤l×&8Ò&8Ñ&:Ô&:ð ð Ñ"��{Ø˜KÒ'Ð'Ý—N’NðN°
ð Nð Nð Nñô ð ð (�EØ—’˜U T™\Ñ*Ô*Ð*Ø˜‘
��ðð		ð 		ð 		ñ 		ô 		ð 		ð 		ð 		ð 		ð 		ð 		øøøð 		ð 		ð 		ð 		ð ˆ}Ðs   ÄA#E2Å2E6Å9E6)	r=   r>   r?   r@   rA   r   rB   rC   rD   r!   )r5   r6   r7   Ú__doc__r¢   Úvocab_files_namesÚmodel_input_namesÚadd_prefix_spacer%   Úpropertyrr   ru   rx   Úintr{   r~   r„   r�   r‘   r*   Ústrr“   r–   r˜   Útupler¨   Ú__classcell__)ro   s   @r   r:   r:   M   s½  ø€ € € € € ðð ð0 *ÐØ$Ð&6Ð7ÐØÐð
 ØØØØØØØØð0(ð 0(ð 0(ð 0(ð 0(ð 0(ðd ð,ð ,ñ „Xð,ð ð,ð ,ñ „Xð,ð ð"ð "ñ „Xð"ð ð!˜Cð !ð !ð !ñ „Xð!ð?ð ?ð ?ðð ð ð4ð 4ð 4ð 4ð 4ð%ð %ð %ð¨t°C¬yð ¸Sð ð ð ð ðIð Ið Ið7ð 7ð 7ðð ¨cð ÀCÈ$ÁJð ÐZ_Ð`cÔZdð ð ð ð ð ð ð ð r&   r:   )r©   r   rŸ   Útransformers.utilsr   r   rF   Útokenization_pythonr   Úutilsr   Ú
get_loggerr5   r£   r¢   r   r   r:   Ú__all__r8   r&   r   ú<module>r·      s  ðð 'Ð &à Ð Ð Ð Ø 	€	€	€	à EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð ÐÑÔð Ø€M€M€Mà 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€à! ;Ð/Ð ðð ð ðð ð ð ð ñ ô ð ð@Xð Xð Xð Xð XÐ)ñ Xô Xð Xðv Ð
€€€r&   