§
    ‚ŠtjÇ  ã                   óÐ  — d Z ddlZddlZddlmZ ddlmZ  ej        e	¦  «        Z
dddœZi d	d
“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd “d!d"“d#d$“d%d&“d'd(“d)d*“i d+d,“d-d.“d/d0“d1d2“d3d4“d5d6“d7d8“d9d:“d;d<“d=d>“d?d@“dAdB“dCdD“dEdF“dGdH“dIdJ“dKdL“¥i dMdN“dOdP“dQdR“dSdT“dUdV“dWdX“dYdZ“d[d\“d]d^“d_d`“dadb“dcdd“dedf“dgdh“didj“dkdl“dmdn“¥dodpdqdrdsœ¥Zdt„ Z G du„ dve¦  «        ZdvgZdS )wz)Tokenization classes for Salesforce CTRL.é    Né   )ÚPreTrainedTokenizer)Úloggingz
vocab.jsonz
merges.txt)Ú
vocab_fileÚmerges_fileÚ	Pregnancyiµ’ ÚChristianityiû  ÚExplaini·Ÿ ÚFitnessiÐ÷  ÚSavingi»ö  ÚAski#j  ÚAssiñv ÚJokeiµ~ Ú	Questionsi6²  ÚThoughtsiÅÁ  ÚRetailivÌ  ÚFeminismiò� ÚWritingiØ.  ÚAtheismiï ÚNetflixiè½  Ú	Computingi×š  ÚOpinioniÍ¨  ÚAlonei§¯  ÚFunnyi%æ  ÚGamingi¦�  ÚHumaniø  ÚIndiai3  ÚJokeriR- ÚDietin�  ÚLegaliS.  ÚNormaniK  ÚTipiñ ÚWeightiwÌ  ÚMoviesiÁ´  ÚRunningi�[  ÚSciencei*  ÚHorrori¡“  Ú
Confessioniœì  ÚFinanceiÚ/  ÚPoliticsiè?  ÚScaryiñí ÚSupportin1  ÚTechnologiesi  ÚTeenageip ÚEventi€  ÚLearnedi„ ÚNotioniòÉ Ú	WikipediaiÏ’  ÚBooksi	  ÚExtracti) ÚConfessionsi-‘ Ú
Conspiracyiœ( ÚLinksiºø  Ú	Narcissusi™K ÚRelationshipiîÕ  ÚRelationshipsiŒ iÇ¢  i   iÄh  i–õ )ÚReviewsÚNewsÚTranslationÚmultilingualc                 óœ   — t          ¦   «         }| d         }| dd…         D ]}|                     ||f¦  «         |}Œt          |¦  «        }|S )z…
    Return set of symbol pairs in a word.

    Word is represented as tuple of symbols (symbols being variable-length strings).
    r   é   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/ctrl/tokenization_ctrl.pyÚ	get_pairsrH   [   s[   € õ ‰EŒE€EØ�Q”€IØ�Q�R�R”ð ð ˆØ�	Š	�9˜dÐ#Ñ$Ô$Ð$Øˆ	ˆ	å�‰JŒJ€EØ€Ló    c                   óf   ‡ — e Zd ZdZeZeZdˆ fd„	Ze	d„ ¦   «         Z
d„ Zd„ Zd„ Zd„ Zd	„ Zd
„ Zˆ xZS )ÚCTRLTokenizera`  
    Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    ú<unk>c           
      ót  •— t          |d¬¦  «        5 }t          j        |¦  «        | _        d d d ¦  «         n# 1 swxY w Y   d„ | j                             ¦   «         D ¦   «         | _        t          |d¬¦  «        5 }|                     ¦   «                              d¦  «        dd…         }d d d ¦  «         n# 1 swxY w Y   d„ |D ¦   «         }t          t          |t          t          |¦  «        ¦  «        ¦  «        ¦  «        | _        i | _        d| _         t          ¦   «         j        d|d	dd
dœ|¤Ž d S )Nzutf-8)Úencodingc                 ó   — i | ]\  }}||“Œ	S © rP   )Ú.0ÚkÚvs      rG   ú
<dictcomp>z*CTRLTokenizer.__init__.<locals>.<dictcomp>‚   s   € Ð>Ð>Ð>¡  A˜˜1Ð>Ð>Ð>rI   ú
r@   éÿÿÿÿc                 óP   — g | ]#}t          |                     ¦   «         ¦  «        ‘Œ$S rP   )ÚtupleÚsplit)rQ   Úmerges     rG   ú
<listcomp>z*CTRLTokenizer.__init__.<locals>.<listcomp>…   s(   € Ð;Ð;Ð;¨5•%˜Ÿš™œÑ&Ô&Ð;Ð;Ð;rI   TÚ	all_zerosÚnone)Ú	unk_tokenÚtoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_patternrP   )ÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadrY   ÚdictÚzipÚrangeÚlenÚ	bpe_ranksÚcacheÚadd_bpe_version_headerÚsuperÚ__init__)	Úselfr   r   r^   ÚkwargsÚvocab_handleÚmerges_handleÚmergesÚ	__class__s	           €rG   rq   zCTRLTokenizer.__init__   s·  ø€ Ý�* wÐ/Ñ/Ô/ð 	3°<Ýœ9 \Ñ2Ô2ˆDŒLð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3à>Ð>¨¬×);Ò);Ñ)=Ô)=Ð>Ñ>Ô>ˆŒÝ�+¨Ð0Ñ0Ô0ð 	<°MØ"×'Ò'Ñ)Ô)×/Ò/°Ñ5Ô5°a¸°dÔ;ˆFð	<ð 	<ð 	<ñ 	<ô 	<ð 	<ð 	<ð 	<ð 	<ð 	<ð 	<øøøð 	<ð 	<ð 	<ð 	<à;Ð;°FÐ;Ñ;Ô;ˆÝ�c &­%µ°F±´Ñ*<Ô*<Ñ=Ô=Ñ>Ô>ˆŒØˆŒ
Ø&*ˆÔ#Ø�‰ŒÔð 	
ØØ#.Ø26Ø#)ð		
ð 	
ð
 ð	
ð 	
ð 	
ð 	
ð 	
s   “9¹=Á =Á=0B9Â9B=Ã B=c                 ó*   — t          | j        ¦  «        S ©N)rl   re   ©rr   s    rG   Ú
vocab_sizezCTRLTokenizer.vocab_size‘   s   € å�4”<Ñ Ô Ð rI   c                 ó0   — t          | j        fi | j        ¤ŽS ry   )ri   re   Úadded_tokens_encoderrz   s    rG   Ú	get_vocabzCTRLTokenizer.get_vocab•   s   € Ý�D”LÐ>Ð> DÔ$=Ð>Ð>Ð>rI   c                 óÜ  ‡ — |‰ j         v r‰ j         |         S t          |¦  «        }t          t          |d d…         ¦  «        |d         dz   gz   ¦  «        }t          |¦  «        }|s|S 	 t	          |ˆ fd„¬¦  «        }|‰ j        vr�n8|\  }}g }d}|t          |¦  «        k     ræ	 |                     ||¦  «        }	|                     |||	…         ¦  «         |	}n-# t          $ r  |                     ||d …         ¦  «         Y n†w xY w||         |k    rC|t          |¦  «        dz
  k     r-||dz            |k    r| 
                    ||z   ¦  «         |dz  }n | 
                    ||         ¦  «         |dz  }|t          |¦  «        k     °æt          |¦  «        }|}t          |¦  «        dk    rnt          |¦  «        }�ŒWd	                     |¦  «        }|d d
…         }|‰ j         |<   |S )NrV   z</w>Tc                 óT   •— ‰j                              | t          d¦  «        ¦  «        S )NÚinf)rm   ÚgetÚfloat)Úpairrr   s    €rG   ú<lambda>z#CTRLTokenizer.bpe.<locals>.<lambda>£   s    ø€ °´×1CÒ1CÀDÍ%ÐPUÉ,Ì,Ñ1WÔ1W€ rI   )Úkeyr   r@   é   ú@@ éüÿÿÿ)rn   rX   ÚlistrH   Úminrm   rl   ÚindexÚextendÚ
ValueErrorÚappendÚjoin)
rr   ÚtokenrC   rD   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   `         rG   ÚbpezCTRLTokenizer.bpe˜   s#  ø€ Ø�D”JÐÐØ”:˜eÔ$Ð$Ý�U‰|Œ|ˆÝ•T˜$˜s ˜sœ)‘_”_¨¨R¬°6Ñ(9Ð':Ñ:Ñ;Ô;ˆÝ˜$‘”ˆàð 	ØˆLð	(Ý˜Ð$WÐ$WÐ$WÐ$WÐXÑXÔXˆFØ˜Tœ^Ð+Ð+ÙØ"‰MˆE�6ØˆHØˆAØ•c˜$‘i”i’-�-ðØŸ
š
 5¨!Ñ,Ô,�Að
 —O’O D¨¨1¨¤IÑ.Ô.Ð.Ø�A�Aøõ "ð ð ð Ø—O’O D¨¨¨¤HÑ-Ô-Ð-Ø�Eðøøøð ˜”7˜eÒ#Ð#¨­C°©I¬I¸©MÒ(9Ð(9¸dÀ1ÀqÁ5¼kÈVÒ>SÐ>SØ—O’O E¨F¡NÑ3Ô3Ð3Ø˜‘F�A�Aà—O’O D¨¤GÑ,Ô,Ð,Ø˜‘F�Að •c˜$‘i”i’-�-õ  ˜X‘”ˆHØˆDÝ�4‰yŒy˜AŠ~ˆ~Øå! $™œ�ñ9	(ð: �zŠz˜$ÑÔˆØ�C�R�CŒyˆØ ˆŒ
�5ÑØˆs   Â(C Ã'DÄDc                 óÎ   — g }t          j        d|¦  «        }|D ]J}|                     t          |                      |¦  «                             d¦  «        ¦  «        ¦  «         ŒK|S )zTokenize a string.z\S+\n?ú )ÚreÚfindallr�   rŠ   r˜   rY   )rr   ÚtextÚsplit_tokensÚwordsr‘   s        rG   Ú	_tokenizezCTRLTokenizer._tokenizeÄ   sf   € àˆå”
˜9 dÑ+Ô+ˆàð 	Bð 	BˆEØ×Ò¥ T§X¢X¨e¡_¤_×%:Ò%:¸3Ñ%?Ô%?Ñ @Ô @ÑAÔAÐAÐAØÐrI   c                 ór   — | j                              || j                              | j        ¦  «        ¦  «        S )z0Converts a token (str) in an id using the vocab.)re   r‚   r^   )rr   r‘   s     rG   Ú_convert_token_to_idz"CTRLTokenizer._convert_token_to_idÎ   s,   € àŒ|×Ò  t¤|×'7Ò'7¸¼Ñ'GÔ'GÑHÔHÐHrI   c                 óB   — | j                              || j        ¦  «        S )z=Converts an index (integer) in a token (str) using the vocab.)rg   r‚   r^   )rr   rŒ   s     rG   Ú_convert_id_to_tokenz"CTRLTokenizer._convert_id_to_tokenÒ   s   € àŒ|×Ò  t¤~Ñ6Ô6Ð6rI   c                 ó|   — d                      |¦  «                             dd¦  «                             ¦   «         }|S )z:Converts a sequence of tokens (string) in a single string.rš   rˆ   Ú )r�   ÚreplaceÚstrip)rr   ÚtokensÚ
out_strings      rG   Úconvert_tokens_to_stringz&CTRLTokenizer.convert_tokens_to_stringÖ   s5   € à—X’X˜fÑ%Ô%×-Ò-¨e°RÑ8Ô8×>Ò>Ñ@Ô@ˆ
ØÐrI   )rL   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚCONTROL_CODESÚcontrol_codesrq   Úpropertyr{   r~   r˜   r    r¢   r¤   r«   Ú__classcell__)rw   s   @rG   rK   rK   k   sÍ   ø€ € € € € ðð ð  *ÐØ!€Mð
ð 
ð 
ð 
ð 
ð 
ð$ ð!ð !ñ „Xð!ð?ð ?ð ?ð*ð *ð *ðXð ð ðIð Ið Ið7ð 7ð 7ðð ð ð ð ð ð rI   rK   )r¯   rc   Úregexr›   Útokenization_pythonr   Úutilsr   Ú
get_loggerr¬   Úloggerr°   r²   rH   rK   Ú__all__rP   rI   rG   ú<module>r¼      sƒ  ðð 0Ð /à €€€à Ð Ð Ð à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð Øðð Ð ð8Ø�ð8à�Dð8ð ˆvð8ð ˆuð	8ð
 ˆeð8ð 
ˆ5ð8ð 
ˆ5ð8ð ˆFð8ð �ð8ð �ð8ð ˆeð8ð �ð8ð ˆuð8ð ˆvð8ð ˆuð8ð  �ð!8ð" ˆuð#8ð 8ð$ ˆUð%8ð& ˆUð'8ð( ˆeð)8ð* ˆTð+8ð, ˆTð-8ð. ˆUð/8ð0 ˆEð18ð2 ˆUð38ð4 ˆdð58ð6 
ˆ5ð78ð8 ˆeð98ð: ˆeð;8ð< ˆuð=8ð> ˆtð?8ð@ ˆeðA8ðB �%ðC8ðD ˆuðE8ð 8ð 8ðF �ðG8ðH ˆVðI8ðJ ˆuðK8ðL �EðM8ðN ˆuðO8ðP ˆUðQ8ðR ˆuðS8ðT ˆfðU8ðV �ðW8ðX ˆTðY8ðZ ˆuð[8ð\ �6ð]8ð^ �%ð_8ð` ˆUða8ðb �ðc8ðd �Eðe8ðf �Vðg8ð 8ðh ØØØðo8ð 8ð 8€ðvð ð ð nð nð nð nð nÐ'ñ nô nð nðn Ð
€€€rI   