§
    ‚ŠtjÉ  ã                   óJ   — d dl mZ d dlmZ d dlmZ d dlmZmZ dede	fd„Z
dS )	é    )ÚPath)ÚAny)ÚTikTokenConverter)ÚTIKTOKEN_VOCAB_FILEÚTOKENIZER_FILEÚencodingÚ
output_dirc                 óì  — t          |¦  «        }|                     d¬¦  «         |dz  t          z  }|t          z  }|j                             dd¬¦  «         t          |                     ¦   «         ¦  «        }t          |                     ¦   «         ¦  «        }	 ddlm} ddl	m
} t          | t
          ¦  «        r || ¦  «        }  || j        |¦  «         nW# t          $ rJ}t          |¦  «        }	d|	                     ¦   «         v rt          d	¦  «        |‚t          d
¦  «        |‚d}~ww xY wt!          || j        | j        ¬¦  «                             ¦   «         }
|
                     |¦  «         dS )a»  
    Converts given `tiktoken` encoding to `PretrainedTokenizerFast` and saves the configuration of converted tokenizer
    on disk.

    Args:
        encoding (`str` or `tiktoken.Encoding`):
            Tokenizer from `tiktoken` library. If `encoding` is `str`, the tokenizer will be loaded with
            `tiktoken.get_encoding(encoding)`.
        output_dir (`str`):
            Save path for converted tokenizer configuration file.
    T)Úexist_okÚtiktoken)Úparentsr   r   )Úget_encoding)Údump_tiktoken_bpeÚblobfilezY`blobfile` is required to save a `tiktoken` file. Install it with `pip install blobfile`.zY`tiktoken` is required to save a `tiktoken` file. Install it with `pip install tiktoken`.N)Ú
vocab_fileÚpatternÚextra_special_tokens)r   Úmkdirr   r   ÚparentÚstrÚabsoluter   r   Útiktoken.loadr   Ú
isinstanceÚ_mergeable_ranksÚImportErrorÚlowerÚ
ValueErrorr   Ú_pat_strÚ_special_tokensÚ	convertedÚsave)r   r	   Ú	save_fileÚtokenizer_fileÚsave_file_absoluteÚoutput_file_absoluter   r   ÚeÚ	error_msgÚ	tokenizers              ú`/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/integrations/tiktoken.pyÚconvert_tiktoken_to_fastr*      s´  € õ �jÑ!Ô!€JØ×Ò˜dÐÑ#Ô#Ð#à˜ZÑ'Õ*=Ñ=€IØ¥.Ñ0€Nð Ô×Ò 4°$ÐÑ7Ô7Ð7å˜Y×/Ò/Ñ1Ô1Ñ2Ô2ÐÝ˜~×6Ò6Ñ8Ô8Ñ9Ô9ÐðØ)Ð)Ð)Ð)Ð)Ð)Ø3Ð3Ð3Ð3Ð3Ð3å�h¥Ñ$Ô$ð 	.Ø#�| HÑ-Ô-ˆHàÐ˜(Ô3Ð5GÑHÔHÐHÐHøÝð ð ð Ý˜‘F”Fˆ	Ø˜ŸšÑ*Ô*Ð*Ð*ÝØkñô àðõ Øgñ
ô 
àð	øøøøðøøøõ "Ø%¨xÔ/@ÐW_ÔWoðñ ô ç‚i�k„kð ð ‡N‚NÐ'Ñ(Ô(Ð(Ð(Ð(s   Â=C Ã
D.Ã$AD)Ä)D.N)Úpathlibr   Útypingr   Ú#transformers.convert_slow_tokenizerr   Ú*transformers.tokenization_utils_tokenizersr   r   r   r*   © ó    r)   ú<module>r1      s   ðØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à AÐ AÐ AÐ AÐ AÐ AØ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ Zð-) sð -)¸ð -)ð -)ð -)ð -)ð -)ð -)r0   