tjdZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m ZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6e6j7e8Z9dZ:dZ;dZe,dz Z,ee e!e"dZ?e:e=dZ@e5e,Gdde/ZAeAZBdS)z Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers see tokenization_utils.py N) defaultdict)Iterable)copyfile)Any)is_offline_mode) AddedToken processors)Encoding) Tokenizer)Decoder)BPEUnigram) BpeTrainerUnigramTrainerWordLevelTrainerWordPieceTrainer) cached_file) SpmConverter)convert_gguf_tokenizer)load_gguf_checkpoint)INIT_TOKENIZER_DOCSTRING BatchEncodingPreTokenizedInputPreTrainedTokenizerBase TextInputTruncationStrategygenerate_merges)PaddingStrategyadd_end_docstringsloggingztokenizer.jsonzspecial_tokens_map.jsonztokenizer_config.jsonztokenizer.modelzadded_tokens.jsonu tokenizer_object ([`tokenizers.Tokenizer`]): A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗 tokenizers](../fast_tokenizers) for more information. tokenizer_file ([`str`]): A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗 tokenizers. )r r WordLevel WordPiece)tokenizer_file vocab_filec)eZdZdZeZdZdZedLdZ fdZ e de fdZ e de fdZdMd ed edzdeefd Zd Ze d Ze dZejdZejdZdZe defdZdeeeffdZe deeeffdZe deeeffdZe deeeffdZeZeZ deeeffdZ!de fdZ"defdZ#e de$fdZ%e de&fdZ' dNde(de dzde dzd e d!e d"e d#e d$e deeee)fe*e(ffd%Z+d&edefd'Z,d(ededzfd)Z-dLd*e*eezdefd+Z.dLd,e defd-Z/dLd.ee*ezd/e dee*ezfd0Z0dOd1ed,edzd2e de*efd3Z1d4e2d5e3d6ed7ed8edzd9edzf d:Z4dde2j5e3j6dd;ddddddddddddfd1e7e8ze*e7ze*e8zde dzde dzde dzd e d!e d"e d#e d$e d?e dzde9f(d@Z:dAe*edefdBZ; dPdCee*ezd/e dDe dzdefdEZ< dQd ee=j>zdFeedGfdHe dzd edzdeedGff dIZ? dRdJZ@e dSdKZAxZBS)TTokenizersBackendaQ Base class for all fast tokenizers (wrapping HuggingFace tokenizers library). Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handles all the shared methods for tokenization and special tokens, as well as methods for downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary. This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). NFc $t|}|dd}|Ltj|r-|t us d|jvs|rtj||d<|S|ntj|rNt|d5}tj |}dddn #1swxYwY| di d}|d vret|}t|d} i| d <|d krg| d <| |d<g|d <tj tj|} ntj|} | j|d<| j|d<| j|d<| j | j|d<| j | j|d<| d} | r| dddkr | d} nt%| t&s| g} | D]C} | ddkr(d| vr$ddl} | | d|d<nD| di d d}|j8t%|t&r"t't/t0|}n|jjdkrFt%|t&r0|r.t%|dt&t0fr d|D}nx|jjdkrdt5|D}nN|jjd ks|jjdkr.t%|t&rdt5|D}||d <t7|dd}d | divr,|r*|jd kr|dd }d |D}||d <|S| d!}| d"}| d }| d }t%|t8rd|d#rOtj|r0d$d%lm}||& |\|d <|d <|St%|t8rtj|r|d'r d$d(lm!}||}|j"|jfi|} d$d)lm#}| |j}|tI|d*r |j%dCi|}n=#tL$r0}tN(d+|jd,|d-Yd}~nd}~wwxYwtI|d.r |j)dCi|}d|vr|t us d|jvr|d d}|d d}| d/pi}||rd0|*D}|*D]a\}}tW|}t9|}| |}|r'||kr!||vr||||<|||<btYj-|j.||1}|||d<|j.j/}|j0dkr|1d2|j2pd3|j3dkr|1d4|j4pd5|j5dkr|1d6|j6pd7nu#tL$rh}tN(d8|d9|d:d$d;lm7}||| d<=} | 8|d<Yd}~nd}~wwxYw|S|At%|t8r,tj|r ||d <|d }|At%|t8r,tj|r ||d <|d }||j|jjd krt%|trd>trttd?t&t8f$fd@ $gdA}!tw}"|!D]+}#|#|vr%|"<$||#g,t{||"B}||d <|S)Dz Build a `tokenizers.Tokenizer` backend from the available serialization files (tokenizer.json, sentencepiece models, tekken.json, vocab/merges). r$N__init__tokenizer_objectutf-8encodingmodeltype)Nrvocabr merges added_tokenspost_processortokenizer_paddingtokenizer_truncation_json_truncation _json_padding normalizerSequence normalizers Precompiledprecompiled_charsmapr_spm_precompiled_charsmaprc,g|]}t|S)tuple).0items h/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/tokenization_utils_tokenizers.py z>TokenizersBackend.convert_to_native_format..s;;;TU4[[;;;r"ci|]\}}|| Sr?r?rAitokens rC z>TokenizersBackend.convert_to_native_format..sCCChaCCCrEr#cTi|]%\}}t|tr|dn||&Sr) isinstancelistrGs rCrJz>TokenizersBackend.convert_to_native_format..s8pppS[STV[E4)@)@KU1XXeQppprEcg|]H}t|tr"t|dnt|IS) )rMstrr@split)rAmerges rCrDz>TokenizersBackend.convert_to_native_format..sKrrrbgZs5K5K]% C 0 0111QVW\Q]Q]rrrrEr% merges_filez tekken.jsonr)MistralConverter)r%.model)SentencePieceExtractor)SLOW_TO_FAST_CONVERTERSconvert_from_spmz,Could not reorder vocab using converter for z due to z/. Falling back to raw SentencePiece extraction.convert_from_spm_modeladded_tokens_decoderci|]\}}|| Sr?r?)rArItoken_ids rCrJz>TokenizersBackend.convert_to_native_format..s&\&\&\?5(x&\&\&\rE)protor0r1 bos_token eos_token unk_tokenzz+Could not extract SentencePiece model from z$ using sentencepiece library due to z%. Falling back to TikToken extractor.)TikTokenConverterextra_special_tokens)r%revaluesreturncg}|D]b}|t|ttfr||@|t |c|SN)rMrNr@extendappendrQ)rf collectedval_iter_special_tokenss rCrnzHTokenizersBackend.convert_to_native_format.._iter_special_tokens)s{') !33C{ !#e}553!(()=)=c)B)BCCCC!((S2222  rE) pad_tokenrcr_ra sep_token cls_token mask_tokenadditional_special_tokensre) skip_tokensr?)>dictpopospathisfiler'__dict__ TokenizerFast from_fileopenjsonloadgetfrom_strdumpsr3padding truncationrMrNbase64 b64decoder.mapr@__name__ enumerategetattrrQendswithconvert_slow_tokenizerrUextract_vocab_merges_from_modelrWextractrXhasattrrY ExceptionloggerwarningrZitemsintrbuild_tokenizer_from_spm_protor^ trainer_specbos_id setdefault bos_pieceeos_id eos_pieceunk_id unk_piecerd convertedrrsetupdater)%clstrust_remote_codekwargs local_kwargsfast_tokenizer_filetokenizer_handletokenizer_json model_typeminimal_tokenizer_json minimal_model tok_from_filenormalizer_configr8rr0r1r%rTrUrW extractorrXconverter_classer[ id_to_tokenr] new_token current_tokenr* proto_specrd converterspecial_tokens_keysrtkeyrns% @rCconvert_to_native_formatz*TokenizersBackend.convert_to_native_formates F|| *../?FF +233 ,)))Zs|-K-KO`-K/>CY]gCgCg% DJDTDT&'=>EE %@A"&&w3377FFEy eT**4 UE!2!233E#y00eT**--gr:::: :zObfkOkOk'0:rrkqrrr)/ X& !%%l33 "&&}55   ))!!(++ j# & & :+>+>}+M+M RTRYR`R`akRlRl @ @ @ @ @ @>*+E+EH *J]J]^fJgJgH F IJJJJJJ32:>> 0y0KKlKK  OOOOOO&=&A&A#,&O&OO&2wPb7c7c2'G'G'W'W,'W'W  NNPs|PP]^PPP3 899N#=3#=#M#M #M#ML &\99,,, #,0N0N(,,Wd;;E)--h==F,8+;+;jc::>rw~~k?Z?Z>%0L "!(+F >ci3 8Je8S8SXbchjnXoXo8S !Xc] !tCy ! ! ! ! ! ! # # # %(EEK* R R,&&&&';';\#=NNr6r7r=r* gguf_filer$r[add_prefix_spaceFr%r0r1 name_or_pathconfigr tokenizertokenizer_configrc i|] \}\}}|| Sr?r?)rArHw_s rCrJz.TokenizersBackend.__init__..qs%CkCkCkYQPVQRTUAqCkCkCkrET)r0r1fuse_unkdropoutr)r0ra9Couldn't instantiate the backend tokenizer from one of: (1) a `tokenizers` library serialization file, (2) a slow tokenizer instance to convert or (3) an equivalent slow tokenizer class to instantiate and convert. You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.r_r`rarbz3The backend tokenizer is not correctly initialized.r5 max_lengthtruncation_side directionstridetruncation_strategystrategyr4ropad_token_type_id pad_type_id padding_sidelengthpad_to_multiple_ofbackend tokenizers add_bos_token add_eos_tokenr3cFh|]}tt|Sr?hashreprrArIs rC z-TokenizersBackend.__init__..s&$^$^$^5T$u++%6%6$^$^$^rEcVg|]%\}}tt|v#|&Sr?r)rAindexrIadded_tokens_decoder_hashs rCrDz.TokenizersBackend.__init__..sA   uDKK  (AAA AAArEc|dSNrr?)xs rCz,TokenizersBackend.__init__..s STUVSWrErc,g|]}t|Sr?rQrs rCrDz.TokenizersBackend.__init__..s;b;b;b5CJJ;b;b;brEc0g|]}|t|Sr?r)rAts rCrDz.TokenizersBackend.__init__..s$WWW1UVWAWWWrE)speciali pre_tokenizerfix_mistral_regex) init_kwargsrr?):rvrcopydeepcopyrwrxryr{r|rrrrlen _tokenizerrMrurr rNr@r ValueErrorrrenable_truncation no_truncationrenable_padding_add_bos_token_add_eos_tokenr3_should_update_post_processorsuperr)r%rsplit_special_tokensencode_special_tokensr[sortedradded_tokens_encoderkeys_special_tokens_maprfrQrk_extra_special_tokensrr add_tokensget_vocab_sizeNotImplementedErrorr_patch_mistral_regexrupdate_post_processor)"selfargsrr6r7r*rrr[rr%r0r1fast_tokenizer gguf_path gguf_param architecturetokenizer_dictradditional_kwargs vocab_dict _truncation_paddingexplicit_bos_eos_in_kwargsr3 tokens_to_addencoderspecial_token_valuerItokensall_named_tokens vocab_sizer __class__s" @rCr)zTokenizersBackend.__init__Hs"::&8$?? ?D99   .555!::&8$??JJ{D11 $jj)94@@%zz*@"EE!::&8%@@ZZ --  7##H%%  '!]+;<>p$/B\p`p  " -DO - < < < < <   lK ,E F F F   / J J J   h H(= > > >   3[5L M M M M O ) ) + + +::1488dDO ? ? ?   2H=Q4R S S S F " " ,F9 %4%>%[/U[B["$jj%@@$jj%@@#ZZ(8$?? ?> <-;DO *-G-q4?KimqKq*""6"""  !(DO 0040I-$^$^DD]$^$^$^!    &';'A'A'C'C X X X   t0557788;b;bTa;b;b;bb$(#;#B#B#D#D : : "*&''w66;NVc;c;c$$%8999/ , ,E5zz((U--G-G$$U+++ }   ! !FWW0H0O0O0Q0QWWW & % %eS))-&ud;;;EEz22- =-SZZ;K-K-K(,  e$$$$ (''' 7799JJ"   JJJ    74?OT#R#R#^ JJ{D ) ) )7d7 $$^T::!,"(**-@$"G"G   DO  . X$/2PTX2X *  - )  & & ( ( ( ( ( ) )s] ]/.]/rgcdS)NTr?rs rCis_fastzTokenizersBackend.is_faststrEcd|jvr]|jddr=t|dr+|jr$tj|jSdSdS)z `bool`: Whether or not the slow tokenizer can be saved. For a sentencepiece based slow tokenizer, this can only be `True` if the original `"sentencepiece.model"` was not deleted. r%rVFT)vocab_files_namesrrr%rwrxryrs rCcan_save_slow_tokenizerz)TokenizersBackend.can_save_slow_tokenizersi 41 1 1d6L\6Z6c6cdl6m6m 1t\** 7t 7w~~do66654rEsave_directoryfilename_prefixctj|s td|ddStj||r|dzndt dz}tj|jtj|krt|j||fS)NzVocabulary path (z) should be a directory-rr%) rwrxisdirrerrorjoinVOCAB_FILES_NAMESabspathr%r)rrrout_vocab_files rCsave_vocabularyz!TokenizersBackend.save_vocabularysw}}^,,  LLT^TTT U U U F oM_s222QbcoQpp   7??4? + +rw~/N/N N N T_n 5 5 5  rEc|j}|j}||jrd|_|j}|j}||jrd|_|jr|dzndd|jrd|zdznd}||jrd|zdzndd |jrd|zdznd}g}|jr|||f|jr|||ftj||| |j _ dS) ze Updates the underlying post processor with the current `bos_token` and `eos_token`. NFz:0 rz$A:0rPz:0z:1z $B:1)singlepairspecial_tokens) r_ bos_token_idrra eos_token_idrrkr TemplateProcessingrr3)rbosr*eosr+r'r(r)s rCrz'TokenizersBackend.update_post_processor s]n( ;4-;!&D n( ;4-;!&D %)%7?S5[[Rww[_[mEucCiRVFVFVsuwwD0BJ39t++DDgkgyRBRUX[R[^bRbRb@BDD   7  ! !3 "5 6 6 6   7  ! !3 "5 6 6 6)3)F^* * * &&&rEc$t|ddS)NrFrrs rCrzTokenizersBackend.add_eos_token$t-u555rEc$t|ddS)NrFr0rs rCrzTokenizersBackend.add_bos_token(r1rEcft|d||dS)Nrobject __setattr__rrvalues rCrzTokenizersBackend.add_eos_token,34!15999 ""$$$$$rEcft|d||dS)Nrr4r7s rCrzTokenizersBackend.add_bos_token1r9rEcg}|jD]j}|t|tr||0t|t r%|t|ddk|jD]g}t|tr||-t|t r%|t|ddh|r||dt|dds |j j | dSdS)a[ Post-initialization hook that runs after the tokenizer is fully set up. This is called by from_pretrained() after loading the tokenizer, which allows us to add any special tokens that may have been passed as AddedToken objects. Child classes should call super()._post_init() if they override this method. NTF)r normalized)r)r) rrfrMrrkrQrrrrr3r)rr token_valuerIs rC _post_initzTokenizersBackend._post_init6sl 3::<< ^ ^K"+z22 ^$$[1111K-- ^$$Z TV[%\%\%\]]]/ X XE%,, X$$U++++E3'' X$$ZtPU%V%V%VWWW  @ OOM$O ? ? ? 48$ ? ? )4?CaCi  & & ( ( ( ( (DjCirEc8|jdS)zP `int`: Size of the base vocabulary (without the added tokens). Fwith_added_tokensrrrs rCrzTokenizersBackend.vocab_sizeVs ---FFFrEc8|jdS)NTr@)r get_vocabrs rCrDzTokenizersBackend.get_vocab]s((4(@@@rEc*|Sri)rDrs rCr0zTokenizersBackend.vocab`s~~rEchdt|jdDS)z Returns the sorted mapping from string to index. The added tokens encoder is cached for performance optimisation in `self._added_tokens_encoder` for the slow tokenizers. c$i|] \}}|j|Sr?contentrAvks rCrJz:TokenizersBackend.added_tokens_encoder..j mmmA 1mmmrEc|dSrr?rBs rCrz8TokenizersBackend.added_tokens_encoder..j dhijdkrErrr[rrs rCrz&TokenizersBackend.added_tokens_encoderds9 nm0I0O0O0Q0QWkWk)l)l)lmmmmrEc4|jS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `dict[str, int]`: The added tokens. )rget_added_tokens_decoderrs rCr[z&TokenizersBackend.added_tokens_decoderls77999rEchdt|jdDS)z Returns the added tokens in the vocabulary as a dictionary of token to index. Returns: `dict[str, int]`: The added tokens. c$i|] \}}|j|Sr?rHrJs rCrJz5TokenizersBackend.get_added_vocab..rMrEc|dSrr?rOs rCrz3TokenizersBackend.get_added_vocab..rPrErrQrs rCget_added_vocabz!TokenizersBackend.get_added_vocab{s9nm0I0O0O0Q0QWkWk)l)l)lmmmmrEcdS)zN Returns True, to avoid expensive `assert tokenizer` gotchas. Tr?rs rC__bool__zTokenizersBackend.__bool__s trEc8|jdS)zD Size of the full vocabulary with the added tokens. Tr@rBrs rC__len__zTokenizersBackend.__len__s---EEErEc|jS)zc `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend. )rrs rCbackend_tokenizerz#TokenizersBackend.backend_tokenizers rEc|jjS)zU `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer. )rdecoderrs rCr_zTokenizersBackend.decoders &&rETr-return_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthverbosec F| d|jv}| d|jv}|r|j |g|jz} n|g} tt} | D]} | d| j|r | d| j|r | d| j|r | d| j|r | d| j |r-| dt| j| | fS)a Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list of encodings, take care of building a batch from overflowing tokens. Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are lists (overflows) of lists (tokens). Output shape: (overflows, sequence length) Ntoken_type_idsattention_mask input_idsspecial_tokens_maskoffset_mappingr) model_input_names overflowingrrNrkidstype_idsrirkoffsetsr) rr-r`rarbrcrdrerf encodings encoding_dictrs rC_convert_encodingz#TokenizersBackend._convert_encodingsV( ! ($48N$N ! ($48N$N ! $ #)=)I! X%99II! I#D))  ; ;A + & - -ae 4 4 4$ C./66qzBBB$ I./66q7GHHH) S34;;A This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. )rnum_special_tokens_to_add)rr(s rCrz+TokenizersBackend.num_special_tokens_to_adds&88>>>rEroskip_special_tokensc<t|tr|j|Sg}|rt |jn t }|D]C}t|}||vr||j|D|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `list[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `list[str]`: The decoded token(s). )rMrrrrall_special_idsrk)rrorr ids_to_skiprs rCconvert_ids_to_tokensz'TokenizersBackend.convert_ids_to_tokenss c3   4?..s33 33FQc$.///CEE  > >EJJE ## MM$/55e<< = = = = rEtextr}c H|jd|||d|S)N)r text_pairr}r?) _encode_plusr)rrr(r}rs rCtokenizezTokenizersBackend.tokenize s2 t lddOallekllssuuurEpadding_strategyrrrrrc |jj |jj}|tjkr |jn<|||j|jd} d} n fd|D} | |kr|jjdi||tj kr||j dSdS|tj kr|nd} | ||n|j |j|j|j|d}||kr|jjdi|dSdS)a Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers library) and restore the tokenizer settings afterwards. The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed section. Args: padding_strategy ([`~utils.PaddingStrategy`]): The kind of padding that will be applied to the input truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]): The kind of truncation that will be applied to the input max_length (`int`): The maximum size of a sequence. stride (`int`): The stride to use when handling overflow. pad_to_multiple_of (`int`, *optional*): If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). padding_side (`str`, *optional*): The side on which the model should have padding applied. Should be selected between ['right', 'left']. Default value is picked from the class attribute of the same name. N)rrrrc>i|]}||dSrir)rArLr s rCrJz@TokenizersBackend.set_truncation_and_padding..Ds)GGG11kooa66GGGrE)rrpad_idrorrr?)rrrrDO_NOT_TRUNCATErr8rrr DO_NOT_PAD no_padding MAX_LENGTHr pad_token_idrorr) rrrrrrrr targetcurrentrr s @rCset_truncation_and_paddingz,TokenizersBackend.set_truncation_and_paddingsiBo0 ?* "4"D D D&--///) /5!1 F"GGGGGGG&  11;;F;;; 9 9 9#**,,,,,$#$47Q#Q#QZZW[F -9-E\\4K\+!^#5&8 F6!!..8888888"!rErris_split_into_wordsreturn_tensorsrc  #d}||std|||std|rAt|ttfo#|o!t|dttf}nt|ttf}|rt|trt d|Pt |t |kr0tdt |dt |d|tt||n|}n |r||fgn|g}t|ttfs t dt|d  ||||| | |j }j j |kr |j _ j ||| }  fd |D}i}|ddD]##fd |D}||#<d|D}r;g}t|D]$\}\}} ||gt |dzz }%||d<|dD]}!|!|t!||| }"|s5| 3s1t!d|"D|"j}"|"S)Ncrt|trdSt|ttfrt |dkrdSt|dtrdSt|dttfrt |ddks!t|ddtrdSt|ddttfrFt |dddkp&t|dddtSdSdSdS)NTrF)rMrQrNr@r)rs rC_is_valid_text_inputz._is_valid_text_inputqs!S!! tAe}-- q66Q;;4!c** !4!tUm44!1Q4yyA~~AaDGS)A)A~#t#AaDGdE];;%"1Q47||q0OJqtAwqz34O4OO$u 5urEztext input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).rzdwhen tokenizing batches of text, `text_pair` must be a list or tuple with the same length as `text`.zbatch length of `text`: z- does not match batch length of `text_pair`: .z:batch_text_or_text_pairs has to be a list or a tuple (got ))rrrrrr)r}is_pretokenizedc Jg|]}|  S))r-r`rarbrcrdrerf)rt) rAr-rarerdrbrcr`rrfs rCrDz2TokenizersBackend._encode_plus..sX   " "!&;&;*C+E'=+ #   rEc0g|]\}}|D]}|Sr?r?)rArBrrrs rCrDz2TokenizersBackend._encode_plus..s.NNN74DINNqQNNNNrEc"g|] \}}|D]}| Sr?r?)rArrBrs rCrDz2TokenizersBackend._encode_plus..s)SSSWQdSSqSSSSrErjoverflow_to_sample_mapping) tensor_typeci|]>\}}|t|dkr#t|dtr|dn|?SrL)rrMrN)rArr8s rCrJz2TokenizersBackend._encode_plus..sW"Uc%jj1nnE!Hd9S9Sn%((Y^rE)rrMrNr@rQ TypeErrorrzipr/rrrr encode_batchr&_eventual_warn_about_too_long_sequencerrrr)$rrrr}rrrrrrrrr`rarbrcrdrerfrrr is_batchedbatch_text_or_text_pairsrrtokens_and_encodingssanitized_tokensstacksanitized_encodingsrrHtoksrrjbatched_outputrs$` ``````` @rCrzTokenizersBackend._encode_plusYs0   ($#D)) W   )=)=i)H)H W   9#D4-88hThjQUVWQX[_afZgFhFhJJ#D4-88J  T)S)) $Tc)nn)D)D *s4yy**I***FOEZtCi,@,@'A'A'A`d $ $?H'Sy(9':':dV $2UDMBB nTRjMkMknnn  ''- 3!1% (    '#'#< ? 04H H H4HDO 1O00 $1/1   & '*1- * *CNNNN&:NNNE$) S ! !SS0DSSS % X)+ & )*> ? ? K K 9D!*qcC[8I4J4J.JJ**=W 9 :)+6 X XI  7 7 :w W W W W&'79LZhiii n4=V4*&4&:&:&<&<( NrErc|jj|jj|nd|S)NrP)r]r_decoder!)rrs rCconvert_tokens_to_stringz*TokenizersBackend.convert_tokens_to_strings@%-9  " * 1 1& 9 9 9&!! rE token_idsclean_up_tokenization_spacesc |ddt|tr|g}t|tr|d}|j||}||n|j}|rgt|jj j dkr0|j s)t d|jj dn||}|S)Nuse_source_tokenizerrj)rr z=Ignoring clean_up_tokenization_spaces=True for BPE tokenizer aE. The clean_up_tokenization post-processing step is designed for WordPiece tokenizers and is destructive for BPE (it strips spaces before punctuation). Set clean_up_tokenization_spaces=False to suppress this warning, or set clean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_output=True to force cleanup anyway.)rvrMrrurrrr/r]r.rGclean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_outputr warning_oncerclean_up_tokenization)rrrrrrs rC_decodezTokenizersBackend._decodes   )4000 i % % $" I i & & /!+.I%%iEX%YY,7 ) (2 % ( 8 T+122;uDDdE##-/---11$77 rE file_names. legacy_formatct|}tj||r|dzndtz}|j|||fz}|S)Nrr)rQrwrxr!TOKENIZER_FILEr]save)rrrrrr$s rC_save_pretrainedz"TokenizersBackend._save_pretrained%sl^,, oM_s222Q__   ##N333>"33 rEc  tj|j}|d}|d} d} |dddkri|dd<g|dd<n|ddd kr^|dd O|dd } |dd| d }  | vr| } d |dd <| d gg|dd<n;|ddd vr i|dd<nt d|ddd7d|dvr-|ddvr|dd|dd<t jtj|g} |D]} | dd}| dd}|ddd kr|sC| dvr| d| d<| td*i| || ||dddkr#d|vr|dd|dd|d<|dddkr#d|vr|dd|dd|d<|ddd kr| | |d<|dt|dddksA|dddkrPd|dvrFtd|ddDr!tj|d<t |dd}|d*|| d|}|||| ,tj}d| vr| dD]}| d|d}fd |D}|| d|d<|D](}|}|t d!)fd"|D| d|d#<d$D]L}|| vrF| |\}} |vr|}|}|t d!||g| |<M| |d<t jtj||j}t*jD]}t/||t/||} |vr|}|j|d}t5|tr-t||j|j|j|jd%&||<|||<|jr|jng}|| |tA|d kr||d'<|d(< |j!d*i|S#tD$rH}d)tG|vr1|d(d|j!d*i|}|_|cYd}~Sd}~wwxYw)+uf Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) as the current one. Args: text_iterator (generator of `list[str]`): The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts if you have everything in memory. vocab_size (`int`): The size of the vocabulary you want for your tokenizer. length (`int`, *optional*): The total number of sequences in the iterator. This is used to provide meaningful progress tracking new_special_tokens (list of `str` or `AddedToken`, *optional*): A list of new special tokens to add to the tokenizer you are training. special_tokens_map (`dict[str, str]`, *optional*): If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special token name to new special token name in this argument. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library. Returns: [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on `text_iterator`. r2r3Nr.r/r r0r1rrrg)r"r#z;This method does not support this type of tokenizer (found z-) only BPE, Unigram, WordLevel and WordPiece.rcridrIcontinuing_subword_prefixend_of_word_suffixr ByteLevelr9 pretokenizersc3.K|]}|ddkVdS)r/rNr?)rA pretokenizers rC z.s@$!(K7rEinitial_alphabet)rr))rtrainerr)rc<g|]}||Sr?r)rArIspecial_tokens_maps rCrDz=TokenizersBackend.train_new_from_iterator..s*![![![5"4"8"8"F"F![![![rEzQAttempted to set a token in the post processor that does not exist in the mappingc:g|]}|Sr?)rv)rArIrs rCrDz=TokenizersBackend.train_new_from_iterator..s)CuCuCuejIDYDYZ_D`D`CuCuCurEro)rsepT) single_wordlstriprstripr<rrer*z7multiple values for keyword argument 'tokenizer_object'r?)$r~loadsrto_strrvrr{rrrkrrjanypre_tokenizers_fastralphabetMODEL_TO_TRAINER_MAPPINGtrain_from_iteratorrvrrrSPECIAL_TOKENS_ATTRIBUTESrrrrMrrrr<rerrrrQ)r text_iteratorrrnew_special_tokensrrrr2r3rcrr) added_tokenrr trainer_classrtrained_tokenizer_jsonrrrIr] special_tokenspecial_token_fullrer new_tokenizerrs ` @rCtrain_new_from_iteratorz)TokenizersBackend.train_new_from_iterator6sDDO$:$:$<$<==%)).99 '++,<== ' "6 *e 3 3/1N7 #G ,02N7 #H - - G $V , 9 9g&x0<'0:*73G4D3Ew'0 G $V ,0J J J/1N7 #G , ,>n]dNeflNm>>>   *~g666w' 48JJJ3EnU\F]^iFj3kN7 #K 0!*4:n+E+EFF ' = =K!ooi66Gd++Ag&v.);;G;!-+i2HL^2^2^);K 11)*:; v vC+,<=cB8LF)5![![![![TZ![![![FLN#34S9(C!'#,#8#8#?#?#+", s##, DvCuCuCuntCuCuCuN#34S9%@@!/ F F  N22-m++F++ +   HCPQFFRR -t444 . 8 8 8 8 +4 ($$$$$$$ s$! V.. X8> Model type one of the mistral models (on older versions) 2) Remote models on the hub from official mistral models >> Tags including `base_model:.*mistralai` rN) lru_cache)version)rhf_api)maxsizemodel_idrgc |}n#t$rYdSwxYw|j0dd|jrdSdS)NFzbase_model:.*mistralairT) model_infortagssearchr!)rr.rres rCis_base_mistralz?TokenizersBackend._patch_mistral_regex..is_base_mistrals~ ++H55   uu z%995rwwuz7J7JKK 45s ! //Tz config.jsonF) cache_dirrIlocal_files_only%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errors _commit_hashr+r,transformers_versionrz5.0.0)mistralmistral3voxtral ministralpixtralrz$The tokenizer you are loading from 'a' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.z[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+isolated)patternbehavior)r use_regex)r functoolsr packagingrtransformers.utils.hubrrrQboolrr}r~rrparsesetattrrrrrpre_tokenizersSplitRegexrrMr9 Metaspacer)rrpretrained_model_name_or_pathrIrrris_localrrrrrrr _config_filemistral_config_detectedf_configrtransformers_model_typersplit_pretokenizercurrent_pretokenizerrrs @@rCrz&TokenizersBackend._patch_mistral_regexs*  ''''''%%%%%%>>>>>>>> 3    c d          00 H ( 4  5% 5*9/:W*X*X 5';-#!16;8=)   L', #',999+Q"illG+++++++++++++++'.{{3I'J'J$*1++l*C*C' (%GMM:N,O,ORYR_R_`gRhRh,h,h  )3?3  )()%gmmrrrurDr0rrr[_added_tokens_encoder_added_tokens_decoderrWrYr[r{r] DecoderFastr_ EncodingFastrrNrtrxrzr~rrrrrrrrrrrrrrrwPathLikerrr __classcell__)rs@rCr'r'SsO  * EJ```[`Da)a)a)a)a)FX    X  ! !c !C$J !Z_`cZd ! ! ! !   466X666X6%%%%%%)))@GCGGGXG A4S>AAAA tCH~   X nd38nnnnXn:d3 ?&;:::X:10nc3hnnnn$ FFFFF =X ''''X'.2-1*/+0',#-(-(-( $d{-( $d{ -( $( -( %) -(!%-(-(-( tCH~tL11 2-(-(-(-(^ 7#7#*777766d3+;&<6WZ6666 ??d?s????*tCyt`cfjknfo`o4vvSvd vtvjnorjsvvvvI9)I90I9 I9  I9  $J I9DjI9I9I9I9\gk#',;,F2D2T!%$))-#'&*-1-1*/+0',#,0)XX++d9o=EV@WWX004 ?BTJ[E\\_ccX! X * X 0 X$JXX"X $JXDjXt X $d{X $d{X$(X %)!X"!%#X$%X&'X(#Tk)X, -XXXXt tCy S    %*48 ))c?)")'+Tk ) ))))^&*&* bk)#s(Od{  t  sCx * CCCCJ BBB[BBBBBrEr')Crrr~rw collectionsrcollections.abcrshutilrtypingrtokenizers.pre_tokenizersrrhuggingface_hubrrrr r rr r{tokenizers.decodersr rtokenizers.modelsr rtokenizers.trainersrrrrrrrrintegrations.ggmlrmodeling_gguf_pytorch_utilsrtokenization_utils_baserrrrrrrutilsrr r! get_loggerrrrSPECIAL_TOKENS_MAP_FILETOKENIZER_CONFIG_FILETIKTOKEN_VOCAB_FILEADDED_TOKENS_FILErr"r'PreTrainedTokenizerFastr?rErCr4s  ######$$$$$$777777++++++--------//////111111666666********^^^^^^^^^^^^......000000555555======@?????????  H % %"3/'( !! (6EXYY,--jjjjj/jj.-j\),rE