tjF<ddlZddlZddlZddlmZddlmZddlmZddl Z ddl m Z ddl m Z ddlmZe jeZd d d d d dZdZedGdde Zdedeeefde jfdZdeddfdZdedeezfdZdgZdS)N)Path)copyfile)Any)PreTrainedTokenizer)logging)requiresz source.spmz target.spmz vocab.jsonztarget_vocab.jsonztokenizer_config.json) source_spm target_spmvocabtarget_vocab_filetokenizer_config_fileu▁) sentencepiece)backendsc eZdZdZeZddgZ d0d eee fdzd dffd Z d Z ded efdZ dZ defdZded eefdZded efdZfdZfdZ d1dededzd effd Zdeed efdZd2d eefdZdZdZed efd Zd2d!ed"edzd eefd#Zd efd$Zd%Z d&Z!d efd'Z"d(ed dfd)Z#d*Z$d+Z% d3d,ed-edzd.ed eefd/Z&xZ'S)4MarianTokenizeraB Construct a Marian tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. Args: source_spm (`str`): [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that contains the vocabulary for the source language. target_spm (`str`): [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that contains the vocabulary for the target language. source_lang (`str`, *optional*): A string representing the source language. target_lang (`str`, *optional*): A string representing the target language. unk_token (`str`, *optional*, defaults to `""`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. eos_token (`str`, *optional*, defaults to `""`): The end of sequence token. pad_token (`str`, *optional*, defaults to `""`): The token used for padding, for example when batching sequences of different lengths. model_max_length (`int`, *optional*, defaults to 512): The maximum sentence length the model accepts. additional_special_tokens (`list[str]`, *optional*, defaults to `["", ""]`): Additional special tokens used by the tokenizer. sp_model_kwargs (`dict`, *optional*): Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set: - `enable_sampling`: Enable subword regularization. - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout. - `nbest_size = {0,1}`: No sampling is performed. - `nbest_size > 1`: samples from the nbest_size results. - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. Examples: ```python >>> from transformers import MarianForCausalLM, MarianTokenizer >>> model = MarianForCausalLM.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> tokenizer = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de") >>> src_texts = ["I am a small frog.", "Tom asked his teacher for advice."] >>> tgt_texts = ["Ich bin ein kleiner Frosch.", "Tom bat seinen Lehrer um Rat."] # optional >>> inputs = tokenizer(src_texts, text_target=tgt_texts, return_tensors="pt", padding=True) >>> outputs = model(**inputs) # should work ``` input_idsattention_maskNFsp_model_kwargsreturnc N| in| |_t|s Jd|| |_t ||_t ||jvrtd| rDt ||_d|j D|_ g|_ n>d|j D|_ d|jD|_ ||_ ||_ ||g|_t||j|_t||j|_|j|_|j|_|d|_t-jd||||| | |j|| d | dS) Nzcannot find spm source z token must be in the vocabci|]\}}|| Sr.0kvs l/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/marian/tokenization_marian.py z,MarianTokenizer.__init__..sIIITQAqIIIci|]\}}|| Srrrs r"r#z,MarianTokenizer.__init__..sBBBTQAqBBBr$cfg|].}|d|d,|/S)>><<) startswithendswith)rr s r" z,MarianTokenizer.__init__..s?2v2v2vall[_N`N`2vefeoeopteueu2v12v2v2vr$F) source_lang target_lang unk_token eos_token pad_tokenmodel_max_lengthrr separate_vocabsr)rrexistsr2 load_jsonencoderstrKeyErrortarget_encoderitemsdecodersupported_language_codesr,r- spm_filesload_spm spm_source spm_target current_spmcurrent_encoder_setup_normalizer_decode_use_source_tokenizersuper__init__)selfr r r r r,r-r.r/r0r1rr2kwargs __class__s r"rEzMarianTokenizer.__init__ks &5%> - -=>> >  w"+,=">">D IIT-@-F-F-H-HIIIDL,.D ) )BBT\-?-?-A-ABBBDL2v2vdl2v2v2vD )&&$j1#:t/CDD":t/CDD?#|    ,1) ##- 0/+  r$c ddlm}||jj|_dS#t t f$r tjdd|_YdSwxYw)Nr)MosesPunctNormalizerz$Recommended: pip install sacremoses.c|SNr)xs r"z3MarianTokenizer._setup_normalizer..sQr$) sacremosesrJr, normalizepunc_normalizer ImportErrorFileNotFoundErrorwarningswarn)rFrJs r"rBz!MarianTokenizer._setup_normalizers} / 7 7 7 7 7 7#7#78H#I#I#SD ./ / / / M@ A A A#.;D  /s $-AArMc4|r||ndS)zHCover moses empty string edge case. They return empty list for '' input!)rQ)rFrMs r"rPzMarianTokenizer.normalizes *+3t##A&&&3r$cR||jvr |j|S|j|jSrL)rAr.)rFtokens r"_convert_token_to_idz$MarianTokenizer._convert_token_to_ids0 D( ( ('. .#DN33r$textcg}|drH|dx}dkr-||d|dz||dzd}||fS)z6Remove language codes like >>fr<< before sentencepiecer'r(N)r)findappend)rFr[codeend_locs r"remove_language_codez$MarianTokenizer.remove_language_codeso ??4  '4&@gR%G%G KK]w{]+ , , ,! &DTzr$c~||\}}|j|t}||zS)N)out_type)rcr@encoder6)rFr[rapiecess r" _tokenizezMarianTokenizer._tokenizes>..t44 d!(((<<f}r$indexc||jvr |j|S|jr|jn|j}||}|r|n|jS)z?Converts an index (integer) in a token (str) using the decoder.)r:rCr>r? IdToPiecer.)rFri spm_modelpieces r"_convert_id_to_tokenz$MarianTokenizer._convert_id_to_tokensU DL <& &'+'H]DOOdo ##E**1uu4>1r$c 8tj|fi|S)ad Convert a list of lists of token ids into a list of strings by calling decode. Args: sequences (`Union[list[int], list[list[int]], np.ndarray, torch.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. If `None`, will default to `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). use_source_tokenizer (`bool`, *optional*, defaults to `False`): Whether or not to use the source tokenizer to decode sequences (only applicable in sequence-to-sequence problems). kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. Returns: `list[str]`: The list of decoded sentences. )rD batch_decode)rF sequencesrGrHs r"rpzMarianTokenizer.batch_decodes$*$uww#I88888r$c 8tj|fi|S)a Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. Args: token_ids (`Union[int, list[int], np.ndarray, torch.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. If `None`, will default to `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). use_source_tokenizer (`bool`, *optional*, defaults to `False`): Whether or not to use the source tokenizer to decode sequences (only applicable in sequence-to-sequence problems). kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. Returns: `str`: The decoded sentence. )rDdecode)rF token_idsrGrHs r"rszMarianTokenizer.decodes#0uww~i226222r$skip_special_tokensclean_up_tokenization_spacesc |j }|d||_tjd|||d|S)zCInternal decode method that handles use_source_tokenizer parameter.use_source_tokenizer)rtrurvr)r2poprCrD_decode)rFrtrurvrGdefault_use_sourcerHs r"rzzMarianTokenizer._decodes`"&!55,2JJ7MOa,b,b)uww  3)E      r$tokenscJ|jr|jn|j}g}d}|D]A}||jvr!||||zdzz }g},||B|||z }|td}|S)zQUses source spm if _decode_use_source_tokenizer is True, and target spm otherwiserW ) rCr>r?all_special_tokens decode_piecesr`replaceSPIECE_UNDERLINEstrip)rFr|sp_modelcurrent_sub_tokens out_stringrYs r"convert_tokens_to_stringz(MarianTokenizer.convert_tokens_to_strings&*&G\4??T_  1 1E///h445GHH5PSVVV %'"""))%0000h,,-?@@@ ''(8#>> !!!r$c8| ||jgzS||z|jgzS)z=Build model inputs from a sequence by appending eos_token_id.) eos_token_id)rF token_ids_0 token_ids_1s r" build_inputs_with_special_tokensz0MarianTokenizer.build_inputs_with_special_tokens&s/  $"3!44 4[(D,=+>>>r$c6|j|_|j|_dSrL)r>r@r5rArFs r"_switch_to_input_modez%MarianTokenizer._switch_to_input_mode-s?#|r$cH|j|_|jr|j|_dSdSrL)r?r@r2r8rArs r"_switch_to_target_modez&MarianTokenizer._switch_to_target_mode1s2?   7#'#6D  7 7r$c*t|jSrL)lenr5rs r" vocab_sizezMarianTokenizer.vocab_size6s4<   r$save_directoryfilename_prefixctj|s td|ddSg}|jrtj||r|dzndtdz}tj||r|dzndtdz}t|j |t|j || || |n_tj||r|dzndtdz}t|j || |ttdtdg|j |j|jgD];\}}} tj||r|dznd|z} tj|tj| krEtj|r&t%|| | | tj|sft'| d 5} | } | | dddn #1swxYwY| | =t-|S) NzVocabulary path (z) should be a directory-rWr r r r wb)ospathisdirloggererrorr2joinVOCAB_FILES_NAMES save_jsonr5r8r`zipr<r>r?abspathisfileropenserialized_model_protowritetuple) rFrr saved_filesout_src_vocab_fileout_tgt_vocab_fileout_vocab_filespm_save_filename spm_orig_pathrl spm_save_pathficontent_spiece_models r"save_vocabularyzMarianTokenizer.save_vocabulary:sw}}^,,  LLT^TTT U U U F   /!#*9A3&&rEVW^E__"" "$*9A3&&rEVWjEkk""  dl$6 7 7 7 d)+= > > >   1 2 2 2   1 2 2 2 2W\\/!Q3!6!6rUfgnUo oN dlN 3 3 3   ~ . . .;> | ,.? .M N N _do .< <  2 2 7 }i GLL/!Q3!6!6rUf fMw}--1O1OOOTVT[TbTbcpTqTqO 666""=1111W^^M22 2-..3"+4+K+K+M+M(HH1222333333333333333""=111[!!!s*J<<K K c*|SrL) get_src_vocabrs r" get_vocabzMarianTokenizer.get_vocabgs!!###r$c0t|jfi|jSrL)dictr5added_tokens_encoderrs r"rzMarianTokenizer.get_src_vocabjsDL>>D$=>>>r$c0t|jfi|jSrL)rr8added_tokens_decoderrs r" get_tgt_vocabzMarianTokenizer.get_tgt_vocabmsD'EE4+DEEEr$c|j}|tgd|S)N)r>r?r@rQr )__dict__copyupdaterfromkeys)rFstates r" __getstate__zMarianTokenizer.__getstate__psH ""$$ MMmmm n n    r$dc|_tdsi_tdsd_fdjD\__j_dS)NrrCFc3BK|]}t|jVdSrL)r=r)rfrFs r" z/MarianTokenizer.__setstate__..s1+f+fRSHQ8L,M,M+f+f+f+f+f+fr$) rhasattrrrCr<r>r?r@rB)rFrs` r" __setstate__zMarianTokenizer.__setstate__ws t.// &#%D t;<< 605D -+f+f+f+fW[We+f+f+f(?      r$cdS)zJust EOSr)rFargsrGs r"num_special_tokens_to_addz)MarianTokenizer.num_special_tokens_to_addsqr$c|t|j|jfd|DS)Nc g|] }|vrdnd S)rrr)rrMall_special_idss r"r+z7MarianTokenizer._special_token_mask..s'>>>QQ/))q>>>r$)setrremove unk_token_id)rFseqrs @r"_special_token_maskz#MarianTokenizer._special_token_masksDd233t0111>>>>#>>>>r$rralready_has_special_tokensc|r||S|||dgzS|||zdgzS)zCGet list where entries are [1] if a token is [eos] or [pad] else 0.Nr)r)rFrrrs r"get_special_tokens_maskz'MarianTokenizer.get_special_tokens_masksb & M++K88 8  ++K88A3> >++K+,EFF!L Lr$) NNNrrrrNF)FNrL)NF)(__name__ __module__ __qualname____doc__rvocab_files_namesmodel_input_namesrr6rrErBrPrZrclistrhintrnrprsboolrzrrrrpropertyrrrrrrrrrrr __classcell__)rHs@r"rr,sz8 8 t*$&6715= = c3h$.=  = = = = = = ~///43434444444cd3i 2#2#222299999.33333:%*48   " '+Tk        ""tCy"S"""" ??QUVYQZ????,,,777 !C!!!X!+"+"c+"C$J+"Z_`cZd+"+"+"+"Z$4$$$$???FFFd !d !t ! ! ! !??? fk M M M.2Tk M^b M c M M M M M M M Mr$rrrrcRtjdi|}|||S)Nr)rSentencePieceProcessorLoad)rrspms r"r=r=s,  . A A A ACHHTNNN Jr$ct|d5}tj||dddddS#1swxYwYdS)Nwr^)indent)rjsondump)datarrs r"rrs dC%A $!$$$$%%%%%%%%%%%%%%%%%%s 6::c~t|d5}tj|cdddS#1swxYwYdS)Nr)rrload)rrs r"r4r4s| dCAy||s 266)rrrTpathlibrshutilrtypingrrtokenization_pythonrutilsrutils.import_utilsr get_loggerrrrrrr6rrr=rrr4__all__rr$r"rs 666666******  H % % ,4   %&&&iMiMiMiMiM)iMiM'&iMX 3c3hM<` %#%$%%%% CD4K  r$