tjzdZddlZddlmZddlmZejeZddiZ dZ Gd d eZ d gZ dS) zTokenization classes for ESM.N)PreTrainedTokenizer)logging vocab_file vocab.txtct|d5}|}d|DcdddS#1swxYwYdS)Nrc6g|]}|S)strip).0ls f/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/esm/tokenization_esm.py z#load_vocab_file..s )))a ))))openread splitlines)rfliness rload_vocab_filers j#  *!##%%))5)))******************s2AAAc eZdZdZeZddgZ dfd Zd ed e fd Z d e d efdZ dZ dZ d e d efdZd ed e fdZ ddeedeedzd eefdZ ddededzded eefdZdZed efdZxZS) EsmTokenizerz& Constructs an ESM tokenizer. input_idsattention_maskc Lt||_tt|j|_dt|jD|_t jd|||||d||j|_| |jdS)Nci|]\}}|| Sr r )r indtoks r z)EsmTokenizer.__init__..5sQQQ(#sS#QQQr) unk_token cls_token pad_token mask_token eos_tokenr ) r all_tokensdict enumerate _id_to_token _token_to_idsuper__init__unique_no_split_tokens _update_trie) selfrr&r'r(r)r*kwargs __class__s rr1zEsmTokenizer.__init__)s**55 4?!;!;<<QQi6P6PQQQ !       '+o# $566666rindexreturncB|j||jSNr.getr&r4r7s r_convert_id_to_tokenz!EsmTokenizer._convert_id_to_tokenE $$UDN;;;rtokencr|j||j|jSr:r/r<r&r4r@s r_convert_token_to_idz!EsmTokenizer._convert_token_to_idH. $$UD,=,A,A$.,Q,QRRRrc *|Sr:)split)r4textr5s r _tokenizezEsmTokenizer._tokenizeKszz||rcl|j}||j|Sr:)r/copyupdateadded_tokens_encoder)r4 base_vocabs r get_vocabzEsmTokenizer.get_vocabNs3&++-- $3444rcr|j||j|jSr:rBrCs r token_to_idzEsmTokenizer.token_to_idSrErcB|j||jSr:r;r=s r id_to_tokenzEsmTokenizer.id_to_tokenVr?rN token_ids_0 token_ids_1c|jg}|jg}||j||zS||z|zS|jtd||z|z|z|zS)Nz=Cannot tokenize multiple sequences when EOS token is not set!) cls_token_id eos_token_id ValueError)r4rTrUclsseps r build_inputs_with_special_tokensz-EsmTokenizer.build_inputs_with_special_tokensYsv ! !   ([(([(3..   &\]] ][ 3&4s::rFalready_has_special_tokensc|r|tdfd|DSdgdgt|zzdgz}||dgt|zdgzz }|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`list[int]`): List of ids of the first sequence. token_ids_1 (`list[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.c*g|]}|jvrdndS)r)all_special_ids)r r@r4s rrz8EsmTokenizer.get_special_tokens_mask..s*WWW%$"666AAAWWWrr`r)rYlen)r4rTrUr]masks` rget_special_tokens_maskz$EsmTokenizer.get_special_tokens_maskgs$ & X& R XWWW;WWW WsqcC ,,,-3  " QC#k***aS0 0D rctj||r|dznddz}t|d5}|d|jdddn #1swxYwY|fS)N-rw )ospathjoinrwriter+)r4save_directoryfilename_prefixrrs rsave_vocabularyzEsmTokenizer.save_vocabularysW\\.O3c?S3H3Hacgr2rss *c " " 0a GGDIIdo.. / / / 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0}s.A55A9<A9c*t|jSr:)rbr+)r4s r vocab_sizezEsmTokenizer.vocab_sizes4?###r)rrrrr r:)NF)__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesr1intstrr>rDrIrOrQrSlistr\boolrdrppropertyrr __classcell__)r6s@rrr!s*$&67 7777778<#<#<<<<S#S#SSSS SSSSSS<<<<<<GK ; ;9 ;379t3C ; c ; ; ; ;fk.2Tk^b c> $C$$$X$$$$$rr) rvrjtokenization_pythonrutilsr get_loggerrsloggerrwrr__all__r rrrs$# 666666  H % %!;/*** m$m$m$m$m$&m$m$m$`  r