
    <h                         S r SSKrSSKrSSKJr  SSKJr  SSKJr  SSK	J
r
  \R                  " \5      r " S	 S
5      rS
/rg)zTokenization classes for RAG.    N)Optional   )BatchEncoding)logging   )	RagConfigc                       \ rS rSrS rS r\S 5       rS rS r	S r
S rS	 r      SS\\   S\\\      S\\   S\\   S\S\\   S\S\4S jjrSrg
)RagTokenizer   c                 >    Xl         X l        U R                   U l        g N)question_encoder	generatorcurrent_tokenizer)selfr   r   s      `/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/rag/tokenization_rag.py__init__RagTokenizer.__init__   s     0"!%!6!6    c                    [         R                  R                  U5      (       a  [        SU S35      e[         R                  " USS9  [         R                  R                  US5      n[         R                  R                  US5      nU R                  R                  U5        U R                  R                  U5        g )NzProvided path (z#) should be a directory, not a fileT)exist_okquestion_encoder_tokenizergenerator_tokenizer)	ospathisfile
ValueErrormakedirsjoinr   save_pretrainedr   )r   save_directoryquestion_encoder_pathgenerator_paths       r   r    RagTokenizer.save_pretrained#   s    77>>.))~.>>abcc
NT2 "^=Y Zn6KL--.CD&&~6r   c                     SSK Jn  UR                  SS 5      nUc  [        R                  " U5      nUR	                  XR
                  SS9nUR	                  XR                  SS9nU " XVS9$ )N   )AutoTokenizerconfigr   )r(   	subfolderr   )r   r   )auto.tokenization_autor'   popr   from_pretrainedr   r   )clspretrained_model_name_or_pathkwargsr'   r(   r   r   s          r   r,   RagTokenizer.from_pretrained,   s     	;Hd+>../LMF(88)2I2IUq 9 
 "11)2B2BNc 2 
	 $4JJr   c                 &    U R                   " U0 UD6$ r   )r   r   argsr/   s      r   __call__RagTokenizer.__call__>   s    %%t6v66r   c                 :    U R                   R                  " U0 UD6$ r   )r   batch_decoder2   s      r   r7   RagTokenizer.batch_decodeA   s    ~~**D;F;;r   c                 :    U R                   R                  " U0 UD6$ r   )r   decoder2   s      r   r:   RagTokenizer.decodeD   s    ~~$$d5f55r   c                 &    U R                   U l        g r   )r   r   r   s    r   _switch_to_input_mode"RagTokenizer._switch_to_input_modeG   s    !%!6!6r   c                 &    U R                   U l        g r   )r   r   r=   s    r   _switch_to_target_mode#RagTokenizer._switch_to_target_modeJ   s    !%r   N	src_texts	tgt_texts
max_lengthmax_target_lengthpaddingreturn_tensors
truncationreturnc           
          [         R                  " S[        5        Uc  U R                  R                  nU " U4SUUUUS.UD6n	Uc  U	$ Uc  U R                  R                  nU " SUSUUUUS.UD6n
U
S   U	S'   U	$ )Nu4  `prepare_seq2seq_batch` is deprecated and will be removed in version 5 of 🤗 Transformers. Use the regular `__call__` method to prepare your inputs and the tokenizer under the `with_target_tokenizer` context manager to prepare your targets. See the documentation of your specific tokenizer for more detailsT)add_special_tokensrH   rE   rG   rI   )text_targetrL   rH   rG   rE   rI   	input_idslabels )warningswarnFutureWarningr   model_max_length)r   rC   rD   rE   rF   rG   rH   rI   r/   model_inputsrO   s              r   prepare_seq2seq_batch"RagTokenizer.prepare_seq2seq_batchM   s     	 	
 //@@J
#)!!
 
 $ $ 6 6 G G 
!#)(!
 
 "(!4Xr   )r   r   r   )NNNlongestNT)__name__
__module____qualname____firstlineno__r   r    classmethodr,   r4   r7   r:   r>   rA   liststrr   intboolr   rV   __static_attributes__rP   r   r   r
   r
      s    7
7 K K"7<670 *.$(+/ (,,9, DI&, SM	,
 $C=, , !, , 
, ,r   r
   )__doc__r   rQ   typingr   tokenization_utils_baser   utilsr   configuration_ragr   
get_loggerrY   loggerr
   __all__rP   r   r   <module>rk      sE    $ 	   4  ( 
		H	%\ \~ 
r   