§
    ‚Štj$  ã                   ó¢   — d Z ddlZddlmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej        e¦  «        Zdd	iZd
„ Z G d„ de¦  «        ZdgZdS )z"Tokenization classes for Splinter.é    N)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)Ú	WordPieceé   )ÚTokenizersBackend)ÚloggingÚ
vocab_filez	vocab.txtc                 ó  — t          j        ¦   «         }t          | dd¬¦  «        5 }|                     ¦   «         }d d d ¦  «         n# 1 swxY w Y   t	          |¦  «        D ]\  }}|                     d¦  «        }|||<   Œ |S )NÚrzutf-8)Úencodingú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens         úp/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/splinter/tokenization_splinter.pyÚ
load_vocabr      sÄ   € ÝÔ#Ñ%Ô%€EÝ	ˆj˜#¨Ð	0Ñ	0Ô	0ð $°FØ×!Ò!Ñ#Ô#ˆð$ð $ð $ñ $ô $ð $ð $ð $ð $ð $ð $øøøð $ð $ð $ð $å! &Ñ)Ô)ð ð ‰ˆˆuØ—’˜TÑ"Ô"ˆØˆˆe‰ˆØ€Ls   ¦AÁAÁAc                   ó°   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 	 	 	 dde	e
e	ef         z  dz  dede	de	de	de	de	de	dededz  fˆ fd„Zed„ ¦   «         Zd„ Zˆ xZS )ÚSplinterTokenizeraú  
    Construct a Splinter tokenizer (backed by HuggingFace's tokenizers library). Based on WordPiece.

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to a vocabulary file.
        tokenizer_file (`str`, *optional*):
            Path to a tokenizers JSON file containing the serialization of a tokenizer.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values.
        question_token (`str`, *optional*, defaults to `"[QUESTION]"`):
            The token used for constructing question representations.
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase`.
        vocab (`str`, `dict` or `list`, *optional*):
            Custom vocabulary dictionary. If not provided, a minimal vocabulary is created.
    Ú	input_idsÚattention_maskNTú[UNK]ú[SEP]ú[PAD]ú[CLS]ú[MASK]ú
[QUESTION]r   Údo_lower_caseÚ	unk_tokenÚ	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚquestion_tokenÚtokenize_chinese_charsÚstrip_accentsc                 óø  •— |�|n]t          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        dddi| _        t          t          | j        t          |¦  «        ¬	¦  «        ¦  «        | _        t          j        d
|	|
|¬¦  «        | j        _        t          j	        ¦   «         | j        _
        t          j        d¬¦  «        | j        _         t          ¦   «         j        d||||||||	|
dœ	|¤Ž || _        |	| _        |
| _        || _        | j        | j        vr|                      | j        gd
¬¦  «         |                      ¦   «          d S )Nr   é   é   r	   é   é   ú.é   )r)   T)Ú
clean_textÚhandle_chinese_charsr0   Ú	lowercasez##)Úprefix)	r)   r*   r+   r,   r-   r.   r(   r/   r0   )Úspecial_tokens© )ÚstrÚ_vocabr   r   Ú
_tokenizerr   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r(   r/   r0   r.   Úall_special_tokensÚ
add_tokensÚupdate_post_processor)Úselfr   r(   r)   r*   r+   r,   r-   r.   r/   r0   ÚkwargsÚ	__class__s               €r   rG   zSplinterTokenizer.__init__Q   s•  ø€ ð  Ð ð ˆEõ �I‘” Ý�I‘” Ý�I‘” Ý�I‘” Ý�J‘” Ý�NÑ#Ô# QØ�Qðð 	Œõ $¥I¨d¬kÅSÈÁ^Ä^Ð$TÑ$TÔ$TÑUÔUˆŒå%0Ô%?ØØ!7Ø'Ø#ð	&
ñ &
ô &
ˆŒÔ"õ )7Ô(GÑ(IÔ(IˆŒÔ%Ý"*Ô"4¸DÐ"AÑ"AÔ"AˆŒÔà�‰ŒÔð 	
ØØØØØ!Ø)Ø'Ø#9Ø'ð	
ð 	
ð ð	
ð 	
ð 	
ð +ˆÔØ&<ˆÔ#Ø*ˆÔØ,ˆÔØÔ dÔ&=Ð=Ð=Ø�OŠO˜TÔ0Ð1À$ˆOÑGÔGÐGØ×"Ò"Ñ$Ô$Ð$Ð$Ð$ó    c                 ó6   — |                       | j        ¦  «        S )N)Úconvert_tokens_to_idsr.   )rK   s    r   Úquestion_token_idz#SplinterTokenizer.question_token_id�   s   € à×)Ò)¨$Ô*=Ñ>Ô>Ð>rN   c           
      óV  — | j         }| j        }| j        }d}| j        }| j        }| j        }|                      d¦  «        }|�|€d S | j        dk    r|› d|› d|› d|› d|› d�
}	n|› d|› d|› d|› d|› d�
}	t          j	        |› d|› d�|	||f||f||f||fg¬¦  «        | j
        _        d S )	Nr6   Úrightz:0 $A:0 ú z:0 $B:1 z:1z:0)ÚsingleÚpairr<   )r,   r*   r.   Úcls_token_idÚsep_token_idrQ   rP   Úpadding_sider   ÚTemplateProcessingr@   Úpost_processor)
rK   ÚclsÚsepÚquestionÚdotrW   rX   rQ   Údot_token_idrV   s
             r   rJ   z'SplinterTokenizer.update_post_processor‘   s"  € ØŒnˆØŒnˆØÔ&ˆØˆØÔ(ˆØÔ(ˆØ Ô2ÐØ×1Ò1°#Ñ6Ô6ˆàˆ;˜#˜+ØˆFàÔ Ò'Ð'ØÐHÐH 8ÐHÐH¨cÐHÐH°CÐHÐHÀÐHÐHÐHˆDˆDàÐHÐH 3ÐHÐH°ÐHÐH¸3ÐHÐHÀÐHÐHÐHˆDå)3Ô)FØÐ*Ð* 3Ð*Ð*Ð*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ð	*
ñ 	*
ô 	*
ˆŒÔ&Ð&Ð&rN   )
NTr"   r#   r$   r%   r&   r'   TN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr>   ÚdictÚintÚboolrG   ÚpropertyrQ   rJ   Ú__classcell__)rM   s   @r   r   r   )   s)  ø€ € € € € ð!ð !ðF *ÐØ$Ð&6Ð7ÐØ€Eð .2Ø"Ø Ø Ø Ø Ø"Ø*Ø'+Ø%)ð:%ð :%à�T˜#˜s˜(”^Ñ# dÑ*ð:%ð ð:%ð ð	:%ð
 ð:%ð ð:%ð ð:%ð ð:%ð ð:%ð !%ð:%ð ˜d‘{ð:%ð :%ð :%ð :%ð :%ð :%ðx ð?ð ?ñ „Xð?ð
ð 
ð 
ð 
ð 
ð 
ð 
rN   r   )rd   r   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr
   Úutilsr   Ú
get_loggerra   Úloggerre   r   r   Ú__all__r=   rN   r   ú<module>ru      sé   ðð )Ð (à Ð Ð Ð à SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ 'Ð 'Ð 'Ð 'Ð 'Ð 'à >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€à! ;Ð/Ð ðð ð ðC
ð C
ð C
ð C
ð C
Ð)ñ C
ô C
ð C
ðL Ð
€€€rN   