§
    ‚Štj  ã                   óŒ   — d dl mZmZmZmZ d dlmZ ddlmZ ddl	m
Z
  e
j        e¦  «        ZddiZ G d„ d	e¦  «        Zd	gZd
S )é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.jsonc                   ó˜   ‡ — e Zd ZdZeZdZddgZeZ		 	 	 	 	 	 	 dde
ee
ef         z  dz  de
ee
         z  dz  de
de
de
de
de
fˆ fd„Zˆ xZS )ÚGemmaTokenizeruá  
    Construct a fast Gemma tokenizer (backed by HuggingFace's tokenizers library).

    This tokenizer uses a BPE model with byte fallback, no prefix space, and a normalizer that replaces
    spaces with "â–�".

    Args:
        tokenizer_file (`str`, optional):
            A tokenizers JSON file containing the serialization of a tokenizer.
        unk_token (`str`, optional, defaults to "<unk>"):
            The unknown token.
        bos_token (`str`, optional, defaults to "<bos>"):
            The beginning of sequence token.
        eos_token (`str`, optional, defaults to "<eos>"):
            The end of sequence token.
        pad_token (`str`, optional, defaults to "<pad>"):
            The padding token.
        mask_token (`str`, optional, defaults to "<mask>"):
            The mask token.
        add_bos_token (`bool`, optional, defaults to True):
            Whether or not to add a `bos_token` at the start of sequences.
        add_eos_token (`bool`, optional, defaults to False):
            Whether or not to add an `eos_token` at the end of sequences.
        vocab (`str` or `dict[str, int]`, optional):
            Custom vocabulary dict. If not provided, a minimal vocabulary is created using the special tokens.
    ÚleftÚ	input_idsÚattention_maskNú<unk>ú<bos>ú<eos>ú<pad>ú<mask>ÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ
mask_tokenc                 ó¬  •— |€Mt          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        dt          |¦  «        di}|| _        |pg | _        t          t	          | j        | j        dt          |¦  «        d d¬¦  «        ¦  «        | _        t          j        dd	d
¬¦  «        | j        _        t          j
        t          j        dd¦  «        t          j        ¦   «         t          j        ¦   «         g¦  «        | j        _        t          j        dd¦  «        | j        _         t#          ¦   «         j        d|||||dœ|¤Ž d S )Nr   é   é   r   é   T)r   r   Úfuse_unkr   ÚdropoutÚbyte_fallbackú Úmerged_with_previousF)ÚpatternÚbehaviorÚinvertu   â–�)r   r   r   r   r   © )ÚstrÚ_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚSplitÚpre_tokenizerr   ÚSequenceÚReplaceÚByteFallbackÚFuseÚdecoderr   Ú
normalizerÚsuperÚ__init__)
Úselfr   r   r   r   r   r   r   ÚkwargsÚ	__class__s
            €új/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/gemma/tokenization_gemma.pyr7   zGemmaTokenizer.__init__;   sa  ø€ ð ˆ=å�I‘” Ý�I‘” Ý�I‘” Ý�I‘” Ý�J‘” ðˆEð ˆŒØ�| ˆŒå#ÝØ”kØ”|ØÝ˜i™.œ.ØØ"ðñ ô ñ	
ô 	
ˆŒõ )7Ô(<ØÐ"8Àð)
ñ )
ô )
ˆŒÔ%õ #+Ô"3ÝÔ˜e SÑ)Ô)­8Ô+@Ñ+BÔ+BÅHÄMÁOÄOÐTñ#
ô #
ˆŒÔõ &1Ô%8¸¸eÑ%DÔ%DˆŒÔ"Ø�‰ŒÔð 	
ØØØØØ!ð	
ð 	
ð ð	
ð 	
ð 	
ð 	
ð 	
ó    )NNr   r   r   r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚpadding_sideÚmodel_input_namesr   Úmodelr*   ÚdictÚintÚlistr7   Ú__classcell__)r:   s   @r;   r   r      sä   ø€ € € € € ðð ð6 *ÐØ€LØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø Ø Ø Ø Ø"ð/
ð /
à�T˜#˜s˜(”^Ñ# dÑ*ð/
ð �d˜3”i‘ $Ñ&ð/
ð ð	/
ð
 ð/
ð ð/
ð ð/
ð ð/
ð /
ð /
ð /
ð /
ð /
ð /
ð /
ð /
ð /
r<   r   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerr=   ÚloggerrA   r   Ú__all__r)   r<   r;   ú<module>rQ      sÄ   ðð HÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ !Ð !Ð !Ð !Ð !Ð !à >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€Ø%Ð'7Ð8Ð ðP
ð P
ð P
ð P
ð P
Ð&ñ P
ô P
ð P
ðf Ð
€€€r<   