§
    ‚ŠtjR  ã                   óœ   — d Z ddlmZmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej        e¦  «        Zddd	d
œZ G d„ de¦  «        ZdgZdS )zTokenization classes for CLIP.é    )ÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filec                   ó”   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 dde	e
e	ef         z  dz  de	ee	         z  dz  d	e	d
e	de	de	fˆ fd„Zd„ Zˆ xZS )ÚCLIPTokenizerav  
    Construct a CLIP tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
    Byte-Pair-Encoding.

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab (`str`, `dict` or `list`, *optional*):
            Vocabulary dict to use for the tokenizer.
        merges (`str` or `list`, *optional*):
            Merges list to use for the BPE tokenizer.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|startoftext|>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The token used for padding, for example when batching sequences of different lengths.
    Ú	input_idsÚattention_maskNú<|endoftext|>ú<|startoftext|>ÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenc                 ó–  •— |�|n.t          |¦  «        dt          |¦  «        dt          |¦  «        di}|pg | _        t          t          || j        d dddt          |¦  «        ¬¦  «        ¦  «        | _        t          j        t          j        ¦   «         t          j        t          d¦  «        d	¦  «        t          j
        ¦   «         g¦  «        | j        _        t          j        t          j        t          d
¦  «        dd¬¦  «        t          j        d¬¦  «        g¦  «        | j        _        t!          j        ¦   «         | j        _         t%          ¦   «         j        d||||dœ|¤Ž t)          j        t          |¦  «        | j        ft          |¦  «        | j        fdd¬¦  «        | j        _        |                      ¦   «          d S )Nr   é   é   Ú z</w>F)r   r   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkr   z\s+ú z[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedT)ÚbehaviorÚinvert)Úadd_prefix_space)r   r   r   r   )ÚsepÚclsr(   Útrim_offsets© )ÚstrÚ_mergesr   r	   Ú
_tokenizerr   ÚSequenceÚNFCÚReplacer   Ú	LowercaseÚ
normalizerr   ÚSplitÚ	ByteLevelÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r   ÚRobertaProcessingÚeos_token_idÚbos_token_idÚpost_processorÚ%_wrap_decode_method_backend_tokenizer)
Úselfr   r   r   r   r   r   ÚkwargsÚ_vocabÚ	__class__s
            €úh/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/clip/tokenization_clip.pyr:   zCLIPTokenizer.__init__8   sá  ø€ ð Ð ð ˆEõ �I‘” Ý�I‘” Ý�I‘” ðð 	ð �| ˆŒå#ÝØØ”|ØØ*,Ø#)ØÝ˜i™.œ.ðñ ô ñ

ô 

ˆŒõ &1Ô%9ÝŒ_ÑÔ¥Ô 3µE¸&±M´MÀ3Ñ GÔ GÍÔI^ÑI`ÔI`Ðañ&
ô &
ˆŒÔ"õ )7Ô(?åÔ$ÝØzñô ð 'Øðñ ô õ Ô(¸%Ð@Ñ@Ô@ð	ñ)
ô )
ˆŒÔ%õ #+Ô"4Ñ"6Ô"6ˆŒÔà�‰ŒÔð 	
ØØØØð		
ð 	
ð
 ð	
ð 	
ð 	
õ *4Ô)EÝ�Y‘” Ô!2Ð3Ý�Y‘” Ô!2Ð3Ø"Øð	*
ñ *
ô *
ˆŒÔ&ð 	×2Ò2Ñ4Ô4Ð4Ð4Ð4ó    c                 óh   ‡‡— | j         j        Š| j         j        j        Šˆˆfd„}|| j         _        d S )Nc                  óh   •—  ‰| i |¤Ž}|                      ‰d¦  «                             ¦   «         }|S )Nr$   )ÚreplaceÚstrip)ÚargsrA   Útextr"   Úorig_decode_methods      €€rD   Únew_decode_methodzNCLIPTokenizer._wrap_decode_method_backend_tokenizer.<locals>.new_decode_method†   s=   ø€ Ø%Ð% tÐ6¨vÐ6Ð6ˆDØ—<’<Ð 2°CÑ8Ô8×>Ò>Ñ@Ô@ˆDØˆKrE   )Úbackend_tokenizerÚdecodeÚmodelr"   )r@   rM   r"   rL   s     @@rD   r?   z3CLIPTokenizer._wrap_decode_method_backend_tokenizer   sR   øø€ Ø!Ô3Ô:Ðð "Ô3Ô9ÔLÐð	ð 	ð 	ð 	ð 	ð 	ð
 ):ˆÔÔ%Ð%Ð%rE   )NNr   r   r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   rP   r-   ÚdictÚintÚlistr:   r?   Ú__classcell__)rC   s   @rD   r   r      sï   ø€ € € € € ðð ð. *ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø(Ø*Ø(Ø(ðE5ð E5à�T˜#˜s˜(”^Ñ# dÑ*ðE5ð �d˜3”i‘ $Ñ&ðE5ð ð	E5ð
 ðE5ð ðE5ð ðE5ð E5ð E5ð E5ð E5ð E5ðN:ð :ð :ð :ð :ð :ð :rE   r   N)rT   Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrQ   ÚloggerrU   r   Ú__all__r,   rE   rD   ú<module>rc      sÚ   ðð %Ð $à ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZØ !Ð !Ð !Ð !Ð !Ð !à >Ð >Ð >Ð >Ð >Ð >Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€à#/ÀÐ`pÐqÐqÐ ðo:ð o:ð o:ð o:ð o:Ð%ñ o:ô o:ð o:ðd Ð
€€€rE   