§
    ‚Štjl  ã                   ól   — d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ  G d„ d¦  «        Zdefd„Zd	S )
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)Úbytes_to_unicode)ÚPreTrainedTokenizerFastc                   ó@   — e Zd ZdZ	 	 	 	 dd„Zdefd„Zd„ Zd	efd
„Z	dS )ÚMistralConverterz'
    A general tiktoken converter.
    Nús(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Fc                 ó>   — || _         || _        || _        || _        d S )N)ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   Úkwargss         ú_/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/integrations/mistral.pyÚ__init__zMistralConverter.__init__   s(   € ð ˆŒ
ØˆŒØ 0ˆÔØ)BˆÔ&Ð&Ð&ó    r   c                 óH  ‡
‡‡— |Š
t          ¦   «         Šˆfd„Šg }i }t          ‰
                     ¦   «         ¦  «        D ]À\  }\  }}|| j        vrª|| ‰|¦  «        <   t	          |¦  «        dk    rŒ3g }t          dt	          |¦  «        ¦  «        D ]=}|d |…         ||d …         }	}|‰
v r#|	‰
v r||	z   ‰
v r|                     ||	|f¦  «         Œ>t          |ˆ
fd„d¬¦  «        }|                     |¦  «         Œ»|||<   ŒÁt          |d„ d¬¦  «        }ˆfd„|D ¦   «         }||fS )Nc                 ól   •— d                      ˆfd„|                      d¦  «        D ¦   «         ¦  «        S )NÚ c                 ó:   •— g | ]}‰t          |¦  «                 ‘ŒS © )Úord)Ú.0ÚcharÚbyte_encoders     €r   ú
<listcomp>zcMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string.<locals>.<listcomp>   s$   ø€ ÐTÐTÐT¸˜L­¨T©¬Ô3ÐTÐTÐTr   zlatin-1)ÚjoinÚdecode)Úbr    s    €r   Útoken_bytes_to_stringzOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string   s6   ø€ Ø—7’7ÐTÐTÐTÐTÀÇÂÈÑ@SÔ@SÐTÑTÔTÑUÔUÐUr   é   c                 ó<   •— ‰| d                  ‰| d                  fS )Nr   r&   r   )ÚxÚ	bpe_rankss    €r   ú<lambda>zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>-   s   ø€ °Y¸qÀ¼t´_ÀiÐPQÐRSÔPTÄoÐ4V€ r   F)ÚkeyÚreversec                 ó   — | d         S )Né   r   )Úvals    r   r*   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>1   s
   € °°A´€ r   c                 óT   •— g | ]$} ‰|d          ¦  «         ‰|d         ¦  «        f‘Œ%S )r   r&   r   )r   r/   r%   s     €r   r!   zDMistralConverter.extract_vocab_merges_from_model.<locals>.<listcomp>2   s?   ø€ ÐcÐcÐcÐUXÐ(Ð(¨¨Q¬Ñ0Ô0Ð2GÐ2GÈÈAÌÑ2OÔ2OÐPÐcÐcÐcr   )	r	   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr)   r    r%   s             @@@r   Úextract_vocab_merges_from_modelz0MistralConverter.extract_vocab_merges_from_model   s—  øøø€ Øˆ	Ý'Ñ)Ô)ˆð	Vð 	Vð 	Vð 	Vð 	Vð ˆØˆÝ"+¨I¯OªOÑ,=Ô,=Ñ">Ô">ð 	#ð 	#ÑˆC‘�%˜Ø˜DÔ:Ð:Ð:Ø69�Ð+Ð+¨EÑ2Ô2Ñ3Ý�u‘:”: ’?�?ØØ�Ý" 1¥c¨%¡j¤jÑ1Ô1ð ?ð ?�EØ',¨V¨e¨V¤}°e¸E¸F¸F´m˜W�GØ )Ð+Ð+°¸9Ð0DÐ0DÈ'ÐT[ÑJ[Ð`iÐIiÐIiØŸš g¨w¸Ð%=Ñ>Ô>Ð>øÝ˜uÐ*VÐ*VÐ*VÐ*VÐ`eÐfÑfÔf�Ø—’˜eÑ$Ô$Ð$Ð$à"��e‘�Ý˜Ð$6Ð$6ÀÐFÑFÔFˆØcÐcÐcÐcÐ\bÐcÑcÔcˆØ�fˆ}Ðr   c                 óÀ   — |                       | j        ¦  «        \  }}t          t          ||d¬¦  «        ¦  «        }t	          |j        d¦  «        rd|j        _        |S )NF)Úfuse_unkÚignore_mergesT)r@   r   r   r   ÚhasattrÚmodelrC   )r   Úvocab_scoresr8   Ú	tokenizers       r   rG   zMistralConverter.tokenizer5   s\   € Ø#×CÒCÀDÄJÑOÔOÑˆ�fÝ�c ,°ÀÐGÑGÔGÑHÔHˆ	Ý�9”? OÑ4Ô4ð 	1Ø,0ˆIŒOÔ)ØÐr   Úreturnc                 ó|  — |                       ¦   «         }t          j        t          j        t	          | j        ¦  «        dd¬¦  «        t          j        | j        d¬¦  «        g¦  «        |_        t          j        ¦   «         |_
        |                     | j        ¦  «         t          j        d¬¦  «        |_        |S )NÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rG   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rG   s     r   Ú	convertedzMistralConverter.converted<   s¨   € Ø—N’NÑ$Ô$ˆ	Ý"0Ô"9åÔ$¥U¨4¬<Ñ%8Ô%8À:ÐV[Ð\Ñ\Ô\ÝÔ(¸$Ô:OÐ[`ÐaÑaÔaðñ#
ô #
ˆ	Ôõ %Ô.Ñ0Ô0ˆ	ÔØ×$Ò$ TÔ%CÑDÔDÐDå#-Ô#7ÀUÐ#KÑ#KÔ#Kˆ	Ô àÐr   )Nr   FN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr@   rG   r   rV   r   r   r   r   r      sŒ   € € € € € ðð ð ð KØØ"&ðCð Cð Cð Cð°Sð ð ð ð ð6ð ð ð˜9ð ð ð ð ð ð r   r   Útokenizer_filec                 óª  — ddl m} ddlm} |                     | ¦  «        }|j        j        j        }t          |j        j        j	        d„ ¬¦  «        }d„ |D ¦   «         }d„ t          |¦  «        D ¦   «         }|                     |¦  «         |}|j        j        j        j        }t          t          |||¬¦  «                             ¦   «         ¬	¦  «        }	|	                     d
|i¦  «         |j        j        |j        j        |j        j        |j        j        dœ}
|
                     ¦   «         D ] \  }}||v r|	                     ||i¦  «         Œ!|	S )z1Convert a "tekken" tokenizer to a fast Tokenizer.r   )ÚSpecialTokens)ÚMistralTokenizerc                 ó   — | d         S )Nr;   r   )r(   s    r   r*   z*convert_tekken_tokenizer.<locals>.<lambda>X   s   € ÐmnÐouÔmv€ r   )r+   c                 ó   — g | ]
}|d          ‘ŒS )Ú	token_strr   )r   r:   s     r   r!   z,convert_tekken_tokenizer.<locals>.<listcomp>Y   s   € ÐAÐAÐA¨%�5˜Ô%ÐAÐAÐAr   c                 ó   — i | ]\  }}||“Œ	S r   r   )r   r9   r:   s      r   ú
<dictcomp>z,convert_tekken_tokenizer.<locals>.<dictcomp>[   s   € ÐKÐKÐK¡j c¨5�u˜cÐKÐKÐKr   )r   r   r   )Útokenizer_objectr   )Ú	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ	unk_token)Ú%mistral_common.tokens.tokenizers.baser^   Ú(mistral_common.tokens.tokenizers.mistralr_   Ú	from_fileÚinstruct_tokenizerrG   Ú_tekken_token2id_nospecialr6   Ú_all_special_tokensr1   ÚupdateÚ_modelÚ_pat_strr
   r   rV   rT   ÚbosÚvalueÚeosÚpadÚunkr2   )r\   r^   r_   Úmistral_tokenizerr   Úsorted_tokensÚall_specialÚspecials_tokensr   rG   ÚMAP_SPECIALÚspecial_keyÚspecial_tokens                r   Úconvert_tekken_tokenizerr   L   sž  € ð DÐCÐCÐCÐCÐCØIÐIÐIÐIÐIÐIð )×2Ò2°>ÑBÔBÐð Ô0Ô:ÔU€EÝÐ,Ô?ÔIÔ]ÐcvÐcvÐwÑwÔw€MØAÐA°=ÐAÑAÔA€KàKÐKµI¸kÑ4JÔ4JÐKÑKÔK€Oà×Ò˜5Ñ!Ô!Ð!Ø€Eð  Ô2Ô<ÔCÔL€Gõ (Ý)Ø°;Èð
ñ 
ô 
ç
Š)‰+Œ+ðñ ô €Ið × Ò Ð"=¸{Ð!KÑLÔLÐLð #Ô&Ô,Ø"Ô&Ô,Ø"Ô&Ô,Ø"Ô&Ô,ð	ð €Kð '2×&7Ò&7Ñ&9Ô&9ð Gð GÑ"ˆ�]Ø˜KÐ'Ð'Ø×(Ò(¨+°}Ð)EÑFÔFÐFøàÐr   N)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Ú#transformers.convert_slow_tokenizerr	   Ú*transformers.tokenization_utils_tokenizersr
   r   r[   r   r   r   r   ú<module>r„      s»   ðØ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MÐ MØ !Ð !Ð !Ð !Ð !Ð !à @Ð @Ð @Ð @Ð @Ð @Ø NÐ NÐ NÐ NÐ NÐ NðAð Að Að Að Añ Aô Að AðH-¨Sð -ð -ð -ð -ð -ð -r   