ó
    úÞ j%   ã                  ó�   • S r SSKJr  SSKrSSKrSSKr " S S5      r S
       SS jjr S
         SS jjrSS jr	S	 r
g)zJThis is an educational implementation of the byte pair encoding algorithm.é    )ÚannotationsNc                  ón   • \ rS rSrSS jrSSS jjrSS jrSS jrSS jr\	SS j5       r
\	S 5       rS	rg
)ÚSimpleBytePairEncodingé   c               ó°   • Xl         X l        UR                  5        VVs0 sH  u  p4XC_M	     snnU l        [        R
                  " U5      U l        gs  snnf )zCreates an Encoding object.N)Úpat_strÚmergeable_ranksÚitemsÚ_decoderÚregexÚcompileÚ_pat)Úselfr   r	   Útoken_bytesÚtokens        ÚO/var/www/html/gaurav/venv/lib/python3.13/site-packages/tiktoken/_educational.pyÚ__init__ÚSimpleBytePairEncoding.__init__   sL   € ð Œà.ÔàFU×F[ÑF[ÔF]Ô^ÑF]Ñ0B°˜Ò+ÑF]Ò^ˆŒÜ—M’M 'Ó*ˆ�	ùó _s    Ac                ó¾   • U R                   R                  U5      n/ nU H9  nUR                  S5      n[        U R                  XbS9nUR                  U5        M;     U$ )zHEncodes a string into tokens.

>>> enc.encode("hello world")
[388, 372]
úutf-8)Ú	visualise)r   ÚfindallÚencodeÚ
bpe_encoder	   Úextend)r   Útextr   ÚwordsÚtokensÚwordÚ
word_bytesÚword_tokenss           r   r   ÚSimpleBytePairEncoding.encode   sZ   € ð —	‘	×!Ñ! $Ó'ˆØˆÛˆDàŸ™ WÓ-ˆJÜ$ T×%9Ñ%9¸:Ñ[ˆKØ�M‰M˜+Ö&ñ	 ð
 ˆó    c                ó:   ^ • SR                  U 4S jU 5       5      $ )zVDecodes a list of tokens into bytes.

>>> enc.decode_bytes([388, 372])
b'hello world'
r#   c              3  ó@   >#   • U H  nTR                   U   v •  M     g 7f©N©r   )Ú.0r   r   s     €r   Ú	<genexpr>Ú6SimpleBytePairEncoding.decode_bytes.<locals>.<genexpr>-   s   øé € ÐA¹&°˜Ÿ™ eÖ,º&ùs   ƒ)Újoin©r   r   s   ` r   Údecode_bytesÚ#SimpleBytePairEncoding.decode_bytes'   s   ø€ ð �x‰xÔA¹&ÓAÓAÐAr#   c                ó@   • U R                  U5      R                  SSS9$ )uØ   Decodes a list of tokens into a string.

Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
the invalid bytes with the replacement character "ï¿½".

>>> enc.decode([388, 372])
'hello world'
r   Úreplace©Úerrors)r-   Údecoder,   s     r   r3   ÚSimpleBytePairEncoding.decode/   s%   € ð × Ñ  Ó(×/Ñ/°À	Ð/ÐJÐJr#   c                óH   • U Vs/ sH  o R                   U   PM     sn$ s  snf )z¡Decodes a list of tokens into a list of bytes.

Useful for visualising how a string is tokenised.

>>> enc.decode_tokens_bytes([388, 372])
[b'hello', b' world']
r'   )r   r   r   s      r   Údecode_tokens_bytesÚ*SimpleBytePairEncoding.decode_tokens_bytes:   s#   € ñ 39Ó9±&¨—‘˜eÔ$±&Ñ9Ð9ùÒ9s   …c                ó(   • [        XUS9n[        X#S9$ )z#Train a BPE tokeniser on some data!)ÚdataÚ
vocab_sizer   ©r   r	   )Ú	bpe_trainr   )Útraining_datar:   r   r	   s       r   ÚtrainÚSimpleBytePairEncoding.trainD   s   € ô $¨ÐW^Ñ_ˆÜ%¨gÑWÐWr#   c                ó”   • [        U [        5      (       a  [        R                  " U 5      n [	        U R
                  U R                  S9$ )Nr;   )Ú
isinstanceÚstrÚtiktokenÚget_encodingr   Ú_pat_strÚ_mergeable_ranks)Úencodings    r   Úfrom_tiktokenÚ$SimpleBytePairEncoding.from_tiktokenJ   s=   € ä�h¤×$Ñ$Ü×,Ò,¨XÓ6ˆHÜ%Ø×%Ñ%°x×7PÑ7Pñ
ð 	
r#   )r   r   r	   r   N)r   rB   r	   údict[bytes, int]ÚreturnÚNone©Úcolour)r   rB   r   ú
str | NonerK   ú	list[int])r   rP   rK   Úbytes)r   rP   rK   rB   )r   rP   rK   úlist[bytes])r=   rB   r:   Úintr   rB   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r-   r3   r6   Ústaticmethodr>   rH   Ú__static_attributes__© r#   r   r   r      sG   † ô+öô Bô	Kô:ð óXó ðXð
 ñ
ó ó
r#   r   c                óä  • U Vs/ sH  n[        U/5      PM     nn U(       a#  US;   a  [        U5        OUS:X  a  [        U5        S nS n[        [	        US S USS  5      5       H2  u  pxU R                  US   US   -   5      n	U	c  M$  Ub  X–:  d  M.  UnU	nM4     Uc  O"Uc   eUS U XE   XES-      -   /-   XES-   S  -   nM¥  U(       a
  [        5         U V
s/ sH  o U
   PM	     nn
U$ s  snf s  sn
f )N©rN   ÚcolorÚsimpleéÿÿÿÿé   r   é   )rQ   Úvisualise_tokensÚprintÚ	enumerateÚzipÚget)r	   Úinputr   ÚbÚpartsÚmin_idxÚmin_rankÚiÚpairÚrankÚpartr   s               r   r   r   S   s*  € ñ "'Ó'¡˜AŒU�A�3ŽZ¡€EÐ'Ø
æØÐ/Ó/Ü  Õ'Ø˜hÓ&Ü�e”ð ˆØˆÜ ¤ U¨3¨B Z°°q°r°Ó!;Ö<‰GˆAØ"×&Ñ& t¨A¡w°°a±Ñ'8Ó9ˆDØÓ XÑ%5¸½Ø�Ø’ñ	 =ð ÑØØÑ"Ð"Ð"ð �h�w� 5¡>°EÀA¹+Ñ4FÑ#FÐ"GÑGÈ%ÐZ[ÑP[ÐP]ÐJ^Ñ^ˆñ/ ö2 ÜŒá05Ó6±¨˜dÔ#±€FÐ6Ø€Mùò= (ùò: 7s   …C(ÃC-c           
     óê  ^• US:  a  [        S5      e0 n[        S5       H  nXT[        U/5      '   M     [        R                  " X 5       VVs/ sH-  ofR                  S5       Vs/ sH  n[        U/5      PM     snPM/     nnn[        U5      U:  GaÎ  [        R                  " 5       mU H)  n	[        U	S S U	SS  5       H  n
TU
==   S-  ss'   M     M+     [        TU4S jS9nUS   US   -   n[        U5      nXÔU'   / nU H¡  n/ nSnU[        U5      S-
  :  aR  Xe   XeS-      4U:X  a  UR                  U5        US	-  nOUR                  Xe   5        US-  nU[        U5      S-
  :  a  MR  U[        U5      S-
  :X  a  UR                  Xe   5        UR                  U5        M£     UnU(       až  [        S
US    SUS    35        [        SU S[        U5       S35        US;   a3  [        S5        [        US S  VVs/ sH  of H  oÝPM     M     snn5        O(US:X  a"  [        S5        US S  H  n[        U5        M     [        S5        [        U5      U:  a  GMÎ  U$ s  snf s  snnf s  snnf )Né   z;vocab_size must be at least 256, so we can encode all bytesr   r_   r`   c                ó   >• TU    $ r&   rZ   )ÚxÚstatss    €r   Ú<lambda>Úbpe_train.<locals>.<lambda>“   s	   ø€ °E¸!²Hr#   )Úkeyr   ra   z The current most common pair is z + zSo we made z our zth tokenr\   z9Now the first fifty words in our training data look like:é2   r^   z:Now the first twenty words in our training data look like:é   Ú
)Ú
ValueErrorÚrangerQ   r   r   r   ÚlenÚcollectionsÚCounterre   ÚmaxÚappendrc   rb   )r9   r:   r   r   Úranksrl   r   rh   r   Úpiecerm   Úmost_common_pairr   r   Ú	new_wordsÚnew_wordrt   s                   @r   r<   r<   w   s‘  ø€ ð �DÓÜÐVÓWÐWØ€EÜ�4Ž[ˆØŒe�Q�C‹jÓñ ô @E¿}º}ÈWÔ?[ô Ù?[°tŸ[™[¨Ô1Ó2Ñ1˜Œ�ˆsŽÑ1Ô2Ñ?[ð 
ñ  ô
 ˆe‹*�zÔ
!ä×#Ò#Ó%ˆÛˆEÜ˜E # 2˜J¨¨a¨b¨	Ö2�Ø�d“˜qÑ •ó 3ñ ô ˜uÔ*<Ñ=ÐØ& qÑ)Ð,<¸QÑ,?Ñ?ˆÜ�E“
ˆà"ˆkÑð ˆ	ÛˆDØˆHØˆAØ”c˜$“i !‘mÓ#Ø‘G˜T a¡%™[Ð)Ð-=Ó=à—O‘O KÔ0Ø˜‘F‘Aà—O‘O D¡GÔ,Ø˜‘F�Að ”c˜$“i !‘mÕ#ð ”C˜“I ‘MÓ!Ø—‘ ¡Ô(Ø×Ñ˜XÖ&ñ ð ˆö ÜÐ4Ð5EÀaÑ5HÐ4IÈÐM]Ð^_ÑM`ÐLaÐbÔcÜ�K ˜}¨E´#°e³*°¸XÐFÔGØÐ/Ó/ÜÐQÔRÜ °E¸#¸2±JÔ!Q±J¨DÊDÀ5¢%ÉD¡%±JÒ!QÕRØ˜hÓ&ÜÐRÔSØ! # 2›J�DÜ˜$–Kñ 'ä�$ŒKôW ˆe‹*�zÖ
!ðZ €Lùòc 	3ùó ùóV "Rs   ÁI)Á%I$Á:I)ÈI/
É$I)c                óD  • S Vs/ sH	  nSU S3PM     nnU  Vs/ sH  o3R                  SSS9PM     nnSnS nU HN  nX%[        U5      -     nX†:X  a  X%S-   [        U5      -     nX†:w  d   eUnU[        U5      -  n[        X‡-   S	S
9  MP     [        S5        g s  snf s  snf )N)é§   é³   é¹   éM   éP   éD   é†   z[48;5;Úmr   r0   r1   r   r`   Ú )Úendz[0m)r3   r}   rc   )	Útoken_valuesrl   Ú
backgroundrs   Úunicode_token_valuesÚrunning_lengthÚ
last_colorr   r]   s	            r   rb   rb   ¼   sÇ   € Ù/OÓPÑ/O¨!�L   1Ó%Ñ/O€JÐPñ JVÓVÉÀAŸH™H W°Y˜HÓ?ÉÐÐVà€NØ€JÛ%ˆØ¬C°
«OÑ;Ñ<ˆØÓØ°Ñ 2´c¸*³oÑEÑFˆEØÓ&Ð&Ð&Øˆ
Øœ#˜e›*Ñ$ˆÜˆe‰m Ô$ñ &ô 
ˆ+Õùò! Qùò Ws
   …BšBc                 ól  • Sn [        [        5       nUR                  5       nS S S 5        [        R	                  WSU S9n[        S5        UR                  S5      nUR                  U5      S:X  d   eUR                  U5      S:X  d   eUR                  U5      SS/:X  d   eU$ ! , (       d  f       Nˆ= f)	NzN's|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+iX  )r:   r   zJThis is the sequence of merges performed in order to encode 'hello world':zhello worlds   hello worlds   hellos    world)
ÚopenÚ__file__Úreadr   r>   rc   r   r3   r-   r6   )Úgpt2_patternÚfr9   Úencr   s        r   Útrain_simple_encodingrž   Ð   s°   € à]ð ô 
ŒhŒ˜1Ø�v‰v‹xˆ÷ 
ô !×
&Ñ
& t¸À\Ð
&Ð
R€Cä	Ð
VÔWØ�Z‰Z˜Ó&€FØ�:‰:�fÓ Ó.Ð.Ð.Ø×Ñ˜FÓ# ~Ó5Ð5Ð5Ø×"Ñ" 6Ó*¨x¸Ð.CÓCÐCÐCà€J÷ 
�ús   ’B%Â%
B3rM   )r	   rJ   rg   rQ   r   rO   rK   rP   )
r9   rB   r:   rS   r   rB   r   rO   rK   rJ   )r’   rR   rK   rL   )Ú__doc__Ú
__future__r   r~   r   rC   r   r   r<   rb   rž   rZ   r#   r   Ú<module>r¡      s“   ðÙ På "ã ã ã ÷D
ñ D
ðP NVð!Ø%ð!Ø.3ð!Ø@Jð!àõ!ðJ GOðBØ
ðBØðBØ),ðBØ9CðBàõBôJó(r#   