ó
    úÞ jÿ  ã                  óŒ   • S SK Jr  S SKrS SKrS SKrS	S jrS
S jrSSS jjr   S           SS jjrSS jr	SSS jjr
g)é    )ÚannotationsNc                óp  • SU ;  a%  [        U SSS9 nUR                  5       sS S S 5        $ U R                  S5      (       a1  SS KnUR	                  U 5      nUR                  5         UR                  $  SS KnUR                  U 5      $ ! , (       d  f       Nk= f! [         a  n[        S5      UeS nAff = f)Nz://Úrbr   ©Ú	buffering)zhttp://zhttps://úOblobfile is not installed. Please install it by running `pip install blobfile`.)
ÚopenÚreadÚ
startswithÚrequestsÚgetÚraise_for_statusÚcontentÚblobfileÚImportErrorÚ
read_bytes)ÚblobpathÚfr   Úrespr   Úes         ÚG/var/www/html/gaurav/venv/lib/python3.13/site-packages/tiktoken/load.pyÚ	read_filer      s¬   € Ø�HÓÜ�(˜D¨AÒ.°!Ø—6‘6“8÷ /Ñ.ð ×ÑÐ2×3Ñ3ãà�|‰|˜HÓ%ˆØ×ÑÔØ�|‰|ÐðÛð
 ×Ñ˜xÓ(Ð(÷# /Õ.ûô ó ÜØ]ó
àð	ûðús#   ’B	Á4B Â	
BÂ
B5Â$B0Â0B5c                óR   • [         R                  " U 5      R                  5       nX!:H  $ ©N)ÚhashlibÚsha256Ú	hexdigest)ÚdataÚexpected_hashÚactual_hashs      r   Ú
check_hashr!      s"   € Ü—.’. Ó&×0Ñ0Ó2€KØÑ'Ð'ó    c                ó®  • SnS[         R                  ;   a  [         R                  S   nO\S[         R                  ;   a  [         R                  S   nO4SS Kn[         R                  R	                  UR                  5       S5      nSnUS:X  a  [        U 5      $ [        R                  " U R                  5       5      R                  5       n[         R                  R	                  X55      n[         R                  R                  U5      (       aQ  [        USSS	9 nUR                  5       nS S S 5        Ub  [        WU5      (       a  W$  [         R                  " U5        [        U 5      n	U(       a"  [        X‘5      (       d  [#        S
U  SU S35      eSS Kn
 [         R&                  " USS9  US-   [)        U
R+                  5       5      -   S-   n[        US5       nUR-                  U	5        S S S 5        [         R.                  " X¶5        U	$ ! , (       d  f       Né= f! [          a     NËf = f! , (       d  f       NG= f! [          a    U(       a  e  U	$ f = f)NTÚTIKTOKEN_CACHE_DIRÚDATA_GYM_CACHE_DIRr   zdata-gym-cacheFÚ r   r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)Úexist_okÚ.z.tmpÚwb)ÚosÚenvironÚtempfileÚpathÚjoinÚ
gettempdirr   r   Úsha1Úencoder   Úexistsr	   r
   r!   ÚremoveÚOSErrorÚ
ValueErrorÚuuidÚmakedirsÚstrÚuuid4ÚwriteÚrename)r   r   Úuser_specified_cacheÚ	cache_dirr,   Ú	cache_keyÚ
cache_pathr   r   Úcontentsr6   Útmp_filenames               r   Úread_file_cachedrB   #   sî  € ØÐØœrŸz™zÓ)Ü—J‘JÐ3Ñ4‰	Ø	¤§¡Ó	+Ü—J‘JÐ3Ñ4‰	ãä—G‘G—L‘L ×!4Ñ!4Ó!6Ð8HÓIˆ	Ø$Ðà�Bƒä˜Ó"Ð"ä—’˜XŸ_™_Ó.Ó/×9Ñ9Ó;€Iä—‘—‘˜iÓ3€JÜ	‡w�w‡~�~�j×!Ñ!Ü�*˜d¨aÒ0°AØ—6‘6“8ˆD÷ 1àÑ ¤J¨t°]×$CÑ$CØˆKð	Ü�IŠI�jÔ!ô ˜Ó"€HÞœZ¨×@Ñ@ÜØ5°h°Z¸{È=È/ð ZHð Ió
ð 	
ó
 ð	Ü
�Š�I¨Ò-Ø! CÑ'¬#¨d¯j©j«lÓ*;Ñ;¸fÑDˆÜ�, Ô%¨Ø�G‰G�HÔ÷ &ä
�	Š	�,Ô+ð €O÷? 1Õ0ûô ó 	Ùð	ú÷ &Õ%ûô ó æØð  ð €OðúsI   ÄHÅH ÆAH> ÇH-Ç,H> È
HÈ
H*È)H*È-
H;È7H> È>IÉIc                óT  ^• [        S5       Vs/ sH6  n[        U5      R                  5       (       d  M#  [        U5      S:w  d  M4  UPM8     nnU Vs0 sH  n[        U5      U_M     snmSn[        S5       H1  nXV;  d  M
  UR                  U5        UT[        SU-   5      '   US-  nM3     [	        U5      S:X  d   e[        X5      R                  5       nUR                  S5      SS  V	s/ sH  n	[        U	R                  5       5      PM     n
n	S
U4S jjn[        U5       VVs0 sH  u  pÅ[        U/5      U_M     nnnA[	        U5      nU
 H  u  pïX}U" U5      U" U5      -   '   US-  nM     SS KnUR                  [        X5      5      nUR                  5        VVs0 sH  u  nnU" U5      U_M     nnnUR                  SS 5        UR                  S	S 5        U(       a"  U H  n[	        U5      S:X  d  M  UU   UU'   M     UU:X  d   eU$ s  snf s  snf s  sn	f s  snnf s  snnf )Né   Ú r   é   Ú
éÿÿÿÿc                ó.   >• [        U4S jU  5       5      $ )Nc              3  ó,   >#   • U H
  nTU   v •  M     g 7fr   © )Ú.0ÚbÚdata_gym_byte_to_bytes     €r   Ú	<genexpr>ÚKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>q   s   øé € Ð=±u°!Ð*¨1Ö-²uùs   ƒ)Úbytes)ÚvaluerN   s    €r   Údecode_data_gymÚ8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymp   s   ø€ ÜÔ=±uÓ=Ó=Ð=r"   s   <|endoftext|>s   <|startoftext|>)rR   r8   ÚreturnrQ   )ÚrangeÚchrÚisprintableÚappendÚlenrB   ÚdecodeÚsplitÚtupleÚ	enumeraterQ   ÚjsonÚloadsÚitemsÚpop)Úvocab_bpe_fileÚencoder_json_fileÚvocab_bpe_hashÚencoder_json_hashÚclobber_one_byte_tokensrM   Úrank_to_intbyteÚnÚvocab_bpe_contentsÚ	merge_strÚ
bpe_mergesrS   ÚiÚ	bpe_ranksÚfirstÚsecondr_   Úencoder_jsonÚkÚvÚencoder_json_loadedrN   s                        @r   Údata_gym_to_mergeable_bpe_ranksru   Y   s(  ø€ ô #(¨¤+ÓX¡+˜Q´°Q³×1CÑ1C×1E“qÌ#ÈaË&ÐTWÉ-—q¡+€OÐXá0?Ó@±¨1œS ›V QšY±Ñ@ÐØ	€AÜ�4Ž[ˆØÕ#Ø×"Ñ" 1Ô%Ø34Ð!¤# d¨Q¡h£-Ñ0Ø�‰FŠAñ	 ô
 ˆÓ 4Ó'Ð'Ð'ô *¨.ÓI×PÑPÓRÐØ<N×<TÑ<TÐUYÓ<ZÐ[\Ð]_Ñ<`ÓaÑ<`¨y”%˜	Ÿ™Ó)Ö*Ñ<`€JÐa÷>ô
 ,5°_Ô+EÔFÑ+E¡4 1”˜�s“˜Q’Ñ+E€IÑFØô 	ˆI‹€AÛ#‰ˆØFG‘/ %Ó(©?¸6Ó+BÑBÑCØ	ˆQ‰Šñ $ó ð
 —:‘:Ô.Ð/@ÓTÓU€LØ=I×=OÑ=OÔ=QÔRÑ=Q±T°Q¸™?¨1Ó-¨qÒ0Ñ=QÐÑRà×ÑÐ,¨dÔ3Ø×ÑÐ.°Ô5æÛ$ˆAÜ�1‹v˜�{Ø2°1Ñ5�	˜!“ñ %ð Ð+Ó+Ð+Ð+àÐùò_ Yùâ@ùò bùó Gùó Ss(   �!H´HÁHÁHÃ+"HÄ%HÆ H$c                óp  •  SS K nUR                  US5       n[        U R	                  5       S S9 HJ  u  pVUR                  [        R                  " U5      S-   [        U5      R                  5       -   S-   5        ML     S S S 5        g ! [         a  n[        S5      UeS nAff = f! , (       d  f       g = f)Nr   r   r)   c                ó   • U S   $ )NrF   rK   )Úxs    r   Ú<lambda>Ú#dump_tiktoken_bpe.<locals>.<lambda>›   s   € À1ÀQÂ4r"   )Úkeyó    ó   
)
r   r   ÚBlobFileÚsortedra   r:   Úbase64Ú	b64encoder8   r1   )rn   Útiktoken_bpe_filer   r   r   ÚtokenÚranks          r   Údump_tiktoken_bper…   “   s£   € ðÛð
 
×	Ñ	Ð,¨dÔ	3°qÜ! )§/¡/Ó"3¹ÔH‰KˆEØ�G‰G”F×$Ò$ UÓ+¨dÑ2´S¸³Y×5EÑ5EÓ5GÑGÈ%ÑOÖPñ I÷ 
4Ð	3øô	 ó ÜØ]ó
àð	ûðú÷ 
4Õ	3ús#   ‚B	 ˜A(B'Â	
B$ÂBÂB$Â'
B5c                ó  • [        X5      n0 nUR                  5        HA  nU(       d  M   UR                  5       u  pV[        U5      U[        R
                  " U5      '   MC     U$ ! [         a  n[        SU< SU  35      UeS nAff = f)NzError parsing line z in )rB   Ú
splitlinesr\   Úintr€   Ú	b64decodeÚ	Exceptionr5   )r‚   r   r@   ÚretÚlinerƒ   r„   r   s           r   Úload_tiktoken_bper�   Ÿ   s�   € äÐ 1ÓA€HØ
€CØ×#Ñ#Ö%ˆÞÙð	[ØŸ*™*›,‰KˆEÜ+.¨t«9ˆC”× Ò  Ó'Ó(ñ &ð €Jøô ó 	[ÜÐ2°4±(¸$Ð?PÐ>QÐRÓSÐYZÐZûð	[ús   «4A%Á%
BÁ/BÂB)r   r8   rU   rQ   )r   rQ   r   r8   rU   Úboolr   )r   r8   r   ú
str | NonerU   rQ   )NNF)rc   r8   rd   r8   re   r�   rf   r�   rg   rŽ   rU   údict[bytes, int])rn   r�   r‚   r8   rU   ÚNone)r‚   r8   r   r�   rU   r�   )Ú
__future__r   r€   r   r*   r   r!   rB   ru   r…   r�   rK   r"   r   Ú<module>r“      su   ðÝ "ã Û Û 	ô)ô,(ö
3ðr "&Ø$(Ø$)ð7Øð7àð7ð ð7ð "ð	7ð
 "ð7ð õ7ôt	Q÷r"   