
    $h                        S SK Jr  S SKrS SKrS SKrS	S jrS
S jrSSS jjr   S           SS jjrSS jr	SSS jjr
g)    )annotationsNc                |   U R                  S5      (       dG  U R                  S5      (       d1   SS KnUR                  U S5       nUR	                  5       sS S S 5        $ SS KnUR                  U 5      nUR                  5         UR                  $ ! [         a  n[        S5      UeS nAff = f! , (       d  f       N]= f)Nzhttp://zhttps://r   Oblobfile is not installed. Please install it by running `pip install blobfile`.rb)	
startswithblobfileImportErrorBlobFilereadrequestsgetraise_for_statuscontent)blobpathr   efr   resps         E/var/www/html/shao/venv/lib/python3.13/site-packages/tiktoken/load.py	read_filer      s    y))(2E2Ej2Q2Q	
 x.!668 /. <<!D<<  	a	 /.s#   B B-
B*B%%B*-
B;c                R    [         R                  " U 5      R                  5       nX!:H  $ N)hashlibsha256	hexdigest)dataexpected_hashactual_hashs      r   
check_hashr      s"    ..&002K''    c                   SnS[         R                  ;   a  [         R                  S   nO\S[         R                  ;   a  [         R                  S   nO4SS Kn[         R                  R	                  UR                  5       S5      nSnUS:X  a  [        U 5      $ [        R                  " U R                  5       5      R                  5       n[         R                  R	                  X55      n[         R                  R                  U5      (       aR  [        US5       nUR                  5       nS S S 5        Ub  [        WU5      (       a  W$  [         R                  " U5        [        U 5      n	U(       a"  [        X5      (       d  [#        S	U  S
U S35      eSS Kn
 [         R&                  " USS9  US-   [)        U
R+                  5       5      -   S-   n[        US5       nUR-                  U	5        S S S 5        [         R.                  " X5        U	$ ! , (       d  f       N= f! [          a     Nf = f! , (       d  f       NG= f! [          a    U(       a  e  U	$ f = f)NTTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRr   zdata-gym-cacheF r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr   r   sha1encoder   existsopenr   r   removeOSError
ValueErroruuidmakedirsstruuid4writerename)r   r   user_specified_cache	cache_dirr)   	cache_key
cache_pathr   r   contentsr4   tmp_filenames               r   read_file_cachedr@       s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!!*d#q668D $ Jt]$C$CK	IIj! "HZ@@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,%GGH &
		,+ O? $#  		 &%     OsI   H	H AH? H.-H? 
H
H+*H+.
H<8H? ?IIc                T  ^ [        S5       Vs/ sH6  n[        U5      R                  5       (       d  M#  [        U5      S:w  d  M4  UPM8     nnU Vs0 sH  n[        U5      U_M     snmSn[        S5       H1  nXV;  d  M
  UR                  U5        UT[        SU-   5      '   US-  nM3     [	        U5      S:X  d   e[        X5      R                  5       nUR                  S5      SS  V	s/ sH  n	[        U	R                  5       5      PM     n
n	S
U4S jjn[        U5       VVs0 sH  u  p[        U/5      U_M     nnnA[	        U5      nU
 H  u  pX}U" U5      U" U5      -   '   US-  nM     SS KnUR                  [        X5      5      nUR                  5        VVs0 sH  u  nnU" U5      U_M     nnnUR                  SS 5        UR                  S	S 5        U(       a"  U H  n[	        U5      S:X  d  M  UU   UU'   M     UU:X  d   eU$ s  snf s  snf s  sn	f s  snnf s  snnf )N    r      
c                .   > [        U4S jU  5       5      $ )Nc              3  ,   >#    U H
  nTU   v   M     g 7fr    ).0bdata_gym_byte_to_bytes     r   	<genexpr>Kdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>n   s     =u!*1-us   )bytes)valuerL   s    r   decode_data_gym8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymm   s    =u===r   s   <|endoftext|>s   <|startoftext|>)rP   r6   returnrO   )rangechrisprintableappendlenr@   decodesplittuple	enumeraterO   jsonloadsitemspop)vocab_bpe_fileencoder_json_filevocab_bpe_hashencoder_json_hashclobber_one_byte_tokensrK   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrQ   i	bpe_ranksfirstsecondr]   encoder_jsonkvencoder_json_loadedrL   s                        @r   data_gym_to_mergeable_bpe_ranksrs   V   s(    #(+X+QQ1C1C1Eq#a&TW-q+OX0?@1SVQY@	A4[#""1%34!#dQh-0FA	 
 4''' *.IPPR<N<T<TUY<Z[\]_<`a<`y%	)*<`Ja>
 ,5_+EF+E41sQ+EIF 	IA#FG/%(?6+BBC	Q $ 
 ::./@TUL=I=O=O=QR=QTQ?1-q0=QR,d3.5$A1v{215	! % ++++_ Y@ b G Ss(   !HHHH+"H%H H$c                p    SS K nUR                  US5       n[        U R	                  5       S S9 HJ  u  pVUR                  [        R                  " U5      S-   [        U5      R                  5       -   S-   5        ML     S S S 5        g ! [         a  n[        S5      UeS nAff = f! , (       d  f       g = f)Nr   r   r&   c                    U S   $ )NrD   rI   )xs    r   <lambda>#dump_tiktoken_bpe.<locals>.<lambda>   s    1Q4r   )key       
)
r   r	   r
   sortedr_   r8   base64	b64encoder6   r.   )rl   tiktoken_bpe_filer   r   r   tokenranks          r   dump_tiktoken_bper      s    
 
		,d	3q!)//"3HKEGGF$$U+d2SY5E5E5GG%OP I 
4	3	  ]
	 
4	3s#   B	 A(B'	
B$BB$'
B5c                   [        X5      n0 nUR                  5        HA  nU(       d  M   UR                  5       u  pV[        U5      U[        R
                  " U5      '   MC     U$ ! [         a  n[        SU< SU  35      UeS nAff = f)NzError parsing line z in )r@   
splitlinesrZ   intr}   	b64decode	Exceptionr3   )r   r   r>   retliner   r   r   s           r   load_tiktoken_bper      s     1AH
C##%	[**,KE+.t9C  '( & J  	[24($?P>QRSYZZ	[s   4A%%
B/BB)r   r6   rS   rO   )r   rO   r   r6   rS   boolr   )r   r6   r   
str | NonerS   rO   )NNF)ra   r6   rb   r6   rc   r   rd   r   re   r   rS   dict[bytes, int])rl   r   r   r6   rS   None)r   r6   r   r   rS   r   )
__future__r   r}   r   r'   r   r   r@   rs   r   r   rI   r   r   <module>r      su    "   	&(
3r "&$($)777 7 "	7
 "7 7t	Qr   