x h8ddlmZmZmZmZddlZddlZddlmZdZ e rddl Z Gdde Z e Z Gdde ZdZd Zy) )absolute_importdivisionprint_functionunicode_literalsNFceZdZdZdZy) MultiTokenzdObject used to monkeypatch ply.lex so that we can return multiple tokens from one lex operation.c:|dj|_||_y)Nr)typetokens)selfr s Y/var/www/html/lionshead-backend/venv/lib/python3.12/site-packages/stone/frontend/lexer.py__init__zMultiToken.__init__s1INN  N)__name__ __module__ __qualname____doc__rrr rr s &rrc `eZdZdZdZdZdZdZdZdZ e dz Z e d z Z e d z Z e d z Z e d z Z e d z Z e dz Z dZ dZ dZ dZ dZdZdZdZdZdZgdZddddddd d!d"d#d$d%d& Ze eej/z Z d'Zd(Zd)Zd*Zd+Zd,Zd-Zd.Zd/Z d0Z!d1Z"d2Z#d3Z$d4Z%d5Z&d6Z'd7Z(d8Z)y9):Lexerz' Lexer. Tokenizes stone files. ))WSIGNORE inclusivec~d|_d|_d|_tjd|_d|_g|_y)Nzstone.stone.lexer)lex tokens_queue cur_indentlogging getLogger_logger last_tokenerrors)r s r rzLexer.__init__"s9 (()<=  rc tjdd|i||_g|_d|_|jj|dzy)z Required by ply.yacc for this to quack (duck typing) like a ply lexer. :param str file_data: Contents of the file to lex. moduler Nr)rrrinput)r file_datakwargss r r%z Lexer.input,sB 771$1&1 y4'(rc|jr,|jjd|_|jS|jj }t |t rQ|jj|j|jjd|_|jS||jdkDr |jri|jjdvrQtdd|jj|jj}|jj||j}tdd|jj|jj}|jj|g|zd|_ |jjd|_|jS||_|jS)zf Returns the next LexToken. Returns None when all tokens have been exhausted. r)NEWLINELINEr)r$DEDENT )rpopr rtoken isinstancerextendr rr _create_tokenlinenolexposappend)r r newline_token dedent_count dedent_tokens r r.z Lexer.token9ss   "//33A6DO,) A!Z(!!((2"&"3"3"7"7":"91!4 OO008KK(5%tTXX__dhhoo)O ))00?#'??L#0 $$JL%%,,l^l-JK&'DO&*&7&7&;&;A&>DO'(DOrc|j| |j}|sy|jjd|0)z;Logs all tokens for human inspection. Useful for debugging.zToken %rN)r%r.rdebug)r datar.s r testz Lexer.testXs; 4JJLE LL  z5 1 r)IDKEYWORDPATHDOT)r+INDENTr))COMMAEQLPARRPAR)BOOLEANFLOATINTEGERNULLSTRING)LBRACKETRBRACKET)LBRACERBRACECOLON)Q)ATz\.z\[z\]=,z\?z\{z\}z\:@)alias annotationannotation_typeattrsby deprecateddocexampleerrorextendsimport namespacepatchroutestructunion union_closed ANNOTATIONANNOTATION_TYPEATTRS DEPRECATEDBYEXTENDSIMPORTPATCHROUTESTRUCTUNION UNION_CLOSED) rVrWrXrZrYr^r_rarbrcrdrec<|jjd|S)z\(r)lexer push_stater r.s r t_LPARz Lexer.t_LPARs z* rc:|jj|S)z\))rs pop_staterus r t_RPARz Lexer.t_RPARs  rc.|jdk(|_|S)z\btrue\b|\bfalse\btrue)valuerus r t_ANY_BOOLEANzLexer.t_ANY_BOOLEANs{{f,  rct|_|S)z\bnull\b) NullTokenr|rus r t_ANY_NULLzLexer.t_ANY_NULLs  rc|j|jvrJ|jdk(r|jr|S|jj |jd|_|S|S)z[a-zA-Z_][a-zA-Z0-9_-]*rWr>)r|KEYWORDSrRESERVEDgetr rus r t_ANY_IDzLexer.t_ANY_IDsR ;;$-- ' 00doo **5;; BEJLLrc|S)z\/[/a-zA-Z0-9_-]*rrus r t_ANY_PATHzLexer.t_ANY_PATHs rc:t|j|_|S)z-?\d+(\.\d*(e-?\d+)?|e-?\d+))floatr|rus r t_ANY_FLOATzLexer.t_ANY_FLOATsEKK(  rc:t|j|_|S)z-?\d+)intr|rus r t_ANY_INTEGERzLexer.t_ANY_INTEGERs%++&  rcd}|jxj|jjdz c_|jdd}d}t dt |D]-}||}|r|dk(rd}n|dk(rd}||z }d}!|d k(rd})||z }/d t |jz}|jDcgc]}|j|dd} }dj| |_|Scc}w) z\"([^\\"]|(\\.))*\"rr$ntr,\ ) rsr2r|countrangelen_indent_level_to_spaces_countr splitlinesreplacejoin) r rescapedsnew_stricindentation_strlinelines_without_indentations r t_ANY_STRINGzLexer.t_ANY_STRINGs !''---- GGAbMq#a&!A!A8A#XA1 9GqLG" =doo NN **,%., LL"a 0, "%.))56 %.s;C/c @|jxj|jjdz c_|jdz }|dk\r|jj |dk(}| xr|jj |dk7}|s|rt dd|j|jt|jzdz }|j|_|j|}|r|S|r"|r|jjd||S|S|dz}|dk\ryy) [#][^\n]*\n+r$rrrr)N) rsr2r|rr3lexdatar1r!_create_tokens_for_next_line_dentr insert)r r.ris_full_line_commentis_partial_line_commentr6 dent_tokenss r t_INITIAL_commentzLexer.t_INITIAL_comment%s ekk//55 LL1 1f#(;;#6#6q#9T#A +?'?(E',{{':':1'='D $#'> -iLL%,,U[[1A"AA"E!G &+kk #"DD!# '&&,"#**11!]C**,, FA)1frc .|jxj|jjdz c_t dd|j|j t |jzdz }|j|_|j|y)rr$r)rN)rsr2r|rr1r3r_check_for_indent)r r.r6s r t_WSIGNORE_commentzLexer.t_WSIGNORE_commentCsn ekk//55%i LL%,,U[[)99A=? #kk  }-rc|jxj|jjdz c_|j |}|r|j j d||S|S)\n+r$r)rsr2r|rrr r)r r6rs r t_INITIAL_NEWLINEzLexer.t_INITIAL_NEWLINELs_""m&9&9&?&?&EE"<<]K     % %a 7  rc|jxj|jjdz c_|j |y)rr$N)rsr2r|rr)r r6s r t_WSIGNORE_NEWLINEzLexer.t_WSIGNORE_NEWLINEVs8""m&9&9&?&?&EE" }-rc |j|}||dk(ry|dkDrdnd}t|d|jdz|jt |j z}|gt |z}|xj|z c_t|S)z Starting from a newline token that isn't followed by another newline token, returns any indent or dedent tokens that immediately follow. If indentation doesn't change, returns None. NrrAr+r,r) _get_next_line_indent_deltar1r2r3rr|absrr)r r6 indent_delta dent_type dent_tokenr s r rz'Lexer._create_tokens_for_next_line_dent[s 77 F  <1#4 ,q 0Hh " t]11A5  3}':':#; ;= L 11 <'&!!rc|j|}||dk(ry|jjd|jjfy)zw Checks that the line following a newline is indented, otherwise a parsing error is generated. Nrz-Line continuation must increment indent by 1.)rr!r4rsr2)r r6rs r rzLexer._check_for_indentpsO 77 F  <1#4 KK  @$$++- .rcP|jdk(sJd|jt|jz}|t|jj k(ry|jj |dj tjdd}|sy|j}t|}|dk(ry|ddk(ryt||z }|dzdkDr2|jjd|jjfy|t|jz }|dzS) aX Returns the change in indentation. The return units are in indentations rather than spaces/tabs. If the next line's indent isn't relevant (e.g. it's a comment), returns None. Since the return value might be 0, the caller should explicitly check the return type, rather than rely on truthiness. r)z3Can only search for a dent starting from a newline.Nrr#zIndent is not divisible by 4.)r r3rr|rsrsplitoslineseplstripr!r4r2rr)r r6 next_line_posrlstripped_linelstripped_line_lengthindentrs r rz!Lexer._get_next_line_indent_deltas$!!Y. B A B.%,,s=3F3F/GG C 3 3 ; ;< <""**=>:@@QOPQR #N 3 A % !  #T22 A:> KK  0-2E2E2L2LM O =doo NN q  rz ch|jjd|jd|jj|j j dt|jdjdz|jjf|jjdy)NzIllegal character %r at line %drzIllegal character %s.ur) rr:r|rsr2r!r4reprrskiprus r t_ANY_errorzLexer.t_ANY_errors < ;;q>5;;+=+= ?  $tEKKN';'B'B3'G G [[   ! "  rN)*rrrrstatesrr%r.r<r t_DOT t_LBRACKET t_RBRACKETt_EQt_COMMAt_Qt_LBRACEt_RBRACEt_COLONt_ATrrtuplevaluesrvryr}rrrrrrrrrrrrrt_ignorerrrr rrs~F )>2F F F F F  F  F  F EJJ DG CHHG DH*#,"& H eHOO%&&F    $ H<.!. "* .#!LHrrcftj}||_||_||_||_|S)z Helper for creating ply.lex.LexToken objects. Unfortunately, LexToken does not have a constructor defined to make settings these values easy. )rLexTokenr r|r2r3) token_typer|r2r3r.s r r1r1s0 LLNEEJEKELEL Lrc |dzS)Nrr)rs r rrs A:r) __future__rrrrrrply.lexr_MYPYtypingobjectrrrr1rrrr rsMRR   H UFUp r