dbfdg 3 G\o#@sdZddlZddlmZddlmZddlmZmZm Z m Z ddl m Z ddl mZedZejd ejZejd ejZejd Zyed d dWn"ek rejdZdZYnBXddlmZejdjejZdZddlZejd=ddlZe`[ejdZejdZ edZ!edZ"edZ#edZ$edZ%edZ&edZ'edZ(edZ)ed Z*ed!Z+ed"Z,ed#Z-ed$Z.ed%Z/ed&Z0ed'Z1ed(Z2ed)Z3ed*Z4ed+Z5ed,Z6ed-Z7ed.Z8ed/Z9ed0Z:ed1Z;ed2Zed5Z?ed6Z@ed7ZAed8ZBed9ZCed:ZDed;ZEed<ZFed=ZGed>ZHed?ZIed@ZJedAZKedBZLedCZMedDZNedEZOedFZPedGZQe!e9e%e(e1e0e4e:e,e6e-e7e+e5e'e2e)e*e.e/e"e&e#e3e$e8dHZReSdIdJe eRDZTejdKdLjUdMdNeVeRdOdPdQDZWeXeGeIeHe;eLeMeNgZYeXe;eOeIeNgZZdRdSZ[dTdUZ\dVdWZ]dXdYZ^dZd[Z_Gd\d]d]e`ZaGd^d_d_ebZceGd`dadae`ZdeGdbdcdce`ZedddeZfGdfdgdge`ZgdS)ha jinja2.lexer ~~~~~~~~~~~~ This module implements a Jinja / Python combination lexer. The `Lexer` class provided by this module is used to do some preprocessing for Jinja. On the one hand it filters out invalid operators like the bitshift operators we don't allow in templates. On the other hand it separates template code and python code in expressions. :copyright: (c) 2017 by the Jinja Team. :license: BSD, see LICENSE for more details. N)deque) itemgetter)implements_iteratorintern iteritems text_type)TemplateSyntaxError)LRUCache2z\s+z7('([^'\\]*(?:\\.[^'\\]*)*)'|"([^"\\]*(?:\\.[^"\\]*)*)")z\d+ufööz evalz[a-zA-Z_][a-zA-Z0-9_]*F) _identifierz[\w{0}]+Tzjinja2._identifierz(?z>=srDz(%s)r<ccs|]}tj|VqdS)N)reescape)r@xr?r?rC srHcCs t| S)N)len)rGr?r?rCsrJ)keycCsL|tkrt|Stdtdtdtdtdtdtdtdt dt d t d t d i j ||S) Nzbegin of commentzend of commentr$zbegin of statement blockzend of statement blockzbegin of print statementzend of print statementzbegin of line statementzend of line statementztemplate data / textzend of template)reverse_operatorsTOKEN_COMMENT_BEGINTOKEN_COMMENT_END TOKEN_COMMENTTOKEN_LINECOMMENTTOKEN_BLOCK_BEGINTOKEN_BLOCK_ENDTOKEN_VARIABLE_BEGINTOKEN_VARIABLE_ENDTOKEN_LINESTATEMENT_BEGINTOKEN_LINESTATEMENT_END TOKEN_DATA TOKEN_EOFget) token_typer?r?rC_describe_token_typesr[cCs|jdkr|jSt|jS)z#Returns a description of the token.r)typevaluer[)tokenr?r?rCdescribe_tokens r_cCs2d|kr&|jdd\}}|dkr*|Sn|}t|S)z0Like `describe_token` but for token expressions.r;r)splitr[)exprr\r]r?r?rCdescribe_token_exprs rccCsttj|S)zsCount the number of newline characters in the string. This is useful for extensions that filter a stream. )rI newline_refindall)r]r?r?rCcount_newlinessrfcCstj}t|jd||jft|jd||jft|jd||jfg}|jdk rp|jt|jdd||jf|jdk r|jt|jdd||jfd d t |d d DS) zACompiles all the rules from the environment into a list of rules.r$blockZvariableNZ linestatementz ^[ \t\v]*r'z(?:^|(?<=\S))[^\S\r\n]*cSsg|]}|ddqS)r`Nr?)r@rGr?r?rCrDsz!compile_rules..T)reverse) rErFrIcomment_start_stringblock_start_stringvariable_start_stringline_statement_prefixappendline_comment_prefixsorted) environmenterulesr?r?rC compile_ruless       rsc@s$eZdZdZefddZddZdS)FailurezjClass that raises a `TemplateSyntaxError` if called. Used by the `Lexer` to specify known errors. cCs||_||_dS)N)message error_class)selfruclsr?r?rC__init__szFailure.__init__cCs|j|j||dS)N)rvru)rwlinenofilenamer?r?rC__call__szFailure.__call__N)__name__ __module__ __qualname____doc__rryr|r?r?r?rCrts rtc@sTeZdZdZfZddedD\ZZZddZ ddZ d d Z d d Z d dZ dS)Tokenz Token class.ccs|]}tt|VqdS)N)propertyr)r@rGr?r?rCrHszToken.cCstj||tt||fS)N)tuple__new__rstr)rxrzr\r]r?r?rCrsz Token.__new__cCs*|jtkrt|jS|jdkr$|jS|jS)Nr)r\rLr])rwr?r?rC__str__s    z Token.__str__cCs2|j|krdSd|kr.|jdd|j|jgkSdS)zTest a token against a token expression. This can either be a token type or ``'token_type:token_value'``. This can only test against string values and types. Tr;r`F)r\rar])rwrbr?r?rCtests  z Token.testcGs x|D]}|j|rdSqWdS)z(Test against multiple token expressions.TF)r)rwiterablerbr?r?rCtest_anys  zToken.test_anycCsd|j|j|jfS)NzToken(%r, %r, %r))rzr\r])rwr?r?rC__repr__ szToken.__repr__N)r}r~rr __slots__rangerzr\r]rrrrrr?r?r?rCrs rc@s(eZdZdZddZddZddZdS) TokenStreamIteratorz`The iterator for tokenstreams. Iterate over the stream until the eof token is reached. cCs ||_dS)N)stream)rwrr?r?rCryszTokenStreamIterator.__init__cCs|S)Nr?)rwr?r?rC__iter__szTokenStreamIterator.__iter__cCs0|jj}|jtkr"|jjtt|j|S)N)rcurrentr\rXclose StopIterationnext)rwr^r?r?rC__next__s    zTokenStreamIterator.__next__N)r}r~rrryrrr?r?r?rCrsrc@s~eZdZdZddZddZddZeZedd d d Z d d Z ddZ dddZ ddZ ddZddZddZddZdS) TokenStreamzA token stream is an iterable that yields :class:`Token`\s. The parser however does not iterate over it but calls :meth:`next` to go one token ahead. The current active token is stored as :attr:`current`. cCs>t||_t|_||_||_d|_tdtd|_ t |dS)NFr`) iter_iterr_pushedrr{closedr TOKEN_INITIALrr)rw generatorrr{r?r?rCry/s zTokenStream.__init__cCst|S)N)r)rwr?r?rCr8szTokenStream.__iter__cCst|jp|jjtk S)N)boolrrr\rX)rwr?r?rC__bool__;szTokenStream.__bool__cCs| S)Nr?)rGr?r?rCrJ?szTokenStream.z Are we at the end of the stream?)doccCs|jj|dS)z Push a token back to the stream.N)rrm)rwr^r?r?rCpushAszTokenStream.pushcCs"t|}|j}|j|||_|S)zLook at the next token.)rrr)rwZ old_tokenresultr?r?rClookEs  zTokenStream.lookr`cCsxt|D] }t|q WdS)zGot n tokens ahead.N)rr)rwnrGr?r?rCskipMszTokenStream.skipcCs|jj|rt|SdS)zqPerform the token test and return the token if it matched. Otherwise the return value is `None`. N)rrr)rwrbr?r?rCnext_ifRs zTokenStream.next_ifcCs|j|dk S)z8Like :meth:`next_if` but only returns `True` or `False`.N)r)rwrbr?r?rCskip_ifYszTokenStream.skip_ifc CsX|j}|jr|jj|_n:|jjtk rTyt|j|_Wntk rR|jYnX|S)z|Go one token ahead and return the old one. Use the built-in :func:`next` instead of calling this directly. ) rrpopleftr\rXrrrr)rwrvr?r?rCr]s zTokenStream.__next__cCs"t|jjtd|_d|_d|_dS)zClose the stream.rNT)rrrzrXrr)rwr?r?rCrlszTokenStream.closec Cst|jj|s^t|}|jjtkr:td||jj|j|jtd|t |jf|jj|j|jz|jSt |XdS)z}Expect a given token type and return it. This accepts the same argument as :meth:`jinja2.lexer.Token.test`. z(unexpected end of template, expected %r.zexpected token %r, got %rN) rrrcr\rXrrzrr{r_r)rwrbr?r?rCexpectrs    zTokenStream.expectN)r`)r}r~rrryrrZ __nonzero__rZeosrrrrrrrrr?r?r?rCr(s  rc CsZ|j|j|j|j|j|j|j|j|j|j |j |j f }t j |}|dkrVt|}|t |<|S)z(Return a lexer which is probably cached.N)rjblock_end_stringrkvariable_end_stringricomment_end_stringrlrn trim_blocks lstrip_blocksnewline_sequencekeep_trailing_newline _lexer_cacherYLexer)rprKZlexerr?r?rC get_lexers" rc@s>eZdZdZddZddZd ddZdd d Zdd d ZdS)ra Class that implements a lexer for a given environment. Automatically created by the environment class, usually you don't have to do that. Note that the lexer is not automatically bound to an environment. Multiple environments can share the same lexer. csdd}tj}ttdfttdfttdftt dft t dft t dfg}t|}|jrTdpVd}i|jr\|d}|d||j}|j|j} || rd|| jdpd7}|j|j} || rd|| jdpd7}|d||j} | j|j} | r d || jdp d} d } d | ||j|||jf} d | ||j| ||jf}| d <|d<nd||j} |j|_|j|_d|ddjd||j| ||j||jfgfdd|Dtdfdf|dtdfgt|d||j||j|fttfdf|dtdfdfgt |d||j||j|ft!dfg|t"|d||j#||j#ft$dfg|t%|d||j| ||j||j|ftt&fdf|dtdfdfgt'|d t(dfg|t)|d!t*t+fdfgi|_,dS)"NcSstj|tjtjBS)N)rEcompileMS)rGr?r?rCrJsz Lexer.__init__..z\n?rr+z^%s(.*)z|%sr`z(?!%s)z^[ \t]*z%s%s(?!%s)|%s\+?z %s%s%s|%s\+?rgr$z%srootz (.*?)(?:%s)r<z4(?P(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))cs&g|]\}}d||j||fqS)z(?P<%s_begin>\s*%s\-|%s))rY)r@rr) prefix_rer?rCrDsz"Lexer.__init__..z#bygroupz.+z(.*?)((?:\-%s\s*|%s)%s)z#popz(.)zMissing end of comment tagz(?:\-%s\s*|%s)%sz \-%s\s*|%sz1(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))zMissing end of raw directivez \s*(\n|$)z(.*?)()(?=\n|$))-rErF whitespace_reTOKEN_WHITESPACEfloat_re TOKEN_FLOAT integer_re TOKEN_INTEGERname_re TOKEN_NAME string_re TOKEN_STRING operator_reTOKEN_OPERATORrsrrrjmatchrigrouprkrrjoinrrWrMrrOrNrtrQrRrSrrTTOKEN_RAW_BEGIN TOKEN_RAW_ENDrUrVTOKEN_LINECOMMENT_BEGINrPTOKEN_LINECOMMENT_ENDrr)rwrpcrqZ tag_rulesZroot_tag_rulesZblock_suffix_reZ no_lstrip_reZ block_diffmZ comment_diffZno_variable_reZ lstrip_reZblock_prefix_reZcomment_prefix_rer?)rrCrys         zLexer.__init__cCstj|j|S)z@Called for strings and template data to normalize it to unicode.)rdrr)rwr]r?r?rC_normalize_newlines$szLexer._normalize_newlinesNcCs&|j||||}t|j|||||S)zCCalls tokeniter + tokenize and wraps it in a token stream. ) tokeniterrwrap)rwsourcerr{staterr?r?rCtokenize(szLexer.tokenizec csjxb|D]X\}}}|tkr"qn2|dkr2d}n"|dkrBd}n|dkrPqn|dkrd|j|}n|dkrr|}n|d krt|}tr|j rtd |||n|d kry$|j|d djd djd}WnHtk r}z*t|j ddj }t||||WYdd}~XnXn:|dkr.t |}n&|dkrBt |}n|dkrTt |}t|||VqWdS)zThis is called with the stream as returned by `tokenize` and wraps every token in a :class:`Token` and converts the value. r%rr&r r"r#r(keywordrzInvalid character in identifierrr`asciibackslashreplacezunicode-escaper;Nrrr)r"r#r)ignored_tokensrr check_ident isidentifierrencodedecode Exceptionrastripintr operatorsr) rwrrr{rzr^r]rqmsgr?r?rCr.sD    "     z Lexer.wrapccs@t|}|j}|jr>|r>x"dD]}|j|r |jdPq Wdj|}d}d}dg} |dk rv|dkrv| j|d nd}|j| d} t|} g} xx| D]j\} }}| j||}|dkrq| r|dkrqt |t rxt |D]\}}|j t kr|||q|d kr`xt|jD]0\}}|dk r|||fV||jd7}PqWtd| q|j|d}|s~|tkr|||fV||jd7}qWn|j}|dkr<|dkr| jdnv|dkr| jdn`|dkr| jdnJ|dkr<| std||||| j}||krs             +  ^