0001 """Parse tree transformation module. 0002 0003 Transforms Python source code into an abstract syntax tree (AST) 0004 defined in the ast module. 0005 0006 The simplest ways to invoke this module are via parse and parseFile. 0007 parse(buf) -> AST 0008 parseFile(path) -> AST 0009 """ 0010 0011 # Original version written by Greg Stein (gstein@lyra.org) 0012 # and Bill Tutt (rassilon@lima.mudlib.org) 0013 # February 1997. 0014 # 0015 # Modifications and improvements for Python 2.0 by Jeremy Hylton and 0016 # Mark Hammond 0017 # 0018 # Some fixes to try to have correct line number on almost all nodes 0019 # (except Module, Discard and Stmt) added by Sylvain Thenault 0020 # 0021 # Portions of this file are: 0022 # Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. 0023 # 0024 # This module is provided under a BSD-ish license. See 0025 # http://www.opensource.org/licenses/bsd-license.html 0026 # and replace OWNER, ORGANIZATION, and YEAR as appropriate. 0027 0028 from compiler.ast import * 0029 import parser 0030 import symbol 0031 import token 0032 import sys 0033 0034 class WalkerError(StandardError): 0035 pass 0036 0037 from consts import CO_VARARGS, CO_VARKEYWORDS 0038 from consts import OP_ASSIGN, OP_DELETE, OP_APPLY 0039 0040 def parseFile(path): 0041 f = open(path, "U") 0042 # XXX The parser API tolerates files without a trailing newline, 0043 # but not strings without a trailing newline. Always add an extra 0044 # newline to the file contents, since we're going through the string 0045 # version of the API. 0046 src = f.read() + "\n" 0047 f.close() 0048 return parse(src) 0049 0050 def parse(buf, mode="exec"): 0051 if mode == "exec" or mode == "single": 0052 return Transformer().parsesuite(buf) 0053 elif mode == "eval": 0054 return Transformer().parseexpr(buf) 0055 else: 0056 raise ValueError("compile() arg 3 must be" 0057 " 'exec' or 'eval' or 'single'") 0058 0059 def asList(nodes): 0060 l = [] 0061 for item in nodes: 0062 if hasattr(item, "asList"): 0063 l.append(item.asList()) 0064 else: 0065 if type(item) is type( (None, None) ): 0066 l.append(tuple(asList(item))) 0067 elif type(item) is type( [] ): 0068 l.append(asList(item)) 0069 else: 0070 l.append(item) 0071 return l 0072 0073 def extractLineNo(ast): 0074 if not isinstance(ast[1], tuple): 0075 # get a terminal node 0076 return ast[2] 0077 for child in ast[1:]: 0078 if isinstance(child, tuple): 0079 lineno = extractLineNo(child) 0080 if lineno is not None: 0081 return lineno 0082 0083 def Node(*args): 0084 kind = args[0] 0085 if nodes.has_key(kind): 0086 try: 0087 return nodes[kind](*args[1:]) 0088 except TypeError: 0089 print nodes[kind], len(args), args 0090 raise 0091 else: 0092 raise WalkerEror, "Can't find appropriate Node type: %s" % str(args) 0093 #return apply(ast.Node, args) 0094 0095 class Transformer: 0096 """Utility object for transforming Python parse trees. 0097 0098 Exposes the following methods: 0099 tree = transform(ast_tree) 0100 tree = parsesuite(text) 0101 tree = parseexpr(text) 0102 tree = parsefile(fileob | filename) 0103 """ 0104 0105 def __init__(self): 0106 self._dispatch = {} 0107 for value, name in symbol.sym_name.items(): 0108 if hasattr(self, name): 0109 self._dispatch[value] = getattr(self, name) 0110 self._dispatch[token.NEWLINE] = self.com_NEWLINE 0111 self._atom_dispatch = {token.LPAR: self.atom_lpar, 0112 token.LSQB: self.atom_lsqb, 0113 token.LBRACE: self.atom_lbrace, 0114 token.BACKQUOTE: self.atom_backquote, 0115 token.NUMBER: self.atom_number, 0116 token.STRING: self.atom_string, 0117 token.NAME: self.atom_name, 0118 } 0119 self.encoding = None 0120 0121 def transform(self, tree): 0122 """Transform an AST into a modified parse tree.""" 0123 if not (isinstance(tree, tuple) or isinstance(tree, list)): 0124 tree = parser.ast2tuple(tree, line_info=1) 0125 return self.compile_node(tree) 0126 0127 def parsesuite(self, text): 0128 """Return a modified parse tree for the given suite text.""" 0129 return self.transform(parser.suite(text)) 0130 0131 def parseexpr(self, text): 0132 """Return a modified parse tree for the given expression text.""" 0133 return self.transform(parser.expr(text)) 0134 0135 def parsefile(self, file): 0136 """Return a modified parse tree for the contents of the given file.""" 0137 if type(file) == type(''): 0138 file = open(file) 0139 return self.parsesuite(file.read()) 0140 0141 # -------------------------------------------------------------- 0142 # 0143 # PRIVATE METHODS 0144 # 0145 0146 def compile_node(self, node): 0147 ### emit a line-number node? 0148 n = node[0] 0149 0150 if n == symbol.encoding_decl: 0151 self.encoding = node[2] 0152 node = node[1] 0153 n = node[0] 0154 0155 if n == symbol.single_input: 0156 return self.single_input(node[1:]) 0157 if n == symbol.file_input: 0158 return self.file_input(node[1:]) 0159 if n == symbol.eval_input: 0160 return self.eval_input(node[1:]) 0161 if n == symbol.lambdef: 0162 return self.lambdef(node[1:]) 0163 if n == symbol.funcdef: 0164 return self.funcdef(node[1:]) 0165 if n == symbol.classdef: 0166 return self.classdef(node[1:]) 0167 0168 raise WalkerEror, ('unexpected node type', n) 0169 0170 def single_input(self, node): 0171 ### do we want to do anything about being "interactive" ? 0172 0173 # NEWLINE | simple_stmt | compound_stmt NEWLINE 0174 n = node[0][0] 0175 if n != token.NEWLINE: 0176 return self.com_stmt(node[0]) 0177 0178 return Pass() 0179 0180 def file_input(self, nodelist): 0181 doc = self.get_docstring(nodelist, symbol.file_input) 0182 if doc is not None: 0183 i = 1 0184 else: 0185 i = 0 0186 stmts = [] 0187 for node in nodelist[i:]: 0188 if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: 0189 self.com_append_stmt(stmts, node) 0190 return Module(doc, Stmt(stmts)) 0191 0192 def eval_input(self, nodelist): 0193 # from the built-in function input() 0194 ### is this sufficient? 0195 return Expression(self.com_node(nodelist[0])) 0196 0197 def decorator_name(self, nodelist): 0198 listlen = len(nodelist) 0199 assert listlen >= 1 and listlen % 2 == 1 0200 0201 item = self.atom_name(nodelist) 0202 i = 1 0203 while i < listlen: 0204 assert nodelist[i][0] == token.DOT 0205 assert nodelist[i + 1][0] == token.NAME 0206 item = Getattr(item, nodelist[i + 1][1]) 0207 i += 2 0208 0209 return item 0210 0211 def decorator(self, nodelist): 0212 # '@' dotted_name [ '(' [arglist] ')' ] 0213 assert len(nodelist) in (3, 5, 6) 0214 assert nodelist[0][0] == token.AT 0215 assert nodelist[-1][0] == token.NEWLINE 0216 0217 assert nodelist[1][0] == symbol.dotted_name 0218 funcname = self.decorator_name(nodelist[1][1:]) 0219 0220 if len(nodelist) > 3: 0221 assert nodelist[2][0] == token.LPAR 0222 expr = self.com_call_function(funcname, nodelist[3]) 0223 else: 0224 expr = funcname 0225 0226 return expr 0227 0228 def decorators(self, nodelist): 0229 # decorators: decorator ([NEWLINE] decorator)* NEWLINE 0230 items = [] 0231 for dec_nodelist in nodelist: 0232 assert dec_nodelist[0] == symbol.decorator 0233 items.append(self.decorator(dec_nodelist[1:])) 0234 return Decorators(items) 0235 0236 def funcdef(self, nodelist): 0237 # -6 -5 -4 -3 -2 -1 0238 # funcdef: [decorators] 'def' NAME parameters ':' suite 0239 # parameters: '(' [varargslist] ')' 0240 0241 if len(nodelist) == 6: 0242 assert nodelist[0][0] == symbol.decorators 0243 decorators = self.decorators(nodelist[0][1:]) 0244 else: 0245 assert len(nodelist) == 5 0246 decorators = None 0247 0248 lineno = nodelist[-4][2] 0249 name = nodelist[-4][1] 0250 args = nodelist[-3][2] 0251 0252 if args[0] == symbol.varargslist: 0253 names, defaults, flags = self.com_arglist(args[1:]) 0254 else: 0255 names = defaults = () 0256 flags = 0 0257 doc = self.get_docstring(nodelist[-1]) 0258 0259 # code for function 0260 code = self.com_node(nodelist[-1]) 0261 0262 if doc is not None: 0263 assert isinstance(code, Stmt) 0264 assert isinstance(code.nodes[0], Discard) 0265 del code.nodes[0] 0266 return Function(decorators, name, names, defaults, flags, doc, code, 0267 lineno=lineno) 0268 0269 def lambdef(self, nodelist): 0270 # lambdef: 'lambda' [varargslist] ':' test 0271 if nodelist[2][0] == symbol.varargslist: 0272 names, defaults, flags = self.com_arglist(nodelist[2][1:]) 0273 else: 0274 names = defaults = () 0275 flags = 0 0276 0277 # code for lambda 0278 code = self.com_node(nodelist[-1]) 0279 0280 return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) 0281 0282 def classdef(self, nodelist): 0283 # classdef: 'class' NAME ['(' testlist ')'] ':' suite 0284 0285 name = nodelist[1][1] 0286 doc = self.get_docstring(nodelist[-1]) 0287 if nodelist[2][0] == token.COLON: 0288 bases = [] 0289 else: 0290 bases = self.com_bases(nodelist[3]) 0291 0292 # code for class 0293 code = self.com_node(nodelist[-1]) 0294 0295 if doc is not None: 0296 assert isinstance(code, Stmt) 0297 assert isinstance(code.nodes[0], Discard) 0298 del code.nodes[0] 0299 0300 return Class(name, bases, doc, code, lineno=nodelist[1][2]) 0301 0302 def stmt(self, nodelist): 0303 return self.com_stmt(nodelist[0]) 0304 0305 small_stmt = stmt 0306 flow_stmt = stmt 0307 compound_stmt = stmt 0308 0309 def simple_stmt(self, nodelist): 0310 # small_stmt (';' small_stmt)* [';'] NEWLINE 0311 stmts = [] 0312 for i in range(0, len(nodelist), 2): 0313 self.com_append_stmt(stmts, nodelist[i]) 0314 return Stmt(stmts) 0315 0316 def parameters(self, nodelist): 0317 raise WalkerEror 0318 0319 def varargslist(self, nodelist): 0320 raise WalkerEror 0321 0322 def fpdef(self, nodelist): 0323 raise WalkerEror 0324 0325 def fplist(self, nodelist): 0326 raise WalkerEror 0327 0328 def dotted_name(self, nodelist): 0329 raise WalkerEror 0330 0331 def comp_op(self, nodelist): 0332 raise WalkerEror 0333 0334 def trailer(self, nodelist): 0335 raise WalkerEror 0336 0337 def sliceop(self, nodelist): 0338 raise WalkerEror 0339 0340 def argument(self, nodelist): 0341 raise WalkerEror 0342 0343 # -------------------------------------------------------------- 0344 # 0345 # STATEMENT NODES (invoked by com_node()) 0346 # 0347 0348 def expr_stmt(self, nodelist): 0349 # augassign testlist | testlist ('=' testlist)* 0350 en = nodelist[-1] 0351 exprNode = self.lookup_node(en)(en[1:]) 0352 if len(nodelist) == 1: 0353 return Discard(exprNode, lineno=exprNode.lineno) 0354 if nodelist[1][0] == token.EQUAL: 0355 nodesl = [] 0356 for i in range(0, len(nodelist) - 2, 2): 0357 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) 0358 return Assign(nodesl, exprNode, lineno=nodelist[1][2]) 0359 else: 0360 lval = self.com_augassign(nodelist[0]) 0361 op = self.com_augassign_op(nodelist[1]) 0362 return AugAssign(lval, op[1], exprNode, lineno=op[2]) 0363 raise WalkerError, "can't get here" 0364 0365 def print_stmt(self, nodelist): 0366 # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) 0367 items = [] 0368 if len(nodelist) == 1: 0369 start = 1 0370 dest = None 0371 elif nodelist[1][0] == token.RIGHTSHIFT: 0372 assert len(nodelist) == 3 \ 0373 or nodelist[3][0] == token.COMMA 0374 dest = self.com_node(nodelist[2]) 0375 start = 4 0376 else: 0377 dest = None 0378 start = 1 0379 for i in range(start, len(nodelist), 2): 0380 items.append(self.com_node(nodelist[i])) 0381 if nodelist[-1][0] == token.COMMA: 0382 return Print(items, dest, lineno=nodelist[0][2]) 0383 return Printnl(items, dest, lineno=nodelist[0][2]) 0384 0385 def del_stmt(self, nodelist): 0386 return self.com_assign(nodelist[1], OP_DELETE) 0387 0388 def pass_stmt(self, nodelist): 0389 return Pass(lineno=nodelist[0][2]) 0390 0391 def break_stmt(self, nodelist): 0392 return Break(lineno=nodelist[0][2]) 0393 0394 def continue_stmt(self, nodelist): 0395 return Continue(lineno=nodelist[0][2]) 0396 0397 def return_stmt(self, nodelist): 0398 # return: [testlist] 0399 if len(nodelist) < 2: 0400 return Return(Const(None), lineno=nodelist[0][2]) 0401 return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 0402 0403 def yield_stmt(self, nodelist): 0404 return Yield(self.com_node(nodelist[1]), lineno=nodelist[0][2]) 0405 0406 def raise_stmt(self, nodelist): 0407 # raise: [test [',' test [',' test]]] 0408 if len(nodelist) > 5: 0409 expr3 = self.com_node(nodelist[5]) 0410 else: 0411 expr3 = None 0412 if len(nodelist) > 3: 0413 expr2 = self.com_node(nodelist[3]) 0414 else: 0415 expr2 = None 0416 if len(nodelist) > 1: 0417 expr1 = self.com_node(nodelist[1]) 0418 else: 0419 expr1 = None 0420 return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) 0421 0422 def import_stmt(self, nodelist): 0423 # import_stmt: import_name | import_from 0424 assert len(nodelist) == 1 0425 return self.com_node(nodelist[0]) 0426 0427 def import_name(self, nodelist): 0428 # import_name: 'import' dotted_as_names 0429 return Import(self.com_dotted_as_names(nodelist[1]), 0430 lineno=nodelist[0][2]) 0431 0432 def import_from(self, nodelist): 0433 # import_from: 'from' dotted_name 'import' ('*' | 0434 # '(' import_as_names ')' | import_as_names) 0435 assert nodelist[0][1] == 'from' 0436 assert nodelist[1][0] == symbol.dotted_name 0437 assert nodelist[2][1] == 'import' 0438 fromname = self.com_dotted_name(nodelist[1]) 0439 if nodelist[3][0] == token.STAR: 0440 # TODO(jhylton): where is the lineno? 0441 return From(fromname, [('*', None)]) 0442 else: 0443 node = nodelist[3 + (nodelist[3][0] == token.LPAR)] 0444 return From(fromname, self.com_import_as_names(node), 0445 lineno=nodelist[0][2]) 0446 0447 def global_stmt(self, nodelist): 0448 # global: NAME (',' NAME)* 0449 names = [] 0450 for i in range(1, len(nodelist), 2): 0451 names.append(nodelist[i][1]) 0452 return Global(names, lineno=nodelist[0][2]) 0453 0454 def exec_stmt(self, nodelist): 0455 # exec_stmt: 'exec' expr ['in' expr [',' expr]] 0456 expr1 = self.com_node(nodelist[1]) 0457 if len(nodelist) >= 4: 0458 expr2 = self.com_node(nodelist[3]) 0459 if len(nodelist) >= 6: 0460 expr3 = self.com_node(nodelist[5]) 0461 else: 0462 expr3 = None 0463 else: 0464 expr2 = expr3 = None 0465 0466 return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) 0467 0468 def assert_stmt(self, nodelist): 0469 # 'assert': test, [',' test] 0470 expr1 = self.com_node(nodelist[1]) 0471 if (len(nodelist) == 4): 0472 expr2 = self.com_node(nodelist[3]) 0473 else: 0474 expr2 = None 0475 return Assert(expr1, expr2, lineno=nodelist[0][2]) 0476 0477 def if_stmt(self, nodelist): 0478 # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 0479 tests = [] 0480 for i in range(0, len(nodelist) - 3, 4): 0481 testNode = self.com_node(nodelist[i + 1]) 0482 suiteNode = self.com_node(nodelist[i + 3]) 0483 tests.append((testNode, suiteNode)) 0484 0485 if len(nodelist) % 4 == 3: 0486 elseNode = self.com_node(nodelist[-1]) 0487 ## elseNode.lineno = nodelist[-1][1][2] 0488 else: 0489 elseNode = None 0490 return If(tests, elseNode, lineno=nodelist[0][2]) 0491 0492 def while_stmt(self, nodelist): 0493 # 'while' test ':' suite ['else' ':' suite] 0494 0495 testNode = self.com_node(nodelist[1]) 0496 bodyNode = self.com_node(nodelist[3]) 0497 0498 if len(nodelist) > 4: 0499 elseNode = self.com_node(nodelist[6]) 0500 else: 0501 elseNode = None 0502 0503 return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) 0504 0505 def for_stmt(self, nodelist): 0506 # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] 0507 0508 assignNode = self.com_assign(nodelist[1], OP_ASSIGN) 0509 listNode = self.com_node(nodelist[3]) 0510 bodyNode = self.com_node(nodelist[5]) 0511 0512 if len(nodelist) > 8: 0513 elseNode = self.com_node(nodelist[8]) 0514 else: 0515 elseNode = None 0516 0517 return For(assignNode, listNode, bodyNode, elseNode, 0518 lineno=nodelist[0][2]) 0519 0520 def try_stmt(self, nodelist): 0521 # 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] 0522 # | 'try' ':' suite 'finally' ':' suite 0523 if nodelist[3][0] != symbol.except_clause: 0524 return self.com_try_finally(nodelist) 0525 0526 return self.com_try_except(nodelist) 0527 0528 def suite(self, nodelist): 0529 # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT 0530 if len(nodelist) == 1: 0531 return self.com_stmt(nodelist[0]) 0532 0533 stmts = [] 0534 for node in nodelist: 0535 if node[0] == symbol.stmt: 0536 self.com_append_stmt(stmts, node) 0537 return Stmt(stmts) 0538 0539 # -------------------------------------------------------------- 0540 # 0541 # EXPRESSION NODES (invoked by com_node()) 0542 # 0543 0544 def testlist(self, nodelist): 0545 # testlist: expr (',' expr)* [','] 0546 # testlist_safe: test [(',' test)+ [',']] 0547 # exprlist: expr (',' expr)* [','] 0548 return self.com_binary(Tuple, nodelist) 0549 0550 testlist_safe = testlist # XXX 0551 testlist1 = testlist 0552 exprlist = testlist 0553 0554 def testlist_gexp(self, nodelist): 0555 if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: 0556 test = self.com_node(nodelist[0]) 0557 return self.com_generator_expression(test, nodelist[1]) 0558 return self.testlist(nodelist) 0559 0560 def test(self, nodelist): 0561 # and_test ('or' and_test)* | lambdef 0562 if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: 0563 return self.lambdef(nodelist[0]) 0564 return self.com_binary(Or, nodelist) 0565 0566 def and_test(self, nodelist): 0567 # not_test ('and' not_test)* 0568 return self.com_binary(And, nodelist) 0569 0570 def not_test(self, nodelist): 0571 # 'not' not_test | comparison 0572 result = self.com_node(nodelist[-1]) 0573 if len(nodelist) == 2: 0574 return Not(result, lineno=nodelist[0][2]) 0575 return result 0576 0577 def comparison(self, nodelist): 0578 # comparison: expr (comp_op expr)* 0579 node = self.com_node(nodelist[0]) 0580 if len(nodelist) == 1: 0581 return node 0582 0583 results = [] 0584 for i in range(2, len(nodelist), 2): 0585 nl = nodelist[i-1] 0586 0587 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 0588 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 0589 n = nl[1] 0590 if n[0] == token.NAME: 0591 type = n[1] 0592 if len(nl) == 3: 0593 if type == 'not': 0594 type = 'not in' 0595 else: 0596 type = 'is not' 0597 else: 0598 type = _cmp_types[n[0]] 0599 0600 lineno = nl[1][2] 0601 results.append((type, self.com_node(nodelist[i]))) 0602 0603 # we need a special "compare" node so that we can distinguish 0604 # 3 < x < 5 from (3 < x) < 5 0605 # the two have very different semantics and results (note that the 0606 # latter form is always true) 0607 0608 return Compare(node, results, lineno=lineno) 0609 0610 def expr(self, nodelist): 0611 # xor_expr ('|' xor_expr)* 0612 return self.com_binary(Bitor, nodelist) 0613 0614 def xor_expr(self, nodelist): 0615 # xor_expr ('^' xor_expr)* 0616 return self.com_binary(Bitxor, nodelist) 0617 0618 def and_expr(self, nodelist): 0619 # xor_expr ('&' xor_expr)* 0620 return self.com_binary(Bitand, nodelist) 0621 0622 def shift_expr(self, nodelist): 0623 # shift_expr ('<<'|'>>' shift_expr)* 0624 node = self.com_node(nodelist[0]) 0625 for i in range(2, len(nodelist), 2): 0626 right = self.com_node(nodelist[i]) 0627 if nodelist[i-1][0] == token.LEFTSHIFT: 0628 node = LeftShift([node, right], lineno=nodelist[1][2]) 0629 elif nodelist[i-1][0] == token.RIGHTSHIFT: 0630 node = RightShift([node, right], lineno=nodelist[1][2]) 0631 else: 0632 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 0633 return node 0634 0635 def arith_expr(self, nodelist): 0636 node = self.com_node(nodelist[0]) 0637 for i in range(2, len(nodelist), 2): 0638 right = self.com_node(nodelist[i]) 0639 if nodelist[i-1][0] == token.PLUS: 0640 node = Add([node, right], lineno=nodelist[1][2]) 0641 elif nodelist[i-1][0] == token.MINUS: 0642 node = Sub([node, right], lineno=nodelist[1][2]) 0643 else: 0644 raise ValueError, "unexpected token: %s" % nodelist[i-1][0] 0645 return node 0646 0647 def term(self, nodelist): 0648 node = self.com_node(nodelist[0]) 0649 for i in range(2, len(nodelist), 2): 0650 right = self.com_node(nodelist[i]) 0651 t = nodelist[i-1][0] 0652 if t == token.STAR: 0653 node = Mul([node, right]) 0654 elif t == token.SLASH: 0655 node = Div([node, right]) 0656 elif t == token.PERCENT: 0657 node = Mod([node, right]) 0658 elif t == token.DOUBLESLASH: 0659 node = FloorDiv([node, right]) 0660 else: 0661 raise ValueError, "unexpected token: %s" % t 0662 node.lineno = nodelist[1][2] 0663 return node 0664 0665 def factor(self, nodelist): 0666 elt = nodelist[0] 0667 t = elt[0] 0668 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) 0669 # need to handle (unary op)constant here... 0670 if t == token.PLUS: 0671 return UnaryAdd(node, lineno=elt[2]) 0672 elif t == token.MINUS: 0673 return UnarySub(node, lineno=elt[2]) 0674 elif t == token.TILDE: 0675 node = Invert(node, lineno=elt[2]) 0676 return node 0677 0678 def power(self, nodelist): 0679 # power: atom trailer* ('**' factor)* 0680 node = self.com_node(nodelist[0]) 0681 for i in range(1, len(nodelist)): 0682 elt = nodelist[i] 0683 if elt[0] == token.DOUBLESTAR: 0684 return Power([node, self.com_node(nodelist[i+1])], 0685 lineno=elt[2]) 0686 0687 node = self.com_apply_trailer(node, elt) 0688 0689 return node 0690 0691 def atom(self, nodelist): 0692 return self._atom_dispatch[nodelist[0][0]](nodelist) 0693 n.lineno = nodelist[0][2] 0694 return n 0695 0696 def atom_lpar(self, nodelist): 0697 if nodelist[1][0] == token.RPAR: 0698 return Tuple(()) 0699 return self.com_node(nodelist[1]) 0700 0701 def atom_lsqb(self, nodelist): 0702 if nodelist[1][0] == token.RSQB: 0703 return List(()) 0704 return self.com_list_constructor(nodelist[1]) 0705 0706 def atom_lbrace(self, nodelist): 0707 if nodelist[1][0] == token.RBRACE: 0708 return Dict(()) 0709 return self.com_dictmaker(nodelist[1]) 0710 0711 def atom_backquote(self, nodelist): 0712 return Backquote(self.com_node(nodelist[1])) 0713 0714 def atom_number(self, nodelist): 0715 ### need to verify this matches compile.c 0716 k = eval(nodelist[0][1]) 0717 return Const(k, lineno=nodelist[0][2]) 0718 0719 def decode_literal(self, lit): 0720 if self.encoding: 0721 # this is particularly fragile & a bit of a 0722 # hack... changes in compile.c:parsestr and 0723 # tokenizer.c must be reflected here. 0724 if self.encoding not in ['utf-8', 'iso-8859-1']: 0725 lit = unicode(lit, 'utf-8').encode(self.encoding) 0726 return eval("# coding: %s\n%s" % (self.encoding, lit)) 0727 else: 0728 return eval(lit) 0729 0730 def atom_string(self, nodelist): 0731 k = '' 0732 for node in nodelist: 0733 k += self.decode_literal(node[1]) 0734 return Const(k, lineno=nodelist[0][2]) 0735 0736 def atom_name(self, nodelist): 0737 return Name(nodelist[0][1], lineno=nodelist[0][2]) 0738 0739 # -------------------------------------------------------------- 0740 # 0741 # INTERNAL PARSING UTILITIES 0742 # 0743 0744 # The use of com_node() introduces a lot of extra stack frames, 0745 # enough to cause a stack overflow compiling test.test_parser with 0746 # the standard interpreter recursionlimit. The com_node() is a 0747 # convenience function that hides the dispatch details, but comes 0748 # at a very high cost. It is more efficient to dispatch directly 0749 # in the callers. In these cases, use lookup_node() and call the 0750 # dispatched node directly. 0751 0752 def lookup_node(self, node): 0753 return self._dispatch[node[0]] 0754 0755 _callers = {} 0756 0757 def com_node(self, node): 0758 # Note: compile.c has handling in com_node for del_stmt, pass_stmt, 0759 # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, 0760 # and compound_stmt. 0761 # We'll just dispatch them. 0762 return self._dispatch[node[0]](node[1:]) 0763 0764 def com_NEWLINE(self, *args): 0765 # A ';' at the end of a line can make a NEWLINE token appear 0766 # here, Render it harmless. (genc discards ('discard', 0767 # ('const', xxxx)) Nodes) 0768 return Discard(Const(None)) 0769 0770 def com_arglist(self, nodelist): 0771 # varargslist: 0772 # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) 0773 # | fpdef ['=' test] (',' fpdef ['=' test])* [','] 0774 # fpdef: NAME | '(' fplist ')' 0775 # fplist: fpdef (',' fpdef)* [','] 0776 names = [] 0777 defaults = [] 0778 flags = 0 0779 0780 i = 0 0781 while i < len(nodelist): 0782 node = nodelist[i] 0783 if node[0] == token.STAR or node[0] == token.DOUBLESTAR: 0784 if node[0] == token.STAR: 0785 node = nodelist[i+1] 0786 if node[0] == token.NAME: 0787 names.append(node[1]) 0788 flags = flags | CO_VARARGS 0789 i = i + 3 0790 0791 if i < len(nodelist): 0792 # should be DOUBLESTAR 0793 t = nodelist[i][0] 0794 if t == token.DOUBLESTAR: 0795 node = nodelist[i+1] 0796 else: 0797 raise ValueError, "unexpected token: %s" % t 0798 names.append(node[1]) 0799 flags = flags | CO_VARKEYWORDS 0800 0801 break 0802 0803 # fpdef: NAME | '(' fplist ')' 0804 names.append(self.com_fpdef(node)) 0805 0806 i = i + 1 0807 if i >= len(nodelist): 0808 break 0809 0810 if nodelist[i][0] == token.EQUAL: 0811 defaults.append(self.com_node(nodelist[i + 1])) 0812 i = i + 2 0813 elif len(defaults): 0814 # XXX This should be a syntax error. 0815 # Treat "(a=1, b)" as "(a=1, b=None)" 0816 defaults.append(Const(None)) 0817 0818 i = i + 1 0819 0820 return names, defaults, flags 0821 0822 def com_fpdef(self, node): 0823 # fpdef: NAME | '(' fplist ')' 0824 if node[1][0] == token.LPAR: 0825 return self.com_fplist(node[2]) 0826 return node[1][1] 0827 0828 def com_fplist(self, node): 0829 # fplist: fpdef (',' fpdef)* [','] 0830 if len(node) == 2: 0831 return self.com_fpdef(node[1]) 0832 list = [] 0833 for i in range(1, len(node), 2): 0834 list.append(self.com_fpdef(node[i])) 0835 return tuple(list) 0836 0837 def com_dotted_name(self, node): 0838 # String together the dotted names and return the string 0839 name = "" 0840 for n in node: 0841 if type(n) == type(()) and n[0] == 1: 0842 name = name + n[1] + '.' 0843 return name[:-1] 0844 0845 def com_dotted_as_name(self, node): 0846 assert node[0] == symbol.dotted_as_name 0847 node = node[1:] 0848 dot = self.com_dotted_name(node[0][1:]) 0849 if len(node) == 1: 0850 return dot, None 0851 assert node[1][1] == 'as' 0852 assert node[2][0] == token.NAME 0853 return dot, node[2][1] 0854 0855 def com_dotted_as_names(self, node): 0856 assert node[0] == symbol.dotted_as_names 0857 node = node[1:] 0858 names = [self.com_dotted_as_name(node[0])] 0859 for i in range(2, len(node), 2): 0860 names.append(self.com_dotted_as_name(node[i])) 0861 return names 0862 0863 def com_import_as_name(self, node): 0864 assert node[0] == symbol.import_as_name 0865 node = node[1:] 0866 assert node[0][0] == token.NAME 0867 if len(node) == 1: 0868 return node[0][1], None 0869 assert node[1][1] == 'as', node 0870 assert node[2][0] == token.NAME 0871 return node[0][1], node[2][1] 0872 0873 def com_import_as_names(self, node): 0874 assert node[0] == symbol.import_as_names 0875 node = node[1:] 0876 names = [self.com_import_as_name(node[0])] 0877 for i in range(2, len(node), 2): 0878 names.append(self.com_import_as_name(node[i])) 0879 return names 0880 0881 def com_bases(self, node): 0882 bases = [] 0883 for i in range(1, len(node), 2): 0884 bases.append(self.com_node(node[i])) 0885 return bases 0886 0887 def com_try_finally(self, nodelist): 0888 # try_fin_stmt: "try" ":" suite "finally" ":" suite 0889 return TryFinally(self.com_node(nodelist[2]), 0890 self.com_node(nodelist[5]), 0891 lineno=nodelist[0][2]) 0892 0893 def com_try_except(self, nodelist): 0894 # try_except: 'try' ':' suite (except_clause ':' suite)* ['else' suite] 0895 #tryexcept: [TryNode, [except_clauses], elseNode)] 0896 stmt = self.com_node(nodelist[2]) 0897 clauses = [] 0898 elseNode = None 0899 for i in range(3, len(nodelist), 3): 0900 node = nodelist[i] 0901 if node[0] == symbol.except_clause: 0902 # except_clause: 'except' [expr [',' expr]] */ 0903 if len(node) > 2: 0904 expr1 = self.com_node(node[2]) 0905 if len(node) > 4: 0906 expr2 = self.com_assign(node[4], OP_ASSIGN) 0907 else: 0908 expr2 = None 0909 else: 0910 expr1 = expr2 = None 0911 clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) 0912 0913 if node[0] == token.NAME: 0914 elseNode = self.com_node(nodelist[i+2]) 0915 return TryExcept(self.com_node(nodelist[2]), clauses, elseNode, 0916 lineno=nodelist[0][2]) 0917 0918 def com_augassign_op(self, node): 0919 assert node[0] == symbol.augassign 0920 return node[1] 0921 0922 def com_augassign(self, node): 0923 """Return node suitable for lvalue of augmented assignment 0924 0925 Names, slices, and attributes are the only allowable nodes. 0926 """ 0927 l = self.com_node(node) 0928 if l.__class__ in (Name, Slice, Subscript, Getattr): 0929 return l 0930 raise SyntaxError, "can't assign to %s" % l.__class__.__name__ 0931 0932 def com_assign(self, node, assigning): 0933 # return a node suitable for use as an "lvalue" 0934 # loop to avoid trivial recursion 0935 while 1: 0936 t = node[0] 0937 if t == symbol.exprlist or t == symbol.testlist or t == symbol.testlist_gexp: 0938 if len(node) > 2: 0939 return self.com_assign_tuple(node, assigning) 0940 node = node[1] 0941 elif t in _assign_types: 0942 if len(node) > 2: 0943 raise SyntaxError, "can't assign to operator" 0944 node = node[1] 0945 elif t == symbol.power: 0946 if node[1][0] != symbol.atom: 0947 raise SyntaxError, "can't assign to operator" 0948 if len(node) > 2: 0949 primary = self.com_node(node[1]) 0950 for i in range(2, len(node)-1): 0951 ch = node[i] 0952 if ch[0] == token.DOUBLESTAR: 0953 raise SyntaxError, "can't assign to operator" 0954 primary = self.com_apply_trailer(primary, ch) 0955 return self.com_assign_trailer(primary, node[-1], 0956 assigning) 0957 node = node[1] 0958 elif t == symbol.atom: 0959 t = node[1][0] 0960 if t == token.LPAR: 0961 node = node[2] 0962 if node[0] == token.RPAR: 0963 raise SyntaxError, "can't assign to ()" 0964 elif t == token.LSQB: 0965 node = node[2] 0966 if node[0] == token.RSQB: 0967 raise SyntaxError, "can't assign to []" 0968 return self.com_assign_list(node, assigning) 0969 elif t == token.NAME: 0970 return self.com_assign_name(node[1], assigning) 0971 else: 0972 raise SyntaxError, "can't assign to literal" 0973 else: 0974 raise SyntaxError, "bad assignment" 0975 0976 def com_assign_tuple(self, node, assigning): 0977 assigns = [] 0978 for i in range(1, len(node), 2): 0979 assigns.append(self.com_assign(node[i], assigning)) 0980 return AssTuple(assigns, lineno=extractLineNo(node)) 0981 0982 def com_assign_list(self, node, assigning): 0983 assigns = [] 0984 for i in range(1, len(node), 2): 0985 if i + 1 < len(node): 0986 if node[i + 1][0] == symbol.list_for: 0987 raise SyntaxError, "can't assign to list comprehension" 0988 assert node[i + 1][0] == token.COMMA, node[i + 1] 0989 assigns.append(self.com_assign(node[i], assigning)) 0990 return AssList(assigns, lineno=extractLineNo(node)) 0991 0992 def com_assign_name(self, node, assigning): 0993 return AssName(node[1], assigning, lineno=node[2]) 0994 0995 def com_assign_trailer(self, primary, node, assigning): 0996 t = node[1][0] 0997 if t == token.DOT: 0998 return self.com_assign_attr(primary, node[2], assigning) 0999 if t == token.LSQB: 1000 return self.com_subscriptlist(primary, node[2], assigning) 1001 if t == token.LPAR: 1002 raise SyntaxError, "can't assign to function call" 1003 raise SyntaxError, "unknown trailer type: %s" % t 1004 1005 def com_assign_attr(self, primary, node, assigning): 1006 return AssAttr(primary, node[1], assigning, lineno=node[-1]) 1007 1008 def com_binary(self, constructor, nodelist): 1009 "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." 1010 l = len(nodelist) 1011 if l == 1: 1012 n = nodelist[0] 1013 return self.lookup_node(n)(n[1:]) 1014 items = [] 1015 for i in range(0, l, 2): 1016 n = nodelist[i] 1017 items.append(self.lookup_node(n)(n[1:])) 1018 return constructor(items, lineno=extractLineNo(nodelist)) 1019 1020 def com_stmt(self, node): 1021 result = self.lookup_node(node)(node[1:]) 1022 assert result is not None 1023 if isinstance(result, Stmt): 1024 return result 1025 return Stmt([result]) 1026 1027 def com_append_stmt(self, stmts, node): 1028 result = self.lookup_node(node)(node[1:]) 1029 assert result is not None 1030 if isinstance(result, Stmt): 1031 stmts.extend(result.nodes) 1032 else: 1033 stmts.append(result) 1034 1035 if hasattr(symbol, 'list_for'): 1036 def com_list_constructor(self, nodelist): 1037 # listmaker: test ( list_for | (',' test)* [','] ) 1038 values = [] 1039 for i in range(1, len(nodelist)): 1040 if nodelist[i][0] == symbol.list_for: 1041 assert len(nodelist[i:]) == 1 1042 return self.com_list_comprehension(values[0], 1043 nodelist[i]) 1044 elif nodelist[i][0] == token.COMMA: 1045 continue 1046 values.append(self.com_node(nodelist[i])) 1047 return List(values, lineno=values[0].lineno) 1048 1049 def com_list_comprehension(self, expr, node): 1050 # list_iter: list_for | list_if 1051 # list_for: 'for' exprlist 'in' testlist [list_iter] 1052 # list_if: 'if' test [list_iter] 1053 1054 # XXX should raise SyntaxError for assignment 1055 1056 lineno = node[1][2] 1057 fors = [] 1058 while node: 1059 t = node[1][1] 1060 if t == 'for': 1061 assignNode = self.com_assign(node[2], OP_ASSIGN) 1062 listNode = self.com_node(node[4]) 1063 newfor = ListCompFor(assignNode, listNode, []) 1064 newfor.lineno = node[1][2] 1065 fors.append(newfor) 1066 if len(node) == 5: 1067 node = None 1068 else: 1069 node = self.com_list_iter(node[5]) 1070 elif t == 'if': 1071 test = self.com_node(node[2]) 1072 newif = ListCompIf(test, lineno=node[1][2]) 1073 newfor.ifs.append(newif) 1074 if len(node) == 3: 1075 node = None 1076 else: 1077 node = self.com_list_iter(node[3]) 1078 else: 1079 raise SyntaxError, \ 1080 ("unexpected list comprehension element: %s %d" 1081 % (node, lineno)) 1082 return ListComp(expr, fors, lineno=lineno) 1083 1084 def com_list_iter(self, node): 1085 assert node[0] == symbol.list_iter 1086 return node[1] 1087 else: 1088 def com_list_constructor(self, nodelist): 1089 values = [] 1090 for i in range(1, len(nodelist), 2): 1091 values.append(self.com_node(nodelist[i])) 1092 return List(values) 1093 1094 if hasattr(symbol, 'gen_for'): 1095 def com_generator_expression(self, expr, node): 1096 # gen_iter: gen_for | gen_if 1097 # gen_for: 'for' exprlist 'in' test [gen_iter] 1098 # gen_if: 'if' test [gen_iter] 1099 1100 lineno = node[1][2] 1101 fors = [] 1102 while node: 1103 t = node[1][1] 1104 if t == 'for': 1105 assignNode = self.com_assign(node[2], OP_ASSIGN) 1106 genNode = self.com_node(node[4]) 1107 newfor = GenExprFor(assignNode, genNode, [], 1108 lineno=node[1][2]) 1109 fors.append(newfor) 1110 if (len(node)) == 5: 1111 node = None 1112 else: 1113 node = self.com_gen_iter(node[5]) 1114 elif t == 'if': 1115 test = self.com_node(node[2]) 1116 newif = GenExprIf(test, lineno=node[1][2]) 1117 newfor.ifs.append(newif) 1118 if len(node) == 3: 1119 node = None 1120 else: 1121 node = self.com_gen_iter(node[3]) 1122 else: 1123 raise SyntaxError, \ 1124 ("unexpected generator expression element: %s %d" 1125 % (node, lineno)) 1126 fors[0].is_outmost = True 1127 return GenExpr(GenExprInner(expr, fors), lineno=lineno) 1128 1129 def com_gen_iter(self, node): 1130 assert node[0] == symbol.gen_iter 1131 return node[1] 1132 1133 def com_dictmaker(self, nodelist): 1134 # dictmaker: test ':' test (',' test ':' value)* [','] 1135 items = [] 1136 for i in range(1, len(nodelist), 4): 1137 items.append((self.com_node(nodelist[i]), 1138 self.com_node(nodelist[i+2]))) 1139 return Dict(items) 1140 1141 def com_apply_trailer(self, primaryNode, nodelist): 1142 t = nodelist[1][0] 1143 if t == token.LPAR: 1144 return self.com_call_function(primaryNode, nodelist[2]) 1145 if t == token.DOT: 1146 return self.com_select_member(primaryNode, nodelist[2]) 1147 if t == token.LSQB: 1148 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) 1149 1150 raise SyntaxError, 'unknown node type: %s' % t 1151 1152 def com_select_member(self, primaryNode, nodelist): 1153 if nodelist[0] != token.NAME: 1154 raise SyntaxError, "member must be a name" 1155 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) 1156 1157 def com_call_function(self, primaryNode, nodelist): 1158 if nodelist[0] == token.RPAR: 1159 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) 1160 args = [] 1161 kw = 0 1162 len_nodelist = len(nodelist) 1163 for i in range(1, len_nodelist, 2): 1164 node = nodelist[i] 1165 if node[0] == token.STAR or node[0] == token.DOUBLESTAR: 1166 break 1167 kw, result = self.com_argument(node, kw) 1168 1169 if len_nodelist != 2 and isinstance(result, GenExpr) \ 1170 and len(node) == 3 and node[2][0] == symbol.gen_for: 1171 # allow f(x for x in y), but reject f(x for x in y, 1) 1172 # should use f((x for x in y), 1) instead of f(x for x in y, 1) 1173 raise SyntaxError, 'generator expression needs parenthesis' 1174 1175 args.append(result) 1176 else: 1177 # No broken by star arg, so skip the last one we processed. 1178 i = i + 1 1179 if i < len_nodelist and nodelist[i][0] == token.COMMA: 1180 # need to accept an application that looks like "f(a, b,)" 1181 i = i + 1 1182 star_node = dstar_node = None 1183 while i < len_nodelist: 1184 tok = nodelist[i] 1185 ch = nodelist[i+1] 1186 i = i + 3 1187 if tok[0]==token.STAR: 1188 if star_node is not None: 1189 raise SyntaxError, 'already have the varargs indentifier' 1190 star_node = self.com_node(ch) 1191 elif tok[0]==token.DOUBLESTAR: 1192 if dstar_node is not None: 1193 raise SyntaxError, 'already have the kwargs indentifier' 1194 dstar_node = self.com_node(ch) 1195 else: 1196 raise SyntaxError, 'unknown node type: %s' % tok 1197 return CallFunc(primaryNode, args, star_node, dstar_node, 1198 lineno=extractLineNo(nodelist)) 1199 1200 def com_argument(self, nodelist, kw): 1201 if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for: 1202 test = self.com_node(nodelist[1]) 1203 return 0, self.com_generator_expression(test, nodelist[2]) 1204 if len(nodelist) == 2: 1205 if kw: 1206 raise SyntaxError, "non-keyword arg after keyword arg" 1207 return 0, self.com_node(nodelist[1]) 1208 result = self.com_node(nodelist[3]) 1209 n = nodelist[1] 1210 while len(n) == 2 and n[0] != token.NAME: 1211 n = n[1] 1212 if n[0] != token.NAME: 1213 raise SyntaxError, "keyword can't be an expression (%s)"%n[0] 1214 node = Keyword(n[1], result, lineno=n[2]) 1215 return 1, node 1216 1217 def com_subscriptlist(self, primary, nodelist, assigning): 1218 # slicing: simple_slicing | extended_slicing 1219 # simple_slicing: primary "[" short_slice "]" 1220 # extended_slicing: primary "[" slice_list "]" 1221 # slice_list: slice_item ("," slice_item)* [","] 1222 1223 # backwards compat slice for '[i:j]' 1224 if len(nodelist) == 2: 1225 sub = nodelist[1] 1226 if (sub[1][0] == token.COLON or \ 1227 (len(sub) > 2 and sub[2][0] == token.COLON)) and \ 1228 sub[-1][0] != symbol.sliceop: 1229 return self.com_slice(primary, sub, assigning) 1230 1231 subscripts = [] 1232 for i in range(1, len(nodelist), 2): 1233 subscripts.append(self.com_subscript(nodelist[i])) 1234 return Subscript(primary, assigning, subscripts, 1235 lineno=extractLineNo(nodelist)) 1236 1237 def com_subscript(self, node): 1238 # slice_item: expression | proper_slice | ellipsis 1239 ch = node[1] 1240 t = ch[0] 1241 if t == token.DOT and node[2][0] == token.DOT: 1242 return Ellipsis() 1243 if t == token.COLON or len(node) > 2: 1244 return self.com_sliceobj(node) 1245 return self.com_node(ch) 1246 1247 def com_sliceobj(self, node): 1248 # proper_slice: short_slice | long_slice 1249 # short_slice: [lower_bound] ":" [upper_bound] 1250 # long_slice: short_slice ":" [stride] 1251 # lower_bound: expression 1252 # upper_bound: expression 1253 # stride: expression 1254 # 1255 # Note: a stride may be further slicing... 1256 1257 items = [] 1258 1259 if node[1][0] == token.COLON: 1260 items.append(Const(None)) 1261 i = 2 1262 else: 1263 items.append(self.com_node(node[1])) 1264 # i == 2 is a COLON 1265 i = 3 1266 1267 if i < len(node) and node[i][0] == symbol.test: 1268 items.append(self.com_node(node[i])) 1269 i = i + 1 1270 else: 1271 items.append(Const(None)) 1272 1273 # a short_slice has been built. look for long_slice now by looking 1274 # for strides... 1275 for j in range(i, len(node)): 1276 ch = node[j] 1277 if len(ch) == 2: 1278 items.append(Const(None)) 1279 else: 1280 items.append(self.com_node(ch[2])) 1281 return Sliceobj(items, lineno=extractLineNo(node)) 1282 1283 def com_slice(self, primary, node, assigning): 1284 # short_slice: [lower_bound] ":" [upper_bound] 1285 lower = upper = None 1286 if len(node) == 3: 1287 if node[1][0] == token.COLON: 1288 upper = self.com_node(node[2]) 1289 else: 1290 lower = self.com_node(node[1]) 1291 elif len(node) == 4: 1292 lower = self.com_node(node[1]) 1293 upper = self.com_node(node[3]) 1294 return Slice(primary, assigning, lower, upper, 1295 lineno=extractLineNo(node)) 1296 1297 def get_docstring(self, node, n=None): 1298 if n is None: 1299 n = node[0] 1300 node = node[1:] 1301 if n == symbol.suite: 1302 if len(node) == 1: 1303 return self.get_docstring(node[0]) 1304 for sub in node: 1305 if sub[0] == symbol.stmt: 1306 return self.get_docstring(sub) 1307 return None 1308 if n == symbol.file_input: 1309 for sub in node: 1310 if sub[0] == symbol.stmt: 1311 return self.get_docstring(sub) 1312 return None 1313 if n == symbol.atom: 1314 if node[0][0] == token.STRING: 1315 s = '' 1316 for t in node: 1317 s = s + eval(t[1]) 1318 return s 1319 return None 1320 if n == symbol.stmt or n == symbol.simple_stmt \ 1321 or n == symbol.small_stmt: 1322 return self.get_docstring(node[0]) 1323 if n in _doc_nodes and len(node) == 1: 1324 return self.get_docstring(node[0]) 1325 return None 1326 1327 1328 _doc_nodes = [ 1329 symbol.expr_stmt, 1330 symbol.testlist, 1331 symbol.testlist_safe, 1332 symbol.test, 1333 symbol.and_test, 1334 symbol.not_test, 1335 symbol.comparison, 1336 symbol.expr, 1337 symbol.xor_expr, 1338 symbol.and_expr, 1339 symbol.shift_expr, 1340 symbol.arith_expr, 1341 symbol.term, 1342 symbol.factor, 1343 symbol.power, 1344 ] 1345 1346 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' 1347 # | 'in' | 'not' 'in' | 'is' | 'is' 'not' 1348 _cmp_types = { 1349 token.LESS : '<', 1350 token.GREATER : '>', 1351 token.EQEQUAL : '==', 1352 token.EQUAL : '==', 1353 token.LESSEQUAL : '<=', 1354 token.GREATEREQUAL : '>=', 1355 token.NOTEQUAL : '!=', 1356 } 1357 1358 _legal_node_types = [ 1359 symbol.funcdef, 1360 symbol.classdef, 1361 symbol.stmt, 1362 symbol.small_stmt, 1363 symbol.flow_stmt, 1364 symbol.simple_stmt, 1365 symbol.compound_stmt, 1366 symbol.expr_stmt, 1367 symbol.print_stmt, 1368 symbol.del_stmt, 1369 symbol.pass_stmt, 1370 symbol.break_stmt, 1371 symbol.continue_stmt, 1372 symbol.return_stmt, 1373 symbol.raise_stmt, 1374 symbol.import_stmt, 1375 symbol.global_stmt, 1376 symbol.exec_stmt, 1377 symbol.assert_stmt, 1378 symbol.if_stmt, 1379 symbol.while_stmt, 1380 symbol.for_stmt, 1381 symbol.try_stmt, 1382 symbol.suite, 1383 symbol.testlist, 1384 symbol.testlist_safe, 1385 symbol.test, 1386 symbol.and_test, 1387 symbol.not_test, 1388 symbol.comparison, 1389 symbol.exprlist, 1390 symbol.expr, 1391 symbol.xor_expr, 1392 symbol.and_expr, 1393 symbol.shift_expr, 1394 symbol.arith_expr, 1395 symbol.term, 1396 symbol.factor, 1397 symbol.power, 1398 symbol.atom, 1399 ] 1400 1401 if hasattr(symbol, 'yield_stmt'): 1402 _legal_node_types.append(symbol.yield_stmt) 1403 1404 _assign_types = [ 1405 symbol.test, 1406 symbol.and_test, 1407 symbol.not_test, 1408 symbol.comparison, 1409 symbol.expr, 1410 symbol.xor_expr, 1411 symbol.and_expr, 1412 symbol.shift_expr, 1413 symbol.arith_expr, 1414 symbol.term, 1415 symbol.factor, 1416 ] 1417 1418 import types 1419 _names = {} 1420 for k, v in symbol.sym_name.items(): 1421 _names[k] = v 1422 for k, v in token.tok_name.items(): 1423 _names[k] = v 1424 1425 def debug_tree(tree): 1426 l = [] 1427 for elt in tree: 1428 if type(elt) == types.IntType: 1429 l.append(_names.get(elt, elt)) 1430 elif type(elt) == types.StringType: 1431 l.append(elt) 1432 else: 1433 l.append(debug_tree(elt)) 1434 return l 1435
Generated by PyXR 0.9.4