def CheckedEval(file_contents):
"""Return the eval of a gyp file.
The gyp file is restricted to dictionaries and lists only, and
repeated keys are not allowed.
Note that this is slower than eval() is.
"""
ast = compiler.parse(file_contents)
assert isinstance(ast, Module)
c1 = ast.getChildren()
assert c1[0] is None
assert isinstance(c1[1], Stmt)
c2 = c1[1].getChildren()
assert isinstance(c2[0], Discard)
c3 = c2[0].getChildren()
assert len(c3) == 1
return CheckNode(c3[0], [])
python类ast()的实例源码
def CheckedEval(file_contents):
"""Return the eval of a gyp file.
The gyp file is restricted to dictionaries and lists only, and
repeated keys are not allowed.
Note that this is slower than eval() is.
"""
ast = compiler.parse(file_contents)
assert isinstance(ast, Module)
c1 = ast.getChildren()
assert c1[0] is None
assert isinstance(c1[1], Stmt)
c2 = c1[1].getChildren()
assert isinstance(c2[0], Discard)
c3 = c2[0].getChildren()
assert len(c3) == 1
return CheckNode(c3[0], [])
def visit_Stmt(self, node):
def _check_del(n):
# del x is just AssName('x', 'OP_DELETE')
# we want to transform it to Delete([Name('x', Del())])
dcls = (_ast.Name, _ast.List, _ast.Subscript, _ast.Attribute)
if isinstance(n, dcls) and isinstance(n.ctx, _ast.Del):
return self._new(_ast.Delete, [n])
elif isinstance(n, _ast.Tuple) and isinstance(n.ctx, _ast.Del):
# unpack last tuple to avoid making del (x, y, z,);
# out of del x, y, z; (there's no difference between
# this two in compiler.ast)
return self._new(_ast.Delete, n.elts)
else:
return n
def _keep(n):
if isinstance(n, _ast.Expr) and n.value is None:
return False
else:
return True
return [s for s in [_check_del(self.visit(n)) for n in node.nodes]
if _keep(s)]
def CheckedEval(file_contents):
"""Return the eval of a gyp file.
The gyp file is restricted to dictionaries and lists only, and
repeated keys are not allowed.
Note that this is slower than eval() is.
"""
ast = compiler.parse(file_contents)
assert isinstance(ast, Module)
c1 = ast.getChildren()
assert c1[0] is None
assert isinstance(c1[1], Stmt)
c2 = c1[1].getChildren()
assert isinstance(c2[0], Discard)
c3 = c2[0].getChildren()
assert len(c3) == 1
return CheckNode(c3[0], [])
def walk(self, ast):
"Validate each node in AST and return True if AST is 'safe'."
self.visit(ast)
return self.errors == []
def get_args(val):
d = {}
args = ast.parse("d(" + val + ")", mode='eval').body.args
i = 1
for arg in args:
if isinstance(arg, ast.Name):
d[str(i)] = literal_eval(arg.id)
else:
d[str(i)] = literal_eval(arg)
i += 1
return d
def get_kwargs(val):
d = {}
args = ast.parse("d(" + val + ")", mode='eval').body.keywords
for arg in args:
d[arg.arg] = literal_eval(arg.value)
return d
def parse_to_list(val):
values = ast.parse("[" + val + "]", mode='eval').body.elts
return [literal_eval(v) for v in values]
def literal_eval(node_or_string):
"""
Safely evaluate an expression node or a string containing a Python
expression. The string or node provided may only consist of the
following Python literal structures: strings, numbers, tuples,
lists, dicts, booleans, and None.
"""
_safe_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, basestring):
node_or_string = compiler.parse(node_or_string, mode='eval')
if isinstance(node_or_string, Expression):
node_or_string = node_or_string.node
def _convert(node):
if isinstance(node, Const) and isinstance(node.value,
(basestring, int, float, long, complex)):
return node.value
elif isinstance(node, Tuple):
return tuple(map(_convert, node.nodes))
elif isinstance(node, compiler.ast.List):
return list(map(_convert, node.nodes))
elif isinstance(node, Dict):
return dict((_convert(k), _convert(v)) for k, v
in node.items)
elif isinstance(node, Name):
if node.name in _safe_names:
return _safe_names[node.name]
elif isinstance(node, UnarySub):
return -_convert(node.expr)
raise ValueError('malformed string')
return _convert(node_or_string)
def __init__(self, indent, lineno, body, final):
Node.__init__(self, indent, lineno)
if isinstance(body, compiler.ast.TryExcept):
self.body = transform(indent, lineno, body)
self.body.has_finally = True
else:
self.body = transform(indent + 1, lineno, body)
self.final = transform(indent + 1, lineno, final)
return
def _extract_args(self, node):
tab = node.argnames[:]
if node.flags & compiler.ast.CO_VARKEYWORDS:
kwarg = tab[-1]
tab = tab[:-1]
else:
kwarg = None
if node.flags & compiler.ast.CO_VARARGS:
vararg = tab[-1]
tab = tab[:-1]
else:
vararg = None
def _tup(t):
if isinstance(t, str):
return self._new(_ast.Name, t, _ast.Store())
elif isinstance(t, tuple):
elts = [_tup(x) for x in t]
return self._new(_ast.Tuple, elts, _ast.Store())
else:
raise NotImplemented
args = []
for arg in tab:
if isinstance(arg, str):
args.append(self._new(_ast.Name, arg, _ast.Param()))
elif isinstance(arg, tuple):
args.append(_tup(arg))
else:
assert False, node.__class__
defaults = [self.visit(d) for d in node.defaults]
return self._new(_ast.arguments, args, vararg, kwarg, defaults)
def visit_CallFunc(self, node):
args = []
keywords = []
for arg in node.args:
if isinstance(arg, compiler.ast.Keyword):
keywords.append(self._new(_ast.keyword, arg.name,
self.visit(arg.expr)))
else:
args.append(self.visit(arg))
return self._new(_ast.Call, self.visit(node.node), args, keywords,
self.visit(node.star_args), self.visit(node.dstar_args))
def find_imports(fn, verbose, ignores):
"Yields a list of the module names the file 'fn' depends on."
ast, _ = parse_python_source(fn)
if ast is None:
raise StopIteration
found_imports = get_ast_imports(ast)
if found_imports is None:
raise StopIteration
dn = dirname(fn)
packroot = None
for modname, rname, lname, lineno, _, _ in found_imports:
islocal = False
names = modname.split('.')
if find_dotted(names, dn):
# This is a local import, we need to find the root in order to
# compute the absolute module name.
if packroot is None:
packroot = find_package_root(fn, ignores)
if not packroot:
logging.warning(
"%d: Could not find package root for local import '%s' from '%s'." %
(lineno, modname, fn))
continue
reldir = dirname(fn)[len(packroot)+1:]
modname = '%s.%s' % (reldir.replace(os.sep, '.'), modname)
islocal = True
if rname is not None:
modname = '%s.%s' % (modname, rname)
yield (modname, lineno, islocal)
def parse_python_source(fn):
"""Parse the file 'fn' and return two things:
1. The AST tree.
2. A list of lines of the source line (typically used for verbose error
messages).
If the file has a syntax error in it, the first argument will be None.
"""
# Read the file's contents to return it.
# Note: we make sure to use universal newlines.
try:
contents = open(fn, 'rU').read()
lines = contents.splitlines()
except (IOError, OSError), e:
logging.error("Could not read file '%s'." % fn)
return None, None
# Convert the file to an AST.
try:
ast = compiler.parse(contents)
except SyntaxError, e:
err = '%s:%s: %s' % (fn, e.lineno or '--', e.msg)
logging.error("Error processing file '%s':\n%s" %
(fn, err))
return None, lines
except TypeError, e:
# Note: this branch untested, applied from a user-submitted patch.
err = '%s: %s' % (fn, str(e))
logging.error("Error processing file '%s':\n%s" %
(fn, err))
return None, lines
return ast, lines
def get_ast_imports(ast):
"""
Given an AST, return a list of module tuples for the imports found, in the
form:
(modname, remote-name, local-name, lineno, pragma)
"""
assert ast is not None
vis = ImportVisitor()
compiler.walk(ast, vis, ImportWalker(vis))
found_imports = vis.finalize()
return found_imports
# **WARNING** This is where all the evil lies. Risk and peril. Watch out.
def printAst(ast, indent=' ', stream=sys.stdout, initlevel=0):
"Pretty-print an AST to the given output stream."
rec_node(ast, initlevel, indent, stream.write)
stream.write('\n')
def safe_eval(code, context = {}, timeout_secs = 5):
"""
Validate source code and make sure it contains no unauthorized
expression/statements as configured via 'unallowed_ast_nodes' and
'unallowed_builtins'. By default this means that code is not
allowed import modules or access dangerous builtins like 'open' or
'eval'. If code is considered 'safe' it will be executed via
'exec' using 'context' as the global environment. More details on
how code is executed can be found in the Python Reference Manual
section 6.14 (ignore the remark on '__builtins__'). The 'context'
enviroment is also validated and is not allowed to contain modules
or builtins. The following exception will be raised on errors:
if 'context' contains unallowed objects =
SafeEvalContextException
if code is didn't validate and is considered 'unsafe' =
SafeEvalCodeException
if code did not execute within the given timelimit =
SafeEvalTimeoutException
"""
ctx_errkeys, ctx_errors = [], []
for (key, obj) in context.items():
if inspect.isbuiltin(obj):
ctx_errkeys.append(key)
ctx_errors.append("key '%s' : unallowed builtin %s" % (key, obj))
if inspect.ismodule(obj):
ctx_errkeys.append(key)
ctx_errors.append("key '%s' : unallowed module %s" % (key, obj))
if ctx_errors:
raise SafeEvalContextException(ctx_errkeys, ctx_errors)
ast = compiler.parse(code)
checker = SafeEvalVisitor()
if checker.walk(ast):
exec_timed(code, context, timeout_secs)
else:
raise SafeEvalCodeException(code, checker.errors)
#----------------------------------------------------------------------
# Basic tests.
#----------------------------------------------------------------------
def find_dependencies(fn, verbose, process_pragmas,
ignore_unused=False,
warning_lambda=logging.warning,
debug_lambda=logging.debug):
"Returns a list of the files 'fn' depends on."
file_errors = []
ast, _ = parse_python_source(fn)
if ast is None:
return [], file_errors
found_imports = get_ast_imports(ast)
if found_imports is None:
return [], file_errors
# Filter out the unused imports if requested.
if ignore_unused:
found_imports, unused_imports = filter_unused_imports(ast, found_imports)
for modname, rname, lname, lineno, level, pragma in unused_imports:
file_errors.append((ERROR_UNUSED, lname))
output_code = (verbose >= 2)
source_lines = None
if output_code:
source_lines = open(fn, 'rU').read().splitlines()
files = []
assert not isdir(fn)
dn = dirname(fn)
seenset = set()
for x in found_imports:
mod, rname, lname, lineno, level, pragma = x
if process_pragmas and pragma == 'OPTIONAL':
if rname is None:
msg = WARNING_OPTIONAL % (lineno, mod)
else:
msg = '%s.%s' % (mod, rname)
logging.warning(msg)
continue
sig = (mod, rname)
if sig in seenset:
continue
seenset.add(sig)
modfile, errors = find_dotted_module(mod, rname, dn, level)
if errors:
file_errors.extend(errors)
for err, name in errors:
if err is ERROR_IMPORT:
efun = warning_lambda
else:
efun = debug_lambda
efun(err % (lineno, name))
if output_code:
efun(ERROR_SOURCE % source_lines[lineno-1].rstrip())
if modfile is None:
continue
files.append(realpath(modfile))
return files, file_errors
