def load_test_cases():
base_path = os.path.dirname(__file__)
test_case_path = os.path.join(base_path, "test_cases")
test_case_files = os.listdir(test_case_path)
test_cases = []
for fname in test_case_files:
if not fname.endswith(".py"):
continue
fullpath = os.path.join(test_case_path, fname)
data = open(fullpath).read()
tree = ast.parse(data, fullpath)
codes, messages = extract_expected_errors(data)
test_cases.append((tree, fullpath, codes, messages))
return test_cases
python类parse()的实例源码
def generate_pyast(code):
"""
Parses the code and creates a structure of lists and dicts only.
:param code: the code as a string
:return: a structure of lists and dicts only, representing the ast of code
"""
import ast
def transform_ast(code_ast):
if isinstance(code_ast, ast.AST):
# noinspection PyProtectedMember
node = {to_camelcase(k): transform_ast(getattr(code_ast, k)) for k in code_ast._fields}
node['node_type'] = to_camelcase(code_ast.__class__.__name__)
return node
elif isinstance(code_ast, list):
return [transform_ast(el) for el in code_ast]
else:
return code_ast
return transform_ast(ast.parse(code))
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def _extract_scripts_from_project(setup_filename='setup.py'):
"""Parse setup.py and return scripts"""
if not os.path.isfile(setup_filename):
return ''
mock_setup = textwrap.dedent('''\
def setup(*args, **kwargs):
__setup_calls__.append((args, kwargs))
''')
parsed_mock_setup = ast.parse(mock_setup, filename=setup_filename)
with open(setup_filename, 'rt') as setup_file:
parsed = ast.parse(setup_file.read())
for index, node in enumerate(parsed.body[:]):
if (not isinstance(node, ast.Expr) or
not isinstance(node.value, ast.Call) or
node.value.func.id != 'setup'):
continue
parsed.body[index:index] = parsed_mock_setup.body
break
fixed = ast.fix_missing_locations(parsed)
codeobj = compile(fixed, setup_filename, 'exec')
local_vars = {}
global_vars = {'__setup_calls__': []}
exec(codeobj, global_vars, local_vars)
_, kwargs = global_vars['__setup_calls__'][0]
return ','.join([os.path.basename(f) for f in kwargs.get('scripts', [])])
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def parse_other_functions(o, otherfuncs, _globals, sigs, external_contracts, origcode, runtime_only=False):
sub = ['seq', initializer_lll]
add_gas = initializer_lll.gas
for _def in otherfuncs:
sub.append(parse_func(_def, _globals, {**{'self': sigs}, **external_contracts}, origcode))
sub[-1].total_gas += add_gas
add_gas += 30
sig = FunctionSignature.from_definition(_def, external_contracts)
sig.gas = sub[-1].total_gas
sigs[sig.name] = sig
if runtime_only:
return sub
else:
o.append(['return', 0, ['lll', sub, 0]])
return o
# Main python parse tree => LLL method
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def get_decorators(cls):
decorators = {}
def visit_FunctionDef(node):
decorators[node.name] = []
for n in node.decorator_list:
name = ''
if isinstance(n, ast.Call):
name = n.func.attr if isinstance(n.func, ast.Attribute) else n.func.id
else:
name = n.attr if isinstance(n, ast.Attribute) else n.id
args = [a.s for a in n.args] if hasattr(n, 'args') else []
decorators[node.name].append((name, args))
node_iter = ast.NodeVisitor()
node_iter.visit_FunctionDef = visit_FunctionDef
_cls = cls if inspect.isclass(cls) else cls.__class__
node_iter.visit(ast.parse(inspect.getsource(_cls)))
return decorators
def get_local_vars(source, namespace):
# local_vars = sys._getframe(depth).f_locals
local_vars_names = set(namespace.keys())
root = ast.parse(source)
required_vars_names = set()
for node in ast.walk(root):
if isinstance(node, ast.Name):
required_vars_names.add(node.id)
builtin_vars_names = set(vars(builtins).keys())
required_local_vars = required_vars_names & local_vars_names
# we might want to add a compiler-ish thing in the future
params = {}
for v in required_local_vars:
params[v] = namespace[v]
return params
def search(func, depth=1):
local_vars = sys._getframe(depth).f_locals
source = get_source_code(func)
tree = ast.parse(source)
child_funcs = []
for node in ast.walk(tree):
if isinstance(node, ast.Call):
if isinstance(node.func, ast.Name):
child_funcs.append(node.func.id)
elif (isinstance(node, ast.Name) and node.id in local_vars and callable(local_vars[node.id]) and node.id not in sys.builtin_module_names):
child_funcs.append(node.id)
child_load_str = ''
for child in child_funcs:
if child in local_vars:
try:
load_string = search(local_vars[child], depth=(depth + 1))
child_load_str += load_string + '\n'
except Exception as e:
pass
load_str = child_load_str + source
return load_str
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def eval_expr(expr):
""" Eval and expression inside a #define using a suppart of python grammar """
def _eval(node):
if isinstance(node, ast.Num):
return node.n
elif isinstance(node, ast.BinOp):
return OPERATORS[type(node.op)](_eval(node.left), _eval(node.right))
elif isinstance(node, ast.UnaryOp):
return OPERATORS[type(node.op)](_eval(node.operand))
elif isinstance(node, ast.BoolOp):
values = [_eval(x) for x in node.values]
return OPERATORS[type(node.op)](**values)
else:
raise TypeError(node)
return _eval(ast.parse(expr, mode='eval').body)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def run_tests(source_state):
orig_code = source_state.code
source_state = test_code(source_state)
source_state.code = orig_code
args = eval(source_state.problem.arguments)
given_code = ast.parse(source_state.problem.given_code)
importNames = getAllImports(source_state.tree) + getAllImports(given_code)
inp = importNames + (list(args.keys()) if type(args) == dict else [])
given_names = [str(x) for x in inp]
imports = getAllImportStatements(source_state.tree) + getAllImportStatements(given_code)
if source_state.tree != None:
(cleaned_state, anon_state, canonical_state) = generate_states(source_state, given_names, imports)
save_states(source_state, cleaned_state, anon_state, canonical_state)
else:
source_state.save()
return source_state
def getMinimalChanges(changes, code, cutoff=None):
# Only do the power set if the # of changes is reasonable
if len(changes) < 8:
changesPowerSet = powerSet(changes)
elif len(changes) < 50:
changesPowerSet = fastPowerSet(changes)
else: # too large, we can't even do the fast power set because it will overwhelm memory
changesPowerSet = [changes]
sortList = list(map(lambda x : (x, sum(len(item.text) + len(item.newText) for item in x)), changesPowerSet))
if cutoff != None:
sortList = list(filter(lambda x : x[1] < cutoff, sortList))
sortList.sort(key=lambda x : x[1])
usedChange = combineSameLocationChanges(changes)
usedCode = applyChanges(code, usedChange)
for (change, l) in sortList:
change = combineSameLocationChanges(change)
tmpSource = applyChanges(code, change)
try:
ast.parse(tmpSource)
usedChange = change
usedCode = tmpSource
break
except:
pass
return (usedChange, usedCode)
def check_ast(argv=None):
files = added_files()
retval = 0
for filename in files:
try:
ast.parse(open(filename, 'rb').read(), filename=filename)
except SyntaxError:
print('{}: failed parsing with {} {}:'.format(
filename,
platform.python_implementation(),
sys.version.partition(' ')[0],
))
print('\n{}'.format(
' ' + traceback.format_exc().replace('\n', '\n '),
))
retval = 1
return retval
def evaluate(self, node, filename=None):
"""
Evaluate a source string or node, using ``filename`` when
displaying errors.
"""
if isinstance(node, string_types):
self.source = node
kwargs = {'mode': 'eval'}
if filename:
kwargs['filename'] = filename
try:
node = ast.parse(node, **kwargs)
except SyntaxError as e:
s = self.get_fragment(e.offset)
raise SyntaxError('syntax error %s' % s)
node_type = node.__class__.__name__.lower()
handler = self.get_handler(node_type)
if handler is None:
if self.source is None:
s = '(source not available)'
else:
s = self.get_fragment(node.col_offset)
raise SyntaxError("don't know how to evaluate %r %s" % (
node_type, s))
return handler(node)
def get_version():
with open(os.path.join('settei', 'version.py')) as f:
tree = ast.parse(f.read(), f.name)
for node in ast.walk(tree):
if not (isinstance(node, ast.Assign) and len(node.targets) == 1):
continue
target, = node.targets
value = node.value
if not (isinstance(target, ast.Name) and
target.id == 'VERSION_INFO' and
isinstance(value, ast.Tuple)):
continue
elts = value.elts
if any(not isinstance(elt, ast.Num) for elt in elts):
continue
return '.'.join(str(elt.n) for elt in elts)
def process(fl, external, genfiles, vendor):
src = open(fl).read()
tree = ast.parse(src, fl)
lst = []
wksp = WORKSPACE(external, genfiles, vendor)
for stmt in ast.walk(tree):
stmttype = type(stmt)
if stmttype == ast.Call:
fn = getattr(wksp, stmt.func.id, "")
if not callable(fn):
continue
path, name = keywords(stmt)
if path.endswith(".git"):
path = path[:-4]
path = pathmap.get(path, path)
tup = fn(name, path)
lst.append(tup)
return lst
def convert_py2rb(s, path='', base_path_count=0, modules=[], mod_paths={}):
"""
Takes Python code as a string 's' and converts this to Ruby.
Example:
>>> convert_py2rb("x[3:]")
'x[3..-1]'
"""
v = RB(path, base_path_count, mod_paths)
for m in modules:
t = ast.parse(m)
v.visit(t)
v.clear()
t = ast.parse(s)
v.visit(t)
return v.read()
def test_literal_num(self):
node = ast.parse("1234")
self.assertEqual("1234", self.fc.visit(node))
node = ast.parse("12.34")
self.assertEqual("12.34", self.fc.visit(node))
node = ast.parse("0x1ABCDEF")
self.assertEqual("28036591", self.fc.visit(node))
# G is not a valid hex digit and should not parse
self.assertRaises(SyntaxError, ast.parse, "0x1ABCDEFG")
node = ast.parse("0b11010101")
self.assertEqual("213", self.fc.visit(node))
# 8 is not a valid binary digit and should not parse
self.assertRaises(SyntaxError, ast.parse, "0b11010181")
def test_add(self):
node = ast.parse("31 + 57")
self.assertEqual("(31 + 57)", self.fc.visit(node))
node = ast.parse("31 + (-57)")
self.assertEqual("(31 + (-57))", self.fc.visit(node))
node = ast.parse("31 + x")
self.assertEqual("(31 + x)", self.fc.visit(node))
node = ast.parse("y + 57")
self.assertEqual("(y + 57)", self.fc.visit(node))
node = ast.parse("x + y")
self.assertEqual("(x + y)", self.fc.visit(node))
node = ast.parse("x + y + z")
self.assertEqual("((x + y) + z)", self.fc.visit(node))
node = ast.parse("x.i + y.i")
self.assertEqual("(x.i + y.i)", self.fc.visit(node))
node = ast.parse("x.a_method(a) + y.a_method(b)")
self.assertEqual("(x.a_method(a) + y.a_method(b))", self.fc.visit(node))
