def visit_For(self, node):
if isinstance(node.target, _ast.Name):
target = node.target.id
if isinstance(node.iter, _ast.Call) and isinstance(node.iter.func, _ast.Name):
if node.iter.func.id == "range":
return self.for_range(node, target)
else:
raise SyntaxError("GPUMAP: Only for ... in range(...) is supported!")
elif isinstance(node.iter, _ast.Name):
if node.iter.id in self.local_vars:
var_type = self.local_vars[node.iter.id]
if isinstance(var_type, str) and var_type.startswith("List<"):
list_type = var_type[var_type.find("<") + 1: var_type.rfind(">")]
return self.for_list(node, list_type, target)
elif isinstance(var_type, str) and var_type.startswith("List_Ptr<"):
list_type = var_type[var_type.find("<") + 1: var_type.rfind(">")]
return self.for_list(node, list_type, target, ptr=True)
else:
raise SyntaxError("GPUMAP: cannot iterate over a non-list type")
else:
raise SyntaxError("GPUMAP: no such variable found: " + node.iter.id)
else:
raise SyntaxError("GPUMAP: Only one variable can be assigned in a for loop!")
python类Name()的实例源码
def _transform_function_arguments(left):
if type(left) is ast.Name:
names = [left]
else:
names = left.elts
# Python3
if hasattr(_ast, 'arg'):
args = [_ast.arg(annotation=None, arg=name.id, col_offset = name.col_offset, lineno=name.lineno) for name in names]
return ast.arguments(args=args, defaults=[], kwonlyargs=[], kw_defaults=[])
# Python 2
arguments = ast.arguments(args=names, defaults=[])
for argument in arguments.args:
argument.ctx = ast.Param()
return arguments
def visit_Compare(self, node):
"""
:type node: _ast.FunctionDef
"""
is_lambda_def = len(node.ops) == 1\
and type(node.ops[0]) is _ast.Gt \
and (type(node.left) is _ast.Tuple or type(node.left) is _ast.Name) \
and all(map(lambda t: type(t) == _ast.Name, getattr(node.left, 'elts', [])))
if not is_lambda_def:
return node
arguments = _transform_function_arguments(node.left)
function_body = node.comparators[0]
lambda_ast_transform = ast.Lambda(args=arguments,
body=function_body,
lineno=node.lineno,
col_offset=node.col_offset)
return lambda_ast_transform
def _convert(node):
if isinstance(node, ast.Str):
return node.s
elif isinstance(node, ast.Num):
return node.n
elif isinstance(node, ast.Tuple):
return tuple(map(_convert, node.elts))
elif isinstance(node, ast.List):
return list(map(_convert, node.elts))
elif isinstance(node, ast.Dict):
return dict((_convert(k), _convert(v)) for k, v
in zip(node.keys, node.values))
elif isinstance(node, ast.Name):
if node.id in SAFE_CONSTANTS:
return SAFE_CONSTANTS[node.id]
raise ValueError('malformed or disallowed expression')
def convert_to_value(item):
if isinstance(item, ast.Str):
return item.s
elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes):
return item.s
elif isinstance(item, ast.Tuple):
return tuple(convert_to_value(i) for i in item.elts)
elif isinstance(item, ast.Num):
return item.n
elif isinstance(item, ast.Name):
result = VariableKey(item=item)
constants_lookup = {
'True': True,
'False': False,
'None': None,
}
return constants_lookup.get(
result.name,
result,
)
elif (not PY33) and isinstance(item, ast.NameConstant):
# None, True, False are nameconstants in python3, but names in 2
return item.value
else:
return UnhandledKeyType()
def NAME(self, node):
"""
Handle occurrence of Name (which can be a load/store/delete access.)
"""
# Locate the name in locals / function / globals scopes.
if isinstance(node.ctx, (ast.Load, ast.AugLoad)):
self.handleNodeLoad(node)
if (node.id == 'locals' and isinstance(self.scope, FunctionScope)
and isinstance(node.parent, ast.Call)):
# we are doing locals() call in current scope
self.scope.usesLocals = True
elif isinstance(node.ctx, (ast.Store, ast.AugStore)):
self.handleNodeStore(node)
elif isinstance(node.ctx, ast.Del):
self.handleNodeDelete(node)
else:
# must be a Param context -- this only happens for names in function
# arguments, but these aren't dispatched through here
raise RuntimeError("Got impossible expression context: %r" % (node.ctx,))
def convert_to_value(item):
if isinstance(item, ast.Str):
return item.s
elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes):
return item.s
elif isinstance(item, ast.Tuple):
return tuple(convert_to_value(i) for i in item.elts)
elif isinstance(item, ast.Num):
return item.n
elif isinstance(item, ast.Name):
result = VariableKey(item=item)
constants_lookup = {
'True': True,
'False': False,
'None': None,
}
return constants_lookup.get(
result.name,
result,
)
elif (not PY33) and isinstance(item, ast.NameConstant):
# None, True, False are nameconstants in python3, but names in 2
return item.value
else:
return UnhandledKeyType()
def NAME(self, node):
"""
Handle occurrence of Name (which can be a load/store/delete access.)
"""
# Locate the name in locals / function / globals scopes.
if isinstance(node.ctx, (ast.Load, ast.AugLoad)):
self.handleNodeLoad(node)
if (node.id == 'locals' and isinstance(self.scope, FunctionScope)
and isinstance(node.parent, ast.Call)):
# we are doing locals() call in current scope
self.scope.usesLocals = True
elif isinstance(node.ctx, (ast.Store, ast.AugStore)):
self.handleNodeStore(node)
elif isinstance(node.ctx, ast.Del):
self.handleNodeDelete(node)
else:
# must be a Param context -- this only happens for names in function
# arguments, but these aren't dispatched through here
raise RuntimeError("Got impossible expression context: %r" % (node.ctx,))
def make_const(arg, lineno=0, col_offset=0):
kw = {'lineno':lineno, 'col_offset':col_offset}
if isinstance(arg, str):
const = _ast.Str(s=arg, **kw)
elif isinstance(arg, (int, float, complex)):
const = _ast.Num(n=arg, **kw)
elif arg is None:
const = _ast.Name(id='None', ctx=_ast.Load(), **kw)
elif isinstance(arg, tuple):
elts = []
for item in arg:
elts.append(make_const(item, **kw))
const = _ast.Tuple(elts=elts, ctx=_ast.Load(), **kw)
else:
const = arg
return const
def decompile_func(func):
'''
Decompile a function into ast.FunctionDef node.
:param func: python function (can not be a built-in)
:return: ast.FunctionDef instance.
'''
if hasattr(func, 'func_code'):
code = func.func_code
else:
code = func.__code__
# For python 3
# defaults = func.func_defaults if sys.version_info.major < 3 else func.__defaults__
# if defaults:
# default_names = code.co_varnames[:code.co_argcount][-len(defaults):]
# else:
# default_names = []
# defaults = [_ast.Name(id='%s_default' % name, ctx=_ast.Load() , lineno=0, col_offset=0) for name in default_names]
ast_node = make_function(code, defaults=[], lineno=code.co_firstlineno)
return ast_node
def _(item, body=None, top=False):
new_body = []
if scan(item.body, _ast.Break):
body.extend(transform_break(item))
return True
elif scan(item.body, _ast.Continue):
body.extend(transform_continue(item))
return True
elif _transform(item.body, body=new_body, top=True):
item.body = new_body
body.append(item)
return True
elif not isinstance(item.iter, _ast.Name):
syntax = ast.parse('tmpfor{} = _'.format(randint(0, 2**32))).body[0]
syntax.value = item.iter
item.iter = syntax.targets[0]
body.extend((syntax, item))
return True
elif top:
body.append(item)
return False
def _transform_multiline_assignment_statements(statements):
assignment_statements = [statement for statement in statements
if type(statement) is _ast.BinOp
and type(statement.op) is _ast.LShift
and type(statement.left) is _ast.Name]
other_statements = [statement for statement in statements if statement not in assignment_statements]
assignments = [ast.Assign(targets=[statement.left], value=statement.right, lineno=statement.lineno, col_offset=statement.col_offset)
for statement in assignment_statements]
for assignment in assignments:
assignment.targets[0].ctx = ast.Store()
return other_statements + assignments
def _get_attr_qual_name(node, aliases):
'''Get a the full name for the attribute node.
This will resolve a pseudo-qualified name for the attribute
rooted at node as long as all the deeper nodes are Names or
Attributes. This will give you how the code referenced the name but
will not tell you what the name actually refers to. If we
encounter a node without a static name we punt with an
empty string. If this encounters something more complex, such as
foo.mylist[0](a,b) we just return empty string.
:param node: AST Name or Attribute node
:param aliases: Import aliases dictionary
:returns: Qualified name referred to by the attribute or name.
'''
if isinstance(node, _ast.Name):
if node.id in aliases:
return aliases[node.id]
return node.id
elif isinstance(node, _ast.Attribute):
name = '%s.%s' % (_get_attr_qual_name(node.value, aliases), node.attr)
if name in aliases:
return aliases[name]
return name
else:
return ""
def get_call_name(node, aliases):
if isinstance(node.func, _ast.Name):
if deepgetattr(node, 'func.id') in aliases:
return aliases[deepgetattr(node, 'func.id')]
return deepgetattr(node, 'func.id')
elif isinstance(node.func, _ast.Attribute):
return _get_attr_qual_name(node.func, aliases)
else:
return ""
def parse_assign_expr(ctx, tree, subparser):
if isinstance(tree, _ast.Assign):
assert len(tree.targets) == 1 and isinstance(tree.targets[0], _ast.Name)
return RowExprAssign(tree.targets[0].id, subparser(ctx, tree.value))
elif isinstance(tree, _ast.Expr):
if isinstance(tree.value, _ast.Name):
return RowExprAssign(tree.value.id, subparser(ctx, tree))
else:
raise Exception("Please assign expression to a variable")
else:
raise Exception("Please assign expression to a variable")
def EXCEPTHANDLER(self, node):
if PY2 or node.name is None:
self.handleChildren(node)
return
# 3.x: the name of the exception, which is not a Name node, but
# a simple string, creates a local that is only bound within the scope
# of the except: block.
for scope in self.scopeStack[::-1]:
if node.name in scope:
is_name_previously_defined = True
break
else:
is_name_previously_defined = False
self.handleNodeStore(node)
self.handleChildren(node)
if not is_name_previously_defined:
# See discussion on https://github.com/PyCQA/pyflakes/pull/59
# We're removing the local name since it's being unbound
# after leaving the except: block and it's always unbound
# if the except: block is never entered. This will cause an
# "undefined name" error raised if the checked code tries to
# use the name afterwards.
#
# Unless it's been removed already. Then do nothing.
try:
del self.scope[node.name]
except KeyError:
pass
def EXCEPTHANDLER(self, node):
if PY2 or node.name is None:
self.handleChildren(node)
return
# 3.x: the name of the exception, which is not a Name node, but
# a simple string, creates a local that is only bound within the scope
# of the except: block.
for scope in self.scopeStack[::-1]:
if node.name in scope:
is_name_previously_defined = True
break
else:
is_name_previously_defined = False
self.handleNodeStore(node)
self.handleChildren(node)
if not is_name_previously_defined:
# See discussion on https://github.com/PyCQA/pyflakes/pull/59
# We're removing the local name since it's being unbound
# after leaving the except: block and it's always unbound
# if the except: block is never entered. This will cause an
# "undefined name" error raised if the checked code tries to
# use the name afterwards.
#
# Unless it's been removed already. Then do nothing.
try:
del self.scope[node.name]
except KeyError:
pass
def pop_assignment(stmnts, name):
for i in range(len(stmnts)):
stmnt = stmnts[i]
if isinstance(stmnt, _ast.Assign) and len(stmnt.targets) == 1 \
and isinstance(stmnt.targets[0], _ast.Name) \
and isinstance(stmnt.targets[0].ctx, _ast.Store):
if stmnt.targets[0].id == name:
stmnts.pop(i)
return stmnt.value
return None
def isNone(node):
if node is None:
return True
elif isinstance(node, _ast.Name) and (node.id == 'None') and isinstance(node.ctx, _ast.Load):
return True
return False
def LOAD_NAME(self, instr):
name = _ast.Name(id=instr.arg, ctx=_ast.Load(), lineno=instr.lineno, col_offset=0)
self.ast_stack.append(name)
def LOAD_FAST(self, instr):
name = _ast.Name(id=instr.arg, ctx=_ast.Load(), lineno=instr.lineno, col_offset=0)
self.ast_stack.append(name)
def LOAD_GLOBAL(self, instr):
name = _ast.Name(id=instr.arg, ctx=_ast.Load(), lineno=instr.lineno, col_offset=0)
self.ast_stack.append(name)
def ROT_TWO(self, instr):
one = self.ast_stack.pop()
two = self.ast_stack.pop()
if self.ilst[0].opname == 'STORE_NAME':
kw = dict(lineno=instr.lineno, col_offset=0)
stores = []
while self.ilst[0].opname == 'STORE_NAME':
stores.append(self.ilst.pop(0))
assert len(stores) <= 3, stores
elts_load = [one, two]
if len(stores) == 3:
elts_load.insert(0, self.ast_stack.pop())
tup_load = _ast.Tuple(elts=elts_load[::-1], ctx=_ast.Load(), **kw)
elts_store = [_ast.Name(id=store.arg, ctx=_ast.Store(), **kw) for store in stores]
tup_store = _ast.Tuple(elts=elts_store, ctx=_ast.Store(), **kw)
assgn = _ast.Assign(value=tup_load, targets=[tup_store], **kw)
self.ast_stack.append(assgn)
# self.ast_stack.append(tup_store)
else:
self.ast_stack.append(one)
self.ast_stack.append(two)
def DELETE_NAME(self, instr):
name = _ast.Name(id=instr.arg, ctx=_ast.Del(), lineno=instr.lineno, col_offset=0)
delete = _ast.Delete(targets=[name], lineno=instr.lineno, col_offset=0)
self.ast_stack.append(delete)
def EXEC_STMT(self, instr):
locals_ = self.ast_stack.pop()
globals_ = self.ast_stack.pop()
expr = self.ast_stack.pop()
if locals_ is globals_:
locals_ = None
if isinstance(globals_, _ast.Name) and getattr(globals_, 'id',) == 'None':
globals_ = None
exec_ = _ast.Exec(body=expr, globals=globals_, locals=locals_, lineno=instr.lineno, col_offset=0)
self.ast_stack.append(exec_)
def _(item, body=None, top=False):
if top and len(item.args) != 0:
if isinstance(item.args[0], _ast.Call):
_transform(item.args[0], body=body, top=top)
if not isinstance(item.args[0], (_ast.Name, _ast.Num)):
syntax = ast.parse('tmpcall{} = b'.format(randint(0, 2**32))).body[0]
syntax.value = item.args[0]
item.args[0] = syntax.targets[0]
body.extend((syntax, item))
return True
return False
def visit_Assign(self, node):
if len(node.targets) > 1:
raise SyntaxError("GPUMAP: multiple assignment not supported")
target_types = map(lambda target: type(target), node.targets)
if tuple in target_types or list in target_types:
raise SyntaxError("GPUMAP: No unpacking allowed")
target = node.targets[0]
# assignment into object field
output = ""
value = self.visit(node.value)
# assignment into variable
if isinstance(target, _ast.Name):
# assignment into new variable
# not sure about the type just yet..
# see if it's a primitive
if target.id not in self.local_vars:
# binops and boolops return primitives
if isinstance(node.value, _ast.Num) or isinstance(node.value, _ast.Compare) or isinstance(node.value, _ast.BinOp) \
or isinstance(node.value, _ast.BoolOp) or isinstance(node.value, _ast.NameConstant):
output += "auto "
# check if referenced list contains primitives
elif isinstance(node.value, _ast.Subscript):
list_name = value[:value.find("[")]
try:
idx = self.func_repr.args.index(list_name)
t = self.func_repr.arg_types[idx]
item_type = t[t.find("<") + 1: t.find(">")]
if item_type in map(lambda t: t.__name__, primitive_map.keys()):
output += "auto "
else:
output += "auto&& "
except:
raise RuntimeError("THIS SHOULD NEVER HAPPEN")
else:
# check if it's an existing variable
try:
idx = self.func_repr.args.index(value)
t = self.func_repr.arg_types[idx]
if t in primitive_map:
output += "auto "
else:
output += "auto&& "
except ValueError:
output += "auto&& "
self.local_vars[target.id] = None
output += self.visit(target)
output += " = " + value
return output
def visit_FunctionDef(self, node):
"""
:type node: _ast.FunctionDef
"""
children = node.body
lambda_assign_children = [child for child in children
if type(child) == _ast.Assign
and len(child.targets) == 1
and type(child.value) == _ast.Compare
and (type(child.value.left) == _ast.Tuple or type(child.value.left) == _ast.Name)
and all(map(lambda t: type(t) == _ast.Name, getattr(child.value.left, 'elts', [])))]
# Support single line lambdas outside of assigns
other_children = [child for child in children if child not in lambda_assign_children]
for child in other_children:
CompareNodeVisitor().visit(child)
for assign_type_child in lambda_assign_children:
arguments = _transform_function_arguments(assign_type_child.value.left)
function_body = assign_type_child.value.comparators[0]
if _is_multiline_lambda(function_body):
all_statements = function_body.elts
return_statement = all_statements[-1]
statements = all_statements[0:len(all_statements) - 1]
statements = _transform_multiline_assignment_statements(statements)
return_statement = _transform_multiline_return_statement(return_statement)
assign_target = assign_type_child.targets[0]
if type(assign_target) is _ast.Attribute:
function_name = assign_target.attr
else:
function_name = assign_target.id
all_transformed_statements = statements + [return_statement]
functiondef_object = ast.FunctionDef(args = arguments,
body=all_transformed_statements,
lineno=assign_type_child.lineno,
name=function_name,
col_offset=assign_type_child.col_offset,
decorator_list=[])
children.insert(0, functiondef_object)
assign_type_child.value = ast.Name(id=functiondef_object.name,
col_offset=functiondef_object.col_offset,
lineno=functiondef_object.lineno,
ctx=ast.Load())
else:
lambda_ast_transform = ast.Lambda(args=arguments,
body=function_body,
lineno=assign_type_child.lineno,
col_offset = assign_type_child.col_offset)
assign_type_child.value = lambda_ast_transform
return node
def _get_literal_value(self, literal):
'''Utility function to turn AST literals into native Python types
:param literal: The AST literal to convert
:return: The value of the AST literal
'''
if isinstance(literal, _ast.Num):
literal_value = literal.n
elif isinstance(literal, _ast.Str):
literal_value = literal.s
elif isinstance(literal, _ast.List):
return_list = list()
for li in literal.elts:
return_list.append(self._get_literal_value(li))
literal_value = return_list
elif isinstance(literal, _ast.Tuple):
return_tuple = tuple()
for ti in literal.elts:
return_tuple = return_tuple + (self._get_literal_value(ti),)
literal_value = return_tuple
elif isinstance(literal, _ast.Set):
return_set = set()
for si in literal.elts:
return_set.add(self._get_literal_value(si))
literal_value = return_set
elif isinstance(literal, _ast.Dict):
literal_value = dict(zip(literal.keys, literal.values))
elif isinstance(literal, _ast.Ellipsis):
# what do we want to do with this?
literal_value = None
elif isinstance(literal, _ast.Name):
literal_value = literal.id
# NOTE(sigmavirus24): NameConstants are only part of the AST in Python
# 3. NameConstants tend to refer to things like True and False. This
# prevents them from being re-assigned in Python 3.
elif six.PY3 and isinstance(literal, _ast.NameConstant):
literal_value = str(literal.value)
# NOTE(sigmavirus24): Bytes are only part of the AST in Python 3
elif six.PY3 and isinstance(literal, _ast.Bytes):
literal_value = literal.s
else:
literal_value = None
return literal_value
def awk_filter_map(data_desc, filter_strs, map_strs):
"""
>>> from tabkit.header import parse_header
>>> awk, desc = awk_filter_map(
... parse_header('# d p e s c m'),
... ['e==157 and (s>100 or s in [15,30,45])'],
... ['ctr=c/s', 'cpm=ctr*m']
... )
>>> print desc
DataDesc([DataField('ctr', 'any'), DataField('cpm', 'any')])
>>> print awk.cmd_line()
LC_ALL=C awk -F $'\\t' 'BEGIN{OFS="\\t";}{if((($3 == 157) && (($4 > 100) || (($4 == 15) || ($4 == 30) || ($4 == 45))))){ctr = ($5 / $4);print(ctr,(ctr * $6));}}'
>>> awk, desc = awk_filter_map(parse_header('# a b'), [], ['__all__'])
>>> print desc
DataDesc([DataField('a', 'any'), DataField('b', 'any')])
"""
ctx = ExprContext(data_desc)
# parse map
for map_expr_str in map_strs:
for node in parse(map_expr_str).body:
if isinstance(node, _ast.Expr) and isinstance(node.value, _ast.Name) and node.value.id == '__all__':
for field in data_desc.fields:
ctx.set_var(field.name, RowExprAssign(field.name, RowExprField(ctx, field.name)))
elif isinstance(node, _ast.Expr) and isinstance(node.value, _ast.Name) and node.value.id == '__rest__':
for field in data_desc.fields:
if not ctx.has_var(field.name):
ctx.set_var(field.name, RowExprAssign(field.name, RowExprField(ctx, field.name)))
else:
expr = parse_rowexpr(ctx, node)
ctx.set_var(expr.target, expr)
# parse filter
nodes = [node for filter_str in filter_strs for node in parse(filter_str).body]
filter_expr = None
if len(nodes) == 0:
pass
elif len(nodes) == 1:
filter_expr = parse_expr(ctx, nodes[0])
else:
filter_expr = RowExprOp('&&', [parse_expr(ctx, node) for node in nodes])
awk_cmd, output_desc = awk_filter_map_from_context(ctx, filter_expr, data_desc.order)
if output_desc:
output_desc.meta = data_desc.meta
return awk_cmd, output_desc or data_desc
