def pack_logging_topics(event_id, args, expected_topics, context):
topics = [event_id]
for pos, expected_topic in enumerate(expected_topics):
typ = expected_topic.typ
arg = args[pos]
value = parse_expr(arg, context)
if isinstance(typ, ByteArrayType) and (isinstance(arg, ast.Str) or (isinstance(arg, ast.Name) and arg.id not in reserved_words)):
if value.typ.maxlen > typ.maxlen:
raise TypeMismatchException("Topic input bytes are to big: %r %r" % (value.typ, typ))
if isinstance(arg, ast.Str):
bytez, bytez_length = string_to_bytes(arg.s)
if len(bytez) > 32:
raise InvalidLiteralException("Can only log a maximum of 32 bytes at a time.")
topics.append(bytes_to_int(bytez + b'\x00' * (32 - bytez_length)))
else:
size = context.vars[arg.id].size
topics.append(byte_array_to_num(value, arg, 'num256', size))
else:
value = unwrap_location(value)
value = base_type_conversion(value, value.typ, typ)
topics.append(value)
return topics
python类Str()的实例源码
def __init__(self, expr, context):
self.expr = expr
self.context = context
self.expr_table = {
LLLnode: self.get_expr,
ast.Num: self.number,
ast.Str: self.string,
ast.NameConstant: self.constants,
ast.Name: self.variables,
ast.Attribute: self.attribute,
ast.Subscript: self.subscript,
ast.BinOp: self.arithmetic,
ast.Compare: self.compare,
ast.BoolOp: self.boolean_operations,
ast.UnaryOp: self.unary_operations,
ast.Call: self.call,
ast.List: self.list_literals,
ast.Dict: self.struct_literals,
ast.Tuple: self.tuple_literals,
}
expr_type = self.expr.__class__
if expr_type in self.expr_table:
self.lll_node = self.expr_table[expr_type]()
else:
raise Exception("Unsupported operator: %r" % ast.dump(self.expr))
def pop_format_context(self, expl_expr):
"""Format the %-formatted string with current format context.
The expl_expr should be an ast.Str instance constructed from
the %-placeholders created by .explanation_param(). This will
add the required code to format said string to .on_failure and
return the ast.Name instance of the formatted string.
"""
current = self.stack.pop()
if self.stack:
self.explanation_specifiers = self.stack[-1]
keys = [ast.Str(key) for key in current.keys()]
format_dict = ast.Dict(keys, list(current.values()))
form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
name = "@py_format" + str(next(self.variable_counter))
self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
return ast.Name(name, ast.Load())
def visit_BinOp(self, node):
node = self.generic_visit(node)
left = node.left
right = node.right
if all(isinstance(value, ast.Num) for value in (left, right)):
if isinstance(node.op, tuple(self._operators.keys())):
val = self._operators[type(node.op)](left.n, right.n)
node = ast.copy_location(ast.Num(n = val), node)
return node
elif all(isinstance(value, ast.Str) for value in (left, right)):
if isinstance(node.op, ast.Add):
val = left.s + right.s
node = ast.copy_location(ast.Str(s = val), node)
return node
return self.fold(node)
#def visit_GeneratorExp(self, node):
# return self.comprehension(node)
def visit_Module(self, node):
for expr in node.body:
if not isinstance(expr, ast.Assign):
continue
if not isinstance(expr.value, (ast.Num, ast.Str)):
continue
if len(expr.targets) != 1:
continue
name = expr.targets[0]
if not isinstance(name, ast.Name):
continue
name = name.id
if not self.is_const_name(name):
continue
if name in self._constants:
self._constants[name] = None
else:
self._constants[name] = expr.value
return self.generic_visit(node)
def basicTypeSpecialFunction(cv):
"""If you're in a number or string (which has no metadata), move up to the AST to make the special functions work."""
if isinstance(cv, SwapVector) or isinstance(cv, MoveVector):
return cv
if (cv.path[0] in [('n', 'Number'), ('s', 'String'), ('id', 'Name'), ('arg', 'Argument'),
('value', 'Name Constant'), ('s', 'Bytes'), ('name', 'Alias')]):
cvCopy = cv.deepcopy()
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
if cv.path[0] == ('n', 'Number'):
cv.newSubtree = ast.Num(cv.newSubtree)
elif cv.path[0] == ('s', 'String'):
cv.newSubtree = ast.Str(cv.newSubtree)
elif cv.path[0] == ('id', 'Name'):
cv.newSubtree = ast.Name(cv.newSubtree, cv.oldSubtree.ctx)
elif cv.path[0] == ('arg', 'Argument'):
cv.newSubtree = ast.arg(cv.newSubtree, cv.oldSubtree.annotation)
elif cv.path[0] == ('value', 'Name Constant'):
cv.newSubtree = ast.NameConstant(cv.newSubtree)
elif cv.path[0] == ('s', 'Bytes'):
cv.newSubtree = ast.Bytes(cv.newSubtree)
elif cv.path[0] == ('name', 'Alias'):
cv.newSubtree = ast.alias(cv.newSubtree, cv.oldSubtree.asname)
cv.path = cv.path[1:]
return cv
def visit_Expr(self, node):
"""
Expr(expr value)
"""
val = self.visit(node.value)
if isinstance(node.value, ast.Str):
"""
<Python> "" * comment
* sub comment ""
<Ruby> # * comment
# * sub comment
"""
comment = val[1:-1]
indent = self.indent_string()
for s in comment.split('\n'):
s = re.sub(r'^%s' % indent, '', s)
self.write("# %s" % s)
else:
self.write(val)
def visit_BinOp(self, node):
if isinstance(node.op, ast.Mod) and isinstance(node.left, ast.Str):
left = self.visit(node.left)
# 'b=%(b)0d and c=%(c)d and d=%(d)d' => 'b=%<b>0d and c=%<c>d and d=%<d>d'
left = re.sub(r"(.+?%)\((.+?)\)(.+?)", r"\1<\2>\3", left)
self._dict_format = True
right = self.visit(node.right)
self._dict_format = False
return "%s %% %s" % (left, right)
left = self.visit(node.left)
right = self.visit(node.right)
if isinstance(node.op, ast.Pow):
return "%s ** %s" % (left, right)
if isinstance(node.op, ast.Div):
if re.search(r"Numo::", left) or re.search(r"Numo::", right):
return "(%s)/(%s)" % (left, right)
else:
return "(%s)/(%s).to_f" % (left, right)
return "(%s)%s(%s)" % (left, self.get_binary_op(node), right)
def get_call_names_helper(node, result):
"""Recursively finds all function names."""
if isinstance(node, ast.Name):
if node.id not in BLACK_LISTED_CALL_NAMES:
result.append(node.id)
return result
elif isinstance(node, ast.Call):
return result
elif isinstance(node, ast.Subscript):
return get_call_names_helper(node.value, result)
elif isinstance(node, ast.Str):
result.append(node.s)
return result
else:
result.append(node.attr)
return get_call_names_helper(node.value, result)
def stateful_wait_for(self, call):
if isinstance(call.func, ast.Attribute):
if call.func.attr in ['wait_for_message', 'wait_for_reaction']:
if self.interactive and not prompt_change(
'A possible change was found to change {} into wait_for.'.format(call.func.attr)
):
return call
event = call.func.attr.split('_')[2]
event = 'message' if event == 'message' else 'reaction_add'
call.func.attr = 'wait_for'
if call.args:
timeout = call.args[0]
call.args = []
call.keywords.append(ast.keyword(arg='timeout', value=timeout))
call.args.insert(0, ast.Str(s=event))
for kw in list(call.keywords):
if kw.arg != 'check' and kw.arg != 'timeout':
call.keywords.remove(kw)
warnings.warn('wait_for change detected. Rewrite removes the author, channel, and content '
'keyword arguments from this method.')
stats_counter['call_changes'] += 1
return call
def test_walk_ast(self):
atok = asttokens.ASTTokens(self.source, parse=True)
def view(node):
return "%s:%s" % (node.__class__.__name__, atok.get_text(node))
scan = [view(n) for n in asttokens.util.walk(atok.tree)]
self.assertEqual(scan, [
"Module:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
"Expr:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
"Call:foo(bar(1 + 2), 'hello' + ', ' + 'world')",
'Name:foo',
'Call:bar(1 + 2)',
'Name:bar',
'BinOp:1 + 2',
'Num:1',
'Num:2',
"BinOp:'hello' + ', ' + 'world'",
"BinOp:'hello' + ', '",
"Str:'hello'",
"Str:', '",
"Str:'world'"
])
def test_replace(self):
self.assertEqual(asttokens.util.replace("this is a test", [(0, 4, "X"), (8, 9, "THE")]),
"X is THE test")
self.assertEqual(asttokens.util.replace("this is a test", []), "this is a test")
self.assertEqual(asttokens.util.replace("this is a test", [(7,7," NOT")]), "this is NOT a test")
source = "foo(bar(1 + 2), 'hello' + ', ' + 'world')"
atok = asttokens.ASTTokens(source, parse=True)
names = [n for n in asttokens.util.walk(atok.tree) if isinstance(n, ast.Name)]
strings = [n for n in asttokens.util.walk(atok.tree) if isinstance(n, ast.Str)]
repl1 = [atok.get_text_range(n) + ('TEST',) for n in names]
repl2 = [atok.get_text_range(n) + ('val',) for n in strings]
self.assertEqual(asttokens.util.replace(source, repl1 + repl2),
"TEST(TEST(1 + 2), val + val + val)")
self.assertEqual(asttokens.util.replace(source, repl2 + repl1),
"TEST(TEST(1 + 2), val + val + val)")
def test_dump(self):
node = ast.parse('spam(eggs, "and cheese")')
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
"keywords=[], starargs=None, kwargs=None))])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Str('and cheese')], [], None, None))])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
)
def _check_query_words(
self, query: ast.Str, parser: Parser,
) -> Generator[Tuple[int, int, str, type], Any, None]:
for token in parser:
word = token.value
if token.is_keyword or token.is_function_name:
if not word.isupper() and word.upper() not in self.excepted_names:
yield(
query.lineno, query.col_offset,
"Q440 keyword {} is not uppercase".format(word),
type(self),
)
if word.upper() in ABBREVIATED_KEYWORDS:
yield(
query.lineno, query.col_offset,
"Q442 avoid abbreviated keywords, {}".format(word),
type(self),
)
elif token.is_name and (not word.islower() or word.endswith('_')):
yield(
query.lineno, query.col_offset,
"Q441 name {} is not valid, must be snake_case, and cannot "
"end with `_`".format(word),
type(self),
)
def parse_args(args):
arg_list = []
for arg in args:
if isinstance(arg, ast.Str):
arg_list.append("%s" % arg.s)
elif isinstance(arg, ast.Name):
value = arg.id
if value == "None":
arg_list.append(None)
else:
arg_list.append(value)
elif isinstance(arg, ast.Num):
arg_list.append(arg.n)
elif isinstance(arg, ast.List):
arg_list.append(parse_args(arg.elts))
elif isinstance(arg, ast.Tuple):
arg_list.append(tuple(parse_args(arg.elts)))
elif isinstance(arg, ast.Attribute):
arg_list.append(str(arg.value.id) + "." + str(arg.attr))
else:
print(arg, type(arg))
return arg_list
def _imports_unicode_literals(contents_text):
try:
ast_obj = ast_parse(contents_text)
except SyntaxError:
return False
for node in ast_obj.body:
# Docstring
if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):
continue
elif isinstance(node, ast.ImportFrom):
if (
node.module == '__future__' and
any(name.name == 'unicode_literals' for name in node.names)
):
return True
elif node.module == '__future__':
continue
else:
return False
else:
return False
def visit_Assign(self, node):
# type: (ast.Assign) -> None
# The LHS gets the inferred type of the RHS.
# We do this post-traversal to let the type inference
# run on the children first.
self.generic_visit(node)
rhs_inferred_type = self._get_inferred_type_for_node(node.value)
if rhs_inferred_type is None:
# Special casing assignment to a string literal.
if isinstance(node.value, ast.Str):
rhs_inferred_type = StringLiteral(node.value.s)
self._set_inferred_type_for_node(node.value, rhs_inferred_type)
for t in node.targets:
if isinstance(t, ast.Name):
self._symbol_table.set_inferred_type(t.id, rhs_inferred_type)
self._set_inferred_type_for_node(node, rhs_inferred_type)
def get_version():
module_path = os.path.join(os.path.dirname(__file__), 'logging_spinner.py')
module_file = open(module_path)
try:
module_code = module_file.read()
finally:
module_file.close()
tree = ast.parse(module_code, module_path)
for node in ast.iter_child_nodes(tree):
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
continue
target, = node.targets
if isinstance(target, ast.Name) and target.id == '__version__':
value = node.value
if isinstance(value, ast.Str):
return value.s
raise ValueError('__version__ is not defined as a string literal')
raise ValueError('could not find __version__')
def get_type(self, node):
if isinstance(node, ast.Num):
return Type.NUMBER
elif isinstance(node, ast.Str):
return Type.STRING
elif isinstance(node, ast.Name):
if self.variables[node.id] is not None:
return self.variables[node.id].var_type
else:
return Type.VOID
elif isinstance(node, ast.BinOp):
if self.get_type(node.left).is_number and self.get_type(node.right).is_number:
return Type.NUMBER
elif self.get_type(node.left).is_string or self.get_type(node.right).is_string:
return Type.STRING
elif isinstance(node, ast.Call):
return self.functions[node.func.id].return_type
else:
return Type.VOID
def test_dump(self):
node = ast.parse('spam(eggs, "and cheese")')
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
"keywords=[], starargs=None, kwargs=None))])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Str('and cheese')], [], None, None))])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
)
def test_dump(self):
node = ast.parse('spam(eggs, "and cheese")')
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
"keywords=[], starargs=None, kwargs=None))])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Str('and cheese')], [], None, None))])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
)
def pop_format_context(self, expl_expr):
"""Format the %-formatted string with current format context.
The expl_expr should be an ast.Str instance constructed from
the %-placeholders created by .explanation_param(). This will
add the required code to format said string to .on_failure and
return the ast.Name instance of the formatted string.
"""
current = self.stack.pop()
if self.stack:
self.explanation_specifiers = self.stack[-1]
keys = [ast.Str(key) for key in current.keys()]
format_dict = ast.Dict(keys, list(current.values()))
form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
name = "@py_format" + str(next(self.variable_counter))
self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
return ast.Name(name, ast.Load())
def _solve(self):
import_aliases = (self.context._context['import_aliases'] if self.context else None)
cursor_node = self.tainted_node.parent
while cursor_node != self.target_node:
test_node = cursor_node
cursor_node = cursor_node.parent
if isinstance(test_node, ast.BinOp):
continue
elif isinstance(test_node, ast.Call):
if isinstance(test_node.func, ast.Attribute) and isinstance(test_node.func.value, ast.Str) and test_node.func.attr == 'format':
return True
function = s_utils.get_call_function(test_node, import_aliases=import_aliases)
if function in ('os.path.abspath', 'os.path.join', 'str'):
continue
elif function == 'os.path.relpath' and s_utils.node_is_child_of_parent(test_node.args[0], self.tainted_node):
continue
elif isinstance(test_node, ast.Subscript):
continue
return False
return True
def test_dump(self):
node = ast.parse('spam(eggs, "and cheese")')
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
"keywords=[], starargs=None, kwargs=None))])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
"Str('and cheese')], [], None, None))])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
"lineno=1, col_offset=0), lineno=1, col_offset=0)])"
)
def get_version():
module_path = os.path.join(os.path.dirname(__file__),
'wikidata', '__init__.py')
module_file = open(module_path)
try:
module_code = module_file.read()
finally:
module_file.close()
tree = ast.parse(module_code, module_path)
for node in ast.iter_child_nodes(tree):
if not isinstance(node, ast.Assign) or len(node.targets) != 1:
continue
target, = node.targets
if isinstance(target, ast.Name) and target.id == '__version__':
value = node.value
if isinstance(value, ast.Str):
return value.s
raise ValueError('__version__ is not defined as a string literal')
raise ValueError('could not find __version__')
def pop_format_context(self, expl_expr):
"""Format the %-formatted string with current format context.
The expl_expr should be an ast.Str instance constructed from
the %-placeholders created by .explanation_param(). This will
add the required code to format said string to .on_failure and
return the ast.Name instance of the formatted string.
"""
current = self.stack.pop()
if self.stack:
self.explanation_specifiers = self.stack[-1]
keys = [ast.Str(key) for key in current.keys()]
format_dict = ast.Dict(keys, list(current.values()))
form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
name = "@py_format" + str(next(self.variable_counter))
self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
return ast.Name(name, ast.Load())
def pop_format_context(self, expl_expr):
"""Format the %-formatted string with current format context.
The expl_expr should be an ast.Str instance constructed from
the %-placeholders created by .explanation_param(). This will
add the required code to format said string to .on_failure and
return the ast.Name instance of the formatted string.
"""
current = self.stack.pop()
if self.stack:
self.explanation_specifiers = self.stack[-1]
keys = [ast.Str(key) for key in current.keys()]
format_dict = ast.Dict(keys, list(current.values()))
form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
name = "@py_format" + str(next(self.variable_counter))
self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
return ast.Name(name, ast.Load())
def specialize_constant(node, value):
if value is None or isinstance(value, bool):
new_node = ast.NameConstant(value=value)
elif isinstance(value, (int, float, complex)):
new_node = ast.Num(n=value)
elif isinstance(value, str):
new_node = ast.Str(s=value)
elif isinstance(value, bytes):
new_node = ast.Bytes(s=value)
elif isinstance(value, tuple):
elts = [specialize_constant(node, elt) for elt in value]
new_node = ast.Tuple(elts=elts, ctx=ast.Load())
else:
raise ValueError("unknown constant: %r" % value)
fatoptimizer.tools.copy_lineno(node, new_node)
return new_node
def convert_to_value(item):
if isinstance(item, ast.Str):
return item.s
elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes):
return item.s
elif isinstance(item, ast.Tuple):
return tuple(convert_to_value(i) for i in item.elts)
elif isinstance(item, ast.Num):
return item.n
elif isinstance(item, ast.Name):
result = VariableKey(item=item)
constants_lookup = {
'True': True,
'False': False,
'None': None,
}
return constants_lookup.get(
result.name,
result,
)
elif (not PY33) and isinstance(item, ast.NameConstant):
# None, True, False are nameconstants in python3, but names in 2
return item.value
else:
return UnhandledKeyType()
def parse_decorator(node: ast.AST):
if isinstance(node, ast.Name):
ret = Decorator()
ret.name = node.id
return ret
elif isinstance(node, ast.Call):
ret = Decorator()
ret.name = node.func.id
for arg in node.args:
if isinstance(arg, ast.Num):
ret.args.append(str(arg.n))
elif isinstance(arg, ast.Str):
ret.args.append(str(arg.n))
elif isinstance(arg, ast.Name):
ret.args.append(arg.id)
else:
v = eval_numeric_constexpr(arg)
if v:
ret.args.append(str(v))
else:
error(loc(node), "Unsupported decorator type")
return ret
else:
error(loc(node), "Supported decorators are Name and Call")
return None
