def restart(self, string):
"""
Restarts parsing with new string
Returns self for chaining.
"""
try:
self.tokens = [
ActionParser.Token(type, string)
for (type, string, trash, trash, trash)
in generate_tokens( iter([string]).next )
if type != TokenType.ENDMARKER
]
except TokenError:
self.tokens = None
self.index = 0
return self
python类ENDMARKER的实例源码
def file_input(self, nodelist):
doc = self.get_docstring(nodelist, symbol.file_input)
if doc is not None:
i = 1
else:
i = 0
stmts = []
for node in nodelist[i:]:
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
self.com_append_stmt(stmts, node)
return Module(doc, Stmt(stmts))
def _find_last_in_line(self, start_token):
try:
newline = self._code.find_token(start_token, token.NEWLINE)
except IndexError:
newline = self._code.find_token(start_token, token.ENDMARKER)
return self._code.prev_token(newline)
def file_input(self, nodelist):
doc = self.get_docstring(nodelist, symbol.file_input)
if doc is not None:
i = 1
else:
i = 0
stmts = []
for node in nodelist[i:]:
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
self.com_append_stmt(stmts, node)
return Module(doc, Stmt(stmts))
def file_input(self, nodelist):
doc = self.get_docstring(nodelist, symbol.file_input)
if doc is not None:
i = 1
else:
i = 0
stmts = []
for node in nodelist[i:]:
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
self.com_append_stmt(stmts, node)
return Module(doc, Stmt(stmts))
def file_input(self, nodelist):
doc = self.get_docstring(nodelist, symbol.file_input)
if doc is not None:
i = 1
else:
i = 0
stmts = []
for node in nodelist[i:]:
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
self.com_append_stmt(stmts, node)
return Module(doc, Stmt(stmts))
def file_input(self, nodelist):
doc = self.get_docstring(nodelist, symbol.file_input)
if doc is not None:
i = 1
else:
i = 0
stmts = []
for node in nodelist[i:]:
if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
self.com_append_stmt(stmts, node)
return Module(doc, Stmt(stmts))
def _simulate_compile_singlemode(self, s):
# Calculate line offsets
lines = [0, 0]
pos = 0
while 1:
pos = s.find('\n', pos)+1
if not pos: break
lines.append(pos)
lines.append(len(s))
oldpos = 0
parenlevel = 0
deflevel = 0
output = []
stmt = []
text = StringIO(s)
tok_gen = tokenize.generate_tokens(text.readline)
for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen:
newpos = lines[srow] + scol
stmt.append(s[oldpos:newpos])
if tok != '':
stmt.append(tok)
oldpos = newpos + len(tok)
# Update the paren level.
if tok in '([{':
parenlevel += 1
if tok in '}])':
parenlevel -= 1
if tok in ('def', 'class') and deflevel == 0:
deflevel = 1
if deflevel and toktype == token.INDENT:
deflevel += 1
if deflevel and toktype == token.DEDENT:
deflevel -= 1
# Are we starting a statement?
if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT,
token.INDENT, token.ENDMARKER) or
tok==':') and parenlevel == 0):
if deflevel == 0 and self._is_expr(stmt[1:-2]):
output += stmt[0]
output.append('__print__((')
output += stmt[1:-2]
output.append('))')
output += stmt[-2:]
else:
output += stmt
stmt = []
return ''.join(output)
def _simulate_compile_singlemode(self, s):
# Calculate line offsets
lines = [0, 0]
pos = 0
while 1:
pos = s.find('\n', pos)+1
if not pos: break
lines.append(pos)
lines.append(len(s))
oldpos = 0
parenlevel = 0
deflevel = 0
output = []
stmt = []
text = StringIO(s)
tok_gen = tokenize.generate_tokens(text.readline)
for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen:
newpos = lines[srow] + scol
stmt.append(s[oldpos:newpos])
if tok != '':
stmt.append(tok)
oldpos = newpos + len(tok)
# Update the paren level.
if tok in '([{':
parenlevel += 1
if tok in '}])':
parenlevel -= 1
if tok in ('def', 'class') and deflevel == 0:
deflevel = 1
if deflevel and toktype == token.INDENT:
deflevel += 1
if deflevel and toktype == token.DEDENT:
deflevel -= 1
# Are we starting a statement?
if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT,
token.INDENT, token.ENDMARKER) or
tok==':') and parenlevel == 0):
if deflevel == 0 and self._is_expr(stmt[1:-2]):
output += stmt[0]
output.append('__print__((')
output += stmt[1:-2]
output.append('))')
output += stmt[-2:]
else:
output += stmt
stmt = []
return ''.join(output)
def _simulate_compile_singlemode(self, s):
# Calculate line offsets
lines = [0, 0]
pos = 0
while 1:
pos = s.find('\n', pos)+1
if not pos: break
lines.append(pos)
lines.append(len(s))
oldpos = 0
parenlevel = 0
deflevel = 0
output = []
stmt = []
text = StringIO(s)
tok_gen = tokenize.generate_tokens(text.readline)
for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen:
newpos = lines[srow] + scol
stmt.append(s[oldpos:newpos])
if tok != '':
stmt.append(tok)
oldpos = newpos + len(tok)
# Update the paren level.
if tok in '([{':
parenlevel += 1
if tok in '}])':
parenlevel -= 1
if tok in ('def', 'class') and deflevel == 0:
deflevel = 1
if deflevel and toktype == token.INDENT:
deflevel += 1
if deflevel and toktype == token.DEDENT:
deflevel -= 1
# Are we starting a statement?
if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT,
token.INDENT, token.ENDMARKER) or
tok==':') and parenlevel == 0):
if deflevel == 0 and self._is_expr(stmt[1:-2]):
output += stmt[0]
output.append('__print__((')
output += stmt[1:-2]
output.append('))')
output += stmt[-2:]
else:
output += stmt
stmt = []
return ''.join(output)
