def get_blocks(tokens):
"""
"""
block = []
level = 0
from pygments.token import Token
for token, value in tokens:
block.append((token,value))
if value == ")":
level += 1
elif value == "(":
level -= 1
if level == 0:
yield block[::-1]
block = []
python类token()的实例源码
def highlight(self, block):
"""Method called on each block to highlight it content"""
tokens = pygments.lex(block, self.python_lexer)
if self.format_rst:
from pygments.token import Token
toks = []
for token in tokens:
if token[0] == Token.String.Doc and len(token[1]) > 6:
toks += pygments.lex(token[1][:3], self.python_lexer)
# parse doc string content by rst lexer
toks += pygments.lex(token[1][3:-3], self.rst_lexer)
toks += pygments.lex(token[1][-3:], self.python_lexer)
elif token[0] == Token.Comment.Single:
toks.append((Token.Comment.Single, token[1][0]))
# parse comment content by rst lexer
# remove the extrat newline added by rst lexer
toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1]
else:
toks.append(token)
tokens = toks
return pygments.format(tokens, self.formatter)
def convert_to_lines(tokens):
"""
Given a stream of tokens, yield lines as strings.
Each output string is guaranteed to end with a newline.
"""
line = []
for token in tokens:
text = get_token_text(token)
line.append(text)
if text == '\n':
yield ''.join(line)
line = []
if line:
line.append('\n')
yield ''.join(line)
def highlight(self, block):
"""Method called on each block to highlight it content"""
tokens = pygments.lex(block, self.python_lexer)
if self.format_rst:
from pygments.token import Token
toks = []
for token in tokens:
if token[0] == Token.String.Doc and len(token[1]) > 6:
toks += pygments.lex(token[1][:3], self.python_lexer)
# parse doc string content by rst lexer
toks += pygments.lex(token[1][3:-3], self.rst_lexer)
toks += pygments.lex(token[1][-3:], self.python_lexer)
elif token[0] == Token.Comment.Single:
toks.append((Token.Comment.Single, token[1][0]))
# parse comment content by rst lexer
# remove the extrat newline added by rst lexer
toks += list(pygments.lex(token[1][1:], self.rst_lexer))[:-1]
else:
toks.append(token)
tokens = toks
return pygments.format(tokens, self.formatter)
def _serialize_token(self, pair):
"""Get string representation of (token, value) pair.
"""
from pygments.token import Token
token, value = pair
# for literals, autocomplete only depends
# on type of argument, not the value
if token in [Token.Literal.Number.Float,
Token.Literal.Number.Integer,
Token.Literal.String]:
return str(token)
else:
return value
def get_autocomplete_stub(lexer, text):
"""
"""
entity = []
from pygments.token import Token
# ignored tokens
ignored = [Token.Comment, Token.Text, Token.Text.Whitespace, Token.Comment.Single]
filtered = lambda pair: pair[0] not in ignored # pair = (token,value)
tokens = filter(filtered, get_tokens_reversed(lexer, text))
blocks = get_blocks(tokens)
block = next(blocks, [])
if len(block) == 1 and block[0][1] == ".":
block = next(blocks, [])
if len(block) > 0 and block[0][1] == "(":
block_ = next(blocks, [])
if len(block_) == 1 and block[0][0] is Token.Name:
return block_ + block
return block
return []
def get_style(style=STYLE):
"""Load Pygments custom style."""
def getattrs(obj, names):
return reduce(getattr, names.split('_'), obj)
return {getattrs(pygments.token, k): (v,) * 2 for k, v in style.items()}
def tokenize(text, lexer=None):
"""
Split text into (token_type, token_text) pairs using the given lexer
When there is no lexer, it will split by words instead.
"""
if lexer is None:
return [(pygments.token.Text, word) for word in split_words(text)]
tokens = lexer.get_tokens(text)
tokens = group_tokens(tokens)
return tokens
def group_tokens(tokens):
"""
Join and separate tokens to be more suitable for diffs.
Transformations:
- Empty tokens are removed
- Text containing newlines is split to have the newline be one token
- Other sequential whitespace tokens are joined
- Token types which contain freeform text (ie. comments, strings) are split into words
"""
for token_type, group in itertools.groupby(tokens, get_token_type):
if any(token_type in type_set for type_set in JOIN_TOKENS):
text = ''.join(get_token_text(token) for token in group)
group = [(token_type, text)]
if any(token_type in type_set for type_set in WORD_TOKENS):
group = (
(token_type, word)
for token in group
for word in split_words(get_token_text(token))
)
# Split by newlines
for token in group:
text_parts = re.split(r'(\n)', get_token_text(token))
for text_part in text_parts:
# Empty tokens are discarded, to avoid confusing
# difflib or highlighting empty regions
if text_part:
yield (token_type, text_part)
def persist_highlighting(tokens):
"""
Given a stream of tokens, yield tokens with additional
START_HIGHLIGHT and END_HIGHLIGHT tokens inserted to persist
highlighting across tokens with a newline '\n' as text.
"""
should_be_highlighting = False
is_highlighting = False
for token in tokens:
token_type = get_token_type(token)
if token_type == 'START_HIGHLIGHT':
assert not should_be_highlighting, 'Multiple attempts to start highlighting'
should_be_highlighting = True
elif token_type == 'END_HIGHLIGHT':
assert should_be_highlighting, 'Attempt to end highlighting while not highlighting'
should_be_highlighting = False
else:
if get_token_text(token) == '\n':
if is_highlighting:
yield ('END_HIGHLIGHT', '')
is_highlighting = False
elif is_highlighting is not should_be_highlighting:
if should_be_highlighting:
yield ('START_HIGHLIGHT', '')
else:
yield ('END_HIGHLIGHT', '')
is_highlighting = should_be_highlighting
yield token
def fill_highlighting_text(tokens, highlight=FORMAT_INVERT, reset=FORMAT_RESET_INVERT):
"""
Given a stream of tokens, yield tokens where highlighting tokens
have formatting text
"""
for token in tokens:
token_type = get_token_type(token)
if token_type == 'START_HIGHLIGHT':
yield ('START_HIGHLIGHT', highlight)
elif token_type == 'END_HIGHLIGHT':
yield ('END_HIGHLIGHT', reset)
else:
yield token
def header_callback(lexer, match):
# highlight correct filetype names specially
if match.group(1) in filetypes or match.group(1) == 'DEFAULT':
yield (match.start(), pygments.token.Keyword, match.group(0))
else:
yield (match.start(), pygments.token.Text, match.group(0))
def key_val_pair(key, value, key_token=pygments.token.Name.Builtin,
value_token=pygments.token.String):
for regex, token in [(value, value_token),
(r'.*?', pygments.token.Name)]:
yield (
r'(%s)([^\S\n]*)(=)([^\S\n]*)(%s)$' % (key, regex),
pygments.lexer.bygroups(
key_token, pygments.token.Text,
pygments.token.Operator, pygments.token.Text, token))
def _on_config_changed(self, junk=None):
# when the font family or size changes, self.textwidget['font']
# also changes because it's a porcupine.textwiddet.ThemedText widget
fontobject = tkfont.Font(name=self.textwidget['font'], exists=True)
font_updates = fontobject.actual()
del font_updates['weight'] # ignore boldness
del font_updates['slant'] # ignore italicness
for (bold, italic), font in self._fonts.items():
# fonts don't have an update() method
for key, value in font_updates.items():
font[key] = value
# http://pygments.org/docs/formatterdevelopment/#styles
# all styles seem to yield all token types when iterated over,
# so we should always end up with the same tags configured
style = pygments.styles.get_style_by_name(config['pygments_style'])
for tokentype, infodict in style:
# this doesn't use underline and border
# i don't like random underlines in my code and i don't know
# how to implement the border with tkinter
key = (infodict['bold'], infodict['italic']) # pep8 line length
kwargs = {'font': self._fonts[key]}
if infodict['color'] is None:
kwargs['foreground'] = '' # reset it
else:
kwargs['foreground'] = '#' + infodict['color']
if infodict['bgcolor'] is None:
kwargs['background'] = ''
else:
kwargs['background'] = '#' + infodict['bgcolor']
self.textwidget.tag_config(str(tokentype), **kwargs)
# make sure that the selection tag takes precedence over our
# token tag
self.textwidget.tag_lower(str(tokentype), 'sel')
# handle things from the highlighting process
def log(self, tokens, stream=sys.stdout):
"""Log the input token stream with the standard Python logging
mecanism.
PARAMETERS
log_fn: the logging function to use
tokens: the input tokens stream
"""
assert self.formatter is not None, 'Internal error'
print >> stream, pygments.format(tokens, self.formatter)
stream.flush()
def status_token(status):
"""Return the token to use for the given test case result status.
RETURNS
a pygments Token
"""
return getattr(Token.TestResult, status, Token.Error)
def format_testcase_diff(diff):
"""Format a testcase output diff.
PARAMETERS
diff: the diff content
RETURNS
a list of pygments' Tokens
"""
def new_line_token():
"""Generate a new line token."""
return Token.Whitespace, '\n'
def indent_token():
"""Generate an indentation space token."""
return Token.Whitespace, ' ' * 4
tokens = []
new_line = True
# Because of logging prefixes, skip the first line to avoid
# misalignment.
tokens.append(new_line_token())
for ttype, value in pygments.lex(diff, DiffLexer()):
for subval in value.split('\n'):
if new_line:
tokens.append(indent_token())
new_line = not subval
if subval:
tokens.append((ttype, subval))
else:
tokens.append(new_line_token())
return tokens
def run(self, edit):
"""
"""
view = self.view
sel = view.sel()
if len(sel) == 0:
return
a,b = sel[0]
query = view.substr(view.word(a)) if a == b else view.substr(sel[0])
if query == "":
return
# run docsetutil command
docset = SwiftKittenEventListener.get_settings(view, "docset")
cmd = self.get_docsetutil_cmd(view, docset, query)
results = check_output(cmd, stderr=STDOUT)
results = str(results, 'utf-8')
if len(results) == 0:
print("No documentation found.")
return
lines = results.splitlines()
# split each line into two paths
pairs = map(lambda line: line.strip().split(" "), lines)
get_lang = lambda a: a.split('/')[0]
get_path = lambda a,b: os.path.join(os.path.dirname(a),
os.path.basename(b))
#
docs = {get_lang(a) : get_path(a,b) for a,b in pairs}
# prefer Swift, Objective-C, C
lang = sorted(docs.keys())[-1]
# construct path to documentation token
path = os.path.join(self.get_tokens_path(docset), docs[lang] + ".xml")
# read documentation file
with open(path, "rb") as f:
xml = f.read()
# convert xml to html
html = str(self.convert_docs_to_html(xml), "utf-8")
#
# TO DO:
# add on_navigate handler
#
# display documentation
view.show_popup(html, max_width=400, max_height=600)
