PythonQLLexer.py 文件源码

def t_NEWLINE(self,t):
    r'\n|\r'
    t.lexer.lineno += len(t.value)
    t.lexer.newlinepos = t.lexer.lexpos
    pos = t.lexer.lexpos
    data = t.lexer.lexdata

    # Consume all the whitespace until we hit something non-white or a newline
    while True:
      if pos >= len(data):
        return t

      if data[pos] in ['\n','\r'] or not re.match('\s',data[pos]):
        break

      pos += 1

    # If this is a line with just whitespace, or we're inside parenthesis,
    # don't return a token

    if data[pos] in ['\n', '\t', '#'] or t.lexer.opened > 0:
      return None

    ws = data[t.lexer.lexpos:pos]

    # Check if we went back to an older identation level, then
    # create some DEDENT tokens
    try:
      idx = t.lexer.indent_stack.index(ws)
      ndedents = len(t.lexer.indent_stack)-idx-1
      for i in range(ndedents):
        t.lexer.indent_stack.pop()
        dedent_tok = lex.LexToken()
        dedent_tok.type = 'DEDENT'
        dedent_tok.value = ''
        dedent_tok.lineno = t.lexer.lineno
        dedent_tok.lexpos = pos
        t.lexer.pushback_token(dedent_tok)

    # Otherwise, check if we have added an identation level and create
    # an IDENT token, or just return a newline

    except:
      last_ident = t.lexer.indent_stack[-1] if t.lexer.indent_stack else ""
      if ws.startswith(last_ident):
        indent_tok = lex.LexToken()
        indent_tok.type = 'INDENT'
        indent_tok.value = ws
        indent_tok.lineno = t.lexer.lineno
        indent_tok.lexpos = pos
        t.lexer.pushback_token(indent_tok)
        t.lexer.indent_stack.append(ws)

      # Current ident doesn't contain the previous ident, identation error!
      else:
        raise Exception("Bad ident at line %d" % t.lexer.lineno )
    return t

  # multi-qoute strings are pretty straightforward