java类org.antlr.v4.runtime.Token的实例源码

public static int getMatchingSymbolStartsLine(Corpus corpus,
                                                  InputDocument doc,
                                                  TerminalNode node)
    {
        TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
        if ( matchingLeftNode != null ) {
            Token matchingLeftToken = matchingLeftNode.getSymbol();
            int i = matchingLeftToken.getTokenIndex();
            if ( i==0 ) return 1; // first token is considered first on line
            Token tokenBeforeMatchingToken = doc.tokens.getPreviousRealToken(i);
//          System.out.printf("doc=%s node=%s, pair=%s, before=%s\n",
//                            new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenBeforeMatchingToken);
            if ( tokenBeforeMatchingToken!=null ) {
                return matchingLeftToken.getLine()>tokenBeforeMatchingToken.getLine() ? 1 : 0;
            }
            else { // matchingLeftToken must be first in file
                return 1;
            }
        }
        return NOT_PAIR;
    }

public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
    tokens.fill();
    CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
    dummy.setChannel(Token.HIDDEN_CHANNEL);
    Token firstRealToken = tokens.getNextRealToken(-1);
    for (int i = 0; i<tokens.size(); i++) {
        if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
        CommonToken t = (CommonToken)tokens.get(i);
        if ( t.getText().matches("\\s+") ) {
            tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
        }
        else {
            t.setLine(0);
            t.setCharPositionInLine(-1);
        }
    }
}

/**
 * Returns location of the end multiline comment symbol.
 *
 * @param comment A token representing a comment
 * @return Location of the end symbol
 */
public static Location getEndOfMultilineComment(Token comment) {
    String commentText = comment.getText();
    if (commentText.charAt(commentText.length() - 1) == '\n') {
        commentText = commentText.substring(0, commentText.length() - 1);
    }
    int numNewlines = 0;
    int lastNewlineIndex = -1;
    for (int i = 0; i < commentText.length(); i++) {
        if (commentText.charAt(i) == '\n') {
            lastNewlineIndex = i;
            numNewlines += 1;
        }
    }
    String lastLine = commentText.substring(lastNewlineIndex + 1);
    return new Location(comment.getLine() + numNewlines,
        numNewlines == 0 ? comment.getCharPositionInLine() + lastLine.length() - 1 : lastLine.length() - 1);
}

/**
 * We want to touch as little ANTR code as possible. We overload this
 * function to pretend the existence of the token vocab parser
 */
@Override
public void importTokensFromTokensFile() {
    if (!tokenVocab.isEmpty()) {
        MemoryTokenVocabParser vparser = new MemoryTokenVocabParser(this, tokenVocab);
        Map<String, Integer> tokens = vparser.load();

        int ret;
        for (String t : tokens.keySet()) {
            if (t.charAt(0) == '\'') {
                ret = defineStringLiteral(t, tokens.get(t));
                if (ret == Token.INVALID_TYPE)
                    throw new IllegalArgumentException("Token must not be INVAlID_TYPE");
            } else {
                ret = defineTokenName(t, tokens.get(t));
                if (ret == Token.INVALID_TYPE)
                    throw new IllegalArgumentException("Token must not be INVAlID_TYPE");
            }
            LOGGER.debug("token {} {}", t, tokens.get(t));
        }
    }
}

@Override
public Object visitBitShiftExpr(KalangParser.BitShiftExprContext ctx) {
    String op;
    Token opStart;
    if(ctx.left!=null){
        op = "<<";
        opStart = ctx.left;
    }else if(ctx.right!=null){
        op = ">>";
        opStart = ctx.right;
    }else if(ctx.uright!=null){
        op = ">>>";
        opStart = ctx.uright;
    }else{
        throw Exceptions.unexceptedValue(ctx);
    }
    return this.createBinaryExpr(op, ctx.expression(0), ctx.expression(1)
            , opStart,ctx.stop, ctx);
}

private static byte[] getBytes(Literal_value_binaryContext rule) {
    Token token = rule.STRING_LITERAL().getSymbol();
    byte[] bytes = new byte[token.getStopIndex() - token.getStartIndex() - 1];
    CharStream cs = token.getInputStream();
    int pos = cs.index();
    cs.seek(token.getStartIndex() + 1);
    int j = 0;
    for (int i = 0; i < bytes.length; i++) {
        int ch = cs.LA(i + 1);
        if (ch == '\\') {
            i++;
            ch = cs.LA(i + 1);
            if (ch == '0') {
                ch = 0;
            }
            else if (ch == 'n') {
                ch = '\n';
            }
            else if (ch == 'r') {
                ch = '\r';
            }
            else if (ch == 'Z') {
                ch = '\032';
            }
        }
        bytes[j] = (byte) ch;
        j++;
    }
    cs.seek(pos);
    if (j != bytes.length) {
        // esacpe characters
        byte[] old = bytes;
        bytes = new byte[j];
        System.arraycopy(old, 0, bytes, 0, j);
    }
    return bytes;
}

private static ArithmeticBinaryExpression.Type getArithmeticBinaryOperator(Token operator) {
  switch (operator.getType()) {
    case SqlBaseLexer.PLUS:
      return ArithmeticBinaryExpression.Type.ADD;
    case SqlBaseLexer.MINUS:
      return ArithmeticBinaryExpression.Type.SUBTRACT;
    case SqlBaseLexer.ASTERISK:
      return ArithmeticBinaryExpression.Type.MULTIPLY;
    case SqlBaseLexer.SLASH:
      return ArithmeticBinaryExpression.Type.DIVIDE;
    case SqlBaseLexer.PERCENT:
      return ArithmeticBinaryExpression.Type.MODULUS;
    default:
      throw new UnsupportedOperationException("Unsupported operator: " + operator.getText());
  }
}

@Override
protected void reportUnwantedToken(Parser recognizer) {
    // change error message from default implementation
    if (inErrorRecoveryMode(recognizer)) {
        return;
    }

    beginErrorCondition(recognizer);

    Token t = recognizer.getCurrentToken();
    String tokenName = getTokenErrorDisplay(t);
    String msg = "extraneous input " + tokenName + " expecting operator";
    recognizer.notifyErrorListeners(t, msg, null);
}

private static boolean insertSemicolon(Token previous, Token next) {
    if (previous == null || next.getType() != PainlessLexer.RBRACK) {
        return false;
    }
    switch (previous.getType()) {
    case PainlessLexer.RBRACK:     // };} would be weird!
    case PainlessLexer.SEMICOLON:  // already have a semicolon, no need to add one
    case PainlessLexer.LBRACK:     // empty blocks don't need a semicolon
        return false;
    default:
        return true;
    }
}

@Override
public Token recoverInline(final Parser recognizer) throws RecognitionException {
    final Token token = recognizer.getCurrentToken();
    final String message = "unexpected token [" + getTokenErrorDisplay(token) + "]" +
        " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";

    Location location = new Location(sourceName, token.getStartIndex());
    throw location.createError(new IllegalArgumentException(message));
}