java类org.antlr.v4.runtime.misc.Utils的实例源码

public static void main(String[] args) throws Exception {
        LangDescriptor[] languages = new LangDescriptor[] {
            JAVA_DESCR,
            JAVA8_DESCR,
            JAVA_GUAVA_DESCR,
            JAVA8_GUAVA_DESCR,
            ANTLR4_DESCR,
            SQLITE_CLEAN_DESCR,
            TSQL_CLEAN_DESCR,
            SQLITE_NOISY_DESCR,
            TSQL_NOISY_DESCR,
//          QUORUM_DESCR,
        };
        List<String> corpusDirs = map(languages, l -> l.corpusDir);
        String[] dirs = corpusDirs.toArray(new String[languages.length]);
        String python = testAllLanguages(languages, dirs, "leave_one_out.pdf");
        String fileName = "python/src/leave_one_out.py";
        Utils.writeFile(fileName, python);
        System.out.println("wrote python code to "+fileName);
    }

public static void format(LangDescriptor language,
                          String testFileName,
                          String outputFileName)
    throws Exception
{
    // load all files up front
    List<String> allFiles = getFilenames(new File(language.corpusDir), language.fileRegex);
    List<InputDocument> documents = load(allFiles, language);
    // if in corpus, don't include in corpus
    final String path = new File(testFileName).getAbsolutePath();
    List<InputDocument> others = filter(documents, d -> !d.fileName.equals(path));
    InputDocument testDoc = parse(testFileName, language);
    Corpus corpus = new Corpus(others, language);
    corpus.train();

    Formatter formatter = new Formatter(corpus, language.indentSize, Formatter.DEFAULT_K,
                                        FEATURES_INJECT_WS, FEATURES_HPOS);
    String output = formatter.format(testDoc, false);

    if ( outputFileName!=null ) {
        Utils.writeFile(outputFileName, output);
    }
    else {
        System.out.print(output);
    }
}

/** Print out a whole tree in LISP form. Arg nodeTextProvider is used on the
 *  node payloads to get the text for the nodes.
 *
 *  @since 4.5.1
 */
public static String toStringTree(Tree t, TreeTextProvider nodeTextProvider) {
    if ( t==null ) return "null";
    String s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
    if ( t.getChildCount()==0 ) return s;
    StringBuilder buf = new StringBuilder();
    buf.append("(");
    s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
    buf.append(s);
    buf.append(' ');
    for (int i = 0; i<t.getChildCount(); i++) {
        if ( i>0 ) buf.append(' ');
        buf.append(toStringTree(t.getChild(i), nodeTextProvider));
    }
    buf.append(")");
    return buf.toString();
}

private String toStringTree(Tree tree, List<String> ruleNames) {
    String s = Utils.escapeWhitespace(getNodeText(tree, ruleNames), false);
    if(tree.getChildCount() == 0) return s;

    StringBuilder buf = new StringBuilder();
    buf.append("(");
    s = Utils.escapeWhitespace(getNodeText(tree, ruleNames), false);
    buf.append(s);
    buf.append(' ');
    for(int i = 0; i < tree.getChildCount(); i++)
    {
        if(i > 0)
            buf.append(' ');
        buf.append(toStringTree(tree.getChild(i), ruleNames));
    }

    buf.append(")");
    return buf.toString();
}

private String toStringTree(@NotNull final Tree t, @Nullable final List<String> ruleNames) {
  if (t.getChildCount() == 0) {
    return Utils.escapeWhitespace(getNodeText(t, ruleNames), true);
  }
  StringBuilder buf = new StringBuilder();
  buf.append(" ( ");
  String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), true);
  buf.append(s);
  buf.append(' ');
  for (int i = 0; i < t.getChildCount(); i++) {
    if (i > 0) {
      buf.append(' ');
    }
    buf.append(toStringTree(t.getChild(i), ruleNames));
  }
  buf.append(" ) ");
  return buf.toString();
}

/** Print out a whole tree in LISP form. {@link #getNodeText} is used on the
 *  node payloads to get the text for the nodes.  Detect
 *  parse trees and extract data appropriately.
 */
public static String toStringTree(@NotNull Tree t, @Nullable List<String> ruleNames) {
    String s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
    if ( t.getChildCount()==0 ) return s;
    StringBuilder buf = new StringBuilder();
    buf.append("(");
    s = Utils.escapeWhitespace(getNodeText(t, ruleNames), false);
    buf.append(s);
    buf.append(' ');
    for (int i = 0; i<t.getChildCount(); i++) {
        if ( i>0 ) buf.append(' ');
        buf.append(toStringTree(t.getChild(i), ruleNames));
    }
    buf.append(")");
    return buf.toString();
}

/**
 * Get a map from token names to token types.
 *
 * <p>Used for XPath and tree pattern compilation.</p>
 */
@NotNull
public Map<String, Integer> getTokenTypeMap() {
    String[] tokenNames = getTokenNames();
    if (tokenNames == null) {
        throw new UnsupportedOperationException("The current recognizer does not provide a list of token names.");
    }

    synchronized (tokenTypeMapCache) {
        Map<String, Integer> result = tokenTypeMapCache.get(tokenNames);
        if (result == null) {
            result = Utils.toMap(tokenNames);
            result.put("EOF", Token.EOF);
            result = Collections.unmodifiableMap(result);
            tokenTypeMapCache.put(tokenNames, result);
        }

        return result;
    }
}

/**
 * Get a map from rule names to rule indexes.
 *
 * <p>Used for XPath and tree pattern compilation.</p>
 */
@NotNull
public Map<String, Integer> getRuleIndexMap() {
    String[] ruleNames = getRuleNames();
    if (ruleNames == null) {
        throw new UnsupportedOperationException("The current recognizer does not provide a list of rule names.");
    }

    synchronized (ruleIndexMapCache) {
        Map<String, Integer> result = ruleIndexMapCache.get(ruleNames);
        if (result == null) {
            result = Collections.unmodifiableMap(Utils.toMap(ruleNames));
            ruleIndexMapCache.put(ruleNames, result);
        }

        return result;
    }
}

public void checkCExec(String filename) throws Exception {
    URL testFolderURL = TestCGen.class.getClassLoader().getResource(SAMPLES_DIR);
    String testFolder = testFolderURL.getPath();
    String workingDir = getWorkingDir();

    String J_pathToFile = testFolder+"/"+filename;
    String C_filename = basename(filename)+".c";

    JTran jTran = new JTran();
    String C_code = jTran.translate(J_pathToFile, C_filename, false, false);

    Utils.writeFile(workingDir+"/"+C_filename, C_code);

    // compile
    String[] cc = {"cc", "-o", basename(filename), C_filename};
    Triple<Integer, String, String> cc_result = exec(cc, getWorkingDir());
    int execCode = cc_result.a;
    String stdout = cc_result.b;
    String stderr = cc_result.c;

    assertEquals("", stdout);
    assertEquals("", stderr);
    assertEquals(0, execCode);

    // execute
    String[] exec_cmd = {"./"+basename(filename)};
    Triple<Integer, String, String> result = exec(exec_cmd, getWorkingDir());
    execCode = result.a;
    stdout = result.b;
    stderr = result.c;

    String expected_output_filename = basename(filename)+".txt";
    String expected_output = readFile(testFolder+"/"+expected_output_filename);

    assertEquals(expected_output, stdout);
    assertEquals("", stderr);
    assertEquals(0, execCode);
}

public static void writePython(LangDescriptor[] languages, List<Integer> ks, Float[][] medians) throws IOException {
    StringBuilder data = new StringBuilder();
    StringBuilder plot = new StringBuilder();
    for (int i = 0; i<languages.length; i++) {
        LangDescriptor language = languages[i];
        List<Float> filteredMedians = BuffUtils.filter(Arrays.asList(medians[i]), m -> m!=null);
        data.append(language.name+'='+filteredMedians+'\n');
        plot.append(String.format("ax.plot(ks, %s, label=\"%s\", marker='%s', color='%s')\n",
                                  language.name, language.name,
                                  nameToGraphMarker.get(language.name),
                                  nameToGraphColor.get(language.name)));
    }

    String python =
        "#\n"+
        "# AUTO-GENERATED FILE. DO NOT EDIT\n" +
        "# CodeBuff %s '%s'\n" +
        "#\n"+
        "import numpy as np\n"+
        "import matplotlib.pyplot as plt\n\n" +
        "%s\n" +
        "ks = %s\n"+
        "fig = plt.figure()\n"+
        "ax = plt.subplot(111)\n"+
        "%s"+
        "ax.tick_params(axis='both', which='major', labelsize=18)\n" +
        "ax.set_xlabel(\"$k$ nearest neighbors\", fontsize=20)\n"+
        "ax.set_ylabel(\"Median error rate\", fontsize=20)\n" +
        "#ax.set_title(\"k Nearest Neighbors vs\\nLeave-one-out Validation Error Rate\")\n"+
        "plt.legend(fontsize=18)\n\n" +
        "fig.savefig('images/vary_k.pdf', format='pdf')\n"+
        "plt.show()\n";
    String code = String.format(python, Tool.version, new Date(), data, ks, plot);

    String fileName = "python/src/vary_k.py";
    Utils.writeFile(fileName, code);
    System.out.println("wrote python code to "+fileName);
}

public static void main(String[] args) throws Exception {
    LangDescriptor[] languages = new LangDescriptor[] {
        JAVA_DESCR,
        JAVA8_DESCR,
        JAVA_GUAVA_DESCR,
    };
    List<String> corpusDirs = map(languages, l -> l.corpusDir);
    String[] dirs = corpusDirs.toArray(new String[languages.length]);
    String python = testAllLanguages(languages, dirs, "all_java_leave_one_out.pdf");
    String fileName = "python/src/all_java_leave_one_out.py";
    Utils.writeFile(fileName, python);
    System.out.println("wrote python code to "+fileName);
}

public static void main(String[] args) throws Exception {
    LangDescriptor[] languages = new LangDescriptor[] {
        SQLITE_NOISY_DESCR,
        SQLITE_CLEAN_DESCR,
        TSQL_NOISY_DESCR,
        TSQL_CLEAN_DESCR,
    };
    List<String> corpusDirs = map(languages, l -> l.corpusDir);
    String[] dirs = corpusDirs.toArray(new String[languages.length]);
    String python = testAllLanguages(languages, dirs, "all_sql_leave_one_out.pdf");
    String fileName = "python/src/all_sql_leave_one_out.py";
    Utils.writeFile(fileName, python);
    System.out.println("wrote python code to "+fileName);
}

public static Triple<Formatter,Float,Float> validate(LangDescriptor language,
                                                         List<InputDocument> documents,
                                                         InputDocument testDoc,
                                                         boolean saveOutput,
                                                         boolean computeEditDistance)
        throws Exception
    {
//      kNNClassifier.resetCache();
        Corpus corpus = new Corpus(documents, language);
        corpus.train();
//      System.out.printf("%d feature vectors\n", corpus.featureVectors.size());
        Formatter formatter = new Formatter(corpus, language.indentSize);
        String output = formatter.format(testDoc, false);
        float editDistance = 0;
        if ( computeEditDistance ) {
            editDistance = normalizedLevenshteinDistance(testDoc.content, output);
        }
        ClassificationAnalysis analysis = new ClassificationAnalysis(testDoc, formatter.getAnalysisPerToken());
//      System.out.println(testDoc.fileName+": edit distance = "+editDistance+", error rate = "+analysis.getErrorRate());
        if ( saveOutput ) {
            File dir = new File(outputDir+"/"+language.name);
            if ( saveOutput ) {
                dir = new File(outputDir+"/"+language.name);
                dir.mkdir();
            }
            Utils.writeFile(dir.getPath()+"/"+new File(testDoc.fileName).getName(), output);
        }
        return new Triple<>(formatter, editDistance, analysis.getErrorRate());
    }

@Override
public void visitTerminal(TerminalNode node) {
    if (builder.length() > 0) {
        builder.append(' ');
    }

    builder.append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false));
}

@Override
public void visitErrorNode(ErrorNode node) {
    if (builder.length() > 0) {
        builder.append(' ');
    }

    builder.append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false));
}

public static void writeFile(String dir, String fileName, String content) {
    try {
        Utils.writeFile(dir+"/"+fileName, content, "UTF-8");
    }
    catch (IOException ioe) {
        System.err.println("can't write file");
        ioe.printStackTrace(System.err);
    }
}

private String process(final Tree t, final List<String> ruleNames) {
    if (t.getChildCount() == 0) return Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
    StringBuilder sb = new StringBuilder();
    sb.append(lead(level));
    level++;
    String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
    sb.append(s + ' ');
    for (int i = 0; i < t.getChildCount(); i++) {
        sb.append(process(t.getChild(i), ruleNames));
    }
    level--;
    sb.append(lead(level));
    return sb.toString();
}

public static void writeFile(String dir, String fileName, String content) {
    try {
        Utils.writeFile(dir+"/"+fileName, content, "UTF-8");
    }
    catch (IOException ioe) {
        System.err.println("can't write file");
        ioe.printStackTrace(System.err);
    }
}

@Override
public void visitTerminal(TerminalNode node) {
    String text = Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false);
    if(text.startsWith(" ") || text.endsWith(" ")){
        text = "'" + text + "'";
    }
    stack.get(node.getParent()).add(text);
}

/**
 * Append the symbol and its position in the input text to the parse tree
 * buffer.
 *
 * @param symbol the symbol which should be appended to the parse tree.
 * @return boolean true if symbol is not null, false otherwise.
 */
protected boolean appendToken(Token symbol) {
  if (symbol != null) {
      buf.append("[");
        String s = Utils.escapeWhitespace(symbol.getText(), false);
        buf.append(s);
        buf.append("]    (line ");
        buf.append(String.valueOf(symbol.getLine()));
        buf.append(":");
        buf.append(String.valueOf(symbol.getCharPositionInLine()));
        buf.append(")");
        return true;
    }
    return false;
}

/**
 * Determine some string representation for the given node and append it to
 * the parse tree buffer.
 *
 * @param t The sub-tree whose current node needs a string representation.
 */
protected void appendNodeText(@NotNull Tree t) {
    if ( ruleNames!=null ) {
        if ( t instanceof RuleNode ) {
            if (appendRule(((RuleNode)t).getRuleContext())) return;
        }
        if ( t instanceof ErrorNode) {
          buf.append("ERROR: ");
          appendToken(((ErrorNode)t).getSymbol());
          return;
        }
        if ( t instanceof TerminalNode) {
            if (appendToken(((TerminalNode)t).getSymbol())) return;
        }
    }
    // no recog for rule names
    Object payload = t.getPayload();
    if ( payload instanceof Token ) {
      if (appendToken((Token)payload)) return;
    }
    if ( payload instanceof RuleContext ) {
      if (appendRule((RuleContext)payload)) return;
    }
    String s = Utils.escapeWhitespace(payload.toString(), false);
    buf.append("unknown payload <<");
   buf.append(s);
   buf.append(">>");
}

/** Output the current symbol with line and character position. */
public void outputSymbol(Token symbol) {
  if ( outputStream != null) {
    outputStream.print(", LT(1)=[");
    String s = Utils.escapeWhitespace(symbol.getText(), false);
    outputStream.print(s);
    outputStream.print("]    (line ");
    outputStream.print(String.valueOf(symbol.getLine()));
    outputStream.print(":");
    outputStream.print(String.valueOf(symbol.getCharPositionInLine()));
    outputStream.println(")");
  }
}

@Override
public String toString() {
    StringBuilder builder = new StringBuilder();
    builder.append("func");

    ParameterModelImpl receiver = getReceiverParameter();
    if (receiver != null) {
        builder.append(" (").append(receiver).append(")");
    }

    builder.append(" ").append(getName());
    builder.append("(");
    builder.append(Utils.join(getParameters().iterator(), ", "));
    builder.append(")");

    @SuppressWarnings("LocalVariableHidesMemberVariable")
    Collection<? extends ParameterModel> returnValues = getReturnValues();
    if (returnValues.size() == 1 && returnValues.iterator().next().getName().equals("_")) {
        builder.append(" ").append(returnValues.iterator().next().getVarType());
    } else if (!returnValues.isEmpty()) {
        builder.append(" (");
        boolean first = true;
        for (ParameterModel parameter : returnValues) {
            if (first) {
                first = false;
            } else {
                builder.append(", ");
            }

            if (!"_".equals(parameter.getName())) {
                builder.append(parameter.getName()).append(' ');
            }

            builder.append(parameter.getVarType());
        }
        builder.append(')');
    }

    return builder.toString();
}

@Override
public boolean equals(Object obj) {
    if (!(obj instanceof TypeModel)) {
        return false;
    }

    TypeModel other = (TypeModel)obj;
    return getKind().equals(other.getKind())
        && Utils.equals(getPackage(), other.getPackage())
        && getName().equals(other.getName());
}

/**
 * @see org.antlr.v4.runtime.tree.Trees.toStringTree(Tree, List<String>)
 */
public static String toStringTree(final Tree t, @Nullable final List<String> ruleNames, final int depth) {
    String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);

    if (t.getChildCount() == 0) {
        return s;
    }

    final StringBuilder buf = new StringBuilder();

    if (depth > 0) {
        buf.append(NEWLINE);
    }

    buf.append(indent(depth));
    buf.append("(");
    s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
    buf.append(s);
    buf.append(' ');

    for (int i = 0; i < t.getChildCount(); i++) {
        if (i > 0) {
            buf.append(' ');
        }

        buf.append(toStringTree(t.getChild(i), ruleNames, depth + 1));
    }

    buf.append(")");

    return buf.toString();
}

@Override
public String toString() {
    String symbol = "";
    if (startIndex >= 0 && startIndex < getInputStream().size()) {
        symbol = getInputStream().getText(Interval.of(startIndex,startIndex));
        symbol = Utils.escapeWhitespace(symbol, false);
    }

    return String.format(Locale.getDefault(), "%s('%s')", LexerNoViableAltException.class.getSimpleName(), symbol);
}

@Override
public void visitTerminal(TerminalNode node) {
    if (builder.length() > 0) {
        builder.append(' ');
    }

    append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false));
}

@Override
public void visitErrorNode(ErrorNode node) {
    if (builder.length() > 0) {
        builder.append(' ');
    }

    append(Utils.escapeWhitespace(Trees.getNodeText(node, ruleNames), false));
}

public static GrammarRootAST parseGrammar(Project project, Tool antlr, String grammarFileName) {
    try {
        String encoding = ConfigANTLRPerGrammar.getProp(project, grammarFileName, ConfigANTLRPerGrammar.PROP_ENCODING, "UTF-8");
        char[] grammarText = Utils.readFile(grammarFileName, encoding);
        String grammarTextS = new String(grammarText).replaceAll("\\r", "");
        ANTLRStringStream in = new ANTLRStringStream(grammarTextS);
        GrammarRootAST t = antlr.parse(grammarFileName, in);
        return t;
    }
    catch (IOException ioe) {
        antlr.errMgr.toolError(ErrorType.CANNOT_OPEN_FILE, ioe, grammarFileName);
    }
    return null;
}

public void checkCGen(String filename) throws Exception {
    URL testFolderURL = TestCGen.class.getClassLoader().getResource(SAMPLES_DIR);
    String testFolder = testFolderURL.getPath();
    String workingDir = getWorkingDir();

    String J_pathToFile = testFolder+"/"+filename;
    String C_filename = basename(filename)+".c";

    JTran jTran = new JTran();
    String C_code = jTran.translate(J_pathToFile, C_filename, false, false);

    Utils.writeFile(workingDir+"/"+C_filename, C_code);

    String[] indent_result_cmd = {
        "indent",
        "-bap", "-bad", "-br", "-nce", "-ncs", "-nprs", "-npcs", "-sai", "-saw",
        "-di1", "-brs", "-blf", "--indent-level4", "-nut", "-sob", "-l200",
        C_filename,
        "-o", C_filename // write on top of itself
    };

    // normalize generated code
    exec(indent_result_cmd, workingDir);

    // format the expected file as well
    String expected_C_CodeFilename = testFolder+"/"+C_filename;
    String[] indent_expected_cmd = {
        "indent",
        "-bap", "-bad", "-br", "-nce", "-ncs", "-nprs", "-npcs", "-sai", "-saw",
        "-di1", "-brs", "-blf", "--indent-level4", "-nut", "-sob", "-l200",
        expected_C_CodeFilename,
        "-o", "expected_"+C_filename
    };
    exec(indent_expected_cmd, workingDir);

    // compare with expected c file
    String[] diff_cmd = {
        "diff", "expected_"+C_filename, C_filename
    };
    Triple<Integer, String, String> result = exec(diff_cmd, workingDir);
    int execCode = result.a;
    String stdout = result.b;
    String stderr = result.c;

    assertEquals("", stdout);
    assertEquals("", stderr);
    assertEquals(0, execCode);
}