Tokenizers.java 文件源码

@Override
public Multiset<String> tokenizeToMultiset(String input) {

    // tokenizeToList is not reused here on purpose. Removing duplicate
    // words early means these don't have to be tokenized multiple
    // times. Increases performance.

    Multiset<String> tokens = HashMultiset.create(input.length());
    tokens.add(input);

    Multiset<String> newTokens = HashMultiset.create(input.length());
    for (Tokenizer t : tokenizers) {
        for (String token : tokens) {
            newTokens.addAll(t.tokenizeToList(token));
        }
        Multiset<String> swap = tokens;
        tokens = newTokens;
        newTokens = swap;
        newTokens.clear();
    }

    return tokens;
}