syntax.py 文件源码

def _test(text=None):
    if text is not None:
        text = "What are the prerequisites for csc369?\n"


    tokenizer = RegexpTokenizer('[\w\d]+')
    word_tokens = tokenizer.tokenize(query)
    # TODO: how to tokenize 'u of t' and 'uoft'
    # TODO: use Bing Spell Check API
    tokens = [Token(tk) for tk in word_tokens]
    ser = RegexpEntityRecognizer()
    ser.process(tokens)