simplePyLex.py 文件源码

def tokenize_file(source_file, language=None, literal_option=3):
    print(source_file)

    code = ""
    try:
        with codecs.open(source_file, "r",encoding='utf-8', errors='ignore') as f:
            code = f.read()
    except UnicodeDecodeError:
        return '', []

    if language is None:
        try:
            lexer = get_lexer_for_filename(source_file)
            language = languageForLexer(lexer)
        except KeyError: # Not a valid extension
            lexer = guess_lexer(code)
            language = languageForLexer(lexer)
    else:
        lexer = get_lexer_by_name(language)

    return tokenize_code(code, lexer, language, literal_option)