reddit_parse.py 文件源码

def raw_data_generator(path):
    if os.path.isdir(path):
        for walk_root, walk_dir, walk_files in os.walk(path):
            for file_name in walk_files:
                file_path = os.path.join(walk_root, file_name)
                if file_path.endswith(FILE_SUFFIX):
                    print("\nReading from {}".format(file_path))
                    with BZ2File(file_path, "r") as raw_data:
                        try:
                            for line in raw_data: yield line
                        except IOError:
                            print("IOError from file {}".format(file_path))
                            continue
                else: print("Skipping file {} (doesn't end with {})".format(file_path, FILE_SUFFIX))
    elif os.path.isfile(path):
        print("Reading from {}".format(path))
        with BZ2File(path, "r") as raw_data:
            for line in raw_data: yield line