process_data.py 文件源码

def import_data(data_dir=DATA_DIR, in_memory=True, extension=".cell"):
    """
    Reads all of the files in the `data_dir` and returns all of the contents in a variable.

    @param data_dir is a string with the name of the data directory
    @param in_memory is a boolean value. If true, it pulls all the data into memory

    @return
        if in_memory == True:
            is a tuple with the following format: ([[size, incoming]], [webpage_label])
                where outgoing is 1 is incoming and -1
        else:
            a tuple with the following format: ([paths], [webpage_label])

    """
    stdout.write("Starting data import\n")

    if in_memory:
        return pull_data_in_memory(data_dir, extension)
    else:
        return get_files(data_dir, extension)