utilities.py 文件源码

def ensure_dataset_exists(files, dirname):
    path = os.path.join("data", dirname)
    rv = [os.path.join(path, f) for f in files]

    logger.info("Retrieving dataset from {}".format(path))
    if not os.path.exists(path):
        # Extract or download data
        try:
            os.makedirs(path)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

        for f, file_path in zip(files, rv):
            data_url = BASE_URL + dirname + "/" + f
            if not os.path.exists(file_path):
                logger.warn("Downloading {}".format(data_url))
                with urllib3.PoolManager().request('GET', data_url, preload_content=False) as r, \
                    open(file_path, 'wb') as w:
                        shutil.copyfileobj(r, w)
    return rv


# Convert data into a stream of never-ending data