fetch.py 文件源码

def fetch(source, destination, allow_file=False):
    """
    Fetch a group of files either from a file archive or version control
    repository.

    Supported URL schemes:
      - file
      - ftp
      - ftps
      - git
      - git+http
      - git+https
      - git+ssh
      - http
      - https

    :param str source: The source URL to retrieve.
    :param str destination: The directory into which the files should be placed.
    :param bool allow_file: Whether or not to permit the file:// URL for processing local resources.
    :return: The destination directory that was used.
    :rtype: str
    """
    source = source.strip()
    if os.path.exists(destination):
        raise ValueError('destination must not be an existing directory')

    parsed_url = urllib.parse.urlparse(source, scheme='file')
    if parsed_url.username is not None:
        # if the username is not none, then the password will be a string
        creds = Creds(parsed_url.username, parsed_url.password or '')
    else:
        creds = Creds(None, None)

    parsed_url = collections.OrderedDict(zip(('scheme', 'netloc', 'path', 'params', 'query', 'fragment'), parsed_url))
    parsed_url['netloc'] = parsed_url['netloc'].split('@', 1)[-1]
    parsed_url['scheme'] = parsed_url['scheme'].lower()


    if parsed_url['scheme'] == 'file':
        if not allow_file:
            raise RuntimeError('file: URLs are not allowed to be processed')
        tmp_path = parsed_url['path']
        if os.path.isdir(tmp_path):
            shutil.copytree(tmp_path, destination, symlinks=True)
        elif os.path.isfile(tmp_path):
            shutil.unpack_archive(tmp_path, destination)
    else:
        tmp_fd, tmp_path = tempfile.mkstemp(suffix='_' + os.path.basename(parsed_url['path']))
        os.close(tmp_fd)
        tmp_file = open(tmp_path, 'wb')
        try:
            _fetch_remote(source, destination, parsed_url, creds, tmp_file, tmp_path)
            if os.stat(tmp_path).st_size:
                shutil.unpack_archive(tmp_path, destination)
        finally:
            tmp_file.close()
            os.remove(tmp_path)
    return