fetcher.py 文件源码

def __init__(self, config):
        self.settings = get_project_settings()
        self.settings.set('DOWNLOAD_MAXSIZE', config.get('max_file_size', 1024 * 1024 * 2))

        self.downloader = HTTPDownloadHandler(self.settings)
        self.proxies = {}
        self.valid_extensions = config.getlist('file_valid_extensions', "jpg, png")
        _proxies = config.items('proxy', ())
        for proxy_type, proxy in _proxies:
            self.proxies[proxy_type] = get_proxy(proxy, proxy_type)