middleware.py 文件源码

def process_exception(self, request, exception, spider):
        if 'proxy' not in request.meta: return

        if isinstance(exception, IgnoreRequest): return # No problem

        mode=request.meta.get('proxy_mode', self.mode) # Possible override
        if mode == 'once': # Try once mode, quit here
            return

        # Simple downvote
        self.pp.set_status(self.map_proxy(request.meta['proxy']), None)
        del request.meta['proxy'] # Will pick new proxy on next request

        # List of conditions when we retry. Some of them may disable the proxy (TBD)
        if type(exception) in (
                ConnectionRefusedError, ConnectError, TimeoutError,
                TCPTimedOutError, NoRouteError, ResponseNeverReceived,
                ResponseFailed, TunnelError ):
            lg.error('{} on %s'.format(type(exception)), request.url)

            return request.replace(dont_filter = True)