downloadermiddlewares.py 文件源码

def process_exception(self, request, exception, spider):

        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
                and not request.meta.get('dont_retry', False):
            return self._retry(request, "%s:%s" % (exception.__class__.__name__, exception), spider)

        else:
            if request.meta.get("callback") == "parse":
                spider.crawler.stats.inc_total_pages(crawlid=request.meta['crawlid'])

            self.logger.error("in retry request error %s" % traceback.format_exc())
            raise IgnoreRequest("%s:%s unhandle error. " % (exception.__class__.__name__, exception))