spider.py 文件源码

def errback(self, failure):
        if failure.check(DNSLookupError):
            host = urlparse(failure.request.url).hostname
            logger.error('DNSLookupError on host[%s]',
                         host)
        elif failure.check(TimeoutError,
                           TCPTimedOutError):
            request = failure.request
            logger.error('TimeoutError on url[%s]',
                         request.url)
        elif failure.check(HttpError):
            response = failure.value.response
            logger.error('HttpError on url[%s, status=%d]',
                         response.url,
                         response.status)
        else:
            logger.error('SpiderError on url[%s, error=%s',
                         failure.request.url,
                         repr(failure))