exception_process.py 文件源码

def process_requset_method_wrapper(func):

    @wraps(func)
    def wrapper_method(*args, **kwds):
        self = args[0]
        request = kwds.get("request")
        spider = kwds.get("spider")
        try:
            return func(*args, **kwds)
        except Exception as e:
            spider.logger.error("error heppened in process_request method of %s in %s. Error:%s, processing %s," % (
            self.__class__.__name__, IP, traceback.format_exc(), request.url))
            spider.crawler.stats.set_failed_download(request.meta, str(e))
            raise IgnoreRequest(e)

    return wrapper_method