spider.py 文件源码

def crawl(cls, spiders, response, *, middleware=None):
        try:
            if middleware:
                await cls._handle_input(response, middleware)
            res = []
            for spider in spiders:
                for r in spider.parse(response):
                    if inspect.iscoroutine(r):
                        r = await r
                    if r:
                        res.append(r)
            if middleware:
                res = await cls._handle_output(response, res, middleware)
        except Exception as e:
            try:
                if middleware:
                    await cls._handle_error(response, e, middleware)
            except Exception:
                pass
            return ()
        else:
            return res