spider.py 文件源码

def start_requests(cls, spiders, start_urls, *, middleware=None):
        try:
            res = []
            for spider in spiders:
                for r in spider.start_requests(start_urls):
                    if inspect.iscoroutine(r):
                        r = await r
                    if r:
                        res.append(r)
            if middleware:
                res = await cls._handle_start_requests(res, middleware)
        except Exception:
            log.warning("Unexpected error occurred when generating start requests", exc_info=True)
            return ()
        else:
            return res