tasks.py 文件源码

def run(cls):
        runner = CrawlerRunner(get_project_settings())

        @defer.inlineCallbacks
        def deferred_crawl():
            for spider, args, kwargs in cls.queue:
                try:
                    yield runner.crawl(spider, *args, **kwargs)
                except KeyError as err:
                    # Log a warning if the scraper name is invalid instead of
                    # causing the job to fail.
                    # NOTE: If there is any other type of error, the job will
                    # fail, and all the jobs that depend on it will fail as
                    # well.
                    logger.warning(err.args[0])

            # XXX: If all the names fail, then trying to run
            # `reactor.stop()` will give an "Unhandled error in
            # Deferred" complaint and hang.  It will also hang in
            # general if no spiders have been run.  I assume there's
            # some twisted-way to handle this, but for now, just log an
            # error.
            if reactor.running:
                reactor.stop()
            else:
                logger.critical("LocalQueue: No valid scraper names found.")

        deferred_crawl()
        reactor.run()