requestor.py 文件源码

def _async_request(self, task):
        """Async request."""

        url = task.get('url')
        if url.startswith('first_task'):
            result = yield gen.maybe_future(self._fake_request(url, task))
        else:
            try:
                if task.get('fetch', {}).get('fetch_type') == 'js':
                    result = yield self._phantomjs_request(url, task)
                else:
                    result = yield self._http_request(url, task)
            except Exception as e:
                logger.exception(e)

        if task.get('process', {}).get('callback'):
            results, follows, db_name, coll_name = self.processor.handle_result(task, result)

            if results:
                # put results to resultdb
                self.processor.put_results(results, db_name, coll_name, task)

            if follows:
                # put new tasks to newtask_queue
                self.put_follows(follows)

        raise gen.Return(result)