test.py 文件源码

def _parse_spider_response(self, spider_response):
        """
        :param spider_response: return of parse spider method
        :return: job item generator
        """



        for response_item in spider_response:
            if isinstance(response_item, Request):
                request = response_item
                file_path = self._dump_format % request.url.replace(self._replace, self._dump_dir)
                if file_path.find('file://') != -1:
                    file_path = file_path.replace('file://', '')
                response = fake_response_from_file(
                        file_path=file_path,
                        request=request,
                        response_class=HtmlResponse
                )
                # If a callback it's a job page request
                if request.callback:
                    for item in request.callback(response):
                        yield item
                # Else it's a next page
                else:
                    for job_item in self._parse_spider_response(self._spider.parse(response)):
                        yield job_item

            elif isinstance(response_item, Item):
                yield response_item