test.py 文件源码

def _crawl(self, start_file_path, fake_url, items=None, connector=None):
        """

        :param start_file_path: file path of start file
        :param fake_url: The fake url for Request
        :param connector: Connector instance
        :param items: List of jobs item to use as "job database". Default is empty list
        :return: list of job items
        """
        if items is None:
            items = []

        if connector is None:
            connector = SpiderTestConnector(items)

        request = Request(url=fake_url)
        start_response = fake_response_from_file(
                start_file_path,
                request=request,
                response_class=HtmlResponse
        )
        self._spider = self._get_prepared_spider()()
        self._spider.set_connector(connector)

        return list(self._parse_spider_response(self._spider.parse(start_response)))