cnmo_forum.py 文件源码

def generate_forum_url_list(self, response):
        all_a_tags = response.xpath('//a/@href').extract()
        forum_dict = {}
        for a_tag in all_a_tags:
            if a_tag.find("forum") != -1:
                if a_tag in forum_dict:
                    forum_dict[a_tag] += 1
                else:
                    forum_dict[a_tag] = 1
        for a_href in forum_dict:
            yield scrapy.Request(
                a_href,
                meta={"page_key": 1},
                dont_filter='true',
                callback=self.get_record_list
            )
        # ???????
        for a_href in self.forum_url:
            yield scrapy.Request(
                a_href,
                meta={"page_key": 1},
                dont_filter='true',
                callback=self.get_record_list
            )