zhiyoo.py 文件源码

def start_requests(self):
        yield scrapy.Request(
            'http://bbs.zhiyoo.com/',
            meta={"page_key": 1, "proxy": MongoClient.get_random_proxy()},
            callback=self.generate_forum
        )
        for index in self.forum_arr:
            yield scrapy.Request(
                'http://bbs.zhiyoo.com/source/module/forum/tab_ajax.php?index=nav_' + str(index),
                meta={"page_key": 1, "proxy": MongoClient.get_random_proxy()},
                callback=self.generate_forum
            )
        # yield scrapy.Request(
        #     'http://bbs.zhiyoo.com/forum-401-1.html',
        #     callback=self.generate_forum_page_list
        # )