python类CrawlSpider()的实例源码

def parse_start_url(self, response):
        """CrawlSpider????start_url??Request?????parse_start_url??"""
        li_list = response.xpath('//*[@id="post_container"]/li')
        for li_div in li_list:
            link = li_div.xpath('.//div[@class="thumbnail"]/a/@href').extract_first()
            yield scrapy.Request(link, callback=self.parse_detail_url)

        next_page = response.xpath('//div[@class="pagination"]/a[@class="next"]/@href').extract_first()
        if next_page:
            yield scrapy.Request(next_page, callback=self.parse_start_url)