suning.py 文件源码

def generate_productlist(self, response):
        product_list = response.xpath("//a[@class='sellPoint']/@href").extract()
        for product_url in product_list:
            yield scrapy.Request(
                'http:' + product_url,
                callback=self.generate_product_detail
            )
        # ?????
        # next_page = response.xpath("//a[@class='cur']/following-sibling::*[1]/@href").extract()[0]
        page_key = int(response.meta['page_key'])
        if page_key < 100:
            yield scrapy.Request(
                response.url.replace('-' + str(page_key) + '.html', '-' + str(page_key + 1) + '.html'),
                meta={"page_key": page_key+1},
                callback=self.generate_productlist
            )