xiaomi_spider.py 文件源码

def parse_page(self, response):
        page = Selector(response)
        lis = page.xpath('//ul[@class="applist"]/li')
        if lis == None:
            return

        url_common = 'http://app.mi.com'

        for li in lis:
            item = XiaomiAppstoreCrawlerItem()
            item['title'] = li.xpath('./h5/a/text()').extract_first().encode('utf-8')
            url = li.xpath('./h5/a/@href').extract_first()
            appid = re.match(r'/detail/(.*)', url).group(1)
            item['appid'] = appid
            # import pudb; pu.db
            req = scrapy.Request(url_common + url, callback=self.parse_details)
            req.meta["item"] = item
            yield req