design_strategy_spider.py 文件源码

def parse_list(self, response):
        selector = Selector(response)
        items_selector = selector.xpath('//div[@id="listITme"]//div[@class="gl-listItem"]')
        for item_selector in items_selector:
            id = item_selector.xpath('a/@href').extract()[0].replace('/strategy/', '')
            # http://guju.com.cn/strategy/strategy_getStrategyInfo_ajax?strategyModel.id=4498
            next_url = (constant.PROTOCOL_HTTP + self.start_url_domain + '/strategy/strategy_getStrategyInfo_ajax?strategyModel.id={id}').format(
                id=id)
            if self.design_strategy_service.is_duplicate_url(next_url):
                log.info("=================???" + next_url + "===========")
                continue
            yield scrapy.Request(next_url, self.parse_content, meta={'id': id})