youku.py 文件源码

def parse(self, response):
        user = response.meta['user']
        count = int(response.xpath('//h3[@node-type="hdTitle"]/following-sibling::span/text()'
                                   ).extract()[0][1:-1].replace(',', ''))

        params = {
            'spm': 'a2hzp.8253869.0.0',
            'order': '1',
            'last_item': '',
            # 'last_vid': re.search(r'last_vid=(\d+)', response.body),
        }
        page, current, num = 1, 0, 50
        while current < count:
            params['page'] = str(page)
            # params['last_pn'] = i
            yield scrapy.FormRequest(url=response.url.split('?')[0], method='GET', meta={'user': user},
                                     formdata=params, callback=self.parse_items)
            current = num * page
            page += 1