scrapy_oschina.py 文件源码

def parse(self, response):
        sel=scrapy.Selector(response)
        links_in_a_page = sel.xpath('//a[@href]')

        for link_sel in links_in_a_page:
            item=OschinaItem()
            link=str(link_sel.re('href="(.*?)"')[0])
            if link:
                if not link.startswith('http'):
                    link=response.url+link
                yield scrapy.Request(link,callback=self.parse)
                item['link']=link
                link_text=link_sel.xpath('text()').extract()
                if link_text:
                    item['link_text']=str(link_text[0].encode('utf-8').strip())
                else:
                    item['link_text']=None

                yield item