epicurious_spider.py 文件源码

def parse(self, response):
    """Parse the recipe list."""
    recipes = LinkExtractor(
        allow=("/recipes/.*/views")
    ).extract_links(response)
    if len(recipes) > 0:
      for recipe_link in recipes:
        yield scrapy.Request(recipe_link.url, callback=self.parse_item)

      base_url, page = response.url.split("=")
      yield scrapy.Request("{}={}".format(base_url, int(page) + 1),
                           callback=self.parse)
    else:
      print "Finished on {}".format(response.url)