def parse(self, response): for item in super(StoppingSpider, self).parse(response): yield item if not self.state.get('was_stopped'): self.state['was_stopped'] = True self.crawler.stop()