pipelines.py 文件源码

def process_item(self, item, spider):
        """check item weather in item_seen
        """
        if item['hash'] in self.item_seen:
            raise DropItem('Duplicate item found: %s' %item)
        else:
            self.item_seen.add(item['hash'])
            return item