pipelines.py 文件源码

def process_item(self, item, spider):
        if isinstance(item, SsptransparenciaBO):
            key = 'bos'
            _id = item['id']
        elif isinstance(item, SsptransparenciaVitima):
            key = 'vitimas'
            _id = '%s::%s' % (item['bo_id'], item['count'])
        elif isinstance(item, SsptransparenciaNatureza):
            key = 'naturezas'
            _id = '%s::%s' % (item['bo_id'], item['count'])

        if _id in self.ids_seen[key]:
            raise DropItem('Duplicate item found: %s' % item)
        else:
            self.ids_seen[key].add(_id)
            return item