pipelines.py 文件源码

def process_item(self, item, spider):
        """return ip is duplicate or not

        :item: crawl item including host port
        :returns: return item or DropItem
        """
        if 'ip' not in item:
            raise DropItem('')
        port = item.get('port', 80)
        host = '%s:%s' % (item['ip'], port)
        if self.conn.sismember(settings.HOST_S, host) or self.dup_in_queue(host):
            raise DropItem('%s, cause duplicate' % (host))
        else:
            return item