pipelines.py 文件源码

def process_item(self, item, spider):
        """save to redis and return item

        :item: crawl item including host port
        :returns: return item or DropItem
        """
        if 'ip' not in item:
            raise DropItem('')
        port = item.get('port', 80)
        host = '%s:%s' % (item['ip'], port)
        self.conn.sadd(self.host_s, host)
        return item