pipelines.py 文件源码

def process_item(self, item, spider):
        if redis_db.hexists(redis_data_dict, item['link']):
            raise DropItem("Duplicate item found: %s" % item)
        else:
#            print item['link']

            cur=self.conn.cursor()
            add_url = """insert into sohuurl(url) VALUES (%s)"""
            data_url=(str(item['link']),)
            cur.execute(add_url,data_url)
            self.conn.commit()
            cur.close()
            return item