youku.py 文件源码

def run(self):
        try:
            self.conn = MySQLdb.connect(host=self.web.host,user=self.web.user,passwd = self.web.passwd,db=self.web.db,charset="utf8",connect_timeout=5)
            print "connet to %s ,%s\n" % (self.web.host,self.web.db)
        except:
            print "can't connet to %s ,%s\n" % (self.web.host,self.web.db)
            return
        i = 0
        try:  
            # python UCS-4 build?????  
            highpoints = re.compile('[\\x00-\\xFF]{2,4}')  
        except re.error:  
            # python UCS-2 build?????  
            highpoints = re.compile('[\uD800-\uDBFF][\uDC00-\uDFFF]')  
        for index,keyword in enumerate(self.web.keyword):
            index = index + 1
            #???????url
            for i in range(1,2):#??????
                myurl = url % (keyword , i)
                #?????????
                food = youkuSGML(self.web.scope)
                #??????????
                try:
                    context = urllib2.urlopen(myurl,timeout=5)
                    content = context.read()
                except:
                    print "can't read from %s " % myurl
                    continue

                food.feed(content)
                self.result += len(food.result)
        #?????????
                for clist in food.result:
                    score = round(random.random(),2)*10
                    scoreer = random.randint(10,100)
                    atime = int(time.time())
                    e = False
                    if clist["title"] == '':
                        continue
                    clist["title"] = MySQLdb.escape_string(clist["title"])
                    sql = "insert into gx_video(`cid`,`intro`,`title`,`picurl`,`playurl`,`score`,`scoreer`,`keywords`,`color`,`actor`,`director`,`content`,`area`,`language`,`year`,`serial`,`addtime`,`hits`,`monthhits`,`weekhits`,`dayhits`,`hitstime`,`stars`,`status`,`up`,`down`,`downurl`,`inputer`,`reurl`,`letter`,`genuine`) values (%d,'',\'%s\',\'%s\',\'%s\',%d,%d,'','','','','','','',0,0,%d,0,0,0,0,0,0,1,0,0,'','','','',0)" % (index,clist["title"],clist["pic"],clist["link"],score,scoreer,atime)
                    print sql
                    try:
                        try:
                            self.conn.ping()
                        except Exception,e:
                            try:
                               self.conn = MySQLdb.connect(host=self.web.host,user=self.web.user,passwd = self.web.passwd,db=self.web.db,charset="utf8",connect_timeout=5)
                               print "Reconnet to %s ,%s\n" % (self.web.host,self.web.db)
                            except:
                               print "can't Reconnet to %s ,%s\n" % (self.web.host,self.web.db)
                               e = True
                        if not e:
                            self.conn.query(sql)
                            self.rfinally += 1
                    except:print sql+"/n"
        print "%s get %d results\n and %s insert successfully" % (self.web.db,self.result,self.rfinally)
        self.conn.close()