sinaSpider.py 文件源码

def start_requests(self):
        #?aims????ID
        #??????finished??
        while self.db.Aims.find_one()!=None:
            ID_item = self.db.Aims.find_one()
            self.db.Aims.delete_one({'ID': ID_item['ID']})
            print '-----------------------------------------'
            print ID_item['ID']
            print '-----------------------------------------'
            ID = str(ID_item['ID'])
            # self.finish_ID.add(ID)
            #??????finish
            if self.db.findin_finished(ID_item):
                print '-----------------------------------------'
                print 'WARNING:  ', ID, ' already finished'
                print '-----------------------------------------'
                self.db.Aims.delete_one(ID_item)
                continue
            else:
            # ????
                url_information0 = "https://m.weibo.cn/api/container/getIndex?type=uid&value=%s" % ID
                print url_information0
                yield Request(url=url_information0, meta={"ID": ID_item['ID']}, callback=self.parseInformation)