weibo_crawler.py 文件源码

def update_timeline(self, task, new_user):
        uid = task['uid']

        user_info = weibo_util.user_info(uid)
        num_weibos = int(user_info['statuses_count'])

        old_weibos = self.get_old_weibos(new_user, uid)
        since_id = 0 if old_weibos == {} else min(old_weibos.keys())
        all_weibos = self.get_all_weibos(uid, num_weibos, max_id=0, since_id=since_id)
        new_weibos = {wid : all_weibos[wid] for wid in all_weibos if wid not in old_weibos}
        deleted_weibos = {wid : old_weibos[wid] for wid in old_weibos if wid not in all_weibos}

        if len(all_weibos) != num_weibos:
            deleted_weibos = self.check_deleted_weibos(deleted_weibos)

        self.update_all(task, user_info, all_weibos, new_weibos, deleted_weibos)
        #logger.log('[x] Crawl finished for user %s.' % (uid))