dbpedia_run_log_http.py 文件源码

def main():
    results = []
    log_file = 'database.log'
    # with open(log_file) as f:
    #   #Spawn pool of workers to execute http queries
    #   pool = Pool()
    #   results = pool.map_async(run_log, f,1)
    #   pool.close()
    #   while not results.ready():
    #       remaining = results._number_left
    #       print "Waiting for", remaining, "tasks to complete..."
    #       sys.stdout.flush()
    #       time.sleep(10)

    with open(log_file) as in_, tqdm(total=40000) as pbar:
        count = 0.
        last_timestamp = datetime.utcnow()
        for l_ in in_:
            count += 1
            res, last_timestamp = run_log(l_, last_timestamp)

            if len(results) > 40000:
                break
            if count == 19:
                count = 0
                pbar.update(19)
                sys.stdout.flush()
            if res != -1.:
                results.append(res)

    with open(log_file + '-test2', 'a') as out:
        for entry in results:
        # for entry in results.get():
            if entry is not None:
                out.write(str(entry))