feed_exp.py 文件源码

def fetch_all(feeds):
    BATCH_SIZE=5
    batches = []
    for feeds_batch in batch_gen(feeds, BATCH_SIZE):
        sem = DeferredSemaphore(len(feeds_batch))
        batch = []
        for feed_ in feeds_batch:
            batch.append(sem.run(fetch_single, feed_meta=feed_))
        batchDef = gatherResults(batch, consumeErrors=False)
        batchDef.addCallback(store_fetched_data)
        batches.append(batchDef)

    # rendez-vous for all feeds that were fetched
    batchesDef = gatherResults(batches, consumeErrors=False)

    batchesDef.addCallbacks(
            clean_up_and_exit,
            errback=lambda x: None,
            )
    return batchesDef