loader.py 文件源码

def load_records(task, dataset_name, query_idx, records):
    """Load a single batch of QUEUE_PAGE records from the given query."""
    dataset = model.get_dataset(dataset_name)
    items = []
    for record in records:
        for item in map_record(dataset.queries[query_idx], record):
            items.append(item)

    try:
        index_items(items)
    except ElasticsearchException as exc:
        time.sleep(30)
        raise task.retry(exc=exc, countdown=30, max_retries=5)

    log.info("[%r] Indexed %s records as %s documents...",
             dataset_name, len(records), len(items))