logclient_operator.py 文件源码

def query_more(fn, offset, size, batch_size, *args):
    """list all data using the fn
    """
    if size < 0:
        expected_total_size = six.MAXSIZE
    else:
        expected_total_size = size
        batch_size = min(size, batch_size)

    response = None
    total_count_got = 0
    complete = False
    while True:
        for _c in range(DEFAULT_QUERY_RETRY_COUNT):
            ret = fn(*args, offset=offset, size=batch_size)
            if ret.is_completed():
                complete = True
                break

            time.sleep(DEFAULT_QUERY_RETRY_INTERVAL)

        if response is None:
            response = ret
        else:
            response.merge(ret)

        # if incompete, exit
        if not complete:
            break

        count = ret.get_count()
        offset += count
        total_count_got += count
        batch_size = min(batch_size, expected_total_size - total_count_got)
        if count == 0 or total_count_got >= expected_total_size:
            break

    return response