data_generator.py 文件源码

def generate_tokens(mean, std_dev, num_tokens):
    tokens = {}
    cnt = 0
    while cnt < num_tokens:
        length = int(round(random.normalvariate(mean,
                                                std_dev)))
        if length < 2:
            continue
        flag = True
        while flag:
            new_token = ''.join(random.choice(string.ascii_lowercase)
                                for i in range(length))
            if tokens.get(new_token) is None:
                tokens[new_token] = True
                flag = False
        cnt += 1
    return list(tokens.keys())