sent-thoughts-words.py 文件源码

def generate_sentence_batch(sents, word2id, max_seqlen, batch_size):
    while True:
        # loop once per epoch
        # shuffle the input
        indices = np.random.permutation(np.arange(len(sents)))
        shuffled_sents = [sents[ix] for ix in indices]
        # convert to list of list of word id        
        sent_wids = [[word2id[word] for word in sent.split()]
                                    for sent in shuffled_sents]
        num_batches = len(shuffled_sents) // batch_size
        for bid in range(num_batches):
            # loop once per batch
            sents_batch = sent_wids[bid * batch_size : (bid + 1) * batch_size]
            sents_batch_padded = sequence.pad_sequences(sents_batch, max_seqlen)
            yield sents_batch_padded, sents_batch_padded            


############################ main ###############################