def decode(self, data, items):
decoded_items = {}
# Split tokens
tokens = tf.string_split([data], delimiter=self.delimiter).values
# Optionally prepend a special token
if self.prepend_token is not None:
tokens = tf.concat([[self.prepend_token], tokens], 0)
# Optionally append a special token
if self.append_token is not None:
tokens = tf.concat([tokens, [self.append_token]], 0)
decoded_items[self.length_feature_name] = tf.size(tokens)
decoded_items[self.tokens_feature_name] = tokens
return [decoded_items[_] for _ in items]
split_tokens_decoder.py 文件源码
python
阅读 31
收藏 0
点赞 0
评论 0
评论列表
文章目录