mappers_test.py 文件源码

def testNGramsWithSpaceSeparator(self):
    string_tensor = tf.constant(['One was Johnny', 'Two was a rat'])
    tokenized_tensor = tf.string_split(string_tensor, delimiter=' ')
    output_tensor = mappers.ngrams(
        tokens=tokenized_tensor,
        ngram_range=(1, 2),
        separator=' ')
    with tf.Session():
      output = output_tensor.eval()
      self.assertAllEqual(
          output.indices,
          [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
           [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6]])
      self.assertAllEqual(output.values, [
          'One', 'One was', 'was', 'was Johnny', 'Johnny',
          'Two', 'Two was', 'was', 'was a', 'a', 'a rat', 'rat'])
      self.assertAllEqual(output.dense_shape, [2, 7])