def __new__(cls, dtype=None, shape=None, tag='', tensor=None):
if tensor is not None:
if dtype is not None:
raise TypeError('Specify only one of tensor and dtype.')
if shape is not None:
raise TypeError('Specify only one of tensor and shape.')
dtype = tensor.dtype
shape = tensor.get_shape().as_list()
elif not (isinstance(dtype, tf.DType) or
isinstance(dtype, six.string_types)):
raise TypeError('%r is not a tf.DType or string' % (dtype,))
dtype = tf.as_dtype(dtype).base_dtype.name
if not all(isinstance(s, numbers.Integral) and s >= 0 for s in shape):
raise TypeError('shape must be non-negative integers: %s' % shape)
shape = tuple(int(s) for s in shape)
if not isinstance(tag, six.string_types):
raise TypeError('A TypeShape tag must be a string; type of %r is %s' %
(tag, type(tag)))
return _TypeShape.__new__(cls, dtype, shape, tag)
python类as_dtype()的实例源码
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def _create_variables(self):
if self.input_type.ndim != 0:
raise TypeError('Embeddings take scalar inputs.')
dtype = tf.as_dtype(self.input_type.dtype)
if not dtype.is_integer: raise TypeError('Embeddings take integer inputs.')
if dtype not in (tf.int32, tf.int64): # only dtypes supported by tf.gather
if np.iinfo(dtype.as_numpy_dtype).max > 2147483647:
# pedantic future-proofing to handle hypothetical tf.uint64
raise TypeError('cannot gather or upcast dtype %s' % dtype)
self._cast = True
else:
self._cast = False
self._weights = tf.get_variable(
'weights', self._weights_shape, initializer=self._initializer,
trainable=self._trainable)
def __init__(self, shape, dtype='float32', name=None):
super(Tensor, self).__init__(input_type=tdt.PyObjectType(),
output_type=tdt.TensorType(shape, dtype),
name=name)
self._dtype = np.dtype(self.output_type.dtype)
if not shape and tf.as_dtype(dtype).is_integer: # memoize scalar ints
self._evaluate = self._evaluate_memoized
def dtype_enum(self):
"""The dtype of this TypeShape as an enum."""
return tf.as_dtype(self.dtype).as_datatype_enum
def _build(self, **kwargs):
"""Build the mask input layer."""
Mask = kwargs[self.name]
assert tf.as_dtype(Mask.dtype).is_bool
M = tf.convert_to_tensor(Mask)
return M, 0.0
def _check_rank_type(self, X, M):
"""Check the rank of the input tensors."""
data_rank = len(X.shape)
mask_rank = len(M.shape)
assert data_rank == 3
assert mask_rank == 2
assert tf.as_dtype(M.dtype).is_bool
def __init__(self, dtype):
self._dtype = tf.as_dtype(dtype)
def __setstate__(self, state):
self._dtype = tf.as_dtype(state)
def __setstate__(self, state):
self._dtype = tf.as_dtype(state['dtype'])
self._is_categorical = state['is_categorical']
self._min_value = state['min_value']
self._max_value = state['max_value']
self._vocabulary_file = state['vocabulary_file']
def __init__(self, images, labels, dtype=tf.float32):
dtype = tf.as_dtype(dtype).base_dtype
if dtype is not tf.float32:
raise TypeError('Invalid image dtype %r, expected float32' %dtype)
assert images.shape[0] == labels.shape[0], ('images.shape: %s labels.shape: %s' % (images.shape, labels.shape))
self._num_examples = images.shape[0]
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def clip_gradients_by_global_norm(gradients_variables, clip_norm=20.):
"""Clips gradients of a multitask loss by their global norm.
Ignores all-zero tensors when computing the global norm.
Args:
gradients_variables: a list of pairs (gradient, variable).
clip_norm: a float Tensor, the global norm to clip on. Default is 20.0.
Returns:
list: A list of pairs of the same type as gradients_variables,.
fixed_global_norm: A 0-D (scalar) Tensor representing the global norm.
"""
gradients, variables = six.moves.zip(*gradients_variables)
def _replace_nonexisting_grad(grad):
if grad is None:
return grad
all_zeros = _is_all_zeros(grad)
return tf.cond(
all_zeros,
lambda: tf.zeros([], dtype=tf.as_dtype(grad.dtype)),
lambda: grad)
nonzero_gradients = [_replace_nonexisting_grad(g) for g in gradients]
fixed_global_norm = tf.global_norm(nonzero_gradients)
gradients, _ = tf.clip_by_global_norm(
gradients, clip_norm, use_norm=fixed_global_norm)
return list(six.moves.zip(gradients, variables)), fixed_global_norm
def __init__(self, images, labels,
dtype=tf.float32, flatten_images=True):
"""Construct a DataSet.
`dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
if (flatten_images):
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def cast_dtype(dtype, target):
"""Changes float dtypes to the target dtype, leaves others unchanged.
Used to map all float values to a target precision. Also casts numpy
dtypes to TensorFlow dtypes.
Parameters
----------
dtype : ``tf.DType`` or :class:`~numpy:numpy.dtype`
Input dtype to be converted
target : ``tf.DType``
Floating point dtype to which all floating types should be converted
Returns
-------
``tf.DType``
Input dtype, converted to ``target`` type if necessary
"""
if not isinstance(dtype, tf.DType):
dtype = tf.as_dtype(dtype)
if dtype.is_floating:
dtype = target
return dtype
input_data.py 文件源码
项目：All-Convnet-Autoencoder-Example
作者: loliverhennigh
项目源码
文件源码
阅读 31
收藏 0
点赞 0
评论 0
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError(
'Invalid image dtype %r, expected uint8 or float32' % dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
self._width = images.shape[1]
self._height = images.shape[2]
self._depth = images.shape[3]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns*depth]
assert images.shape[3] == IMAGE_DEPTH
images = images.reshape(
images.shape[0],
images.shape[1] * images.shape[2] * images.shape[3])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(np.float32)
images = np.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
images = images.reshape(images.shape[0],
images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(numpy.float32)
images = numpy.multiply(images, 1.0 / 255.0)
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def _process_health_pill_value(self,
wall_time,
step,
device_name,
output_slot,
node_name,
tensor_proto,
node_name_set=None):
"""Creates a HealthPillEvent containing various properties of a health pill.
Args:
wall_time: The wall time in seconds.
step: The session run step of the event.
device_name: The name of the node's device.
output_slot: The numeric output slot.
node_name: The name of the node (without the output slot).
tensor_proto: A tensor proto of data.
node_name_set: An optional set of node names that are relevant. If not
provided, no filtering by relevance occurs.
Returns:
An event_accumulator.HealthPillEvent. Or None if one could not be created.
"""
if node_name_set and node_name not in node_name_set:
# This event is not relevant.
return None
# Since we seek health pills for a specific step, this function
# returns 1 health pill per node per step. The wall time is the
# seconds since the epoch.
elements = list(tf.make_ndarray(tensor_proto))
return HealthPillEvent(
wall_time=wall_time,
step=step,
device_name=device_name,
output_slot=output_slot,
node_name=node_name,
dtype=repr(tf.as_dtype(elements[12])),
shape=elements[14:],
value=elements)
def variable(value, dtype=None, borrow=None, broadcastable=None, name=None, settable=True):
if dtype is None:
dtype = np.float32
x = tf.Variable(value, dtype=tf.as_dtype(dtype), name=name)
x._statestream_settable = settable
if settable:
x._assign_placeholder = tf.placeholder(dtype, shape=value.shape)
x._assign_op = x.assign(x._assign_placeholder)
return x
def scalar(value, dtype=np.float32, borrow=None, name=None, settable=True):
if dtype is None:
dtype = np.float32
x = tf.Variable(value, dtype=tf.as_dtype(dtype), name=name)
x._statestream_settable = settable
if settable:
x._assign_placeholder = tf.placeholder(dtype, shape=x.get_shape().as_list())
x._assign_op = x.assign(x._assign_placeholder)
return x
def set_value(x, value):
if x._statestream_settable:
value = np.asarray(value, dtype=x.dtype.base_dtype.name)
tf_dtype = tf.as_dtype(x.dtype.name.split('_')[0])
tf_get_session().run(x._assign_op, feed_dict={x._assign_placeholder: value})
else:
raise TypeError("Tried to set / assign non-settable tensorflow variable: " + str(x.name))
def zeros(shape, dtype=None, name=None):
if dtype is None:
dtype = np.float32
return tf.zeros(shape=shape, dtype=tf.as_dtype(dtype), name=name)
def ones(shape, dtype=None, name=None):
if dtype is None:
dtype = np.float32
return tf.ones(shape=shape, dtype=tf.as_dtype(dtype), name=name)
def __init__(self, images, labels, fake_data=False, one_hot=False,
dtype=tf.float32, trim_flag=False):
"""Construct a DataSet.
one_hot arg is used only if fake_data is true. `dtype` can be either
`uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
`[0, 1]`.
"""
dtype = tf.as_dtype(dtype).base_dtype
if dtype not in (tf.uint8, tf.float32):
raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
dtype)
if fake_data:
self._num_examples = 10000
self.one_hot = one_hot
else:
assert images.shape[0] == labels.shape[0], (
'images.shape: %s labels.shape: %s' % (images.shape,
labels.shape))
self._num_examples = images.shape[0]
# Convert shape from [num examples, rows, columns, depth]
# to [num examples, rows*columns] (assuming depth == 1)
assert images.shape[3] == 1
# images = images.reshape(images.shape[0],
# images.shape[1] * images.shape[2])
if dtype == tf.float32:
# Convert from [0, 255] -> [0.0, 1.0].
images = images.astype(np.float32)
images = np.multiply(images, 1.0 / 255.0)
# log.info(str(images.max()))
log.info(str(images.shape)) # (50000, 28, 28, 1)
log.info(str(labels.shape))
# if trim_flag:
# images = images[:500]
# labels = labels[:500]
# add generated data
'''gen_data = np.load('mnist-gen')
images = np.concatenate((images, gen_data['image']))
labels = np.concatenate((labels, gen_data['label']))'''
self._num_examples = images.shape[0]
log.info('using %d data for training' % self._num_examples )
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
