def prepare_oae_PU3(known_transisitons):
print("discriminate the correct transitions and the other transitions generated by OAE,",
" filtered by the learned state discriminator",
sep="\n")
N = known_transisitons.shape[1] // 2
y = generate_oae_action(known_transisitons)
print("removing invalid successors (sd3)")
ind = np.where(np.squeeze(combined(y[:,N:])) > 0.5)[0]
y = y[ind]
if len(known_transisitons) > 100:
y = y[:len(known_transisitons)] # undersample
print("valid:",len(known_transisitons),"mixed:",len(y),)
print("creating binary classification labels")
return (default_networks['PUDiscriminator'], *prepare_binary_classification_data(known_transisitons, y))
################################################################
# training parameters
python类squeeze()的实例源码
def prepare_oae_PU4(known_transisitons):
print("Learn from pre + action label",
"*** INCOMPATIBLE MODEL! ***",
sep="\n")
N = known_transisitons.shape[1] // 2
y = generate_oae_action(known_transisitons)
ind = np.where(np.squeeze(combined(y[:,N:])) > 0.5)[0]
y = y[ind]
actions = oae.encode_action(known_transisitons, batch_size=1000).round()
positive = np.concatenate((known_transisitons[:,:N], np.squeeze(actions)), axis=1)
actions = oae.encode_action(y, batch_size=1000).round()
negative = np.concatenate((y[:,:N], np.squeeze(actions)), axis=1)
# random.shuffle(negative)
# negative = negative[:len(positive)]
# normalize
return (default_networks['PUDiscriminator'], *prepare_binary_classification_data(positive, negative))
def prepare_oae_PU5(known_transisitons):
print("Learn from pre + suc + action label",
"*** INCOMPATIBLE MODEL! ***",
sep="\n")
N = known_transisitons.shape[1] // 2
y = generate_oae_action(known_transisitons)
ind = np.where(np.squeeze(combined(y[:,N:])) > 0.5)[0]
y = y[ind]
actions = oae.encode_action(known_transisitons, batch_size=1000).round()
positive = np.concatenate((known_transisitons, np.squeeze(actions)), axis=1)
actions = oae.encode_action(y, batch_size=1000).round()
negative = np.concatenate((y, np.squeeze(actions)), axis=1)
# random.shuffle(negative)
# negative = negative[:len(positive)]
# normalize
return (default_networks['PUDiscriminator'], *prepare_binary_classification_data(positive, negative))
def sumlogs(x, axis=None, out=None):
"""Sum of vector where numbers are represented by their logarithms.
Calculates np.log(np.sum(np.exp(x), axis=axis)) in such a fashion that it
works even when elements have large magnitude.
"""
maxx = x.max(axis=axis, keepdims=True)
xnorm = x - maxx
np.exp(xnorm, out=xnorm)
out = np.sum(xnorm, axis=axis, out=out)
if isinstance(out, np.ndarray):
np.log(out, out=out)
else:
out = np.log(out)
out += np.squeeze(maxx)
return out
def run_bottleneck_on_image(sess, image_data, image_data_tensor,
bottleneck_tensor):
"""Runs inference on an image to extract the 'bottleneck' summary layer.
Args:
sess: Current active TensorFlow Session.
image_data: String of raw JPEG data.
image_data_tensor: Input data layer in the graph.
bottleneck_tensor: Layer before the final softmax.
Returns:
Numpy array of bottleneck values.
"""
bottleneck_values = sess.run(
bottleneck_tensor,
{image_data_tensor: image_data})
bottleneck_values = np.squeeze(bottleneck_values)
return bottleneck_values
def extract_image_plane(snic, axes, ele_pos):
"""extract 2D imaging plane node IDs
Extract a 2D matrix of the imaging plane node IDs based on the
elevation position (mesh coordinates).
:param snic: sorted node IDs and coordinates
:param axes: spatial axes
:param ele_pos: elevation position for plane of interest
:returns: image_plane (node IDs)
"""
import numpy as np
ele0 = np.min(np.where(axes[0] >= ele_pos))
image_plane = np.squeeze(snic[ele0, :, :]).astype(int)
return image_plane
def test_basic(self):
from numpy.random import rand
a = rand(20, 10, 10, 1, 1)
b = rand(20, 1, 10, 1, 20)
c = rand(1, 1, 20, 10)
assert_array_equal(np.squeeze(a), np.reshape(a, (20, 10, 10)))
assert_array_equal(np.squeeze(b), np.reshape(b, (20, 10, 20)))
assert_array_equal(np.squeeze(c), np.reshape(c, (20, 10)))
# Squeezing to 0-dim should still give an ndarray
a = [[[1.5]]]
res = np.squeeze(a)
assert_equal(res, 1.5)
assert_equal(res.ndim, 0)
assert_equal(type(res), np.ndarray)
def k_nearest_approx(self, vec, k):
"""Get the k nearest neighbors of a vector (in terms of cosine similarity).
:param (np.array) vec: query vector
:param (int) k: number of top neighbors to return
:return (list[tuple[str, float]]): a list of (word, cosine similarity) pairs, in descending order
"""
if not hasattr(self, 'lshf'):
self.lshf = self._init_lsh_forest()
# TODO(kelvin): make this inner product score, to be consistent with k_nearest
distances, neighbors = self.lshf.kneighbors(vec, n_neighbors=k, return_distance=True)
scores = np.subtract(1, distances)
nbr_score_pairs = self.score_map(np.squeeze(neighbors), np.squeeze(scores))
return sorted(nbr_score_pairs.items(), key=lambda x: x[1], reverse=True)
def k_nearest_approx(self, vec, k):
"""Get the k nearest neighbors of a vector (in terms of cosine similarity).
:param (np.array) vec: query vector
:param (int) k: number of top neighbors to return
:return (list[tuple[str, float]]): a list of (word, cosine similarity) pairs, in descending order
"""
if not hasattr(self, 'lshf'):
self.lshf = self._init_lsh_forest()
# TODO(kelvin): make this inner product score, to be consistent with k_nearest
distances, neighbors = self.lshf.kneighbors(vec, n_neighbors=k, return_distance=True)
scores = np.subtract(1, distances)
nbr_score_pairs = self.score_map(np.squeeze(neighbors), np.squeeze(scores))
return sorted(nbr_score_pairs.items(), key=lambda x: x[1], reverse=True)
def new_image(self, image, diag=False):
if isinstance(image, str):
self.image_file = image
self.image = np.array(PIL.Image.open(image))
else:
self.image_file = None
self.image = image
# Get the image into the right format.
if self.image.dtype != np.uint8:
raise TypeError('Image %s dtype is not unsigned 8 bit integer, image.dtype is %s.'%(
'"%s"'%self.image_file if self.image_file is not None else 'argument',
self.image.dtype))
self.image = np.squeeze(self.image)
if len(self.image.shape) == 2:
self.image = np.dstack([self.image] * 3)
self.preprocess_edges()
self.randomize_view()
if diag:
plt.figure('Image')
plt.title('Image')
plt.imshow(self.image, interpolation='nearest')
plt.show()
def plot(self, image, filename, save_sample):
""" Plot an image."""
image = np.minimum(image, 1)
image = np.maximum(image, -1)
image = np.squeeze(image)
# Scale to 0..255.
imin, imax = image.min(), image.max()
image = (image - imin) * 255. / (imax - imin) + .5
image = image.astype(np.uint8)
if save_sample:
try:
Image.fromarray(image).save(filename)
except Exception as e:
print("Warning: could not sample to ", filename, ". Please check permissions and make sure the path exists")
print(e)
GlobalViewer.update(image)
def grid_vis(X, nh, nw): #[buggy]
if X.shape[0] == 1:
return X[0]
# nc = 3
if X.ndim == 3:
X = X[..., np.newaxis]
if X.shape[-1] == 1:
X = np.tile(X, [1,1,1,3])
h, w = X[0].shape[:2]
if X.dtype == np.uint8:
img = np.ones((h * nh, w * nw, 3), np.uint8) * 255
else:
img = np.ones((h * nh, w * nw, 3), X.dtype)
for n, x in enumerate(X):
j = n // nw
i = n % nw
img[j * h:j * h + h, i * w:i * w + w, :] = x
img = np.squeeze(img)
return img
def sim_getCorrelation(We,words,f, weight4ind, scoring_function, params):
f = open(f,'r')
lines = f.readlines()
golds = []
seq1 = []
seq2 = []
for i in lines:
i = i.split("\t")
p1 = i[0]; p2 = i[1]; score = float(i[2])
X1, X2 = data_io.getSeqs(p1,p2,words)
seq1.append(X1)
seq2.append(X2)
golds.append(score)
x1,m1 = data_io.prepare_data(seq1)
x2,m2 = data_io.prepare_data(seq2)
m1 = data_io.seq2weight(x1, m1, weight4ind)
m2 = data_io.seq2weight(x2, m2, weight4ind)
scores = scoring_function(We,x1,x2,m1,m2, params)
preds = np.squeeze(scores)
return pearsonr(preds,golds)[0], spearmanr(preds,golds)[0]
def getCorrelation(model,words,f, params=[]):
f = open(f,'r')
lines = f.readlines()
preds = []
golds = []
seq1 = []
seq2 = []
for i in lines:
i = i.split("\t")
p1 = i[0]; p2 = i[1]; score = float(i[2])
X1, X2 = data_io.getSeqs(p1,p2,words)
seq1.append(X1)
seq2.append(X2)
golds.append(score)
x1,m1 = data_io.prepare_data(seq1)
x2,m2 = data_io.prepare_data(seq2)
if params and params.weightfile:
m1 = data_io.seq2weight(x1, m1, params.weight4ind)
m2 = data_io.seq2weight(x2, m2, params.weight4ind)
scores = model.scoring_function(x1,x2,m1,m2)
preds = np.squeeze(scores)
return pearsonr(preds,golds)[0], spearmanr(preds,golds)[0]
def save_np_image(image, output_file, save_format='jpeg'):
"""Saves an image to disk.
Args:
image: 3-D numpy array of shape [image_size, image_size, 3] and dtype
float32, with values in [0, 1].
output_file: str, output file.
save_format: format for saving image (eg. jpeg).
"""
image = np.uint8(image * 255.0)
buf = io.BytesIO()
scipy.misc.imsave(buf, np.squeeze(image, 0), format=save_format)
buf.seek(0)
f = tf.gfile.GFile(output_file, 'w')
f.write(buf.getvalue())
f.close()
def run_bottleneck_on_image(sess, image_data, image_data_tensor,
bottleneck_tensor):
"""Runs inference on an image to extract the 'bottleneck' summary layer.
Args:
sess: Current active TensorFlow Session.
image_data: String of raw JPEG data.
image_data_tensor: Input data layer in the graph.
bottleneck_tensor: Layer before the final softmax.
Returns:
Numpy array of bottleneck values.
"""
bottleneck_values = sess.run(
bottleneck_tensor,
{image_data_tensor: image_data})
bottleneck_values = np.squeeze(bottleneck_values)
return bottleneck_values
def standardize(data_train, data_test):
"""
Standardize a dataset to have zero mean and unit standard deviation.
:param data_train: 2-D Numpy array. Training data.
:param data_test: 2-D Numpy array. Test data.
:return: (train_set, test_set, mean, std), The standardized dataset and
their mean and standard deviation before processing.
"""
std = np.std(data_train, 0, keepdims=True)
std[std == 0] = 1
mean = np.mean(data_train, 0, keepdims=True)
data_train_standardized = (data_train - mean) / std
data_test_standardized = (data_test - mean) / std
mean, std = np.squeeze(mean, 0), np.squeeze(std, 0)
return data_train_standardized, data_test_standardized, mean, std
def effective_sample_size(samples, burn_in=100):
"""
Compute the effective sample size of a chain of vector samples, using the
algorithm in Stan. Users should flatten their samples as vectors if not so.
:param samples: A 2-D numpy array of shape ``(M, D)``, where ``M`` is the
number of samples, and ``D`` is the number of dimensions of each
sample.
:param burn_in: The number of discarded samples.
:return: A 1-D numpy array. The effective sample size.
"""
current_ess = np.inf
esses = []
for d in range(samples.shape[1]):
ess = effective_sample_size_1d(np.squeeze(samples[burn_in:, d]))
assert ess >= 0
if ess > 0:
current_ess = min(current_ess, ess)
esses.append(ess)
return current_ess
def tru_plot9(X,labels,t,plot_suffix,clust_names,clust_color, plot_loc):
"""
From clustering_on_transcript_compatibility_counts, see github for MIT license
"""
unique_labels = np.unique(labels)
plt.figure(figsize=(15,10))
for i in unique_labels:
ind = np.squeeze(labels == i)
plt.scatter(X[ind,0],X[ind,1],c=clust_color[i],s=36,edgecolors='gray',
lw = 0.5, label=clust_names[i])
plt.legend(loc='upper right',bbox_to_anchor=(1.1, 1))
plt.legend(loc='upper right',bbox_to_anchor=(1.19, 1.01))
plt.title(t)
plt.xlim([-20,20])
plt.ylim([-20,20])
plt.axis('off')
plt.savefig(plot_loc+ 't-SNE_plot_tru_plot9_'+ plot_suffix +'.pdf', bbox_inches='tight')
# Plot function with Zeisel's colors corresponding to labels
