def plot_density_map(x, y, xbins, ybins, Nlevels=4, cbar=True, weights=None):
Z = np.histogram2d(x, y, bins=(xbins, ybins), weights=weights)[0].astype(float).T
# central values
lt = get_centers_from_bins(xbins)
lm = get_centers_from_bins(ybins)
cX, cY = np.meshgrid(lt, lm)
X, Y = np.meshgrid(xbins, ybins)
im = plt.pcolor(X, Y, Z, cmap=plt.cm.Blues)
plt.contour(cX, cY, Z, levels=nice_levels(Z, Nlevels), cmap=plt.cm.Greys_r)
if cbar:
cb = plt.colorbar(im)
else:
cb = None
plt.xlim(xbins[0], xbins[-1])
plt.ylim(ybins[0], ybins[-1])
try:
plt.tight_layout()
except Exception as e:
print(e)
return plt.gca(), cb
python类tight_layout()的实例源码
two_sigma_financial_modelling.py 文件源码
项目:PortfolioTimeSeriesAnalysis
作者: MizioAnd
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def predicted_vs_actual_y_xgb(self, xgb, best_nrounds, xgb_params, x_train_split, x_test_split, y_train_split,
y_test_split, title_name):
# Split the training data into an extra set of test
# x_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train)
dtrain_split = xgb.DMatrix(x_train_split, label=y_train_split)
dtest_split = xgb.DMatrix(x_test_split)
print(np.shape(x_train_split), np.shape(x_test_split), np.shape(y_train_split), np.shape(y_test_split))
gbdt = xgb.train(xgb_params, dtrain_split, best_nrounds)
y_predicted = gbdt.predict(dtest_split)
plt.figure(figsize=(10, 5))
plt.scatter(y_test_split, y_predicted, s=20)
rmse_pred_vs_actual = self.rmse(y_predicted, y_test_split)
plt.title(''.join([title_name, ', Predicted vs. Actual.', ' rmse = ', str(rmse_pred_vs_actual)]))
plt.xlabel('Actual y')
plt.ylabel('Predicted y')
plt.plot([min(y_test_split), max(y_test_split)], [min(y_test_split), max(y_test_split)])
plt.tight_layout()
def plot(self, fontsize=16):
"""Create the barplot from the stats file"""
from sequana.lazy import pylab
from sequana.lazy import pandas as pd
pylab.clf()
df = pd.DataFrame(self._parse_data()['rules'])
ts = df.ix['mean-runtime']
total_time = df.ix['mean-runtime'].sum()
#ts['total'] = self._parse_data()['total_runtime'] / float(self.N)
ts['total'] = total_time
ts.sort_values(inplace=True)
ts.plot.barh(fontsize=fontsize)
pylab.grid(True)
pylab.xlabel("Seconds (s)", fontsize=fontsize)
try:
pylab.tight_layout()
except:
pass
def predicted_vs_actual_sale_price(self, x_train, y_train, title_name):
# Split the training data into an extra set of test
x_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train)
print(np.shape(x_train_split), np.shape(x_test_split), np.shape(y_train_split), np.shape(y_test_split))
lasso = LassoCV(alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1,
0.3, 0.6, 1],
max_iter=50000, cv=10)
# lasso = RidgeCV(alphas=[0.0001, 0.0003, 0.0006, 0.001, 0.003, 0.006, 0.01, 0.03, 0.06, 0.1,
# 0.3, 0.6, 1], cv=10)
lasso.fit(x_train_split, y_train_split)
y_predicted = lasso.predict(X=x_test_split)
plt.figure(figsize=(10, 5))
plt.scatter(y_test_split, y_predicted, s=20)
rmse_pred_vs_actual = self.rmse(y_predicted, y_test_split)
plt.title(''.join([title_name, ', Predicted vs. Actual.', ' rmse = ', str(rmse_pred_vs_actual)]))
plt.xlabel('Actual Sale Price')
plt.ylabel('Predicted Sale Price')
plt.plot([min(y_test_split), max(y_test_split)], [min(y_test_split), max(y_test_split)])
plt.tight_layout()
def predicted_vs_actual_sale_price_xgb(self, xgb_params, x_train, y_train, seed, title_name):
# Split the training data into an extra set of test
x_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train)
dtrain_split = xgb.DMatrix(x_train_split, label=y_train_split)
dtest_split = xgb.DMatrix(x_test_split)
res = xgb.cv(xgb_params, dtrain_split, num_boost_round=1000, nfold=4, seed=seed, stratified=False,
early_stopping_rounds=25, verbose_eval=10, show_stdv=True)
best_nrounds = res.shape[0] - 1
print(np.shape(x_train_split), np.shape(x_test_split), np.shape(y_train_split), np.shape(y_test_split))
gbdt = xgb.train(xgb_params, dtrain_split, best_nrounds)
y_predicted = gbdt.predict(dtest_split)
plt.figure(figsize=(10, 5))
plt.scatter(y_test_split, y_predicted, s=20)
rmse_pred_vs_actual = self.rmse(y_predicted, y_test_split)
plt.title(''.join([title_name, ', Predicted vs. Actual.', ' rmse = ', str(rmse_pred_vs_actual)]))
plt.xlabel('Actual Sale Price')
plt.ylabel('Predicted Sale Price')
plt.plot([min(y_test_split), max(y_test_split)], [min(y_test_split), max(y_test_split)])
plt.tight_layout()
def plotAccuracyGraph(X, Y, Xlabel='Variable', Ylabel='Accuracy', graphTitle="Test Accuracy Graph", filename="graph.pdf"):
""" Plots and saves accuracy graphs """
try:
timestamp = int(time.time())
fig = P.figure(figsize=(8,5))
# Set the graph's title
P.title(graphTitle, fontname='monospace')
# Set the axes labels
P.xlabel(Xlabel, fontsize=12, fontname='monospace')
P.ylabel(Ylabel, fontsize=12, fontname='monospace')
# Add horizontal and vertical lines to the graph
P.grid(color='DarkGray', linestyle='--', linewidth=0.1, axis='both')
# Add the data to the graph
P.plot(X, Y, 'r-*', linewidth=1.0)
# Save figure
prettyPrint("Saving figure to ./%s" % filename)#(graphTitle.replace(" ","_"), timestamp))
P.tight_layout()
fig.savefig("./%s" % filename)#(graphTitle.replace(" ", "_"), timestamp))
except Exception as e:
prettyPrint("Error encountered in \"plotAccuracyGraph\": %s" % e, "error")
return False
return True
def view_waveforms_clusters(data, halo, threshold, templates, amps_lim, n_curves=200, save=False):
nb_templates = templates.shape[1]
n_panels = numpy.ceil(numpy.sqrt(nb_templates))
mask = numpy.where(halo > -1)[0]
clust_idx = numpy.unique(halo[mask])
fig = pylab.figure()
square = True
center = len(data[0] - 1)//2
for count, i in enumerate(xrange(nb_templates)):
if square:
pylab.subplot(n_panels, n_panels, count + 1)
if (numpy.mod(count, n_panels) != 0):
pylab.setp(pylab.gca(), yticks=[])
if (count < n_panels*(n_panels - 1)):
pylab.setp(pylab.gca(), xticks=[])
subcurves = numpy.where(halo == clust_idx[count])[0]
for k in numpy.random.permutation(subcurves)[:n_curves]:
pylab.plot(data[k], '0.5')
pylab.plot(templates[:, count], 'r')
pylab.plot(amps_lim[count][0]*templates[:, count], 'b', alpha=0.5)
pylab.plot(amps_lim[count][1]*templates[:, count], 'b', alpha=0.5)
xmin, xmax = pylab.xlim()
pylab.plot([xmin, xmax], [-threshold, -threshold], 'k--')
pylab.plot([xmin, xmax], [threshold, threshold], 'k--')
#pylab.ylim(-1.5*threshold, 1.5*threshold)
ymin, ymax = pylab.ylim()
pylab.plot([center, center], [ymin, ymax], 'k--')
pylab.title('Cluster %d' %i)
if nb_templates > 0:
pylab.tight_layout()
if save:
pylab.savefig(os.path.join(save[0], 'waveforms_%s' %save[1]))
pylab.close()
else:
pylab.show()
del fig
def view_raw_templates(file_name, n_temp=2, square=True):
N_e, N_t, N_tm = templates.shape
if not numpy.iterable(n_temp):
if square:
idx = numpy.random.permutation(numpy.arange(N_tm//2))[:n_temp**2]
else:
idx = numpy.random.permutation(numpy.arange(N_tm//2))[:n_temp]
else:
idx = n_temp
import matplotlib.colors as colors
my_cmap = pylab.get_cmap('winter')
cNorm = colors.Normalize(vmin=0, vmax=N_e)
scalarMap = pylab.cm.ScalarMappable(norm=cNorm, cmap=my_cmap)
pylab.figure()
for count, i in enumerate(idx):
if square:
pylab.subplot(n_temp, n_temp, count + 1)
if (numpy.mod(count, n_temp) != 0):
pylab.setp(pylab.gca(), yticks=[])
if (count < n_temp*(n_temp - 1)):
pylab.setp(pylab.gca(), xticks=[])
else:
pylab.subplot(len(idx), 1, count + 1)
if count != (len(idx) - 1):
pylab.setp(pylab.gca(), xticks=[])
for j in xrange(N_e):
colorVal = scalarMap.to_rgba(j)
pylab.plot(templates[j, :, i], color=colorVal)
pylab.title('Template %d' %i)
pylab.tight_layout()
pylab.show()
def view_whitening(data):
pylab.subplot(121)
pylab.imshow(data['spatial'], interpolation='nearest')
pylab.title('Spatial')
pylab.xlabel('# Electrode')
pylab.ylabel('# Electrode')
pylab.colorbar()
pylab.subplot(122)
pylab.title('Temporal')
pylab.plot(data['temporal'])
pylab.xlabel('Time [ms]')
x, y = pylab.xticks()
pylab.xticks(x, (x-x[-1]//2)//10)
pylab.tight_layout()
two_sigma_financial_modelling.py 文件源码
项目:PortfolioTimeSeriesAnalysis
作者: MizioAnd
项目源码
文件源码
阅读 24
收藏 0
点赞 0
评论 0
def predicted_vs_actual_y_input_model(self, model, x_train_split, x_test_split, y_train_split, y_test_split,
title_name):
# Split the training data into an extra set of test
# x_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train)
print(np.shape(x_train_split), np.shape(x_test_split), np.shape(y_train_split), np.shape(y_test_split))
model.fit(x_train_split, y_train_split)
y_predicted = model.predict(x_test_split)
plt.figure(figsize=(10, 5))
plt.scatter(y_test_split, y_predicted, s=20)
rmse_pred_vs_actual = self.rmse(y_predicted, y_test_split)
plt.title(''.join([title_name, ', Predicted vs. Actual.', ' rmse = ', str(rmse_pred_vs_actual)]))
plt.xlabel('Actual y')
plt.ylabel('Predicted y')
plt.plot([min(y_test_split), max(y_test_split)], [min(y_test_split), max(y_test_split)])
plt.tight_layout()
def test_dT_impact(xvals, f, nmpc, sim, start=0.1, end=0.8, step=0.02, ymax=200,
sample_size=100, label=None):
"""Used to generate Figure XX of the paper."""
c = raw_input("Did you remove iter/time caps in IPOPT settings? [y/N] ")
if c.lower() not in ['y', 'yes']:
print "Then go ahead and do it."
return
stats = [Statistics() for _ in xrange(len(xvals))]
fails = [0. for _ in xrange(len(xvals))]
pylab.ion()
pylab.clf()
for (i, dT) in enumerate(xvals):
f(dT)
for _ in xrange(sample_size):
nmpc.on_tick(sim)
if 'Solve' in nmpc.nlp.return_status:
stats[i].add(nmpc.nlp.solve_time)
else: # max CPU time exceeded, infeasible problem detected, ...
fails[i] += 1.
yvals = [1000 * ts.avg if ts.avg is not None else 0. for ts in stats]
yerr = [1000 * ts.std if ts.std is not None else 0. for ts in stats]
pylab.bar(
xvals, yvals, width=step, yerr=yerr, color='y', capsize=5,
align='center', error_kw={'capsize': 5, 'elinewidth': 5})
pylab.xlim(start - step / 2, end + step / 2)
pylab.ylim(0, ymax)
pylab.grid(True)
if label is not None:
pylab.xlabel(label, fontsize=24)
pylab.ylabel('Comp. time (ms)', fontsize=20)
pylab.tick_params(labelsize=16)
pylab.twinx()
yfails = [100. * fails[i] / sample_size for i in xrange(len(xvals))]
pylab.plot(xvals, yfails, 'ro', markersize=12)
pylab.plot(xvals, yfails, 'r--', linewidth=3)
pylab.xlim(start - step / 2, end + step / 2)
pylab.ylabel("Failure rate [%]", fontsize=20)
pylab.tight_layout()
def predicted_vs_actual_sale_price_input_model(self, model, x_train, y_train, title_name):
# Split the training data into an extra set of test
x_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train)
print(np.shape(x_train_split), np.shape(x_test_split), np.shape(y_train_split), np.shape(y_test_split))
model.fit(x_train_split, y_train_split)
y_predicted = model.predict(x_test_split)
plt.figure(figsize=(10, 5))
plt.scatter(y_test_split, y_predicted, s=20)
rmse_pred_vs_actual = self.rmse(y_predicted, y_test_split)
plt.title(''.join([title_name, ', Predicted vs. Actual.', ' rmse = ', str(rmse_pred_vs_actual)]))
plt.xlabel('Actual Sale Price')
plt.ylabel('Predicted Sale Price')
plt.plot([min(y_test_split), max(y_test_split)], [min(y_test_split), max(y_test_split)])
plt.tight_layout()
def adjust_layout(self) :
x0, x1, y0, y1 = plt.axis()
plot_margin_x = 0.01 * float(x1)
plot_margin_y = 0.01 * float(y1)
plt.axis((x0 - plot_margin_x, x1 + plot_margin_x, y0, y1 + plot_margin_y))
plt.tight_layout()
def tight_layout():
from matplotlib import get_backend
from pylab import gcf
if get_backend().lower() in ['agg', 'macosx']:
gcf().set_tight_layout(True)
else:
plt.tight_layout()
def post_callback(self, *args, **kwargs):
self.kwargs.update(kwargs)
tight_layout(**self.kwargs)
def plot_marginals(state_space,p,name,t,labels = False,interactive = False):
import matplotlib
import matplotlib.pyplot as pl
if interactive == True:
pl.ion()
pl.clf()
pl.suptitle("time: "+ str(t)+" units")
#print("time : "+ str(t))
D = state_space.shape[1]
for i in range(D):
marg_X = np.unique(state_space[:,i])
A = np.where(marg_X[:,np.newaxis] == state_space[:,i].T[np.newaxis,:],1,0)
marg_p = np.dot(A,p)
pl.subplot(int(D/2)+1,2,i+1)
pl.plot(marg_X,marg_p)
pl.yticks(np.linspace(np.amin(marg_p), np.amax(marg_p), num=3))
pl.axvline(np.sum(marg_X*marg_p),color= 'r')
pl.axvline(marg_X[np.argmax(marg_p)],color='g')
if labels == False:
pl.xlabel("Specie: " + str(i+1))
else:
pl.xlabel(labels[i])
if interactive == True:
pl.draw()
else:
pl.tight_layout()
pl.show()
def show_comparison(expt_base, algorithms, subset, ymin=None, ymax=None):
expt_names = ['/'.join([expt_base, alg]) for alg in algorithms]
labels = [ALGORITHM_LABELS[alg] for alg in algorithms]
plot_log_probs(expt_names, subset=subset, labels=labels)
pylab.xlim(1e1, 1e5)
if ymin is None:
ymin, ymax = get_ylim(expt_base)
pylab.ylim(ymin, ymax)
pylab.xlabel('Wall clock time (seconds)', fontsize='x-large')
pylab.ylabel('{} log-probabilities'.format({'train': 'Training', 'test': 'Test'}[subset]),
fontsize='x-large')
pylab.tight_layout()
def view_synthetic_templates(self, indices=None, time=None, nn=100, hf_dist=45, a_dist=1.0):
if indices is None:
indices = range(self.nb_cells)
if not numpy.iterable(indices):
indices = [indices]
scaling = None
pylab.figure()
for i in indices:
template = self._get_synthetic_template(i, time, nn, hf_dist, a_dist)
template = template.toarray()
width = template.shape[1]
xmin, xmax = self.probe.field_of_view['x_min'], self.probe.field_of_view['x_max']
ymin, ymax = self.probe.field_of_view['y_min'], self.probe.field_of_view['y_max']
if scaling is None:
scaling= 10*numpy.max(numpy.abs(template))
colorVal = self._scalarMap_synthetic.to_rgba(i)
for count, i in enumerate(xrange(self.nb_channels)):
x, y = self.probe.positions[:, i]
xpadding = ((x - xmin)/(float(xmax - xmin) + 1))*(2*width)
ypadding = ((y - ymin)/(float(ymax - ymin) + 1))*scaling
pylab.plot(xpadding + numpy.arange(width), ypadding + template[i, :], color=colorVal)
pylab.tight_layout()
pylab.setp(pylab.gca(), xticks=[], yticks=[])
pylab.xlim(xmin, 3*width)
pylab.show()
def view_circus_templates(self, indices=None):
if indices is None:
indices = range(self.nb_templates)
if not numpy.iterable(indices):
indices = [indices]
data = self.template_store.get(indices, ['templates', 'norms'])
width = self.template_store.width
templates = data.pop('templates').T
norms = data.pop('norms')
scaling = None
pylab.figure()
for count, i in enumerate(indices):
template = templates[count].toarray().reshape(self.nb_channels, width) * norms[count]
xmin, xmax = self.probe.field_of_view['x_min'], self.probe.field_of_view['x_max']
ymin, ymax = self.probe.field_of_view['y_min'], self.probe.field_of_view['y_max']
if scaling is None:
scaling= 10*numpy.max(numpy.abs(template))
colorVal = self._scalarMap_circus.to_rgba(i)
for count, i in enumerate(xrange(self.nb_channels)):
x, y = self.probe.positions[:, i]
xpadding = ((x - xmin)/(float(xmax - xmin) + 1))*(2*width)
ypadding = ((y - ymin)/(float(ymax - ymin) + 1))*scaling
pylab.plot(xpadding + numpy.arange(width), ypadding + template[i, :], color=colorVal)
pylab.tight_layout()
pylab.setp(pylab.gca(), xticks=[], yticks=[])
pylab.xlim(xmin, 3*width)
pylab.show()
def main():
from argparse import ArgumentParser
p = ArgumentParser()
p.add_argument('--grammar', choices=('both', 'medium', 'big'))
p.add_argument('--rollout', choices=('CP', 'DP'))
args = p.parse_args()
CP = ('evalb_avg', 'pops')
DP = ('expected_recall_avg', 'mask')
GRAMMARS = ['medium', 'big'] if args.grammar == 'both' else [args.grammar]
ACC, RUN = DP if args.rollout == 'DP' else CP
pl.ion()
fig1, ax1 = pl.subplots(nrows=3, #sharex=True,
ncols=2, figsize=(10,10))
for i in range(3):
for j in range(2):
ax1[i,j].grid(False)
fig2, ax2 = pl.subplots(nrows=1, #sharex=True,
ncols=2, figsize=(10,5))
for i, GRAMMAR in enumerate(GRAMMARS):
plot(GRAMMAR, ACC, RUN, ax=ax1[:,i], col=i)
plot2(GRAMMAR, ACC, RUN, ax=ax2[i], col=i)
fig1.tight_layout()
fig2.tight_layout()
pl.ioff()
pl.show()
def show_group_frontiers(groups, name, XMAX, YMIN, baseline=None):
colors = cycle(['m','g','b','y','c','k'])
ax = pl.figure().add_subplot(111)
groups = list(groups)
print
print yellow % name
print yellow % '============'
for color, (group_name, r) in zip(colors, groups):
print '%s; color: %s; len %s' % (group_name, color, len(r))
for color, (group_name, r) in zip(colors, groups):
show_frontier(r.dev_runtime, r.dev_accuracy, label='%s (dev)' % group_name,
c=color, alpha=0.5, ax=ax, XMAX=XMAX, YMIN=YMIN, lw=LW)
# show_frontier(r.train_runtime, r.train_accuracy, label='%s (train)' % group_name,
# c=color, linestyle=':', alpha=0.5, ax=ax, XMAX=XMAX, YMIN=YMIN, lw=LW)
ax.set_xlabel('runtime')
ax.set_ylabel('accuracy')
if baseline is not None:
# show_frontier(baseline.train_runtime, baseline.train_accuracy, ax=pl.gca(),
# c='r', linestyle=':', alpha=0.25, label='baseline (train)', XMAX=XMAX, YMIN=YMIN, lw=LW)
show_frontier(baseline.dev_runtime, baseline.dev_accuracy, ax=pl.gca(),
c='r', alpha=0.25, label='baseline (dev)', XMAX=XMAX, YMIN=YMIN, lw=LW)
ax.set_title('Frontiers grouped-by %s' % name)
#ax.set_xlim(0.35, .85)
#ax.set_ylim(.74, .83)
if 0:
ax.legend(loc='best')
pl.tight_layout()
else:
# Shink current axis by 20%
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.85, box.height])
ax.figure.canvas.draw()
def view_templates(file_name, temp_id=0, best_elec=None, templates=None):
params = CircusParser(file_name)
N_e = params.getint('data', 'N_e')
N_total = params.getint('data', 'N_total')
sampling_rate = params.getint('data', 'sampling_rate')
do_temporal_whitening = params.getboolean('whitening', 'temporal')
do_spatial_whitening = params.getboolean('whitening', 'spatial')
spike_thresh = params.getfloat('detection', 'spike_thresh')
file_out_suff = params.get('data', 'file_out_suff')
N_t = params.getint('detection', 'N_t')
nodes, edges = get_nodes_and_edges(params)
chunk_size = N_t
N_total = params.getint('data', 'N_total')
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
if templates is None:
templates = load_data(params, 'templates')
clusters = load_data(params, 'clusters')
probe = params.probe
positions = {}
for i in probe['channel_groups'].keys():
positions.update(probe['channel_groups'][i]['geometry'])
xmin = 0
xmax = 0
ymin = 0
ymax = 0
scaling = 10*numpy.max(numpy.abs(templates[:,temp_id].toarray().reshape(N_e, N_t)))
for i in xrange(N_e):
if positions[i][0] < xmin:
xmin = positions[i][0]
if positions[i][0] > xmax:
xmax = positions[i][0]
if positions[i][1] < ymin:
ymin = positions[i][0]
if positions[i][1] > ymax:
ymax = positions[i][1]
if best_elec is None:
best_elec = clusters['electrodes'][temp_id]
elif best_elec == 'auto':
best_elec = numpy.argmin(numpy.min(templates[:, :, temp_id], 1))
pylab.figure()
for count, i in enumerate(xrange(N_e)):
x, y = positions[i]
xpadding = ((x - xmin)/(float(xmax - xmin) + 1))*(2*N_t)
ypadding = ((y - ymin)/(float(ymax - ymin) + 1))*scaling
if i == best_elec:
c='r'
elif i in inv_nodes[edges[nodes[best_elec]]]:
c='k'
else:
c='0.5'
pylab.plot(xpadding + numpy.arange(0, N_t), ypadding + templates[i, :, temp_id], color=c)
pylab.tight_layout()
pylab.setp(pylab.gca(), xticks=[], yticks=[])
pylab.xlim(xmin, 3*N_t)
pylab.show()
return best_elec
def view_masks(file_name, t_start=0, t_stop=1, n_elec=0):
params = CircusParser(file_name)
data_file = params.get_data_file()
data_file.open()
N_e = params.getint('data', 'N_e')
N_t = params.getint('detection', 'N_t')
N_total = params.nb_channels
sampling_rate = params.rate
do_temporal_whitening = params.getboolean('whitening', 'temporal')
do_spatial_whitening = params.getboolean('whitening', 'spatial')
spike_thresh = params.getfloat('detection', 'spike_thresh')
file_out_suff = params.get('data', 'file_out_suff')
nodes, edges = get_nodes_and_edges(params)
chunk_size = (t_stop - t_start)*sampling_rate
padding = (t_start*sampling_rate, t_start*sampling_rate)
inv_nodes = numpy.zeros(N_total, dtype=numpy.int32)
inv_nodes[nodes] = numpy.argsort(nodes)
safety_time = params.getint('clustering', 'safety_time')
if do_spatial_whitening:
spatial_whitening = load_data(params, 'spatial_whitening')
if do_temporal_whitening:
temporal_whitening = load_data(params, 'temporal_whitening')
thresholds = load_data(params, 'thresholds')
data = data_file.get_data(0, chunk_size, padding=padding, nodes=nodes)
data_shape = len(data)
data_file.close()
peaks = {}
indices = inv_nodes[edges[nodes[n_elec]]]
if do_spatial_whitening:
data = numpy.dot(data, spatial_whitening)
if do_temporal_whitening:
data = scipy.ndimage.filters.convolve1d(data, temporal_whitening, axis=0, mode='constant')
for i in xrange(N_e):
peaks[i] = algo.detect_peaks(data[:, i], thresholds[i], valley=True, mpd=0)
pylab.figure()
for count, i in enumerate(indices):
pylab.plot(count*5 + data[:, i], '0.25')
#xmin, xmax = pylab.xlim()
pylab.scatter(peaks[i], count*5 + data[peaks[i], i], s=10, c='r')
for count, i in enumerate(peaks[n_elec]):
pylab.axvspan(i - safety_time, i + safety_time, facecolor='r', alpha=0.5)
pylab.ylim(-5, len(indices)*5 )
pylab.xlabel('Time [ms]')
pylab.ylabel('Electrode')
pylab.tight_layout()
pylab.setp(pylab.gca(), yticks=[])
pylab.show()
return peaks
def plot_rels(data, labels=None, colors=None, outfile="rels", latent=None, alpha=0.8, title=''):
ns, n = data.shape
if labels is None:
labels = map(str, range(n))
ncol = 5
nrow = int(np.ceil(float(n * (n - 1) / 2) / ncol))
fig, axs = pylab.subplots(nrow, ncol)
fig.set_size_inches(5 * ncol, 5 * nrow)
pairs = list(combinations(range(n), 2))
if colors is not None:
colors = (colors - np.min(colors)) / (np.max(colors) - np.min(colors))
for ax, pair in zip(axs.flat, pairs):
diff_x = max(data[:, pair[0]]) - min(data[:, pair[0]])
diff_y = max(data[:, pair[1]]) - min(data[:, pair[1]])
ax.set_xlim([min(data[:, pair[0]]) - 0.05 * diff_x, max(data[:, pair[0]]) + 0.05 * diff_x])
ax.set_ylim([min(data[:, pair[1]]) - 0.05 * diff_y, max(data[:, pair[1]]) + 0.05 * diff_y])
ax.scatter(data[:, pair[0]], data[:, pair[1]], c=colors, cmap=pylab.get_cmap("jet"),
marker='.', alpha=alpha, edgecolors='none', vmin=0, vmax=1)
ax.set_xlabel(shorten(labels[pair[0]]))
ax.set_ylabel(shorten(labels[pair[1]]))
for ax in axs.flat[axs.size - 1:len(pairs) - 1:-1]:
ax.scatter(data[:, 0], data[:, 1], marker='.')
fig.suptitle(title, fontsize=16)
pylab.rcParams['font.size'] = 12 #6
# pylab.draw()
# fig.set_tight_layout(True)
pylab.tight_layout()
pylab.subplots_adjust(top=0.95)
for ax in axs.flat[axs.size - 1:len(pairs) - 1:-1]:
ax.set_visible(False)
filename = outfile + '.png'
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
fig.savefig(outfile + '.png')
pylab.close('all')
return True
# Hierarchical graph visualization utilities
def _plot_features(out_dir, signal, sampling_rate, logmel, delta, delta_delta, specgram, filename):
try:
os.makedirs(out_dir)
except:
pass
sampling_interval = 1.0 / sampling_rate
times = np.arange(len(signal)) * sampling_interval
pylab.clf()
plt.rcParams['font.size'] = 18
pylab.figure(figsize=(len(signal) / 2000, 16))
ax1 = pylab.subplot(511)
pylab.plot(times, signal)
pylab.title("Waveform")
pylab.xlabel("Time [sec]")
pylab.ylabel("Amplitude")
pylab.xlim([0, len(signal) * sampling_interval])
ax2 = pylab.subplot(512)
specgram = np.log(specgram)
pylab.pcolormesh(np.arange(0, specgram.shape[0]), np.arange(0, specgram.shape[1]) * 8000 / specgram.shape[1], specgram.T, cmap=pylab.get_cmap("jet"))
pylab.title("Spectrogram")
pylab.xlabel("Time [sec]")
pylab.ylabel("Frequency [Hz]")
pylab.colorbar()
ax3 = pylab.subplot(513)
pylab.pcolormesh(np.arange(0, logmel.shape[0]), np.arange(1, 41), logmel.T, cmap=pylab.get_cmap("jet"))
pylab.title("Log mel filter bank features")
pylab.xlabel("Frame")
pylab.ylabel("Filter number")
pylab.colorbar()
ax4 = pylab.subplot(514)
pylab.pcolormesh(np.arange(0, delta.shape[0]), np.arange(1, 41), delta.T, cmap=pylab.get_cmap("jet"))
pylab.title("Deltas")
pylab.xlabel("Frame")
pylab.ylabel("Filter number")
pylab.colorbar()
ax5 = pylab.subplot(515)
pylab.pcolormesh(np.arange(0, delta_delta.shape[0]), np.arange(1, 41), delta_delta.T, cmap=pylab.get_cmap("jet"))
pylab.title("Delta-deltas")
pylab.xlabel("Frame")
pylab.ylabel("Filter number")
pylab.colorbar()
pylab.tight_layout()
pylab.savefig(os.path.join(out_dir, filename), bbox_inches="tight")
def _plotVirtualTime(accessList, archName, fgname, rd_bin_width = 250):
"""
Plot data access based on virtual time
"""
print "converting to num arrary for _plotVirtualTime"
stt = time.time()
x, y, id, ad, yd = _raListToVTimeNA(accessList)
print ("Converting to num array takes %d seconds" % (time.time() - stt))
fig = pl.figure()
ax = fig.add_subplot(211)
ax.set_xlabel('Access sequence number (%s to %s)' % (id.split('T')[0], ad.split('T')[0]), fontsize = 9)
ax.set_ylabel('Observation sequence number', fontsize = 9)
ax.set_title('%s archive activity ' % (archName), fontsize=10)
ax.tick_params(axis='both', which='major', labelsize=8)
ax.tick_params(axis='both', which='minor', labelsize=6)
ax.plot(x, y, color = 'b', marker = 'x', linestyle = '',
label = 'access', markersize = 3)
#legend = ax.legend(loc = 'upper left', shadow=True, prop={'size':7})
ax1 = fig.add_subplot(212)
ax1.set_xlabel('Access sequence number (%s to %s)' % (id.split('T')[0], ad.split('T')[0]), fontsize = 9)
ax1.set_ylabel('Reuse distance (in-between accesses)', fontsize = 9)
ax1.tick_params(axis='both', which='major', labelsize=8)
ax1.tick_params(axis='both', which='minor', labelsize=6)
ax1.plot(x, yd, color = 'k', marker = '+', linestyle = '',
label = 'reuse distance', markersize = 3)
pl.tight_layout()
fig.savefig(fgname)
pl.close(fig)
y1d = yd[~np.isnan(yd)]
num_bin = (max(y1d) - min(y1d)) / rd_bin_width
hist, bins = np.histogram(y1d, bins = num_bin)
width = 0.7 * (bins[1] - bins[0])
center = (bins[:-1] + bins[1:]) / 2
fig1 = pl.figure()
#fig1.suptitle('Histogram of data transfer rate from Pawsey to MIT', fontsize=14)
ax2 = fig1.add_subplot(111)
ax2.set_title('Reuse distance Histogram for %s' % archName, fontsize = 10)
ax2.set_ylabel('Frequency', fontsize = 9)
ax2.set_xlabel('Reuse distance (# of observation)', fontsize = 9)
ax2.tick_params(axis='both', which='major', labelsize=8)
ax2.tick_params(axis='both', which='minor', labelsize=6)
pl.bar(center, hist, align='center', width=width)
fileName, fileExtension = os.path.splitext(fgname)
fig1.savefig('%s_rud_hist%s' % (fileName, fileExtension))
pl.close(fig1)
def plotReductionGraph(dataSamples, dataLabels, classNames, dimension=2, graphTitle="Test Graph", filename="reduction.pdf"):
""" Plots data sample visualization graphs """
try:
timestamp = int(time.time())
colors = ['DarkRed', 'DarkGreen', 'DarkBlue', 'DarkOrange', 'DarkMagenta', 'DarkCyan', 'Gray', 'Black']
randomColor = lambda: random.randint(0,255)
markers = ['*', 'o', 'v', '^', 's', 'd', 'D', 'p', 'h', 'H', '<', '>', '.', ',', '|', '_']
fig = P.figure(figsize=(8,5))
if dimension == 3:
ax = fig.add_subplot(111, projection='3d')
P.title(graphTitle, fontname='monospace')
if dimension == 2:
P.xlabel('x1', fontsize=12, fontname='monospace')
P.ylabel('x2', fontsize=12, fontname='monospace')
else:
ax.set_xlabel('x1', fontsize=12, fontname='monospace')
ax.set_ylabel('x2', fontsize=12, fontname='monospace')
ax.set_zlabel('x3', fontsize=12, fontname='monospace')
P.grid(color='DarkGray', linestyle='--', linewidth=0.1, axis='both')
for c in range(len(classNames)):
X,Y,Z = [], [], []
for labelIndex in range(len(dataLabels)):
if c == dataLabels[labelIndex]:
X.append(dataSamples[labelIndex,:].tolist()[0])
Y.append(dataSamples[labelIndex,:].tolist()[1])
if dimension == 3:
Z.append(dataSamples[labelIndex,:].tolist()[2])
# Plot points of that class
#P.plot(Y, X, color='#%02X%02X%02X' % (randomColor(), randomColor(), randomColor()), marker=markers[c], markeredgecolor='None', markersize=4.0, linestyle='None', label=classNames[c])
if dimension == 2:
P.plot(Y, X, color=colors[c % len(colors)], marker=markers[c % len(markers)], markersize=5.0, linestyle='None', label=classNames[c])
else:
ax.scatter(X,Y,Z,c=colors[c % len(colors)], marker=markers[c % len(markers)])
if dimension == 2:
#P.legend([x.split(",")[-1] for x in classNames], fontsize='xx-small', numpoints=1, fancybox=True)
P.legend([x for x in classNames], fontsize='xx-small', numpoints=1, fancybox=True)
else:
ax.legend([x for x in classNames], fontsize='xx-small', numpoints=1, fancybox=True)
prettyPrint("Saving results to ./%s" % filename)#(graphTitle, timestamp))
P.tight_layout()
fig.savefig("./%s" % filename)#(graphTitle, timestamp))
except Exception as e:
prettyPrint("Error encountered in \"plotReductionGraph\": %s" % e, "error")
return False
return True