def __init__(self, p, max_shift, bins_per_semitone,
target_type='chords_maj_min'):
"""
Augmenter that shifts by semitones a spectrum with logarithmically
spaced frequency bins.
:param p: percentage of data to be shifted
:param max_shift: maximum number of semitones to shift
:param bins_per_semitone: number of spectrogram bins per semitone
:param target_type: specifies target type
"""
self.p = p
self.max_shift = max_shift
self.bins_per_semitone = bins_per_semitone
if target_type == 'chords_maj_min':
self.adapt_targets = self._adapt_targets_chords_maj_min
elif target_type == 'chroma':
self.adapt_targets = self._adapt_targets_chroma
python类shift()的实例源码
def __call__(self, batch_iterator):
"""
:param batch_iterator: data iterator that yields the data to be
augmented
:return: augmented data/target pairs
"""
for data, targets in batch_iterator:
batch_size = len(data)
shifts = np.random.rand(batch_size) * 2 * self.max_shift - \
self.max_shift
# zero out shifts for 1-p percentage
no_shift = random.sample(range(batch_size),
int(batch_size * (1 - self.p)))
shifts[no_shift] = 0
new_data = np.empty_like(data)
for i in range(batch_size):
new_data[i] = shift(
data[i], (shifts[i] * self.bins_per_semitone, 0))
yield new_data, targets
def get_optflux_Eran (P, P_noshift, D, S, V):
"""Function that calculates optimal flux and corresponding error based
on the PSF [P], the PSF shifted by the fractional pixel shift
[P_shift], data [D], sky [S] and variance [V]. All are assumed to
be in electrons rather than counts. These can be 1- or
2-dimensional arrays with the same shape, while the sky can also
be a scalar. See Eqs. 36 and 37 of Zackay & Ofek 2017, ApJ, 836,
187.
"""
# and optimal flux and its error
denominator = np.sum((P_noshift*P)/V)
optflux = np.sum((P_noshift*(D-S)/V)) / denominator
optfluxerr = 1./np.sqrt(denominator)
return optflux, optfluxerr
################################################################################
def load_transform(image_path, angle=0., s=(0,0), size=(20,20)):
#Load the image
original = imread(image_path, flatten=True)
#Rotate the image
rotated = np.maximum(np.minimum(rotate(original, angle=angle, cval=1.), 1.), 0.)
#Shift the image
shifted = shift(rotated, shift=s)
#Resize the image
resized = np.asarray(imresize(rotated, size=size), dtype=np.float32) / 255 #Note here we coded manually as np.float32, it should be tf.float32
#Invert the image
inverted = 1. - resized
max_value = np.max(inverted)
if max_value > 0:
inverted /= max_value
return inverted
def load_transform(image_path, angle=0., s=(0,0), size=(20,20)):
#Load the image
original = imread(image_path, flatten=True)
#Rotate the image
rotated = np.maximum(np.minimum(rotate(original, angle=angle, cval=1.), 1.), 0.)
#Shift the image
shifted = shift(rotated, shift=s)
#Resize the image
resized = np.asarray(imresize(rotated, size=size), dtype=np.float32) / 255 #Note here we coded manually as np.float32, it should be tf.float32
#Invert the image
inverted = 1. - resized
max_value = np.max(inverted)
if max_value > 0:
inverted /= max_value
return inverted
def translate_transform_batch(x):
t = (np.random.rand(x.shape[0], 2) - 0.5) * 4
x_out = np.empty_like(x)
for i in range(x.shape[0]):
# Super slow but whatever...
shift(x[i, 0], shift=t[i], output=x_out[i, 0], order=3, mode='reflect')
shift(x[i, 1], shift=t[i], output=x_out[i, 1], order=3, mode='reflect')
shift(x[i, 2], shift=t[i], output=x_out[i, 2], order=3, mode='reflect')
return x_out
def _shift_fft(array, shift_value):
Ndim = array.ndim
dims = array.shape
dtype = array.dtype.kind
if (dtype != 'f'):
raise ValueError('Array must be float')
shifted = array
if (Ndim == 1):
Nx = dims[0]
x_ramp = np.arange(Nx, dtype=array.dtype) - Nx//2
tilt = (2*np.pi/Nx) * (shift_value[0]*x_ramp)
cplx_tilt = np.cos(tilt) + 1j*np.sin(tilt)
cplx_tilt = fft.fftshift(cplx_tilt)
narray = fft.fft(fft.ifft(array) * cplx_tilt)
shifted = narray.real
elif (Ndim == 2):
Nx = dims[0]
Ny = dims[1]
x_ramp = np.outer(np.full(Nx, 1.), np.arange(Ny, dtype=array.dtype)) - Nx//2
y_ramp = np.outer(np.arange(Nx, dtype=array.dtype), np.full(Ny, 1.)) - Ny//2
tilt = (2*np.pi/Nx) * (shift_value[0]*x_ramp+shift_value[1]*y_ramp)
cplx_tilt = np.cos(tilt) + 1j*np.sin(tilt)
cplx_tilt = fft.fftshift(cplx_tilt)
narray = fft.fft2(fft.ifft2(array) * cplx_tilt)
shifted = narray.real
else:
raise ValueError('This function can shift only 1D or 2D arrays')
return shifted
def _shift_interp_builtin(array, shift_value, mode='constant', cval=0):
shifted = ndimage.shift(array, np.flip(shift_value, 0), order=3, mode=mode, cval=cval)
return shifted
def _shift_roll(array, shift_value):
Ndim = array.ndim
if (Ndim == 1):
shifted = np.roll(array, shift_value[0])
elif (Ndim == 2):
shifted = np.roll(np.roll(array, shift_value[0], axis=1), shift_value[1], axis=0)
else:
raise ValueError('This function can shift only 1D or 2D arrays')
return shifted
def expend_training_data(images, labels):
expanded_images = []
expanded_labels = []
j = 0 # counter
for x, y in zip(images, labels):
j = j+1
if j%100==0:
print ('expanding data : %03d / %03d' % (j,numpy.size(images,0)))
# register original data
expanded_images.append(x)
expanded_labels.append(y)
# get a value for the background
# zero is the expected value, but median() is used to estimate background's value
bg_value = numpy.median(x) # this is regarded as background's value
image = numpy.reshape(x, (-1, 28))
for i in range(4):
# rotate the image with random degree
angle = numpy.random.randint(-15,15,1)
new_img = ndimage.rotate(image,angle,reshape=False, cval=bg_value)
# shift the image with random distance
shift = numpy.random.randint(-2, 2, 2)
new_img_ = ndimage.shift(new_img,shift, cval=bg_value)
# register new training data
expanded_images.append(numpy.reshape(new_img_, 784))
expanded_labels.append(y)
# images and labels are concatenated for random-shuffle at each epoch
# notice that pair of image and label should not be broken
expanded_train_total_data = numpy.concatenate((expanded_images, expanded_labels), axis=1)
numpy.random.shuffle(expanded_train_total_data)
return expanded_train_total_data
# Prepare MNISt data
def shift_augmentation(X, h_range, w_range):
progbar = Progbar(X.shape[0]) # progress bar for augmentation status tracking
X_shift = np.copy(X)
size = X.shape[2:]
for i in range(len(X)):
h_random = np.random.rand() * h_range * 2. - h_range
w_random = np.random.rand() * w_range * 2. - w_range
h_shift = int(h_random * size[0])
w_shift = int(w_random * size[1])
for j in range(X.shape[1]):
X_shift[i, j] = ndimage.shift(X[i, j], (h_shift, w_shift), order=0)
progbar.add(1)
return X_shift
def sub_load_data(data, img_size, aug):
img_name, dataset = data
img = misc.imread(dataset+'images/'+img_name+'.bmp', mode='L')
seg = misc.imread(dataset+'seg_labels/'+img_name+'.png', mode='L')
try:
ali = misc.imread(dataset+'ori_labels/'+img_name+'.bmp', mode='L')
except:
ali = np.zeros_like(img)
mnt = np.array(mnt_reader(dataset+'mnt_labels/'+img_name+'.mnt'), dtype=float)
if any(img.shape != img_size):
# random pad mean values to reach required shape
if np.random.rand()<aug:
tra = np.int32(np.random.rand(2)*(np.array(img_size)-np.array(img.shape)))
else:
tra = np.int32(0.5*(np.array(img_size)-np.array(img.shape)))
img_t = np.ones(img_size)*np.mean(img)
seg_t = np.zeros(img_size)
ali_t = np.ones(img_size)*np.mean(ali)
img_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = img
seg_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = seg
ali_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = ali
img = img_t
seg = seg_t
ali = ali_t
mnt = mnt+np.array([tra[1],tra[0],0])
if np.random.rand()<aug:
# random rotation [0 - 360] & translation img_size / 4
rot = np.random.rand() * 360
tra = (np.random.rand(2)-0.5) / 2 * img_size
img = ndimage.rotate(img, rot, reshape=False, mode='reflect')
img = ndimage.shift(img, tra, mode='reflect')
seg = ndimage.rotate(seg, rot, reshape=False, mode='constant')
seg = ndimage.shift(seg, tra, mode='constant')
ali = ndimage.rotate(ali, rot, reshape=False, mode='reflect')
ali = ndimage.shift(ali, tra, mode='reflect')
mnt_r = point_rot(mnt[:, :2], rot/180*np.pi, img.shape, img.shape)
mnt = np.column_stack((mnt_r+tra[[1, 0]], mnt[:, 2]-rot/180*np.pi))
# only keep mnt that stay in pic & not on border
mnt = mnt[(8<=mnt[:,0])*(mnt[:,0]<img_size[1]-8)*(8<=mnt[:, 1])*(mnt[:,1]<img_size[0]-8), :]
return img, seg, ali, mnt
def sub_load_data(data, img_size, aug):
img_name, dataset = data
img = misc.imread(dataset+'images/'+img_name+'.bmp', mode='L')
seg = misc.imread(dataset+'seg_labels/'+img_name+'.png', mode='L')
try:
ali = misc.imread(dataset+'ori_labels/'+img_name+'.bmp', mode='L')
except:
ali = np.zeros_like(img)
mnt = np.array(mnt_reader(dataset+'mnt_labels/'+img_name+'.mnt'), dtype=float)
if any(img.shape != img_size):
# random pad mean values to reach required shape
if np.random.rand()<aug:
tra = np.int32(np.random.rand(2)*(np.array(img_size)-np.array(img.shape)))
else:
tra = np.int32(0.5*(np.array(img_size)-np.array(img.shape)))
img_t = np.ones(img_size)*np.mean(img)
seg_t = np.zeros(img_size)
ali_t = np.ones(img_size)*np.mean(ali)
img_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = img
seg_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = seg
ali_t[tra[0]:tra[0]+img.shape[0],tra[1]:tra[1]+img.shape[1]] = ali
img = img_t
seg = seg_t
ali = ali_t
mnt = mnt+np.array([tra[1],tra[0],0])
if np.random.rand()<aug:
# random rotation [0 - 360] & translation img_size / 4
rot = np.random.rand() * 360
tra = (np.random.rand(2)-0.5) / 2 * img_size
img = ndimage.rotate(img, rot, reshape=False, mode='reflect')
img = ndimage.shift(img, tra, mode='reflect')
seg = ndimage.rotate(seg, rot, reshape=False, mode='constant')
seg = ndimage.shift(seg, tra, mode='constant')
ali = ndimage.rotate(ali, rot, reshape=False, mode='reflect')
ali = ndimage.shift(ali, tra, mode='reflect')
mnt_r = point_rot(mnt[:, :2], rot/180*np.pi, img.shape, img.shape)
mnt = np.column_stack((mnt_r+tra[[1, 0]], mnt[:, 2]-rot/180*np.pi))
# only keep mnt that stay in pic & not on border
mnt = mnt[(8<=mnt[:,0])*(mnt[:,0]<img_size[1]-8)*(8<=mnt[:, 1])*(mnt[:,1]<img_size[0]-8), :]
return img, seg, ali, mnt
def load_transform(image_path, angle=0., s=(0, 0), size=(20, 20)):
# Load the image
original = imread(image_path, flatten=True)
# Rotate the image
rotated = np.maximum(np.minimum(rotate(original, angle=angle, cval=1.), 1.), 0.)
# Shift the image
shifted = shift(rotated, shift=s)
# Resize the image
resized = np.asarray(scipy.misc.imresize(rotated, size=size), dtype=theano.config.floatX) / 255.
# Invert the image
inverted = 1. - resized
max_value = np.max(inverted)
if max_value > 0.:
inverted /= max_value
return inverted
def transform_image(image_path, angle=0., s=(0,0), size=(20,20)):
original = imread(image_path, flatten=True)
rotated = np.maximum(np.minimum(rotate(original, angle=angle, cval=1.), 1.), 0.)
shifted = shift(rotated, shift=s)
resized = np.asarray(imresize(rotated, size=size), dtype=np.float32)/255
inverted = 1. - resized
max_value = np.max(inverted)
if max_value > 0:
inverted /= max_value
return inverted
def nudge_images(X, y):
# Having a larger dataset shows more clearly the behavior of the
# methods, but we multiply the size of the dataset only by 2, as the
# cost of the hierarchical clustering methods are strongly
# super-linear in n_samples
shift = lambda x: ndimage.shift(x.reshape((8, 8)),
.3 * np.random.normal(size=2),
mode='constant',
).ravel()
X = np.concatenate([X, np.apply_along_axis(shift, 1, X)])
Y = np.concatenate([y, y], axis=0)
return X, Y
