python类nanmean()的实例源码

def _print_train_val(self):
        """ Print training and validation information """

        ClassificationSW._print_train_val(self)

        cur_iter = self._cur_iter
        cur_round = self._cur_round

        # display training errors
        if cur_iter % cfg.TRAIN.TRAIN_FREQ == 0:
            err_train = self._err_mean
            print 'Round {}, Iteration {}: training error = {}'.format(cur_round, cur_iter, err_train.mean())
            # if self._model_params.model is not None:
            #     print 'err_corr: {}'.format(self._err_corr)

        # display validation errors
        if cur_iter % cfg.TRAIN.VAL_FREQ == 0:
            # perform validation
            err_val = np.zeros((cfg.TRAIN.VAL_SIZE * cfg.TRAIN.IMS_PER_BATCH, self._num_classes))
            for i in xrange(cfg.TRAIN.VAL_SIZE * cfg.TRAIN.IMS_PER_BATCH):
                self._solver.test_nets[0].forward()
                err_val[i,:] = (self._solver.test_nets[0].blobs['error'].data > 0.5)
            err_val = np.nanmean(err_val, axis=0)
            print 'Round {}, Iteration {}: validation error = {}'.format(cur_round, cur_iter, np.nanmean(err_val))

def _print_train_val(self):
        """ Print training and validation information """
        # evaluate training performance

        ClassificationSW._print_train_val(self)

        cur_iter = self._cur_iter
        cur_round = self._cur_round

        # display training errors
        if cur_iter % cfg.TRAIN.TRAIN_FREQ == 0:
            err_train = self._err_mean
            print 'Round {}, Iteration {}: training error = {}'.format(cur_round, cur_iter, err_train.mean())

        # display validation errors
        if cur_iter % cfg.TRAIN.VAL_FREQ == 0:
            # perform validation
            err_val = np.zeros((cfg.TRAIN.VAL_SIZE * cfg.TRAIN.IMS_PER_BATCH, ))
            for i in xrange(cfg.TRAIN.VAL_SIZE * cfg.TRAIN.IMS_PER_BATCH):
                self._solver.test_nets[0].forward()
                err_val[i,:] = 1.0 - self._solver.test_nets[0].blobs['acc'].data
            err_val = np.nanmean(err_val, axis=0)
            print 'Round {}, Iteration {}: validation error = {}'.format(cur_round, cur_iter, np.nanmean(err_val))

def on_step_begin(self, step, logs):
        if self.step % self.interval == 0:
            if len(self.episode_rewards) > 0:
                metrics = np.array(self.metrics)
                assert metrics.shape == (self.interval, len(self.metrics_names))
                formatted_metrics = ''
                if not np.isnan(metrics).all():  # not all values are means
                    means = np.nanmean(self.metrics, axis=0)
                    assert means.shape == (len(self.metrics_names),)
                    for name, mean in zip(self.metrics_names, means):
                        formatted_metrics += ' - {}: {:.3f}'.format(name, mean)

                formatted_infos = ''
                if len(self.infos) > 0:
                    infos = np.array(self.infos)
                    if not np.isnan(infos).all():  # not all values are means
                        means = np.nanmean(self.infos, axis=0)
                        assert means.shape == (len(self.info_names),)
                        for name, mean in zip(self.info_names, means):
                            formatted_infos += ' - {}: {:.3f}'.format(name, mean)
                print('{} episodes - episode_reward: {:.3f} [{:.3f}, {:.3f}]{}{}'.format(len(self.episode_rewards), np.mean(self.episode_rewards), np.min(self.episode_rewards), np.max(self.episode_rewards), formatted_metrics, formatted_infos))
                print('')
            self.reset()
            print('Interval {} ({} steps performed)'.format(self.step // self.interval + 1, self.step))

def on_episode_end(self, episode, logs):
        duration = timeit.default_timer() - self.starts[episode]

        metrics = self.metrics[episode]
        if np.isnan(metrics).all():
            mean_metrics = np.array([np.nan for _ in self.metrics_names])
        else:
            mean_metrics = np.nanmean(metrics, axis=0)
        assert len(mean_metrics) == len(self.metrics_names)

        data = list(zip(self.metrics_names, mean_metrics))
        data += list(logs.items())
        data += [('episode', episode), ('duration', duration)]
        for key, value in data:
            if key not in self.data:
                self.data[key] = []
            self.data[key].append(value)

        if self.interval is not None and episode % self.interval == 0:
            self.save_data()

        # Clean up.
        del self.metrics[episode]
        del self.starts[episode]

def corr(data):  
  ns = data.shape[0];
  nt = data.shape[1];

  pairs = make_pairs(ns);
  npp = len(pairs);

  mean = np.nanmean(data, axis = 0);
  var = np.nanvar(data - mean, axis = 0);

  c = np.zeros(nt);
  for p in pairs:
    c += np.nanmean( (data[p[0]] - mean) * (data[p[1]] - mean), axis = 0) / var;
  c /= npp;

  return c;

def load_stage_binned(strain = 'n2', wid  = all, dtype = 'speed', nbins_per_stage = 10, function = np.nanmean, nout = 1):
  """Load data an bin at different stages"""
  if isinstance(wid, int):
    wids = [wid];
  else:
    wids = wid;

  data = load(strain = strain, wid = wids, dtype = dtype);
  sbins = stage_bins(strain = strain, wid = wids, nbins_per_stage = nbins_per_stage);
  bdata = bin_data(data, sbins, function = function, nout = nout);

  if isinstance(wid, int):
    bdata = bdata[0];

  return bdata;



############################################################################
### Accessing aligned data
############################################################################

def binned_average(x, bin_size = 10, function = np.nanmean):
  """Binned average of a signal"""
  n = len(x);
  r = n % bin_size;
  if r != 0:
    xp = np.pad(x, (0, bin_size - r), mode = 'constant', constant_values = np.nan);
  else:
    xp = x;
  #print xp.shape
  s = len(xp) / bin_size;
  xp.shape = (s,bin_size);
  return function(xp, axis = 1);










############################################################################
### Aligning Data
############################################################################

def plotTimeVsLvls(ax, runs, *args, **kwargs):
    """Plots Time vs TOL of @runs, as
    returned by MIMCDatabase.readRunData()
    ax is in instance of matplotlib.axes
    """
    ax.set_xlabel(r'$\ell$')
    ax.set_ylabel('Time (s)')
    ax.set_yscale('log')
    fnNorm = kwargs.pop("fnNorm")
    if "__calc_moments" in kwargs:
        _, _, Tl, M, _ = kwargs.pop("__calc_moments")
    else:
        _, _, Tl, M, _ = __calc_moments(runs,
                                        seed=kwargs.pop('seed', None),
                                        direction=kwargs.pop('direction', None), fnNorm=fnNorm)
    ax.xaxis.set_major_locator(MaxNLocator(integer=True))

    min_tl = np.nanpercentile(Tl, 5, axis=1)
    med = np.nanmean(Tl, axis=1)
    max_tl = np.nanpercentile(Tl, 95, axis=1)
    line = ax.errorbar(np.arange(0, len(Tl)),
                       med, yerr=[med-min_tl, max_tl-med],
                       *args, **kwargs)
    return line[0].get_xydata(), [line]

def r(self):
        """
        Pearson correlation of the fitted Variogram

        :return:
        """
        # get the experimental and theoretical variogram and cacluate means
        experimental, model = self.__model_deviations()
        mx = np.nanmean(experimental)
        my = np.nanmean(model)

        # claculate the single pearson correlation terms
        term1 = np.nansum(np.fromiter(map(lambda x, y: (x-mx) * (y-my), experimental, model), np.float))

        t2x = np.nansum(np.fromiter(map(lambda x: (x-mx)**2, experimental), np.float))
        t2y = np.nansum(np.fromiter(map(lambda y: (y-my)**2, model), np.float))

        return term1 / (np.sqrt(t2x * t2y))

def shifts_from_picked_coordinate(self, locs, coordinate):
        ''' Calculates the shift from each channel to each other along a given coordinate. '''
        n_channels = len(locs)
        # Calculating center of mass for each channel and pick
        coms = []
        for channel_locs in locs:
            coms.append([])
            for group_locs in channel_locs:
                group_com = np.mean(getattr(group_locs, coordinate))
                coms[-1].append(group_com)
        # Calculating image shifts
        d = np.zeros((n_channels, n_channels))
        for i in range(n_channels-1):
            for j in range(i+1, n_channels):
                    d[i, j] = np.nanmean([cj - ci for ci, cj in zip(coms[i], coms[j])])
        return d

def per_class_acc(predictions, label_tensor):
    labels = label_tensor
    size = predictions.shape[0]
    num_class = predictions.shape[3]
    hist = np.zeros((num_class, num_class))
    for i in range(size):
      hist += fast_hist(labels[i].flatten(), predictions[i].argmax(2).flatten(), num_class)
    acc_total = np.diag(hist).sum() / hist.sum()
    print ('accuracy = %f'%np.nanmean(acc_total))
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
    print ('mean IU  = %f'%np.nanmean(iu))
    for ii in range(num_class):
        if float(hist.sum(1)[ii]) == 0:
          acc = 0.0
        else:
          acc = np.diag(hist)[ii] / float(hist.sum(1)[ii])
        print("    class # %d accuracy = %f "%(ii,acc))

def do_seg_tests(net, iter, save_format, dataset, layer='score', gt='label'):
    n_cl = net.blobs[layer].channels
    if save_format:
        save_format = save_format.format(iter)
    hist, loss = compute_hist(net, save_format, dataset, layer, gt)
    # mean loss
    print '>>>', datetime.now(), 'Iteration', iter, 'loss', loss
    # overall accuracy
    acc = np.diag(hist).sum() / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'overall accuracy', acc
    # per-class accuracy
    acc = np.diag(hist) / hist.sum(1)
    print '>>>', datetime.now(), 'Iteration', iter, 'mean accuracy', np.nanmean(acc)
    # per-class IU
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
    print '>>>', datetime.now(), 'Iteration', iter, 'mean IU', np.nanmean(iu)
    freq = hist.sum(1) / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'fwavacc', \
            (freq[freq > 0] * iu[freq > 0]).sum()
    return hist

def do_seg_tests(net, iter, save_format, dataset, layer='score', gt='label'):
    n_cl = net.blobs[layer].channels
    if save_format:
        save_format = save_format.format(iter)
    hist, loss = compute_hist(net, save_format, dataset, layer, gt)
    # mean loss
    print '>>>', datetime.now(), 'Iteration', iter, 'loss', loss
    # overall accuracy
    acc = np.diag(hist).sum() / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'overall accuracy', acc
    # per-class accuracy
    acc = np.diag(hist) / hist.sum(1)
    print '>>>', datetime.now(), 'Iteration', iter, 'mean accuracy', np.nanmean(acc)
    # per-class IU
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
    print '>>>', datetime.now(), 'Iteration', iter, 'mean IU', np.nanmean(iu)
    freq = hist.sum(1) / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'fwavacc', \
            (freq[freq > 0] * iu[freq > 0]).sum()
    return hist

def do_seg_tests(net, iter, save_format, dataset, layer='score', gt='label'):
    n_cl = net.blobs[layer].channels
    if save_format:
        save_format = save_format.format(iter)
    hist, loss = compute_hist(net, save_format, dataset, layer, gt)
    # mean loss
    print '>>>', datetime.now(), 'Iteration', iter, 'loss', loss
    # overall accuracy
    acc = np.diag(hist).sum() / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'overall accuracy', acc
    # per-class accuracy
    acc = np.diag(hist) / hist.sum(1)
    print '>>>', datetime.now(), 'Iteration', iter, 'mean accuracy', np.nanmean(acc)
    # per-class IU
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
    print '>>>', datetime.now(), 'Iteration', iter, 'mean IU', np.nanmean(iu)
    freq = hist.sum(1) / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'fwavacc', \
            (freq[freq > 0] * iu[freq > 0]).sum()
    return hist

def do_seg_tests(net, iter, save_format, dataset, layer='score', gt='label'):
    n_cl = net.blobs[layer].channels
    if save_format:
        save_format = save_format.format(iter)
    hist, loss = compute_hist(net, save_format, dataset, layer, gt)
    # mean loss
    print '>>>', datetime.now(), 'Iteration', iter, 'loss', loss
    # overall accuracy
    acc = np.diag(hist).sum() / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'overall accuracy', acc
    # per-class accuracy
    acc = np.diag(hist) / hist.sum(1)
    print '>>>', datetime.now(), 'Iteration', iter, 'mean accuracy', np.nanmean(acc)
    # per-class IU
    iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist))
    print '>>>', datetime.now(), 'Iteration', iter, 'mean IU', np.nanmean(iu)
    freq = hist.sum(1) / hist.sum()
    print '>>>', datetime.now(), 'Iteration', iter, 'fwavacc', \
            (freq[freq > 0] * iu[freq > 0]).sum()
    return hist

def htmt(self):

        htmt_ = pd.DataFrame(pd.DataFrame.corr(self.data_),
                             index=self.manifests, columns=self.manifests)

        mean = []
        allBlocks = []
        for i in range(self.lenlatent):
            block_ = self.Variables['measurement'][
                self.Variables['latent'] == self.latent[i]]
            allBlocks.append(list(block_.values))
            block = htmt_.ix[block_, block_]
            mean_ = (block - np.diag(np.diag(block))).values
            mean_[mean_ == 0] = np.nan
            mean.append(np.nanmean(mean_))

        comb = [[k, j] for k in range(self.lenlatent)
                for j in range(self.lenlatent)]

        comb_ = [(np.sqrt(mean[comb[i][1]] * mean[comb[i][0]]))
                 for i in range(self.lenlatent ** 2)]

        comb__ = []
        for i in range(self.lenlatent ** 2):
            block = (htmt_.ix[allBlocks[comb[i][1]],
                              allBlocks[comb[i][0]]]).values
#            block[block == 1] = np.nan
            comb__.append(np.nanmean(block))

        htmt__ = np.divide(comb__, comb_)
        where_are_NaNs = np.isnan(htmt__)
        htmt__[where_are_NaNs] = 0

        htmt = pd.DataFrame(np.tril(htmt__.reshape(
            (self.lenlatent, self.lenlatent)), k=-1), index=self.latent, columns=self.latent)

        return htmt

def imputeSNPs(X):
    snpsMean = np.nanmean(X, axis=0)
    isNan = np.isnan(X)
    for i,m in enumerate(snpsMean): X[isNan[:,i], i] = m

    return X

def test_ignore_nan(self):
        """ Test that NaNs are handled correctly """
        stream = [np.random.random(size = (16,12)) for _ in range(5)]
        for s in stream:
            s[randint(0, 15), randint(0,11)] = np.nan

        with catch_warnings():
            simplefilter('ignore')
            from_iaverage = last(iaverage(stream, ignore_nan = True))  
        from_numpy = np.nanmean(np.dstack(stream), axis = 2)
        self.assertTrue(np.allclose(from_iaverage, from_numpy))

def test_against_numpy_nanmean(self):
        """ Test results against numpy.mean"""
        source = [np.random.random((16, 12, 5)) for _ in range(10)]
        for arr in source:
            arr[randint(0, 15), randint(0, 11), randint(0, 4)] = np.nan
        stack = np.stack(source, axis = -1)
        for axis in (0, 1, 2, None):
            with self.subTest('axis = {}'.format(axis)):
                from_numpy = np.nanmean(stack, axis = axis)
                out = last(imean(source, axis = axis, ignore_nan = True))
                self.assertSequenceEqual(from_numpy.shape, out.shape)
                self.assertTrue(np.allclose(out, from_numpy))

def trainAndCrossValidate(self):
        num_folds = min(self.num_cross_folds, len(self.crossVal_X))
        accs = []
        aucs = []
        f1s = []
        precisions = []
        recalls = []
        for f in range(num_folds):
            val_X = self.crossVal_X[f]
            val_Y = self.crossVal_y[f]
            train_folds_X = [self.crossVal_X[x] for x in range(num_folds) if x != f]
            train_folds_Y = [self.crossVal_y[x] for x in range(num_folds) if x != f]
            train_X = train_folds_X[0]
            train_Y = train_folds_Y[0]
            for i in range(1,len(train_folds_X)):
                train_X = np.concatenate((train_X,train_folds_X[i]))
                train_Y = np.concatenate((train_Y,train_folds_Y[i]))

            self.train_X = train_X
            self.train_y = train_Y
            self.val_X = val_X
            self.val_y = val_Y
            acc, auc, f1, precision, recall = self.trainAndValidate()
            accs.append(acc)
            aucs.append(auc)
            f1s.append(f1)
            precisions.append(precision)
            recalls.append(recall)
        if PRINT_CROSS_VAL_FOLDS: print "\t\tPer-fold cross-validation accuracy: ", accs
        return np.nanmean(accs), np.nanmean(aucs), np.nanmean(f1s), np.nanmean(precisions), np.nanmean(recalls)