def plot_validation_curve(estimators, X, y, cv=10, **kwargs):
figsize = (6.4 * len(estimators), 4.8)
fig, axes = plt.subplots(nrows=1, ncols=len(estimators), figsize=figsize)
param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
if len(estimators) == 1:
axes = [axes]
for ax, estimator in zip(axes, estimators):
train_scores, test_scores = validation_curve(
estimator=estimator,
X=X,
y=y,
param_name='clf__C',
param_range=param_range,
cv=cv,
**kwargs)
xlabel = 'Parameter C'
_plot_curve(ax, param_range, train_scores, test_scores, xlabel, 'log')
ax.set_title(pipeline_name(estimator))
# fig.tight_layout(pad=1.08, h_pad=None, w_pad=None, rect=None)
return fig
python类validation_curve()的实例源码
def validation_crv(estimator, X, y, title, n_jobs=1):
param_range = np.logspace(-6, -1, 5)
train_scores, test_scores = validation_curve(
estimator, X, y, param_name="max_features", param_range=param_range,
cv=10, scoring="accuracy", n_jobs=n_jobs)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.title(title)
plt.xlabel("$\gamma$")
plt.ylabel("Score")
plt.ylim(0.0, 1.1)
lw = 2
plt.semilogx(param_range, train_scores_mean, label="Training score",
color="darkorange", lw=lw)
plt.fill_between(param_range, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.2,
color="darkorange", lw=lw)
plt.semilogx(param_range, test_scores_mean, label="Cross-validation score",
color="navy", lw=lw)
plt.fill_between(param_range, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.2,
color="navy", lw=lw)
plt.legend(loc="best")
return plt
def test_validation_curve():
X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
n_redundant=0, n_classes=2,
n_clusters_per_class=1, random_state=0)
param_range = np.linspace(0, 1, 10)
with warnings.catch_warnings(record=True) as w:
train_scores, test_scores = validation_curve(
MockEstimatorWithParameter(), X, y, param_name="param",
param_range=param_range, cv=2
)
if len(w) > 0:
raise RuntimeError("Unexpected warning: %r" % w[0].message)
assert_array_almost_equal(train_scores.mean(axis=1), param_range)
assert_array_almost_equal(test_scores.mean(axis=1), 1 - param_range)
def validation_curve(estimator, epochs, y, param_name, param_range, cv=None):
"""Validation curve on epochs.
Parameters
----------
estimator : object that implements "fit" and "predict" method.
the estimator whose Validation curve must be found
epochs : instance of mne.Epochs.
The epochs.
y : array
The labels.
param_name : str
Name of the parameter that will be varied.
param_range : array
The values of the parameter that will be evaluated.
cv : int, cross-validation generator or an iterable, optional
Determines the cross-validation strategy.
Returns
-------
train_scores : array
The scores in the training set
test_scores : array
The scores in the test set
"""
from sklearn.model_selection import validation_curve
if not isinstance(estimator, GlobalAutoReject):
msg = 'No guarantee that it will work on this estimator.'
raise NotImplementedError(msg)
BaseEpochs = _get_epochs_type()
if not isinstance(epochs, BaseEpochs):
raise ValueError('Only accepts MNE epochs objects.')
data_picks = _handle_picks(epochs.info, picks=None)
X = epochs.get_data()[:, data_picks, :]
n_epochs, n_channels, n_times = X.shape
estimator.n_channels = n_channels
estimator.n_times = n_times
train_scores, test_scores = \
validation_curve(estimator, X.reshape(n_epochs, -1), y=y,
param_name="thresh", param_range=param_range,
cv=cv, n_jobs=1, verbose=0)
return train_scores, test_scores
