4
$\begingroup$

What is the most efficient way to do repeated cross-validation in sklearn? I know with R and the caret package, in the trainControl function, I just need to set the method to 'repeatedcv' (see 5.3 Basic Parameter Tuning). What is the equivalent approach in python? I read the documentation of kfold carefully I can couldn't find a seamless approach that does repeated cv, pools the results, and returns the best model along with the cv results in the same way that the R caret package does.

$\endgroup$

2 Answers 2

3
$\begingroup$

It looks like a RepeatedKFold and RepeatedStratifiedKFold class was added to sklearn. Here is an example of how to apply it in practice:

#Assumes sklearn version 0.19.0

#Load Data
###############################################################################
from sklearn import datasets

iris = datasets.load_iris()
X = iris.data
y = iris.target

#quickly making the target binary classification to simplify the example
y = 1*(y==0)

#Training and Testing Split
###############################################################################
from sklearn.model_selection import train_test_split
my_rand_state=0

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, 
                                              random_state=my_rand_state)

#Define simple model
###############################################################################
from sklearn.linear_model import LogisticRegression
log_clf=LogisticRegression()
C=[0.001 , 0.01, 10, 100,1000]

#Simple pre-processing estimators
###############################################################################
from sklearn.preprocessing import StandardScaler
std_scale=StandardScaler()

#Defining the CV method: Using the Repeated Stratified K Fold
###############################################################################
from sklearn.model_selection import RepeatedStratifiedKFold
n_folds=10
n_repeats=30

skfold = RepeatedStratifiedKFold(n_splits=n_folds,n_repeats=n_repeats,
                                 random_state=my_rand_state)

#Creating simple pipeline and defining the gridsearch
###############################################################################
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
n_jobs=4

log_clf_pipe = Pipeline(steps=[('scale',std_scale),('clf',log_clf)])
log_clf_est = GridSearchCV(estimator=log_clf_pipe,cv=skfold,
              scoring='roc_auc',n_jobs=n_jobs,
              param_grid=dict(clf__C=C))

#Fit the Model & Plot the Results
###############################################################################
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt

log_clf_est.fit(X_train,y_train)

#ploting results
log_fpr, log_tpr, _ = roc_curve(y_test, 
                    log_clf_est.predict_proba(X_test)[:,1])
log_roc_auc = auc(log_fpr, log_tpr)

plt.plot(log_fpr, log_tpr, color='seagreen', linestyle='--',
         label='LOG (area = %0.2f)' % log_roc_auc, lw=2)
plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='k',
         label='Luck')

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curves of Model')
plt.legend(loc="lower right")
plt.show()
$\endgroup$
1
  • $\begingroup$ can you extend your answer please? $\endgroup$
    – Ferdi
    Sep 1, 2017 at 19:10
0
$\begingroup$

I think you can also use something like the followings for nested loop classification.. using the iris data & kernel SVC as an example..

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_iris
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
import numpy as np

iris = load_iris()
X = iris.data
y = iris.target

pipe_svc = Pipeline([('scl', StandardScaler()),
            ('clf', SVC(random_state=1))])

param_range = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]

param_grid = [{'clf__C': param_range, 'clf__gamma': param_range, 'clf__kernel': ['rbf']}]

N_outer=10
N_inner=10

scores=[]
for i in range(N_outer):
    k_fold_outer = StratifiedKFold(n_splits=10,shuffle=True,random_state=i)
    for j in range(N_inner):
        k_fold_inner = StratifiedKFold(n_splits=10,shuffle=True,random_state=j)
        gs = GridSearchCV(estimator=pipe_svc, param_grid=param_grid, scoring='accuracy', cv=k_fold_inner)
        score=cross_val_score(estimator=gs,X=X,y=y,cv=k_fold_outer)
        scores.append(score)

np.shape(scores)
$\endgroup$
0

Not the answer you're looking for? Browse other questions tagged or ask your own question.