After a bit of resaerchresearch I found out that I can use pipeline
in order to avoid data leakage.
This is how my code looks like now:
def produce_learning_curve(self, model, model_name, nb_splits, output_folder, parameters=None, nb_repeats=None):
X_train, y_train = self.X_train, self.y_train
pipe = Pipeline([
('sc', StandardScaler()),
('model', model(**parameters, random_state=42))
])
print('Inside Learning curve')
print('X_train: ', self.X_train.shape)
print('y_train: ', self.y_train.shape)
if nb_repeats is None:
cv = StratifiedKFold(n_splits=nb_splits, random_state=42)
else:
cv = RepeatedStratifiedKFold(n_splits=nb_splits, n_repeats=nb_repeats, random_state=42)
# train_sizes, train_scores, test_scores = learning_curve(model(**parameters), X_train, y_train, cv=cv, scoring='accuracy') # calculate learning curve values
train_sizes, train_scores, test_scores = learning_curve(pipe, X_train, y_train, cv=cv, scoring='accuracy') # calculate learning curve values
# mean of the results of the training and testing
train_scores_mean = np.mean(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
plt.figure()
plt.xlabel("Number of Training Samples")
plt.ylabel("Accuracy")
plt.plot(train_sizes, train_scores_mean, label="training")
plt.plot(train_sizes, test_scores_mean, label="validation")
plt.legend()
if not os.path.exists(output_folder):
os.makedirs(output_folder)
plt.savefig(output_folder + '%s_learning_curve.png' % model_name)
plt.close()
```