Consider a 3
class data, say, Iris data.
Suppose we want do binary SVM classification for this multiclass data using Python's sklearn. So we have the following three binary classification problems: {class1, class2}, {class1, class3}, {class2, class3}
.
For each of the above problem, we can get a *classification accuracy, say, 60%, 70%, 80%.
I have the following question:
How to combine the results of these 3
binary classifiers and get a result equivalent to a multiclass classifier, i.e., how to get the final classification accuracy, precision, recall, f1-score and a 3x3 confusion matrix after combining accuracies, precisions, recalls, f1-scores and 2x2 confusion matrices obtained from binary classifiers?
import pandas as pd
import numpy as np
from sklearn.model_selection import KFold
from sklearn import svm, datasets
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix
import time
import os
tic = time.clock()
# Import data
iris = datasets.load_iris()
X = iris.data
Y = iris.target
# Now, suppose we have three separate sets {data1, target1}, {data2, target2}, {data3, target3}
# for binaray classification.
#dataset = [{data1 + data2, target1 + target2}, {data1+ data3, target1 + target3}, {data2 + data3, target2 + target3}]
for d in dataset:
#Import any pair, say, {data1 + data2, target1 + target2}. We will import 3 pairs one-by-one for 3 different binary classification problems.
#data = data1 + data2
#label = target1 + target2
K = 10 #Number of folds
for i in range(K):
kf = KFold(n_splits=K, random_state=None, shuffle=False)
cv = list(kf.split(data1))
trainIndex, testIndex = cv[i][0], cv[i][1]
trainData, testData = data.iloc[trainIndex], data.iloc[testIndex]
trainData_label, testData_label = data_labe.iloc[trainIndex], data_labe.iloc[testIndex]
# So now, we have Train, Test, Train_label, Test_label
clf = []
clf = svm.SVC(kernel='rbf')
clf.fit(Train, Train_label)
predicted_label = clf.predict(Test)
Accuracy_Score = accuracy_score(Test_label, predicted_label)
Precision_Score = precision_score(Test_label, predicted_label, average="macro")
Recall_Score = recall_score(Test_label, predicted_label, average="macro")
F1_Score = f1_score(Test_label, predicted_label, average="macro")
print('Average Accuracy: %0.2f +/- (%0.1f) %%' % (Accuracy_Score.mean()*100, Accuracy_Score.std()*100))
print('Average Precision: %0.2f +/- (%0.1f) %%' % (Precision_Score.mean()*100, Precision_Score.std()*100))
print('Average Recall: %0.2f +/- (%0.1f) %%' % (Recall_Score.mean()*100, Recall_Score.std()*100))
print('Average F1-Score: %0.2f +/- (%0.1f) %%' % (F1_Score.mean()*100, F1_Score.std()*100))
CM = confusion_matrix(Test_label, predicted_label)
print('-------------------------------------------------------------------------------')
toc = time.clock()
print("Total time to run the complete code = ", toc-tic)
Also, how would we combine the accuracies if we also did 10-fold cross-validation?