I think I have managed to rewrite the Bootstrap 0.632+ from R to Java using the Weka Java API. The original R function can be found in the bootpred method inside the bootstrap package (link).
As you can see from the source code, I have used the corrected $\hat{R'}$ and $\hat{Err}^{(1)'}$ with the final (corrected) equation (32) from the original article.
However, despite correcting for abnormal values, I sometimes still get negative error rate, which is of course impossible and therefore invalid. Also, I have noticed that the difference between the 0.632 and the 0.632+ method is minimal, if any.
If someone finds any errors in my source code, I would be really grateful if you could point them out.
public class Bootstrap632plus extends AbstractPerformance {
private final int repeats;
private double Err632;
private double resub;
public Bootstrap632plus(Instances instances, int repeats) {
super(instances);
this.repeats = repeats;
}
public double getErr632() {
return Err632;
}
public double getResub() {
return resub;
}
@Override
public double getErrorRate(final MachineLearningAlgorithm machineLearningAlgorithm, final Random seed) throws Exception {
// First component
double err = predictionError(machineLearningAlgorithm);
this.resub = err;
// Error rates
List<Double> errorRates = Collections.synchronizedList(new ArrayList<>());
// GAMA related stuff
final int numClasses = instances.numClasses();
AtomicIntegerArray p_l = new AtomicIntegerArray(numClasses);
AtomicIntegerArray q_l = new AtomicIntegerArray(numClasses);
// Bootstrap iterations
seed.ints(repeats).parallel().forEach(randomSeed -> {
// Get error rate
Evaluation evaluation = bootstrapIteration(machineLearningAlgorithm, randomSeed);
errorRates.add(evaluation.errorRate());
/*
GAMA VARIABLE
Confusion matrix:
- first dimension (rows): real distribution for first class
- second dimension (columns): predicted distribution for first class
p_l = observed proportions of responses where y_i equals l
- sum by l-th row (first dimension)
q_l = observer proportions of predicted responses where y_i equals l
- sum by l-th column (second dimension)
GAMA = SUM_by_l(p_l * (1 - q_l))
*/
double[][] confusionMatrix = evaluation.confusionMatrix();
for(int l = 0; l < numClasses; l++) {
int p_tmp = 0, q_tmp = 0;
for(int n = 0; n < numClasses; n++) {
// Sum for l-th class
p_tmp += confusionMatrix[l][n];
q_tmp += confusionMatrix[n][l];
}
// Add data for l-th class
p_l.addAndGet(l, p_tmp);
q_l.addAndGet(l, q_tmp);
}
});
// Second component
double Err1 = errorRates.stream().mapToDouble(i -> i).average().orElse(0);
// Plain 0.632 bootstrap
Err632 = .368*err + .632*Err1;
// GAMA
final double observations = instances.size() * repeats;
double gama = 0;
for(int l = 0; l < numClasses; l++) {
// Normalize numbers -> divide by number of all observations (repeats * dataset size)
gama += ((double)p_l.get(l) / observations) * (1 - ((double)q_l.get(l) / observations));
}
// Relative overfitting rate (R)
double R = (Err1 - err) / (gama - err);
// Modified variables (according to original journal article)
double Err1_ = Double.min(Err1, gama);
double R_ = R;
// R can fall out of [0, 1] -> set it to 0
if(!(Err1 > err && gama > err)) {
R_ = 0;
}
// The 0.632+ bootstrap (as used in original article)
double Err632plus = Err632 + (Err1_ - err) * (.368 * .632 * R_) / (1 - .368 * R_);
return Err632plus;
}
/**
* Prediction error: first component of the 0.632+ bootstrap.
* Train the classifier on the whole dataset and then also test it on the whole dataset.
*
* @param machineLearningAlgorithm Specified machine learning algorithm
* @return prediction error [0, 1]
* @throws Exception exception
*/
private double predictionError(final MachineLearningAlgorithm machineLearningAlgorithm) throws Exception {
// Train
Classifier classifier = ClassifierFactory.instantiate(machineLearningAlgorithm);
classifier.buildClassifier(instances);
// Test
Evaluation evaluation = new Evaluation(instances);
evaluation.evaluateModel(classifier, instances);
// Return error rate
return evaluation.errorRate();
}
/**
* One iteration of the Leave-one-out Bootstrap Cross-Validation.
* @return
* @throws Exception
*/
private Evaluation bootstrapIteration(final MachineLearningAlgorithm machineLearningAlgorithm, final int randomSeed) {
try {
final int SIZE = instances.size();
final Random r = new Random(randomSeed);
// Custom sampling (100%, with replacement)
List<Instance> TRAIN = new ArrayList<>(SIZE); // Empty list (add one-by-one)
List<Instance> TEST = new ArrayList<>(instances); // Full (remove one-by-one)
for(int i = 0; i < SIZE; i++) {
// Random select instance
Instance instance = instances.get(r.nextInt(SIZE));
// Add to TRAIN, remove from TEST
TRAIN.add(instance);
TEST.remove(instance);
}
// Train
Instances trainSet = new Instances(instances, TRAIN.size());
trainSet.addAll(TRAIN);
Classifier classifier = ClassifierFactory.instantiate(machineLearningAlgorithm);
classifier.buildClassifier(trainSet);
// Test set
Instances testSet = new Instances(instances, TEST.size());
testSet.addAll(TEST);
// Test
Evaluation evaluation = new Evaluation(instances);
evaluation.evaluateModel(classifier, testSet);
// Return the evaluation (for further processing)
return evaluation;
} catch(Exception e) {
throw new RuntimeException(e);
}
}
}
