Return to Revisions

Bridgeburners's comment is correct: in this model, softmax always predicts 1 (because there's only one output node). I changed to sigmoid activation as I had used originally, then found another problem: under sigmoid, model.predict returns probabilities, not classes, so the predictions need to thresholded first. With these bugs fixed, and epochs cranked up to 300, I'm now getting 99% accuracy.

import numpy as np

X = np.array([[c == '1' for c in line] for line in [
    '1101111111011111111111111111111100011010111110010011111111111111011111110111111111111101111111111110',
    '0011111001000001011010001011000100010001110011011101111111110000111111001001001001110011011101111000',
    '0111101111010000000000111110001101000010110000101111010111100110010110000110011000001010000000000110']])
X = X.T
n_cases = X.shape[0]

y = np.sum(X, axis = 1) > 1

np.random.seed(123)
n_folds = 10
folds = np.repeat(range(n_folds), np.ceil(n_cases / n_folds))[
    range(n_cases)]
np.random.shuffle(folds)

y_pred = np.zeros_like(y)

for fold_i in range(n_folds):
    print("Fold", fold_i)

    import os; os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
    from tensorflow.keras.models import Sequential
    from tensorflow.keras.layers import Dense, Activation

    model = Sequential([
        Dense(10, activation = "relu"),
        Dense(1, activation = "sigmoid")])
    model.compile(
        optimizer = "rmsprop",
        loss = "binary_crossentropy")
    model.fit(
        X[folds != fold_i],
        y[folds != fold_i],
        verbose = False,
        epochs = 300,
        batch_size = np.sum(folds != fold_i) // 5 + 1)

    print("Training accuracy:", np.mean(
       y[folds != fold_i] == (model.predict(X[folds != fold_i])[:,0] > .5)))

    y_pred[folds == fold_i] = (model.predict(X[folds == fold_i])[:,0] > .5)

print(np.mean(y_pred == y))