0
$\begingroup$

I implemented a neural network in Keras and Tensorflow to make predictions on the MNIST dataset. I used the same architecture for both Keras and Tensorflow. While the code in Keras gives me always an accuracy on the test set of more than 95%, the code in Tensorflow gives me about 70-80%. Moreover, on the Tensorflow code, if I put num_layers = 5, the accuracy drops to 20%. Can anyone tell me what is wrong with my code ?

# Code in Keras
import keras

# Data
(X_train, y_train), (X_test, y_test) = keras.datasets.mnist.load_data()
X_train = X_train.reshape(-1, 28*28).astype('float32') / 255
X_test = X_test.reshape(-1, 28*28).astype('float32') / 255
y_train = keras.utils.to_categorical(y_train, num_classes = 10)
y_test = keras.utils.to_categorical(y_test, num_classes = 10)

# Model
model = keras.models.Sequential()
model.add(keras.layers.Dense(128, input_dim = 28*28, activation = 'relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(128, activation = 'relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(10, activation = 'softmax'))

# Training
model.compile(loss = 'categorical_crossentropy',
              optimizer = keras.optimizers.Adam(lr = 0.001),
              metrics = ['accuracy'])
history = model.fit(X_train, y_train, epochs = 5, batch_size = 128,
                    validation_data = (X_test, y_test), verbose = 1)

and

# Code in Tensorflow
import tensorflow as tf
from math import floor

# Data
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.reshape(-1, 28*28).astype('float32') / 255
X_test = X_test.reshape(-1, 28*28).astype('float32') / 255
y_train = tf.keras.utils.to_categorical(y_train, num_classes = 10)
y_test = tf.keras.utils.to_categorical(y_test, num_classes = 10)

# Training parameters
num_classes = 10
num_neurones = 128
num_layers = 5
dropout_kprob = 1 - 0.3
batch_size = 128
epochs = 5
learning_rate = 0.001

# tf Graph input
X = tf.placeholder("float", [None, 28*28])
Y = tf.placeholder("float", [None, num_classes])
dropout = tf.placeholder(tf.float32)

# Weights
weights = {}
biases = {}
for i in range(num_layers):
    if (i == 0):
        weights['W_layer_' + str(i)] = tf.Variable(tf.random_normal([28*28, num_neurones]))
        biases['b_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones]))
    else:
        weights['W_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones, num_neurones]))
        biases['b_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones]))
weights['out'] = tf.Variable(tf.random_normal([num_neurones, num_classes]))
biases['out'] = tf.Variable(tf.random_normal([num_classes]))

# Model
def model(X):
    prev_X = X
    for i in range(num_layers):
        prev_X = tf.matmul(prev_X, weights['W_layer_' + str(i)]) + biases['b_layer_' + str(i)]
        prev_X = tf.nn.relu(prev_X)
        prev_X = tf.nn.dropout(prev_X, dropout)
    prev_X = tf.matmul(prev_X, weights['out']) + biases['out']
    return prev_X

# Prediction
logits = model(X)
pred = tf.nn.softmax(logits)

# Loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = Y))
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
train_op = optimizer.minimize(loss_op)

# Evaluation
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Training
init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    for i in range(epochs):
        for j in range(floor(len(X_train)/batch_size)):
            batch_x = X_train[j * batch_size : (j + 1) * batch_size]
            batch_y = y_train[j * batch_size : (j + 1) * batch_size]
            sess.run(train_op, feed_dict = {X: batch_x, Y: batch_y, dropout: dropout_kprob})
        test_loss, test_acc = sess.run([loss_op, accuracy], feed_dict = {X: X_test, Y: y_test, dropout: 1})
        print("Epoch " + str(i) + ", Test Loss = " + "{:.4f}".format(test_loss) + \
              ", Testing Accuracy = " + "{:.3f}".format(test_acc))
$\endgroup$
5
  • $\begingroup$ It looks ok, but keep_prob argument in tf.nn.dropout seems to be deprecated. $\endgroup$ Jul 28, 2019 at 18:03
  • $\begingroup$ Thanks for pointing out the dropout. In the tensorflow code, if I set num_layers to 5, it gave a bad accuracy of 20%. But if I change "dropout: 1" to "dropout: dropout: dropout_kprob" in one of the last lines of the code, I have an accuracy of 85%. Usually, it is better to desactivate the dropout during the evaluation, isn't it the right way to do it in tensorflow? Anyway, if I remove completely the dropout from both Tensorflow and Keras code, I still have a significant better accuracy of 10% in the Keras code when num_layers = 5. $\endgroup$ Jul 28, 2019 at 19:58
  • 1
    $\begingroup$ Maybe try to initialize layers with lower standard deviation, or add batch norm $\endgroup$ Jul 28, 2019 at 22:36
  • $\begingroup$ agree -- init looks weird $\endgroup$
    – shimao
    Jul 29, 2019 at 4:29
  • $\begingroup$ Thanks a lot, it was indeed the problem. $\endgroup$ Jul 29, 2019 at 7:58

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.