I implemented a neural network in Keras and Tensorflow to make predictions on the MNIST dataset. I used the same architecture for both Keras and Tensorflow. While the code in Keras gives me always an accuracy on the test set of more than 95%, the code in Tensorflow gives me about 70-80%. Moreover, on the Tensorflow code, if I put num_layers = 5, the accuracy drops to 20%. Can anyone tell me what is wrong with my code ?
# Code in Keras
import keras
# Data
(X_train, y_train), (X_test, y_test) = keras.datasets.mnist.load_data()
X_train = X_train.reshape(-1, 28*28).astype('float32') / 255
X_test = X_test.reshape(-1, 28*28).astype('float32') / 255
y_train = keras.utils.to_categorical(y_train, num_classes = 10)
y_test = keras.utils.to_categorical(y_test, num_classes = 10)
# Model
model = keras.models.Sequential()
model.add(keras.layers.Dense(128, input_dim = 28*28, activation = 'relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(128, activation = 'relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(10, activation = 'softmax'))
# Training
model.compile(loss = 'categorical_crossentropy',
optimizer = keras.optimizers.Adam(lr = 0.001),
metrics = ['accuracy'])
history = model.fit(X_train, y_train, epochs = 5, batch_size = 128,
validation_data = (X_test, y_test), verbose = 1)
and
# Code in Tensorflow
import tensorflow as tf
from math import floor
# Data
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.reshape(-1, 28*28).astype('float32') / 255
X_test = X_test.reshape(-1, 28*28).astype('float32') / 255
y_train = tf.keras.utils.to_categorical(y_train, num_classes = 10)
y_test = tf.keras.utils.to_categorical(y_test, num_classes = 10)
# Training parameters
num_classes = 10
num_neurones = 128
num_layers = 5
dropout_kprob = 1 - 0.3
batch_size = 128
epochs = 5
learning_rate = 0.001
# tf Graph input
X = tf.placeholder("float", [None, 28*28])
Y = tf.placeholder("float", [None, num_classes])
dropout = tf.placeholder(tf.float32)
# Weights
weights = {}
biases = {}
for i in range(num_layers):
if (i == 0):
weights['W_layer_' + str(i)] = tf.Variable(tf.random_normal([28*28, num_neurones]))
biases['b_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones]))
else:
weights['W_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones, num_neurones]))
biases['b_layer_' + str(i)] = tf.Variable(tf.random_normal([num_neurones]))
weights['out'] = tf.Variable(tf.random_normal([num_neurones, num_classes]))
biases['out'] = tf.Variable(tf.random_normal([num_classes]))
# Model
def model(X):
prev_X = X
for i in range(num_layers):
prev_X = tf.matmul(prev_X, weights['W_layer_' + str(i)]) + biases['b_layer_' + str(i)]
prev_X = tf.nn.relu(prev_X)
prev_X = tf.nn.dropout(prev_X, dropout)
prev_X = tf.matmul(prev_X, weights['out']) + biases['out']
return prev_X
# Prediction
logits = model(X)
pred = tf.nn.softmax(logits)
# Loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = Y))
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
train_op = optimizer.minimize(loss_op)
# Evaluation
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Training
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in range(epochs):
for j in range(floor(len(X_train)/batch_size)):
batch_x = X_train[j * batch_size : (j + 1) * batch_size]
batch_y = y_train[j * batch_size : (j + 1) * batch_size]
sess.run(train_op, feed_dict = {X: batch_x, Y: batch_y, dropout: dropout_kprob})
test_loss, test_acc = sess.run([loss_op, accuracy], feed_dict = {X: X_test, Y: y_test, dropout: 1})
print("Epoch " + str(i) + ", Test Loss = " + "{:.4f}".format(test_loss) + \
", Testing Accuracy = " + "{:.3f}".format(test_acc))