# CNN accuracy very low, what is wrong with this script?

I'm building an image recognition system that classifies cars from 10 different models. The input data is a folder with 100 images for each model.

I am having difficulties because the model stagnates at 16% accuracy when tries to predict the test set. My question is: are there any major pitfalls in my script that are causing the model to work so bad?

I modified a script that I found online for a CNN and that is the model that I use:

from scipy import misc
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from datetime import datetime
from scipy.signal import convolve2d
from sklearn.utils import shuffle
import os

def y2indicator(y):
N = len(y)
ind = np.zeros((N, 10))
for i in xrange(N):
ind[i, y[i]] = 1
return ind

def error_rate(p, t):
return np.mean(p != t)

def convpool(X, W, b):
# just assume pool size is (2,2) because we need to augment it with 1s
conv_out = tf.nn.conv2d(X, W, strides=[1, 1, 1, 1], padding='SAME')
pool_out = tf.nn.max_pool(conv_out, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
return tf.nn.relu(pool_out)

def init_filter(shape, poolsz):
w = np.random.randn(*shape) / np.sqrt( np.prod(shape[:-1]) + shape[-1]*np.prod(shape[:-2] / np.prod(poolsz) ))
return w.astype(np.float32)

img_folder = 'cars'
makes = [d for d in os.listdir(img_folder) if os.path.isdir(os.path.join(img_folder, d))]

# index for model classification
i = -1
# dict to put X & y data of all images into it
data = {'X': [], 'y': []}
for make in makes:
make_path = os.path.join(img_folder, make)
models = [d for d in os.listdir(make_path) if os.path.isdir(os.path.join(make_path, d))]
for model in models:
i += 1
model_path = os.path.join(img_folder, make, model)
jpegs = [img for img in os.listdir(os.path.join(img_folder, make, model)) if 'jpg' in img]
for j, jpg in enumerate(jpegs):
if j < 100:
img_pixels = img_pixels.flatten() / np.float32(255)
if img_pixels.shape == (67500,):
data['X'].append(img_pixels)
data['y'].append(i)
else:
break

print data['X'][0].shape
print len(data['X'])
print len(data['y'])
print data['X'][0]
Ytrain = y2indicator(data['y'])
print Ytrain.shape

X, Y = shuffle(data['X'], data['y'])

Xtrain = np.array(X[:-100])
Ytrain = Y[:-100]
Ytrain_ind = y2indicator(Ytrain)

Xtest = X[-100:]
Ytest = Y[-100:]
Ytest_ind = y2indicator(Ytest)

Ytrain_ind.shape
print Xtrain[0].shape == (225, 300, 3)
print type(Xtrain)
print type(Ytrain_ind)

max_iter = 1000
print_period = 10

lr = 0.01
reg = 0.01

N, D = Xtrain.shape
batch_sz = 500
n_batches = N / batch_sz

# add an extra layer just for fun
M1 = 2000
M2 = 200
K = 10
W1_init = np.random.randn(D, M1) / 28
b1_init = np.zeros(M1)
W2_init = np.random.randn(M1, M2) / np.sqrt(M1)
b2_init = np.zeros(M2)
W3_init = np.random.randn(M2, K) / np.sqrt(M2)
b3_init = np.zeros(K)

# define variables and expressions
X = tf.placeholder(tf.float32, shape=(None, D), name='X')
T = tf.placeholder(tf.float32, shape=(None, K), name='T')
W1 = tf.Variable(W1_init.astype(np.float32))
b1 = tf.Variable(b1_init.astype(np.float32))
W2 = tf.Variable(W2_init.astype(np.float32))
b2 = tf.Variable(b2_init.astype(np.float32))
W3 = tf.Variable(W3_init.astype(np.float32))
b3 = tf.Variable(b3_init.astype(np.float32))

# define the model
Z1 = tf.nn.relu( tf.matmul(X, W1) + b1 )
Z2 = tf.nn.relu( tf.matmul(Z1, W2) + b2 )
Yish = tf.matmul(Z2, W3) + b3

# cost function and optimizer
cost = tf.reduce_sum(tf.nn.softmax_cross_entropy_with_logits(labels=T, logits=Yish))

train_op = tf.train.RMSPropOptimizer(0.0001, decay=0.99, momentum=0.9).minimize(cost)

# we'll use this to calculate the error rate
predict_op = tf.argmax(Yish, 1)

t0 = datetime.now()
LL = []
init = tf.global_variables_initializer()
with tf.Session() as session:
session.run(init)
for i in xrange(max_iter):
session.run(train_op, feed_dict={X: Xtrain, T: Ytrain_ind})
if i % print_period > -1:
test_cost = session.run(cost, feed_dict={X: Xtest, T: Ytest_ind})
prediction = session.run(
predict_op, feed_dict={X: Xtest})
err = error_rate(prediction, Ytest)
print "Cost / err at iteration i=%d: %.3f / %.3f" % (i, test_cost, err)
LL.append(test_cost)
print "Elapsed time:", (datetime.now() - t0)


Thanks for your help and sorry for the ignorance!

• my question is similar to this one: stats.stackexchange.com/questions/198463/… I don't know how it violates the rules... – Javier Ventajas Hernández Apr 9 '17 at 19:26
• @MichaelChernick this is not debugging per se. His script isn't erroring, it just has a high error rate. He's asking what parameters settings might improve his error. If you'd classify (what a pun) that as debugging, pretty sure 20% of the questions on this site would have to be closed. – Thomas W Apr 9 '17 at 19:37
• Maybe the main problem is the low number of training data, that is 100 samples per class. – Hossein Apr 12 '17 at 12:37

As far as I can see, while you do define a convolution function convpool, you do not use it. Your entire model is this:

# define the model
Z1 = tf.nn.relu( tf.matmul(X, W1) + b1 )
Z2 = tf.nn.relu( tf.matmul(Z1, W2) + b2 )
Yish = tf.matmul(Z2, W3) + b3


This is not convolution, this is just matrix multiplication, with very large matrices. Your very small data size (100 samples per class) is not likely to be sufficient to train a CNN, and even more so an entire matrix to multiply the data.

I would consider doing 2-3 CNN layers with max-pooling, followed by a single fully connected layer to reduce the dimension to that of the output (i.e. number of classes), and see how it goes.

It would also be helpful to see the training error and testing error vs. minibatch iteration / epochs, to see what happens (if you overfit, underfit, etc.).