# How to train an identity deep neural network?

I am new to machine learning and I wanted to get a feel of neural networks by constructing an identity DNN. It takes an one hot vector say [1,0,0,0] and should output the same one hot vector [1,0,0,0]. I am stuck with this seemingly trivial problem for quite a few hours. No matter what I do, the network doesnt seem to solve this simple problem.

Here is the code for tensor flow. I checked the graph. The algorithm appears fine.

import tensorflow as tf
import random

nodeCount = [10,20,10]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 100
TRAIN_SIZE = 1000

x = tf.placeholder(tf.float32,[None,10],name='x')
y = tf.placeholder(tf.float32,name='y')

def model():
layerCount = len(nodeCount)
layer = [None for _ in range(layerCount)]
weights = [None for _ in range(layerCount - 1)]
biases = [None for _ in range(layerCount - 1)]
layer = x
for i in range(layerCount-1):
with tf.name_scope('layers/layer_'+str(i)):
with tf.name_scope('weights/weight_'+str(i)):
name = 'weights/'+str(i)
weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
tf.histogram_summary(name,weights[i])
with tf.name_scope('biases/bias_'+str(i)):
name = 'biases/'+str(i)
biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
tf.histogram_summary(name,biases[i])

layer[i+1] = tf.nn.sigmoid(layer[i+1])

return layer[i+1]

def getNextBatch():
v = [0 for _ in range(10)]
v = 1

r = [None for _ in range(BATCH_SIZE)]
for i in range(BATCH_SIZE):
random.shuffle(v)
r[i] = v.copy()
return r,r

def trainNN():
_y = model()

with tf.name_scope('Cost_Function'):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_y,y))

with tf.name_scope('Learning_Rate'):
learning_rate = tf.Variable(1e-4,dtype=tf.float32)

with tf.name_scope('Optimizer'):

with tf.name_scope('summaries'):
tf.histogram_summary('outputs/_y',_y)
tf.scalar_summary('cost',cost)
tf.histogram_summary('inputs/x',x)
tf.histogram_summary('outputs/y',y)

epochs = 10

with tf.Session() as sess:
merged = tf.merge_all_summaries()
tw = tf.train.SummaryWriter(SUMMARY_DIR+'/graph',sess.graph)
sess.run(tf.initialize_all_variables())

for epoch in range(epochs):
epochLoss = 0

for i in range(TRAIN_SIZE):
bx,by = getNextBatch()
fd = {x:bx,y:by}
run_options = tf.RunOptions()
summary,c = sess.run([merged, cost],
feed_dict=fd,
options=run_options,
epochLoss += c

print('Epoch ',epoch,'/',epochs,':',epochLoss)

tw.close()

trainNN()


The problem was probably with the learning rate hyper parameter. When I switched to the AdamOptimizer, the problem fixed itself.

Here is the new code for future reference.

import tensorflow as tf
import numpy as np
import random

vLength = 8

nodeCount = [vLength,8,8,vLength]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 10
TRAIN_SIZE = 100

def model(x_t):
layerCount = len(nodeCount)
layer = [None for _ in range(layerCount)]
weights = [None for _ in range(layerCount - 1)]
biases = [None for _ in range(layerCount - 1)]
layer = x_t
for i in range(layerCount-1):
with tf.name_scope('layers/layer_'+str(i)):
with tf.name_scope('weights/weight_'+str(i)):
name = 'weights/'+str(i)
weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
tf.histogram_summary(name,weights[i])
with tf.name_scope('biases/bias_'+str(i)):
name = 'biases/'+str(i)
biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
tf.histogram_summary(name,biases[i])

if(i != layerCount-2):
layer[i+1] = tf.nn.tanh(layer[i+1])
else:
layer[i+1] = tf.nn.softmax(layer[i+1])

return layer[i+1]

def getNextBatch():
v = [0.0 for _ in range(vLength)]
v = 1.0

r = [None for _ in range(BATCH_SIZE)]
for i in range(BATCH_SIZE):
random.shuffle(v)
r[i] = v.copy()
return r,r

m = None
def trainNN():
x_t = tf.placeholder(tf.float32,[None,vLength],'input')
y_t = tf.placeholder(tf.float32, [None, vLength],'actual')

m = model(x_t)

with tf.name_scope('Cost_Function'):
cost = tf.reduce_mean(-tf.reduce_sum(y_t * tf.log(m)))

with tf.name_scope('Learning_Rate'):
learning_rate = tf.Variable(0.5,dtype=tf.float32)

with tf.name_scope('Optimizer'):

with tf.name_scope('summaries'):
tf.histogram_summary('outputs/actual',y_t)
tf.scalar_summary('cost',cost)
tf.histogram_summary('inputs/x',x_t)
tf.histogram_summary('outputs/predicted',m)

with tf.name_scope('testing'):
correct = tf.equal(tf.argmax(y_t,1),tf.argmax(m,1))
accuracy = tf.reduce_mean(tf.cast(correct,'float'))
tf.scalar_summary('accuracy',accuracy)

epochs = 10

with tf.Session() as sess:
merged = tf.merge_all_summaries()

sess.run(tf.initialize_all_variables())

for epoch in range(epochs):
tw = tf.train.SummaryWriter(SUMMARY_DIR+'/epoch'+str(epoch),sess.graph)
epochLoss = 0
c = 0
for i in range(TRAIN_SIZE):
bx,by = getNextBatch()
fd = {x_t:bx,y_t:by}
#optimizer.run(fd)

run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
summary,c,_ = sess.run([merged,cost,optimizer],
feed_dict=fd,
options=run_options,