How to train an identity deep neural network? I am new to machine learning and I wanted to get a feel of neural networks by constructing an identity DNN. It takes an one hot vector say [1,0,0,0] and should output the same one hot vector [1,0,0,0]. I am stuck with this seemingly trivial problem for quite a few hours. No matter what I do, the network doesnt seem to solve this simple problem.
Here is the code for tensor flow. I checked the graph. The algorithm appears fine.
import tensorflow as tf
import random

nodeCount = [10,20,10]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 100
TRAIN_SIZE = 1000

x = tf.placeholder(tf.float32,[None,10],name='x')
y = tf.placeholder(tf.float32,name='y')

def model():    
    layerCount = len(nodeCount)
    layer = [None for _ in range(layerCount)]
    weights = [None for _ in range(layerCount - 1)]
    biases = [None for _ in range(layerCount - 1)]
    layer[0] = x
    for i in range(layerCount-1):
        with tf.name_scope('layers/layer_'+str(i)):
            with tf.name_scope('weights/weight_'+str(i)):
                name = 'weights/'+str(i)
                weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,weights[i])
            with tf.name_scope('biases/bias_'+str(i)):
                name = 'biases/'+str(i)
                biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,biases[i])

            layer[i+1] = tf.add(tf.matmul(layer[i],weights[i]),biases[i])
            layer[i+1] = tf.nn.sigmoid(layer[i+1])

    return layer[i+1]

def getNextBatch():
    v = [0 for _ in range(10)]
    v[0] = 1

    r = [None for _ in range(BATCH_SIZE)]
    for i in range(BATCH_SIZE):
        random.shuffle(v)
        r[i] = v.copy()
    return r,r


def trainNN():
    _y = model()    

    with tf.name_scope('Cost_Function'):
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_y,y))        

    with tf.name_scope('Learning_Rate'):
        learning_rate = tf.Variable(1e-4,dtype=tf.float32)

    with tf.name_scope('Optimizer'):
        optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

    with tf.name_scope('summaries'):
        tf.histogram_summary('outputs/_y',_y)
        tf.scalar_summary('cost',cost)
        tf.histogram_summary('inputs/x',x)
        tf.histogram_summary('outputs/y',y)       

    epochs = 10

    with tf.Session() as sess:
        merged = tf.merge_all_summaries()
        tw = tf.train.SummaryWriter(SUMMARY_DIR+'/graph',sess.graph)
        sess.run(tf.initialize_all_variables())

        for epoch in range(epochs):
            epochLoss = 0

            for i in range(TRAIN_SIZE):
                bx,by = getNextBatch()
                fd = {x:bx,y:by}
                run_metadata = tf.RunMetadata()
                run_options = tf.RunOptions()
                summary,c = sess.run([merged, cost],
                              feed_dict=fd,
                              options=run_options,
                              run_metadata=run_metadata)
                tw.add_summary(summary,i)
                epochLoss += c

            print('Epoch ',epoch,'/',epochs,':',epochLoss)

        tw.close()            

trainNN()

 A: The problem was probably with the learning rate hyper parameter.
When I switched to the AdamOptimizer, the problem fixed itself.
Here is the new code for future reference.
import tensorflow as tf
import numpy as np
import random

vLength = 8

nodeCount = [vLength,8,8,vLength]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 10
TRAIN_SIZE = 100

def model(x_t):    
    layerCount = len(nodeCount)
    layer = [None for _ in range(layerCount)]
    weights = [None for _ in range(layerCount - 1)]
    biases = [None for _ in range(layerCount - 1)]
    layer[0] = x_t
    for i in range(layerCount-1):
        with tf.name_scope('layers/layer_'+str(i)):
            with tf.name_scope('weights/weight_'+str(i)):
                name = 'weights/'+str(i)
                weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,weights[i])
            with tf.name_scope('biases/bias_'+str(i)):
                name = 'biases/'+str(i)
                biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,biases[i])

            layer[i+1] = tf.add(tf.matmul(layer[i],weights[i]),biases[i])
            if(i != layerCount-2):
                layer[i+1] = tf.nn.tanh(layer[i+1])
            else:
                layer[i+1] = tf.nn.softmax(layer[i+1])

    return layer[i+1]

def getNextBatch():
    v = [0.0 for _ in range(vLength)]
    v[0] = 1.0

    r = [None for _ in range(BATCH_SIZE)]
    for i in range(BATCH_SIZE):
        random.shuffle(v)
        r[i] = v.copy()
    return r,r

m = None
def trainNN():
    x_t = tf.placeholder(tf.float32,[None,vLength],'input')
    y_t = tf.placeholder(tf.float32, [None, vLength],'actual')

    m = model(x_t)    

    with tf.name_scope('Cost_Function'):
        cost = tf.reduce_mean(-tf.reduce_sum(y_t * tf.log(m)))

    with tf.name_scope('Learning_Rate'):
        learning_rate = tf.Variable(0.5,dtype=tf.float32)

    with tf.name_scope('Optimizer'):
        #optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
        optimizer = tf.train.AdamOptimizer().minimize(cost)

    with tf.name_scope('summaries'):
        tf.histogram_summary('outputs/actual',y_t)
        tf.scalar_summary('cost',cost)
        tf.histogram_summary('inputs/x',x_t)
        tf.histogram_summary('outputs/predicted',m)

    with tf.name_scope('testing'):
        correct = tf.equal(tf.argmax(y_t,1),tf.argmax(m,1))
        accuracy = tf.reduce_mean(tf.cast(correct,'float'))
        tf.scalar_summary('accuracy',accuracy)

    epochs = 10

    with tf.Session() as sess:
        merged = tf.merge_all_summaries()

        sess.run(tf.initialize_all_variables())

        for epoch in range(epochs):
            tw = tf.train.SummaryWriter(SUMMARY_DIR+'/epoch'+str(epoch),sess.graph)
            epochLoss = 0
            c = 0
            for i in range(TRAIN_SIZE):
                bx,by = getNextBatch()    
                fd = {x_t:bx,y_t:by}
                #optimizer.run(fd)        

                run_metadata = tf.RunMetadata()
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                summary,c,_ = sess.run([merged,cost,optimizer],
                              feed_dict=fd,
                              options=run_options,
                              run_metadata=run_metadata)
                if(i%BATCH_SIZE == 0):
                    tw.add_summary(summary,i)
                    tw.add_run_metadata(run_metadata,'step%d'%i)

                epochLoss += c       

            tx,ty = getNextBatch()
            print('Epoch ',epoch,'/',epochs,':',epochLoss/TRAIN_SIZE,' ',accuracy.eval({x_t:tx,y_t:ty}))

        tw.close()            

trainNN()

