0
$\begingroup$

I am new to machine learning and I wanted to get a feel of neural networks by constructing an identity DNN. It takes an one hot vector say [1,0,0,0] and should output the same one hot vector [1,0,0,0]. I am stuck with this seemingly trivial problem for quite a few hours. No matter what I do, the network doesnt seem to solve this simple problem.

Here is the code for tensor flow. I checked the graph. The algorithm appears fine.

import tensorflow as tf
import random

nodeCount = [10,20,10]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 100
TRAIN_SIZE = 1000

x = tf.placeholder(tf.float32,[None,10],name='x')
y = tf.placeholder(tf.float32,name='y')

def model():    
    layerCount = len(nodeCount)
    layer = [None for _ in range(layerCount)]
    weights = [None for _ in range(layerCount - 1)]
    biases = [None for _ in range(layerCount - 1)]
    layer[0] = x
    for i in range(layerCount-1):
        with tf.name_scope('layers/layer_'+str(i)):
            with tf.name_scope('weights/weight_'+str(i)):
                name = 'weights/'+str(i)
                weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,weights[i])
            with tf.name_scope('biases/bias_'+str(i)):
                name = 'biases/'+str(i)
                biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,biases[i])

            layer[i+1] = tf.add(tf.matmul(layer[i],weights[i]),biases[i])
            layer[i+1] = tf.nn.sigmoid(layer[i+1])

    return layer[i+1]

def getNextBatch():
    v = [0 for _ in range(10)]
    v[0] = 1

    r = [None for _ in range(BATCH_SIZE)]
    for i in range(BATCH_SIZE):
        random.shuffle(v)
        r[i] = v.copy()
    return r,r


def trainNN():
    _y = model()    

    with tf.name_scope('Cost_Function'):
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_y,y))        

    with tf.name_scope('Learning_Rate'):
        learning_rate = tf.Variable(1e-4,dtype=tf.float32)

    with tf.name_scope('Optimizer'):
        optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

    with tf.name_scope('summaries'):
        tf.histogram_summary('outputs/_y',_y)
        tf.scalar_summary('cost',cost)
        tf.histogram_summary('inputs/x',x)
        tf.histogram_summary('outputs/y',y)       

    epochs = 10

    with tf.Session() as sess:
        merged = tf.merge_all_summaries()
        tw = tf.train.SummaryWriter(SUMMARY_DIR+'/graph',sess.graph)
        sess.run(tf.initialize_all_variables())

        for epoch in range(epochs):
            epochLoss = 0

            for i in range(TRAIN_SIZE):
                bx,by = getNextBatch()
                fd = {x:bx,y:by}
                run_metadata = tf.RunMetadata()
                run_options = tf.RunOptions()
                summary,c = sess.run([merged, cost],
                              feed_dict=fd,
                              options=run_options,
                              run_metadata=run_metadata)
                tw.add_summary(summary,i)
                epochLoss += c

            print('Epoch ',epoch,'/',epochs,':',epochLoss)

        tw.close()            

trainNN()
$\endgroup$
1
$\begingroup$

The problem was probably with the learning rate hyper parameter. When I switched to the AdamOptimizer, the problem fixed itself.

Here is the new code for future reference.

import tensorflow as tf
import numpy as np
import random

vLength = 8

nodeCount = [vLength,8,8,vLength]

SUMMARY_DIR = '/media/mint/D/summaries'

BATCH_SIZE = 10
TRAIN_SIZE = 100

def model(x_t):    
    layerCount = len(nodeCount)
    layer = [None for _ in range(layerCount)]
    weights = [None for _ in range(layerCount - 1)]
    biases = [None for _ in range(layerCount - 1)]
    layer[0] = x_t
    for i in range(layerCount-1):
        with tf.name_scope('layers/layer_'+str(i)):
            with tf.name_scope('weights/weight_'+str(i)):
                name = 'weights/'+str(i)
                weights[i] = tf.Variable(tf.random_normal([nodeCount[i],nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,weights[i])
            with tf.name_scope('biases/bias_'+str(i)):
                name = 'biases/'+str(i)
                biases[i] = tf.Variable(tf.random_normal([nodeCount[i+1]]),name=name)
                tf.histogram_summary(name,biases[i])

            layer[i+1] = tf.add(tf.matmul(layer[i],weights[i]),biases[i])
            if(i != layerCount-2):
                layer[i+1] = tf.nn.tanh(layer[i+1])
            else:
                layer[i+1] = tf.nn.softmax(layer[i+1])

    return layer[i+1]

def getNextBatch():
    v = [0.0 for _ in range(vLength)]
    v[0] = 1.0

    r = [None for _ in range(BATCH_SIZE)]
    for i in range(BATCH_SIZE):
        random.shuffle(v)
        r[i] = v.copy()
    return r,r

m = None
def trainNN():
    x_t = tf.placeholder(tf.float32,[None,vLength],'input')
    y_t = tf.placeholder(tf.float32, [None, vLength],'actual')

    m = model(x_t)    

    with tf.name_scope('Cost_Function'):
        cost = tf.reduce_mean(-tf.reduce_sum(y_t * tf.log(m)))

    with tf.name_scope('Learning_Rate'):
        learning_rate = tf.Variable(0.5,dtype=tf.float32)

    with tf.name_scope('Optimizer'):
        #optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
        optimizer = tf.train.AdamOptimizer().minimize(cost)

    with tf.name_scope('summaries'):
        tf.histogram_summary('outputs/actual',y_t)
        tf.scalar_summary('cost',cost)
        tf.histogram_summary('inputs/x',x_t)
        tf.histogram_summary('outputs/predicted',m)

    with tf.name_scope('testing'):
        correct = tf.equal(tf.argmax(y_t,1),tf.argmax(m,1))
        accuracy = tf.reduce_mean(tf.cast(correct,'float'))
        tf.scalar_summary('accuracy',accuracy)

    epochs = 10

    with tf.Session() as sess:
        merged = tf.merge_all_summaries()

        sess.run(tf.initialize_all_variables())

        for epoch in range(epochs):
            tw = tf.train.SummaryWriter(SUMMARY_DIR+'/epoch'+str(epoch),sess.graph)
            epochLoss = 0
            c = 0
            for i in range(TRAIN_SIZE):
                bx,by = getNextBatch()    
                fd = {x_t:bx,y_t:by}
                #optimizer.run(fd)        

                run_metadata = tf.RunMetadata()
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                summary,c,_ = sess.run([merged,cost,optimizer],
                              feed_dict=fd,
                              options=run_options,
                              run_metadata=run_metadata)
                if(i%BATCH_SIZE == 0):
                    tw.add_summary(summary,i)
                    tw.add_run_metadata(run_metadata,'step%d'%i)

                epochLoss += c       

            tx,ty = getNextBatch()
            print('Epoch ',epoch,'/',epochs,':',epochLoss/TRAIN_SIZE,' ',accuracy.eval({x_t:tx,y_t:ty}))

        tw.close()            

trainNN()
| cite | improve this answer | |
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.