Why does cost function increases over time? (OpenAI cartpole) I was trying the cartpole environment from OpenAI gym. The cost function goes up with time and the reward function goes down. I have no clue why it happens and how to solve it.
Top: Cost, bottom: Reward
import numpy as np
import tensorflow as tf
import gym
import matplotlib.pyplot as plt

env = gym.make('CartPole-v0')

episodes = 1000
batchSize = 50
simulationSteps = 200
maxExperienceSize = 400
trainingSessions = 10
skip = 10
decimals = 2

# Hyper parameters
epsilon = 0.0
learningRate = 1e-4
gamma = .9

# Handles
state = tf.placeholder(tf.float32,[None,4],name='state')
action = tf.placeholder(tf.float32,[None,1],name='action')
nextQ = tf.placeholder(tf.float32,[None,1],name='nextQ')

# Model
def model(state,action):    
    stateAction = tf.concat(1,[state,action],name='stateAction')

    weights1 = tf.Variable(tf.random_normal([5,3]),name='weights1')
    weights2 = tf.Variable(tf.random_normal([3,1]),name='weights2')

    activations1 = tf.nn.relu(tf.matmul(stateAction,weights1),name='activations1')
    activations2 = tf.nn.relu(tf.matmul(activations1,weights2),name='activations2')

    return activations2

prediction = model(state,action)

cost = tf.reduce_sum(tf.squared_difference(prediction,nextQ))
optimizer = tf.train.AdamOptimizer(learningRate).minimize(cost)

experience = []
avgRewards = []
costs = []

def getQValues(sess,s):
    fd = {state:[s],action:[[0.0]]}
    q0 = sess.run([prediction],fd)

    fd = {state:[s],action:[[1.0]]}
    q1 = sess.run([prediction],fd)

    return [q0[0],q1[0]]

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for episode in range(episodes):
        env.reset()
        totalReward = 0
        resets = 1

        for t in range(simulationSteps):
            if len(experience) > maxExperienceSize:
                break

            lastState = tuple(np.around(env.state,decimals))

            # Take a random action
            a = env.action_space.sample()

            # Otherwise query the neural network
            if np.random.uniform() > np.exp(-epsilon):
                q = getQValues(sess,env.state)
                a = np.argmax(q)

            observation, reward, done, info = env.step(a)
            totalReward += reward
            qPrime = getQValues(sess,env.state)
            newQ = reward + gamma * max(qPrime)

            # Experience table
            row = (lastState,(a,),tuple(np.around(newQ[0],decimals)))            
            experience.append(row)

            if done:
                env.reset()
                resets += 1           

        # Remove duplicates
        experience = np.vstack({tuple(row) for row in experience})

        totalCost = 0
        for _ in range(trainingSessions):
            # Shuffle the experience
            np.random.shuffle(experience)

            # Get a batch
            batch = np.array(experience[:batchSize])         

            states = np.array(list(batch[:,0]))
            actions = np.array(list(batch[:,1]))    
            nextQs = np.array(list(batch[:,2]))

            fd = {state:states,action:actions,nextQ:nextQs}
            c,_ = sess.run([cost,optimizer],fd)

            totalCost += c

        experience = list(experience)

        # Forget half the experience
        if len(experience) >= maxExperienceSize:
            experience = experience[:int(maxExperienceSize/2)]

        epsilon += 2/(episodes)

        if episode % skip == 0:
            costs.append(totalCost/trainingSessions/batchSize)
            avgRewards.append(totalReward/resets)

            print(str(episode)+'\t'+str(costs[-1])+'\t'+str(avgRewards[-1])+'\t'
                  +str(len(experience)))



plt.figure(1)
plt.subplot(211)
plt.plot(costs)

plt.subplot(212)
plt.plot(avgRewards)
plt.show()

I decreased the learning rate from 1e-3 to 1e-4 but no luck.
EDIT 1:
Adding a bias term solved the cost issue but the reward just wont go up.

 A: I looked through multiple solved examples and none of them try to solve it using this method. They use other methods like policy gradient or some kind of hill climbing. There is no problem with the code, but the problem is actually with the intuition. Q-learning works when the model satisfies the Markov Property. 
The markov property states that the current state should have all information on how much reward can be acquired. For the reward function to converge, the reward that can be obtained from each state should have a definite final value. That is not the case here. Let us take an example. Lets say when the needle is at 89 degrees, (90 degrees being straight up) it can go right to 91 degrees and then go back to 89 degrees. So, the number of times a particular state is traversed determines the total expected reward. Therefore, the reward function will not converge as there is no upper bound for the expected reward, from any state. 
EDIT
Having a system of discounted rewards on losing works because now we have a tight upper and lower bounds of rewards. The agent gets a -1 on losing so it will try to stay as far as possible from a bad state. The ideal state or the upper bound of reward is now 0. Here is the modified code.
import numpy as np
import tensorflow as tf
import gym
import matplotlib.pyplot as plt
import time

env = gym.make('CartPole-v0')

episodes = 500
batchSize = 100
simulationSteps = 500
maxExperienceSize = 600
trainingSessions = 6
skip = 10

# Hyper parameters
epsilon = 0.0
#learningRate = 1e-3
gamma = .9

# Handles
state = tf.placeholder(tf.float32,[None,4],name='state')
action = tf.placeholder(tf.float32,[None,1],name='action')
nextQ = tf.placeholder(tf.float32,[None,1],name='nextQ')

# Model
def model(state,action):    
    stateAction = tf.concat(1,[state,action],name='stateAction')

    weights1 = tf.Variable(tf.random_normal([5,3],0.1,1e-2),name='weights1')
    biases1 = tf.Variable(tf.random_normal([3],0.1,1e-2),name='biases1')

    weights2 = tf.Variable(tf.random_normal([3,1],0.1,1e-2),name='weights2')
    biases2 = tf.Variable(tf.random_normal([1],0.1,1e-2),name='biases2')

    activations1 = tf.nn.relu(tf.matmul(stateAction,weights1)+biases1,name='activations1')
    #activations2 = tf.nn.relu(tf.matmul(activations1,weights2)+biases2,name='activations2')
    activations2 = tf.matmul(activations1,weights2)+biases2

    return activations2

prediction = model(state,action)

cost = tf.reduce_sum(tf.squared_difference(prediction,nextQ))
optimizer = tf.train.AdamOptimizer().minimize(cost)

experience0 = []
experience1 = []
avgRewards = []
costs = []

def getQValues(sess,s):
    fd = {state:[s],action:[[0.0]]}
    q0 = sess.run([prediction],fd)

    fd = {state:[s],action:[[1.0]]}
    q1 = sess.run([prediction],fd)

    return np.array([float(q0[0][0]),float(q1[0][0])])

def softmax(x):
    return np.exp(x) / np.sum(np.exp(x), axis=0)

start = time.clock()
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for episode in range(episodes):
        env.reset()
        totalReward = 0
        resets = 1       

        rows = []
        for t in range(simulationSteps):
            #if len(experience) > maxExperienceSize:
            #    break

            lastState = tuple(env.state)

            # Take a random action
            a = env.action_space.sample()

            q = getQValues(sess,env.state)
            # Otherwise query the neural network
            if np.random.uniform() > np.exp(-epsilon):
            #if False:                
                a = np.argmax(q)
                #a = np.random.choice(2,p=softmax(q))

            observation, reward, done, info = env.step(a)
            totalReward += reward

            rows.append([lastState,(a,),(0,)])

            if episode == episodes-1:
                print(q)
                env.render()
                time.sleep(.1)

            if done:
                env.reset()
                resets += 1

                for i in range(len(rows)-1,0,-1):
                    if i == len(rows)-1:
                        newQ = -1
                    else:
                        newQ = gamma * newQ

                    rows[i][2] = (newQ,)
                    #print(rows[i])

                # Experience table
                for row in rows:
                    forgetIndex = int(np.random.uniform(0,maxExperienceSize//2+1))
                    if a == 0:
                        experience0.append(tuple(row))
                        if len(experience0) > maxExperienceSize//2:
                            experience0.pop(forgetIndex)
                    else:
                        experience1.append(tuple(row))
                        if len(experience1) > maxExperienceSize//2:
                            experience1.pop(forgetIndex)
                rows = []

        # Remove duplicates
        #experience = np.vstack({tuple(row) for row in experience})

        # Get equal amounts of both experience
        dataSize = min(maxExperienceSize,len(experience0),len(experience1))
        experience = experience0[:dataSize] + experience1[:dataSize]        

        totalCost = 0
        for _ in range(trainingSessions):
            # Shuffle the experience
            np.random.shuffle(experience)

            # Get a batch
            batch = np.array(experience[:batchSize])         

            states = np.array(list(batch[:,0]))
            actions = np.array(list(batch[:,1]))    
            nextQs = np.array(list(batch[:,2]))

            fd = {state:states,action:actions,nextQ:nextQs}
            c,_ = sess.run([cost,optimizer],fd)

            totalCost += c

        if episode % skip == 0:
            elapsed = (time.clock() - start)/skip
            start = time.clock()
            print(elapsed * (episodes - episode)/60,' mins left')
            costs.append(totalCost/trainingSessions/batchSize)
            avgRewards.append(totalReward/resets)                
            print(len(experience0),len(experience1))
            print(str(episode)+'\t'+str(costs[-1])+'\t'+str(avgRewards[-1])+'\t'
                  +str(len(experience)))


        '''# Forget half the experience
        if len(experience0) >= maxExperienceSize/2:
            experience0 = experience0[:maxExperienceSize//4]

        if len(experience1) >= maxExperienceSize/2:
            experience1 = experience1[:maxExperienceSize//4]'''

        epsilon += 2/(episodes)

env.close()
plt.figure(1)
plt.subplot(211)
plt.plot(costs)

plt.subplot(212)
plt.plot(avgRewards)
plt.show()

It is quite slow. Here is the new graph, top cost, bottom reward.

