k-armed bandit experiment using an epoch-greedy method I'm new to reinforcement learning (and machine learning in general) and have attempted to recreate the k-armed bandit experiment in section 2.2, Action-value methods, of "Reinforcement Learning: An Introduction" by Sutton & Barto. The expected graph is supposed to look like this:
However, mine looks like this
I would like to know if the code below is not accurately duplicating the experiment and if so, what I missed or should add to get similar results.
# coding: utf-8

# In[44]:

import numpy as np
from matplotlib import pyplot as plt

get_ipython().magic('matplotlib inline')

# Returns the action-value (sample-average) for each action at the current time step
def Qt(actions):
    results = [0.0 if actions[i][1] == 0 else actions[i][0] / float(actions[i][1]) for i in range(len(actions))]
    return results


# The reward for selecting an action
# Selected around true_values[action_index] with unit variance
def get_reward(true_values, action_index):
    estimated = np.random.normal(true_values[action_index], size=1)[0]
    return estimated


# k - no or arms
# epsilon - is the probability for exploration
#           if epsilon = 0, then purely greedy
# iters - no. of iterations/runs
def epoch_greedy(k, epsilon, iterations):
    true_values = np.random.normal(size=10)

    # actions[i] is the ith action
    # actions[i][0] is the sum of rewards for action i
    # actions[i][1] is the no. of times action i has been taken
    actions = [[0.0, 0] for _ in range(k)]

    # Cummulative reward
    total = 0.0
    rewards = []

    for _ in range(iterations):
        prob = np.random.rand(1)

        # greedy (exploit current knowledge)
        if prob > epsilon:
            action_index = np.argmax(Qt(actions))
        # explore
        else:
            action_index = np.random.randint(0, k)

        reward = get_reward(true_values, action_index)

        # Update
        total += reward
        rewards.append(total/1000.0)

        action = actions[action_index]
        action[0] += reward
        action[1] += 1

    return rewards


# In[ ]:

# Run the k-armed bandit experiment using k-arms
# for iters iterations epoch times
# Returns the mean reward for each iteration across
# epochs executions
def run_experiment(k, epsilon, iters, epochs):
    rewards = []
    for i in range(epochs):
        rewards.append(epoch_greedy(k, epsilon, iters))
        if (i % 50) == 0:
            print('Epoch #{}, k={}, epsilon={}, iters={}'.format(i, k, epsilon, iters))
    print('Done... (epsilon={})\n'.format(epsilon))

    # Compute the mean reward for each iteration
    means = np.mean(np.array(rewards), axis=0)

    return means


# In[42]:

e_0_01 = run_experiment(10, 0.01, 1000, 2000)
e_0_1 = run_experiment(10, 0.1, 1000, 2000)
e_0 = run_experiment(10, 0, 1000, 2000)


# In[43]:

x_axis = range(1, 1001)
plt.plot(x_axis, e_0_01, c='blue')
plt.plot(x_axis, e_0_1, c='red')
plt.plot(x_axis, e_0, c='green')
plt.xlabel('Steps')
plt.ylabel('Average reward')
plt.show()

 A: In epoch_greedy(), I was storing the cumulative reward as opposed to the instantaneous results for each iteration. After updating the code, I was able to get a similar graph: 
import numpy as np
from matplotlib import pyplot as plt


# Returns the action-value (sample-average) for each action at the current time step
def Qt(actions):
    results = [0.0 if actions[i][1] == 0 else actions[i][0] / float(actions[i][1]) for i in range(len(actions))]
    return results


# The reward for selecting an action
# Selected around true_values[action_index] with unit variance
def get_reward(true_values, action_index):
    estimated = np.random.normal(true_values[action_index], size=1)[0]
    return estimated


# k - no or amrs
# epsilon - is the probability for exploration
#           if epsilon = 0, then purely greedy
# iters - no. of iterations/runs
def epoch_greedy(k, epsilon, iterations):
    true_values = np.random.normal(size=k)

    # actions[i] is the ith action
    # actions[i][0] is the sum of rewards for action i
    # actions[i][1] is the no. of times action i has been taken
    actions = [[0.0, 0] for _ in range(k)]

    rewards = []

    for _ in range(iterations):
        prob = np.random.rand(1)

        # greedy (exploit current knowledge)
        if prob > epsilon:
            # action_index = np.argmax(Qt(actions))
            # See: http://stackoverflow.com/questions/42071597/numpy-argmax-random-tie-breaking
            action_values = np.array(Qt(actions))
            action_index = np.random.choice(np.flatnonzero(action_values == action_values.max()))
        # explore
        else:
            action_index = np.random.randint(0, k)

        reward = get_reward(true_values, action_index)

        # Update
        rewards.append(reward)

        action = actions[action_index]
        action[0] += reward
        action[1] += 1

    return rewards


# Run the k-armed bandit experiment using k-arms
# for iters iterations epoch times
# Returns the mean reward for each iteration across
# epochs executions
def run_experiment(k, epsilon, iters, epochs):
    rewards = []
    for i in range(epochs):
        rewards.append(epoch_greedy(k, epsilon, iters))
        if (i % 50) == 0:
            print('Epoch #{}, k={}, epsilon={}, iters={}'.format(i, k, epsilon, iters))
    print('Done... (epsilon={})\n'.format(epsilon))

    # Compute the mean reward for each iteration
    means = np.mean(np.array(rewards), axis=0)

    return means


e_0_01 = run_experiment(10, 0.01, 1000, 2000)
e_0_1 = run_experiment(10, 0.1, 1000, 2000)
e_0 = run_experiment(10, 0, 1000, 2000)

x_axis = range(1, 1001)
plt.plot(x_axis, e_0_01, c='blue', label='e = 0.01')
plt.plot(x_axis, e_0_1, c='red', label='e = 0.1')
plt.plot(x_axis, e_0, c='green', label='e = 0')
plt.xlabel('Steps')
plt.ylabel('Average reward')
plt.legend()
plt.show()

