0
$\begingroup$

I am simulating a mouse to find a cheese on an empty table. I randomly put a cheese on the table and let the mouse find the cheese without falling off the table. The problem is, in test part, agent goes into an infinite loop if i don't provide any epsilon values. If I provide an epsilon value, agent can fall off the table. The table is discrete but the cheese is random. So i need the mouse to learn the table first. Say the agent at 322. Agent finds the largest Q and moves to the left which is 321. This time, largest value is 322. And it keeps going...

I couldn't find any problems. Is my approach wrong or missing something?

Here's the code i have written:

import numpy as np
import random
import matplotlib.pyplot as plt
import pickle

class Env:
    def __init__(self):
        self.grid_size = 10 # 10x10 matrix
        self.observation_size = self.grid_size * self.grid_size
        self.action_size = 4 # 0: left 1: right 2: up 3: down

    def reset(self):
        self.grid = []
        self.holes = []
        self.target = []
        self.x = np.random.randint(0, self.grid_size)
        while True:
            self.y = np.random.randint(0, self.grid_size)
            if self.x != self.y:
                break
        self.state = (self.y * self.grid_size) + self.x
        while True:
            self.target_x = np.random.randint(0, self.grid_size)
            while True:
                self.target_y = np.random.randint(0, self.grid_size)
                if self.target_y != self.target_x:
                    break
            self.target = (self.target_y * self.grid_size) + self.target_x
            if self.target != self.state:
                break
        return self.state

    def step(self, action):
        reward = 0
        done = False
        # 0: left 1: right 2: up 3: down
        x = self.x
        y = self.y
        if action == 0:
            self.x = self.x - 1
        elif action == 1:
            self.x = self.x + 1
        elif action == 2:
            self.y = self.y + 1
        elif action == 3:
            self.y = self.y - 1
        self.state = (self.y * self.grid_size) + self.x
        if self.x < 0 or self.x >= self.grid_size:
            if self.x < 0:
                reward = -100
            else:
                reward = -100
            #self.x = x
            done = True
        elif self.y < 0 or self.y >= self.grid_size:
            if self.y < 0:
                reward = -100
            else:
                reward = -100
            #self.y = y
            done = True
        elif self.state == self.target:
            reward = 100
            done = True
        else:
            reward = 1
        return self.state, reward, done

def train(env, alpha, gamma, epsilon, epoch):
    print("Training...")
    qt = np.zeros([env.observation_size * env.observation_size, env.action_size])
    error_list = []
    reward_list = []
    errors = 0
    rewards = 0
    for ep in range(epoch):
        state = env.reset()
        while True:
            if random.uniform(0, 1) < epsilon:
                action = np.random.randint(env.action_size)
            else:
                action = np.argmax(qt[state])
            next_state, reward, done = env.step(action)
            # Q function
            old_value = qt[state, action]
            next_max = np.max(qt[next_state])
            next_value = old_value + alpha * (reward + gamma * next_max - old_value)
            qt[state, action] = next_value
            state = next_state
            rewards += reward
            if reward < 0:
                errors += abs(reward)
            if done:
                break
        error_list.append(errors)
        reward_list.append(rewards)
        print("Episode: {}, Rewards: {}, Errors: {}".format(ep, rewards, errors))
    fig, ax = plt.subplots(2, 1)
    ax[0].plot(reward_list)
    ax[1].plot(error_list)
    plt.show()
    return qt

def test(env, qt, epsilon):
    print()
    print("Testing...")
    error_list = []
    reward_list = []
    rewards = 0
    errors = 0
    state = env.reset()
    steps = 0
    while True:
        steps += 1
        """
        if random.uniform(0, 1) < epsilon:
            action = np.random.randint(env.action_size)
        else:
            action = np.argmax(qt[state])
        r = np.random.randint(0, 2)
        """
        action = np.argmax(qt[state])
        next_state, reward, done = env.step(action)
        state = next_state
        rewards += reward
        print(state, action)
        if reward <= -100:
            errors += 1
        if done:
            break
    error_list.append(errors)
    reward_list.append(rewards)
    print("Steps: {}, Rewards: {}, Errors: {}".format(steps, rewards, errors))
    print("Target pos:", env.target_x, env.target_y)
    print("State:", env.x, env.y)

alpha = 0.1 # learning rate
gamma = 0.99 # discount rate
epsilon = 0.1 # explore rate
epoch = 100000 # epoch

load_file = True
fl = "./grid.pck"

env = Env()
if not load_file:
    qt = train(env, alpha, gamma, epsilon, epoch)
    with open(fl, 'wb') as fp:
        pickle.dump(qt, fp)
else:
    with open(fl, 'rb') as fp:
        qt = pickle.load(fp)
test(env, qt, epsilon)
$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.