# Q-learning agent stucks in an infinite loop

I am simulating a mouse to find a cheese on an empty table. I randomly put a cheese on the table and let the mouse find the cheese without falling off the table. The problem is, in test part, agent goes into an infinite loop if i don't provide any epsilon values. If I provide an epsilon value, agent can fall off the table. The table is discrete but the cheese is random. So i need the mouse to learn the table first. Say the agent at 322. Agent finds the largest Q and moves to the left which is 321. This time, largest value is 322. And it keeps going...

I couldn't find any problems. Is my approach wrong or missing something?

Here's the code i have written:

import numpy as np
import random
import matplotlib.pyplot as plt
import pickle

class Env:
def __init__(self):
self.grid_size = 10 # 10x10 matrix
self.observation_size = self.grid_size * self.grid_size
self.action_size = 4 # 0: left 1: right 2: up 3: down

def reset(self):
self.grid = []
self.holes = []
self.target = []
self.x = np.random.randint(0, self.grid_size)
while True:
self.y = np.random.randint(0, self.grid_size)
if self.x != self.y:
break
self.state = (self.y * self.grid_size) + self.x
while True:
self.target_x = np.random.randint(0, self.grid_size)
while True:
self.target_y = np.random.randint(0, self.grid_size)
if self.target_y != self.target_x:
break
self.target = (self.target_y * self.grid_size) + self.target_x
if self.target != self.state:
break
return self.state

def step(self, action):
reward = 0
done = False
# 0: left 1: right 2: up 3: down
x = self.x
y = self.y
if action == 0:
self.x = self.x - 1
elif action == 1:
self.x = self.x + 1
elif action == 2:
self.y = self.y + 1
elif action == 3:
self.y = self.y - 1
self.state = (self.y * self.grid_size) + self.x
if self.x < 0 or self.x >= self.grid_size:
if self.x < 0:
reward = -100
else:
reward = -100
#self.x = x
done = True
elif self.y < 0 or self.y >= self.grid_size:
if self.y < 0:
reward = -100
else:
reward = -100
#self.y = y
done = True
elif self.state == self.target:
reward = 100
done = True
else:
reward = 1
return self.state, reward, done

def train(env, alpha, gamma, epsilon, epoch):
print("Training...")
qt = np.zeros([env.observation_size * env.observation_size, env.action_size])
error_list = []
reward_list = []
errors = 0
rewards = 0
for ep in range(epoch):
state = env.reset()
while True:
if random.uniform(0, 1) < epsilon:
action = np.random.randint(env.action_size)
else:
action = np.argmax(qt[state])
next_state, reward, done = env.step(action)
# Q function
old_value = qt[state, action]
next_max = np.max(qt[next_state])
next_value = old_value + alpha * (reward + gamma * next_max - old_value)
qt[state, action] = next_value
state = next_state
rewards += reward
if reward < 0:
errors += abs(reward)
if done:
break
error_list.append(errors)
reward_list.append(rewards)
print("Episode: {}, Rewards: {}, Errors: {}".format(ep, rewards, errors))
fig, ax = plt.subplots(2, 1)
ax.plot(reward_list)
ax.plot(error_list)
plt.show()
return qt

def test(env, qt, epsilon):
print()
print("Testing...")
error_list = []
reward_list = []
rewards = 0
errors = 0
state = env.reset()
steps = 0
while True:
steps += 1
"""
if random.uniform(0, 1) < epsilon:
action = np.random.randint(env.action_size)
else:
action = np.argmax(qt[state])
r = np.random.randint(0, 2)
"""
action = np.argmax(qt[state])
next_state, reward, done = env.step(action)
state = next_state
rewards += reward
print(state, action)
if reward <= -100:
errors += 1
if done:
break
error_list.append(errors)
reward_list.append(rewards)
print("Steps: {}, Rewards: {}, Errors: {}".format(steps, rewards, errors))
print("Target pos:", env.target_x, env.target_y)
print("State:", env.x, env.y)

alpha = 0.1 # learning rate
gamma = 0.99 # discount rate
epsilon = 0.1 # explore rate
epoch = 100000 # epoch