I want to train a recurrent policy gradient which predicts action probabilities based on prior environment states. However, I am unable to backpropagate during the "update policy" step, in which the running rewards are scaled, normalized, summed, and used to update the model. I understand that training an rnn in this context is unusual, because the RNN has to be manually unrolled, as the environment state depends on the last prediction in the sequence of predictions.
My unrolling scheme is:
- sample environment state
- state, hidden_in -> RNN cell -> hidden_out
- hidden_out -> linear function -> action probability
- hidden_out -> hidden_in
But I get this error during the backward step:
RuntimeError: grad can be implicitly created only for scalar outputs
Here is the code for the model and policy update:
import torch.nn as nn class Policy(nn.Module): def __init__(self, state_space, action_space, hidden_size, n_layers, dropout_rate, gamma): super(Policy, self).__init__() self.input_size = state_space.shape self.output_size = action_space.n self.hidden_size = hidden_size self.n_layers = n_layers self.rnn = nn.GRUCell(input_size=self.input_size, hidden_size=self.hidden_size) self.relu = nn.LeakyReLU() self.linear = nn.Linear(self.hidden_size, self.output_size) self.softmax = nn.Softmax(dim=-1) self.dropout_rate = dropout_rate self.dropout = nn.Dropout(p=self.dropout_rate) self.gamma = gamma # history self.hidden_history = None self.policy_history = None self.reward_episode = None self.reward_episode_local = None self.reset_episode() # Overall reward and loss history self.reward_history = list() self.reward_history_local = list() self.loss_history = list() def reset_episode(self): # Episode policy and reward history self.hidden_history = list() self.policy_history = list() self.reward_episode = list() self.reward_episode_local = list() def forward(self, x): size = x.shape x = x.view([1, size]) # batch size = 1 if len(self.hidden_history) > 0: h_0 = self.hidden_history[-1] else: h_0 = None x = self.rnn(x, h_0) self.hidden_history.append(x) x = self.relu(x) x = self.linear(x) x = self.softmax(x) return x
def update_policy(policy, optimizer, e): R = 0 rewards =  # Discount future rewards back to the present using gamma for r in reversed(policy.reward_episode): R = r + policy.gamma * R rewards.insert(0, R) # Scale rewards rewards = torch.FloatTensor(rewards) # Normalize rewards rewards = (rewards - rewards.mean()) / (rewards.std() + float(np.finfo(np.float32).eps)) # Calculate loss policy_history = torch.stack(policy.policy_history) loss = (torch.mul(policy_history, rewards).mul(-1), -1) # Update network weights optimizer.zero_grad() loss.backward() optimizer.step() # Save and intialize episode history counters policy.loss_history.append(loss.data) policy.reward_history.append(np.sum(policy.reward_episode)) policy.reset_episode()
I am open to recommendations as to how to implement an RNN policy gradient, but primarily I would like to understand what is the cause of this error.