I want to train a recurrent policy gradient which predicts action probabilities based on prior environment states. However, I am unable to backpropagate during the "update policy" step, in which the running rewards are scaled, normalized, summed, and used to update the model. I understand that training an rnn in this context is unusual, because the RNN has to be manually unrolled, as the environment state depends on the last prediction in the sequence of predictions.

My unrolling scheme is:

  1. sample environment state
  2. state, hidden_in -> RNN cell -> hidden_out
  3. hidden_out -> linear function -> action probability
  4. hidden_out -> hidden_in

But I get this error during the backward step:

RuntimeError: grad can be implicitly created only for scalar outputs

Here is the code for the model and policy update:

import torch.nn as nn


class Policy(nn.Module):
    def __init__(self, state_space, action_space, hidden_size, n_layers, dropout_rate, gamma):
        super(Policy, self).__init__()
        self.input_size = state_space.shape[0]
        self.output_size = action_space.n
        self.hidden_size = hidden_size
        self.n_layers = n_layers

        self.rnn = nn.GRUCell(input_size=self.input_size,
                              hidden_size=self.hidden_size)

        self.relu = nn.LeakyReLU()
        self.linear = nn.Linear(self.hidden_size, self.output_size)
        self.softmax = nn.Softmax(dim=-1)

        self.dropout_rate = dropout_rate
        self.dropout = nn.Dropout(p=self.dropout_rate)
        self.gamma = gamma

        # history
        self.hidden_history = None
        self.policy_history = None
        self.reward_episode = None
        self.reward_episode_local = None

        self.reset_episode()

        # Overall reward and loss history
        self.reward_history = list()
        self.reward_history_local = list()
        self.loss_history = list()

    def reset_episode(self):
        # Episode policy and reward history
        self.hidden_history = list()
        self.policy_history = list()
        self.reward_episode = list()
        self.reward_episode_local = list()

    def forward(self, x):
        size = x.shape[0]
        x = x.view([1, size])   # batch size = 1

        if len(self.hidden_history) > 0:
            h_0 = self.hidden_history[-1]
        else:
            h_0 = None

        x = self.rnn(x, h_0)
        self.hidden_history.append(x)

        x = self.relu(x)
        x = self.linear(x)
        x = self.softmax(x)

        return x

...

def update_policy(policy, optimizer, e):
    R = 0
    rewards = []

    # Discount future rewards back to the present using gamma
    for r in reversed(policy.reward_episode):
        R = r + policy.gamma * R
        rewards.insert(0, R)

    # Scale rewards
    rewards = torch.FloatTensor(rewards)

    # Normalize rewards
    rewards = (rewards - rewards.mean()) / (rewards.std() + float(np.finfo(np.float32).eps))

    # Calculate loss
    policy_history = torch.stack(policy.policy_history)
    loss = (torch.mul(policy_history, rewards).mul(-1), -1)

    # Update network weights
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # Save and intialize episode history counters
    policy.loss_history.append(loss.data[0])
    policy.reward_history.append(np.sum(policy.reward_episode))
    policy.reset_episode()

I am open to recommendations as to how to implement an RNN policy gradient, but primarily I would like to understand what is the cause of this error.

up vote 1 down vote accepted

Turns out this was a simple bug in which the shape of my calculated rewards (aka loss) did not match the shape of my output layer...

Your Answer

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.

Not the answer you're looking for? Browse other questions tagged or ask your own question.