Bug fixes for A2C and PPO

This commit is contained in:
Antonin Raffin 2019-10-28 14:27:32 +01:00
parent b150167bdd
commit 799e30ff3d
3 changed files with 39 additions and 31 deletions

View file

@ -51,7 +51,7 @@ class A2C(PPO):
_init_setup_model=True): _init_setup_model=True):
super(A2C, self).__init__(policy, env, learning_rate=learning_rate, super(A2C, self).__init__(policy, env, learning_rate=learning_rate,
n_steps=n_steps, batch_size=n_steps, n_epochs=1, n_steps=n_steps, batch_size=None, n_epochs=1,
gamma=gamma, gae_lambda=gae_lambda, ent_coef=ent_coef, gamma=gamma, gae_lambda=gae_lambda, ent_coef=ent_coef,
vf_coef=vf_coef, max_grad_norm=max_grad_norm, vf_coef=vf_coef, max_grad_norm=max_grad_norm,
tensorboard_log=tensorboard_log, policy_kwargs=policy_kwargs, tensorboard_log=tensorboard_log, policy_kwargs=policy_kwargs,
@ -72,21 +72,25 @@ class A2C(PPO):
lr=self.learning_rate, alpha=0.99, lr=self.learning_rate, alpha=0.99,
eps=self.rms_prop_eps, weight_decay=0) eps=self.rms_prop_eps, weight_decay=0)
def train(self, gradient_steps, batch_size=64): def train(self, gradient_steps, batch_size=None):
for gradient_step in range(gradient_steps): # A2C with gradient_steps > 1 does not make sense
# approx_kl_divs = [] assert gradient_steps == 1
# Sample replay buffer # We do not use minibatches for A2C
for replay_data in self.rollout_buffer.get(batch_size): assert batch_size is None
for rollout_data in self.rollout_buffer.get(batch_size=None):
# Unpack # Unpack
obs, action, _, _, advantage, return_batch = replay_data obs, action, _, _, advantage, return_batch = rollout_data
if isinstance(self.action_space, spaces.Discrete): if isinstance(self.action_space, spaces.Discrete):
# Convert discrete action for float to long # Convert discrete action for float to long
action = action.long().flatten() action = action.long().flatten()
# TODO: avoid second computation of everything because of the gradient
values, log_prob, entropy = self.policy.get_policy_stats(obs, action) values, log_prob, entropy = self.policy.get_policy_stats(obs, action)
values = values.flatten() values = values.flatten()
# Normalize advantage (not present in the original implementation) # Normalize advantage (not present in the original implementation)
if self.normalize_advantage: if self.normalize_advantage:
advantage = (advantage - advantage.mean()) / (advantage.std() + 1e-8) advantage = (advantage - advantage.mean()) / (advantage.std() + 1e-8)
@ -97,7 +101,7 @@ class A2C(PPO):
value_loss = F.mse_loss(return_batch, values) value_loss = F.mse_loss(return_batch, values)
# Entropy loss favor exploration # Entropy loss favor exploration
entropy_loss = th.mean(entropy) entropy_loss = -th.mean(entropy)
loss = policy_loss + self.ent_coef * entropy_loss + self.vf_coef * value_loss loss = policy_loss + self.ent_coef * entropy_loss + self.vf_coef * value_loss

View file

@ -145,7 +145,7 @@ class RolloutBuffer(BaseBuffer):
if self.pos == self.buffer_size: if self.pos == self.buffer_size:
self.full = True self.full = True
def get(self, batch_size): def get(self, batch_size=None):
assert self.full assert self.full
indices = th.randperm(self.buffer_size * self.n_envs) indices = th.randperm(self.buffer_size * self.n_envs)
# Prepare the data # Prepare the data
@ -155,6 +155,10 @@ class RolloutBuffer(BaseBuffer):
self.__dict__[tensor] = self.swap_and_flatten(self.__dict__[tensor]) self.__dict__[tensor] = self.swap_and_flatten(self.__dict__[tensor])
self.generator_ready = True self.generator_ready = True
# Return everything, don't create minibatches
if batch_size is None:
batch_size = self.buffer_size * self.n_envs
start_idx = 0 start_idx = 0
while start_idx < self.buffer_size * self.n_envs: while start_idx < self.buffer_size * self.n_envs:
yield self._get_samples(indices[start_idx:start_idx + batch_size]) yield self._get_samples(indices[start_idx:start_idx + batch_size])

View file

@ -205,7 +205,7 @@ class PPO(BaseRLModel):
# Entropy loss favor exploration # Entropy loss favor exploration
entropy_loss = th.mean(entropy) entropy_loss = -th.mean(entropy)
loss = policy_loss + self.ent_coef * entropy_loss + self.vf_coef * value_loss loss = policy_loss + self.ent_coef * entropy_loss + self.vf_coef * value_loss