Re-sample noise matrix for PPO

This commit is contained in:
Antonin Raffin 2019-12-20 11:28:20 +01:00
parent e894f1f11b
commit 161c608f9c

View file

@ -210,6 +210,12 @@ class PPO(BaseRLModel):
# Convert discrete action for float to long # Convert discrete action for float to long
action = action.long().flatten() action = action.long().flatten()
# Re-sample the noise matrix because the log_std has changed
# TODO: investigate why there is no issue with the gradient
# if that line is commented (as in SAC)
if self.use_sde:
self.policy.reset_noise(batch_size)
values, log_prob, entropy = self.policy.evaluate_actions(obs, action) values, log_prob, entropy = self.policy.evaluate_actions(obs, action)
values = values.flatten() values = values.flatten()
# Normalize advantage # Normalize advantage