mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-17 08:56:08 +00:00
Add classic advantage computation
This commit is contained in:
parent
c15b4bda1e
commit
69a348276e
1 changed files with 34 additions and 14 deletions
|
|
@ -113,10 +113,15 @@ class RolloutBuffer(BaseBuffer):
|
||||||
self.generator_ready = False
|
self.generator_ready = False
|
||||||
super(RolloutBuffer, self).reset()
|
super(RolloutBuffer, self).reset()
|
||||||
|
|
||||||
def compute_returns_and_advantage(self, last_value, dones=False):
|
def compute_returns_and_advantage(self, last_value, dones=False, use_gae=True):
|
||||||
"""
|
"""
|
||||||
From PPO2
|
From Stable-Baselines PPO2
|
||||||
|
:param last_value: (th.Tensor)
|
||||||
|
:param dones: ([bool])
|
||||||
|
:param use_gae: (bool) Whether to use Generalized Advantage Estimation
|
||||||
|
or normal advantage for advantage computation.
|
||||||
"""
|
"""
|
||||||
|
if use_gae:
|
||||||
last_gae_lam = 0
|
last_gae_lam = 0
|
||||||
for step in reversed(range(self.buffer_size)):
|
for step in reversed(range(self.buffer_size)):
|
||||||
if step == self.buffer_size - 1:
|
if step == self.buffer_size - 1:
|
||||||
|
|
@ -129,6 +134,21 @@ class RolloutBuffer(BaseBuffer):
|
||||||
last_gae_lam = delta + self.gamma * self.gae_lambda * next_non_terminal * last_gae_lam
|
last_gae_lam = delta + self.gamma * self.gae_lambda * next_non_terminal * last_gae_lam
|
||||||
self.advantages[step] = last_gae_lam
|
self.advantages[step] = last_gae_lam
|
||||||
self.returns = self.advantages + self.values
|
self.returns = self.advantages + self.values
|
||||||
|
else:
|
||||||
|
# Discounted return with value bootstrap
|
||||||
|
# Note: this is equivalent to GAE computation
|
||||||
|
# with gae_lambda = 1.0
|
||||||
|
last_return = 0.0
|
||||||
|
for step in reversed(range(self.buffer_size)):
|
||||||
|
if step == self.buffer_size - 1:
|
||||||
|
next_non_terminal = th.FloatTensor(1.0 - dones)
|
||||||
|
next_value = last_value.clone().cpu().flatten()
|
||||||
|
last_return = self.rewards[step] + next_non_terminal * next_value
|
||||||
|
else:
|
||||||
|
next_non_terminal = 1.0 - self.dones[step + 1]
|
||||||
|
last_return = self.rewards[step] + self.gamma * last_return * next_non_terminal
|
||||||
|
self.returns[step] = last_return
|
||||||
|
self.advantages = self.returns - self.values
|
||||||
|
|
||||||
def add(self, obs, action, reward, done, value, log_prob):
|
def add(self, obs, action, reward, done, value, log_prob):
|
||||||
if len(log_prob.shape) == 0:
|
if len(log_prob.shape) == 0:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue