saymrwulf/stable-baselines3
1
0
Fork 0
mirror of https://github.com/saymrwulf/stable-baselines3.git synced 2026-09-16 22:20:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert previous changes in SAC + SDE

Browse source
This commit is contained in:
Antonin Raffin 2019-12-02 14:06:30 +01:00
parent 4e39a0627c
commit 8e9802784c
1 changed files with 6 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
torchy_baselines/sac/sac.py
View file

@ -175,8 +175,10 @@ class SAC(BaseRLModel):
# or sample again the noise matrix
# otherwise the intermediate step `std = th.exp(log_std)`
# is lost and we cannot backpropagate through again
# anyway, we need to sample because `log_std` may have changed between two gradient steps
if self.use_sde:
self.actor.reset_noise(batch_size=batch_size)
# self.actor.reset_noise(batch_size=batch_size)
self.actor.reset_noise()
# Action by the current actor for the sampled state
action_pi, log_prob = self.actor.action_log_prob(obs)
@ -201,8 +203,8 @@ class SAC(BaseRLModel):
with th.no_grad():
if self.use_sde:
self.actor.reset_noise(batch_size=batch_size)
# if self.use_sde:
# self.actor.reset_noise(batch_size=batch_size)
# Select action according to policy
next_action, next_log_prob = self.actor.action_log_prob(next_obs)
# Compute the target Q value
@ -232,10 +234,7 @@ class SAC(BaseRLModel):
# Optimize the actor
self.actor.optimizer.zero_grad()
# Cf comment above, otherwise pytorch raises an error
# ("Trying to backward through the graph a second time")
# retain_graph = True if self.use_sde and gradient_steps > 1 else False
actor_loss.backward(retain_graph=False)
actor_loss.backward()
self.actor.optimizer.step()
# Update target networks