saymrwulf/stable-baselines3
1
0
Fork 0
mirror of https://github.com/saymrwulf/stable-baselines3.git synced 2026-09-16 22:20:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert previous changes in SAC + SDE

Browse source
This commit is contained in:
Antonin Raffin 2019-12-02 14:06:30 +01:00
parent 4e39a0627c
commit 8e9802784c
1 changed files with 6 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
torchy_baselines/sac/sac.py
View file

@ -175,8 +175,10 @@ class SAC(BaseRLModel):
# or sample again the noise matrix # or sample again the noise matrix
# otherwise the intermediate step `std = th.exp(log_std)` # otherwise the intermediate step `std = th.exp(log_std)`
# is lost and we cannot backpropagate through again # is lost and we cannot backpropagate through again
# anyway, we need to sample because `log_std` may have changed between two gradient steps
if self.use_sde: if self.use_sde:
self.actor.reset_noise(batch_size=batch_size) # self.actor.reset_noise(batch_size=batch_size)
self.actor.reset_noise()
# Action by the current actor for the sampled state # Action by the current actor for the sampled state
action_pi, log_prob = self.actor.action_log_prob(obs) action_pi, log_prob = self.actor.action_log_prob(obs)
@ -201,8 +203,8 @@ class SAC(BaseRLModel):
with th.no_grad(): with th.no_grad():
if self.use_sde: # if self.use_sde:
self.actor.reset_noise(batch_size=batch_size) # self.actor.reset_noise(batch_size=batch_size)
# Select action according to policy # Select action according to policy
next_action, next_log_prob = self.actor.action_log_prob(next_obs) next_action, next_log_prob = self.actor.action_log_prob(next_obs)
# Compute the target Q value # Compute the target Q value
@ -232,10 +234,7 @@ class SAC(BaseRLModel):
# Optimize the actor # Optimize the actor
self.actor.optimizer.zero_grad() self.actor.optimizer.zero_grad()
# Cf comment above, otherwise pytorch raises an error actor_loss.backward()
# ("Trying to backward through the graph a second time")
# retain_graph = True if self.use_sde and gradient_steps > 1 else False
actor_loss.backward(retain_graph=False)
self.actor.optimizer.step() self.actor.optimizer.step()
# Update target networks # Update target networks