mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-16 08:49:07 +00:00
Fix squash output unscaling when using gSDE (#1652)
* prevents squash_output if not use_sde, see #1592 * update changelog * add unscaling of actions taken during training * add test regarding squashing and unquashing * avoids try-except block * format Gymnasium code with black * makes mypy pass * makes pytype pass * sort imports * makes error message in assert statement clearer Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org> * improves code commenting * replaces full env with wrapper * Cleanup code * Reformat --------- Co-authored-by: PatrickHelm <patrick.helm@gmx.net> Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org> Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>
This commit is contained in:
parent
84163b468c
commit
16c6a886db
4 changed files with 63 additions and 6 deletions
|
|
@ -20,10 +20,13 @@ New Features:
|
|||
|
||||
Bug Fixes:
|
||||
^^^^^^^^^^
|
||||
- Prevents using squash_output and not use_sde in ActorCritcPolicy (@PatrickHelm)
|
||||
- Performs unscaling of actions in collect_rollout in OnPolicyAlgorithm (@PatrickHelm)
|
||||
- Moves VectorizedActionNoise into ``_setup_learn()`` in OffPolicyAlgorithm (@PatrickHelm)
|
||||
- Prevents out of bound error on Windows if no seed is passed (@PatrickHelm)
|
||||
- Calls ``callback.update_locals()`` before ``callback.on_rollout_end()`` in OnPolicyAlgorithm (@PatrickHelm)
|
||||
|
||||
|
||||
Deprecations:
|
||||
^^^^^^^^^^^^^
|
||||
|
||||
|
|
@ -35,6 +38,7 @@ Others:
|
|||
- Fixed ``stable_baselines3/common/vec_env/vec_video_recorder.py`` type hints
|
||||
- Fixed ``stable_baselines3/common/save_util.py`` type hints
|
||||
|
||||
|
||||
Documentation:
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
|
|
|
|||
|
|
@ -171,9 +171,16 @@ class OnPolicyAlgorithm(BaseAlgorithm):
|
|||
|
||||
# Rescale and perform action
|
||||
clipped_actions = actions
|
||||
# Clip the actions to avoid out of bound error
|
||||
|
||||
if isinstance(self.action_space, spaces.Box):
|
||||
clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high)
|
||||
if self.policy.squash_output:
|
||||
# Unscale the actions to match env bounds
|
||||
# if they were previously squashed (scaled in [-1, 1])
|
||||
clipped_actions = self.policy.unscale_action(clipped_actions)
|
||||
else:
|
||||
# Otherwise, clip the actions to avoid out of bound error
|
||||
# as we are sampling from an unbounded Gaussian distribution
|
||||
clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high)
|
||||
|
||||
new_obs, rewards, dones, infos = env.step(clipped_actions)
|
||||
|
||||
|
|
|
|||
|
|
@ -489,6 +489,8 @@ class ActorCriticPolicy(BasePolicy):
|
|||
|
||||
self.log_std_init = log_std_init
|
||||
dist_kwargs = None
|
||||
|
||||
assert not (squash_output and not use_sde), "squash_output=True is only available when using gSDE (use_sde=True)"
|
||||
# Keyword arguments for gSDE distribution
|
||||
if use_sde:
|
||||
dist_kwargs = {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import gymnasium as gym
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch as th
|
||||
from torch.distributions import Normal
|
||||
|
|
@ -59,20 +61,62 @@ def test_sde_check():
|
|||
PPO("MlpPolicy", "CartPole-v1", use_sde=True)
|
||||
|
||||
|
||||
def test_only_sde_squashed():
|
||||
with pytest.raises(AssertionError, match="use_sde=True"):
|
||||
PPO("MlpPolicy", "Pendulum-v1", use_sde=False, policy_kwargs=dict(squash_output=True))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_class", [SAC, A2C, PPO])
|
||||
@pytest.mark.parametrize("use_expln", [False, True])
|
||||
def test_state_dependent_noise(model_class, use_expln):
|
||||
@pytest.mark.parametrize("squash_output", [False, True])
|
||||
def test_state_dependent_noise(model_class, use_expln, squash_output):
|
||||
kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64}
|
||||
|
||||
policy_kwargs = dict(log_std_init=-2, use_expln=use_expln, net_arch=[64])
|
||||
|
||||
if model_class in [A2C, PPO]:
|
||||
policy_kwargs["squash_output"] = squash_output
|
||||
elif not squash_output:
|
||||
pytest.skip("SAC can only use squashed output")
|
||||
|
||||
env = StoreActionEnvWrapper(gym.make("Pendulum-v1"))
|
||||
model = model_class(
|
||||
"MlpPolicy",
|
||||
"Pendulum-v1",
|
||||
env,
|
||||
use_sde=True,
|
||||
seed=None,
|
||||
seed=1,
|
||||
verbose=1,
|
||||
policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]),
|
||||
policy_kwargs=policy_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
model.learn(total_timesteps=255)
|
||||
buffer = model.replay_buffer if model_class == SAC else model.rollout_buffer
|
||||
# Check that only scaled actions are stored
|
||||
assert (buffer.actions <= model.action_space.high).all()
|
||||
assert (buffer.actions >= model.action_space.low).all()
|
||||
if squash_output:
|
||||
# Pendulum action range is [-2, 2]
|
||||
# we check that the action are correctly unscaled
|
||||
if buffer.actions.max() > 0.5:
|
||||
assert np.max(env.actions) > 1.0
|
||||
if buffer.actions.max() < -0.5:
|
||||
assert np.min(env.actions) < -1.0
|
||||
model.policy.reset_noise()
|
||||
if model_class == SAC:
|
||||
model.policy.actor.get_std()
|
||||
|
||||
|
||||
class StoreActionEnvWrapper(gym.Wrapper):
|
||||
"""
|
||||
Keep track of which actions were sent to the env.
|
||||
"""
|
||||
|
||||
def __init__(self, env):
|
||||
super().__init__(env)
|
||||
# defines list for tracking actions
|
||||
self.actions = []
|
||||
|
||||
def step(self, action):
|
||||
# appends list for tracking actions
|
||||
self.actions.append(action)
|
||||
return super().step(action)
|
||||
|
|
|
|||
Loading…
Reference in a new issue