From 16c6a886db529bdff2a1d084126f60888c132695 Mon Sep 17 00:00:00 2001 From: PatrickHelm <91552435+PatrickHelm@users.noreply.github.com> Date: Fri, 1 Sep 2023 17:58:15 +0200 Subject: [PATCH] Fix squash output unscaling when using gSDE (#1652) * prevents squash_output if not use_sde, see #1592 * update changelog * add unscaling of actions taken during training * add test regarding squashing and unquashing * avoids try-except block * format Gymnasium code with black * makes mypy pass * makes pytype pass * sort imports * makes error message in assert statement clearer Co-authored-by: Antonin RAFFIN * improves code commenting * replaces full env with wrapper * Cleanup code * Reformat --------- Co-authored-by: PatrickHelm Co-authored-by: Antonin RAFFIN Co-authored-by: Antonin Raffin --- docs/misc/changelog.rst | 4 ++ .../common/on_policy_algorithm.py | 11 +++- stable_baselines3/common/policies.py | 2 + tests/test_sde.py | 52 +++++++++++++++++-- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index ae9ffcd..d5b0a26 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -20,10 +20,13 @@ New Features: Bug Fixes: ^^^^^^^^^^ +- Prevents using squash_output and not use_sde in ActorCritcPolicy (@PatrickHelm) +- Performs unscaling of actions in collect_rollout in OnPolicyAlgorithm (@PatrickHelm) - Moves VectorizedActionNoise into ``_setup_learn()`` in OffPolicyAlgorithm (@PatrickHelm) - Prevents out of bound error on Windows if no seed is passed (@PatrickHelm) - Calls ``callback.update_locals()`` before ``callback.on_rollout_end()`` in OnPolicyAlgorithm (@PatrickHelm) + Deprecations: ^^^^^^^^^^^^^ @@ -35,6 +38,7 @@ Others: - Fixed ``stable_baselines3/common/vec_env/vec_video_recorder.py`` type hints - Fixed ``stable_baselines3/common/save_util.py`` type hints + Documentation: ^^^^^^^^^^^^^^ diff --git a/stable_baselines3/common/on_policy_algorithm.py b/stable_baselines3/common/on_policy_algorithm.py index f038df6..a3f47b2 100644 --- a/stable_baselines3/common/on_policy_algorithm.py +++ b/stable_baselines3/common/on_policy_algorithm.py @@ -171,9 +171,16 @@ class OnPolicyAlgorithm(BaseAlgorithm): # Rescale and perform action clipped_actions = actions - # Clip the actions to avoid out of bound error + if isinstance(self.action_space, spaces.Box): - clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high) + if self.policy.squash_output: + # Unscale the actions to match env bounds + # if they were previously squashed (scaled in [-1, 1]) + clipped_actions = self.policy.unscale_action(clipped_actions) + else: + # Otherwise, clip the actions to avoid out of bound error + # as we are sampling from an unbounded Gaussian distribution + clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high) new_obs, rewards, dones, infos = env.step(clipped_actions) diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py index 21d2034..a4b462b 100644 --- a/stable_baselines3/common/policies.py +++ b/stable_baselines3/common/policies.py @@ -489,6 +489,8 @@ class ActorCriticPolicy(BasePolicy): self.log_std_init = log_std_init dist_kwargs = None + + assert not (squash_output and not use_sde), "squash_output=True is only available when using gSDE (use_sde=True)" # Keyword arguments for gSDE distribution if use_sde: dist_kwargs = { diff --git a/tests/test_sde.py b/tests/test_sde.py index 4fc16fc..aec8135 100644 --- a/tests/test_sde.py +++ b/tests/test_sde.py @@ -1,3 +1,5 @@ +import gymnasium as gym +import numpy as np import pytest import torch as th from torch.distributions import Normal @@ -59,20 +61,62 @@ def test_sde_check(): PPO("MlpPolicy", "CartPole-v1", use_sde=True) +def test_only_sde_squashed(): + with pytest.raises(AssertionError, match="use_sde=True"): + PPO("MlpPolicy", "Pendulum-v1", use_sde=False, policy_kwargs=dict(squash_output=True)) + + @pytest.mark.parametrize("model_class", [SAC, A2C, PPO]) @pytest.mark.parametrize("use_expln", [False, True]) -def test_state_dependent_noise(model_class, use_expln): +@pytest.mark.parametrize("squash_output", [False, True]) +def test_state_dependent_noise(model_class, use_expln, squash_output): kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64} + + policy_kwargs = dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]) + + if model_class in [A2C, PPO]: + policy_kwargs["squash_output"] = squash_output + elif not squash_output: + pytest.skip("SAC can only use squashed output") + + env = StoreActionEnvWrapper(gym.make("Pendulum-v1")) model = model_class( "MlpPolicy", - "Pendulum-v1", + env, use_sde=True, - seed=None, + seed=1, verbose=1, - policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]), + policy_kwargs=policy_kwargs, **kwargs, ) model.learn(total_timesteps=255) + buffer = model.replay_buffer if model_class == SAC else model.rollout_buffer + # Check that only scaled actions are stored + assert (buffer.actions <= model.action_space.high).all() + assert (buffer.actions >= model.action_space.low).all() + if squash_output: + # Pendulum action range is [-2, 2] + # we check that the action are correctly unscaled + if buffer.actions.max() > 0.5: + assert np.max(env.actions) > 1.0 + if buffer.actions.max() < -0.5: + assert np.min(env.actions) < -1.0 model.policy.reset_noise() if model_class == SAC: model.policy.actor.get_std() + + +class StoreActionEnvWrapper(gym.Wrapper): + """ + Keep track of which actions were sent to the env. + """ + + def __init__(self, env): + super().__init__(env) + # defines list for tracking actions + self.actions = [] + + def step(self, action): + # appends list for tracking actions + self.actions.append(action) + return super().step(action)