Fix squash output unscaling when using gSDE (#1652)

* prevents squash_output if not use_sde, see #1592

* update changelog

* add unscaling of actions taken during training

* add test regarding squashing and unquashing

* avoids try-except block

* format Gymnasium code with black

* makes mypy pass

* makes pytype pass

* sort imports

* makes error message in assert statement clearer

Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>

* improves code commenting

* replaces full env with wrapper

* Cleanup code

* Reformat

---------

Co-authored-by: PatrickHelm <patrick.helm@gmx.net>
Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>
This commit is contained in:
PatrickHelm 2023-09-01 17:58:15 +02:00 committed by GitHub
parent 84163b468c
commit 16c6a886db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 63 additions and 6 deletions

View file

@ -20,10 +20,13 @@ New Features:
Bug Fixes:
^^^^^^^^^^
- Prevents using squash_output and not use_sde in ActorCritcPolicy (@PatrickHelm)
- Performs unscaling of actions in collect_rollout in OnPolicyAlgorithm (@PatrickHelm)
- Moves VectorizedActionNoise into ``_setup_learn()`` in OffPolicyAlgorithm (@PatrickHelm)
- Prevents out of bound error on Windows if no seed is passed (@PatrickHelm)
- Calls ``callback.update_locals()`` before ``callback.on_rollout_end()`` in OnPolicyAlgorithm (@PatrickHelm)
Deprecations:
^^^^^^^^^^^^^
@ -35,6 +38,7 @@ Others:
- Fixed ``stable_baselines3/common/vec_env/vec_video_recorder.py`` type hints
- Fixed ``stable_baselines3/common/save_util.py`` type hints
Documentation:
^^^^^^^^^^^^^^

View file

@ -171,9 +171,16 @@ class OnPolicyAlgorithm(BaseAlgorithm):
# Rescale and perform action
clipped_actions = actions
# Clip the actions to avoid out of bound error
if isinstance(self.action_space, spaces.Box):
clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high)
if self.policy.squash_output:
# Unscale the actions to match env bounds
# if they were previously squashed (scaled in [-1, 1])
clipped_actions = self.policy.unscale_action(clipped_actions)
else:
# Otherwise, clip the actions to avoid out of bound error
# as we are sampling from an unbounded Gaussian distribution
clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high)
new_obs, rewards, dones, infos = env.step(clipped_actions)

View file

@ -489,6 +489,8 @@ class ActorCriticPolicy(BasePolicy):
self.log_std_init = log_std_init
dist_kwargs = None
assert not (squash_output and not use_sde), "squash_output=True is only available when using gSDE (use_sde=True)"
# Keyword arguments for gSDE distribution
if use_sde:
dist_kwargs = {

View file

@ -1,3 +1,5 @@
import gymnasium as gym
import numpy as np
import pytest
import torch as th
from torch.distributions import Normal
@ -59,20 +61,62 @@ def test_sde_check():
PPO("MlpPolicy", "CartPole-v1", use_sde=True)
def test_only_sde_squashed():
with pytest.raises(AssertionError, match="use_sde=True"):
PPO("MlpPolicy", "Pendulum-v1", use_sde=False, policy_kwargs=dict(squash_output=True))
@pytest.mark.parametrize("model_class", [SAC, A2C, PPO])
@pytest.mark.parametrize("use_expln", [False, True])
def test_state_dependent_noise(model_class, use_expln):
@pytest.mark.parametrize("squash_output", [False, True])
def test_state_dependent_noise(model_class, use_expln, squash_output):
kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64}
policy_kwargs = dict(log_std_init=-2, use_expln=use_expln, net_arch=[64])
if model_class in [A2C, PPO]:
policy_kwargs["squash_output"] = squash_output
elif not squash_output:
pytest.skip("SAC can only use squashed output")
env = StoreActionEnvWrapper(gym.make("Pendulum-v1"))
model = model_class(
"MlpPolicy",
"Pendulum-v1",
env,
use_sde=True,
seed=None,
seed=1,
verbose=1,
policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]),
policy_kwargs=policy_kwargs,
**kwargs,
)
model.learn(total_timesteps=255)
buffer = model.replay_buffer if model_class == SAC else model.rollout_buffer
# Check that only scaled actions are stored
assert (buffer.actions <= model.action_space.high).all()
assert (buffer.actions >= model.action_space.low).all()
if squash_output:
# Pendulum action range is [-2, 2]
# we check that the action are correctly unscaled
if buffer.actions.max() > 0.5:
assert np.max(env.actions) > 1.0
if buffer.actions.max() < -0.5:
assert np.min(env.actions) < -1.0
model.policy.reset_noise()
if model_class == SAC:
model.policy.actor.get_std()
class StoreActionEnvWrapper(gym.Wrapper):
"""
Keep track of which actions were sent to the env.
"""
def __init__(self, env):
super().__init__(env)
# defines list for tracking actions
self.actions = []
def step(self, action):
# appends list for tracking actions
self.actions.append(action)
return super().step(action)