From 49813d8c68c3642164081984227823ffc33e66d1 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Wed, 25 May 2022 10:24:21 -0400 Subject: [PATCH] Update doc and add check for unbounded action space (#918) --- docs/guide/integrations.rst | 3 +++ docs/misc/changelog.rst | 2 ++ stable_baselines3/common/base_class.py | 5 +++++ stable_baselines3/common/env_checker.py | 5 +++++ tests/test_envs.py | 12 ++++++++++-- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/guide/integrations.rst b/docs/guide/integrations.rst index 7f21bd3..98cf84a 100644 --- a/docs/guide/integrations.rst +++ b/docs/guide/integrations.rst @@ -48,11 +48,14 @@ Hugging Face 🤗 The Hugging Face Hub 🤗 is a central place where anyone can share and explore models. It allows you to host your saved models 💾. You can see the list of stable-baselines3 saved models here: https://huggingface.co/models?other=stable-baselines3 +Most of them are available via the RL Zoo. Official pre-trained models are saved in the SB3 organization on the hub: https://huggingface.co/sb3 We wrote a tutorial on how to use 🤗 Hub and Stable-Baselines3 here: https://colab.research.google.com/drive/1GI0WpThwRHbl-Fu2RHfczq6dci5GBDVE#scrollTo=q4cz-w9MdO7T +For up to date instructions (for instance for using ``package_to_hub()``), please take a look at the Huggingface SB3 package README: https://github.com/huggingface/huggingface_sb3 + Installation ------------- diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 4bcecc4..c473fc0 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -26,6 +26,7 @@ Bug Fixes: - Fixed a bug with special characters in the tensorboard log name (@quantitative-technologies) - Fixed a bug in ``DummyVecEnv``'s and ``SubprocVecEnv``'s seeding function. None value was unchecked (@ScheiklP) - Fixed a bug where ``EvalCallback`` would crash when trying to synchronize ``VecNormalize`` stats when observation normalization was disabled +- Added a check for unbounded actions Deprecations: ^^^^^^^^^^^^^ @@ -42,6 +43,7 @@ Documentation: - Added link to PPO ICLR blog post - Added remark about breaking Markov assumption and timeout handling - Added doc about MLFlow integration via custom logger (@git-thor) +- Updated Huggingface integration doc Release 1.5.0 (2022-03-25) diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py index bdba909..e16814c 100644 --- a/stable_baselines3/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -185,6 +185,11 @@ class BaseAlgorithm(ABC): if self.use_sde and not isinstance(self.action_space, gym.spaces.Box): raise ValueError("generalized State-Dependent Exploration (gSDE) can only be used with continuous actions.") + if isinstance(self.action_space, gym.spaces.Box): + assert np.all( + np.isfinite(np.array([self.action_space.low, self.action_space.high])) + ), "Continuous action space must have a finite lower and upper bound" + @staticmethod def _wrap_env(env: GymEnv, verbose: int = 0, monitor_wrapper: bool = True) -> VecEnv: """ " diff --git a/stable_baselines3/common/env_checker.py b/stable_baselines3/common/env_checker.py index c4e5669..ed07e7e 100644 --- a/stable_baselines3/common/env_checker.py +++ b/stable_baselines3/common/env_checker.py @@ -274,6 +274,11 @@ def check_env(env: gym.Env, warn: bool = True, skip_render_check: bool = True) - "cf https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html" ) + if isinstance(action_space, spaces.Box): + assert np.all( + np.isfinite(np.array([action_space.low, action_space.high])) + ), "Continuous action space must have a finite lower and upper bound" + if isinstance(action_space, spaces.Box) and action_space.dtype != np.dtype(np.float32): warnings.warn( f"Your action space has dtype {action_space.dtype}, we recommend using np.float32 to avoid cast errors." diff --git a/tests/test_envs.py b/tests/test_envs.py index 671e2a5..1cec17a 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -141,6 +141,8 @@ def test_non_default_spaces(new_obs_space): spaces.Box(low=1, high=-1, shape=(2,), dtype=np.float32), # Same boundaries spaces.Box(low=1, high=1, shape=(2,), dtype=np.float32), + # Unbounded action space + spaces.Box(low=-np.inf, high=1, shape=(2,), dtype=np.float32), # Almost good, except for one dim spaces.Box(low=np.array([-1, -1, -1]), high=np.array([1, 1, 0.99]), dtype=np.float32), ], @@ -156,8 +158,14 @@ def test_non_default_action_spaces(new_action_space): # Change the action space env.action_space = new_action_space - with pytest.warns(UserWarning): - check_env(env) + # Unbounded action space throws an error, + # the rest only warning + if not np.all(np.isfinite(env.action_space.low)): + with pytest.raises(AssertionError), pytest.warns(UserWarning): + check_env(env) + else: + with pytest.warns(UserWarning): + check_env(env) def check_reset_assert_error(env, new_reset_return):