Update doc and add check for unbounded action space (#918)

docs/guide/integrations.rst

@@ -48,11 +48,14 @@ Hugging Face 🤗
 The Hugging Face Hub 🤗 is a central place where anyone can share and explore models. It allows you to host your saved models 💾.
 
 You can see the list of stable-baselines3 saved models here: https://huggingface.co/models?other=stable-baselines3
+Most of them are available via the RL Zoo.
 
 Official pre-trained models are saved in the SB3 organization on the hub: https://huggingface.co/sb3
 
 We wrote a tutorial on how to use 🤗 Hub and Stable-Baselines3 here: https://colab.research.google.com/drive/1GI0WpThwRHbl-Fu2RHfczq6dci5GBDVE#scrollTo=q4cz-w9MdO7T
 
+For up to date instructions (for instance for using ``package_to_hub()``), please take a look at the Huggingface SB3 package README: https://github.com/huggingface/huggingface_sb3
+
 Installation
 -------------
 

docs/misc/changelog.rst

@@ -26,6 +26,7 @@ Bug Fixes:
 - Fixed a bug with special characters in the tensorboard log name (@quantitative-technologies)
 - Fixed a bug in ``DummyVecEnv``'s and ``SubprocVecEnv``'s seeding function. None value was unchecked (@ScheiklP)
 - Fixed a bug where ``EvalCallback`` would crash when trying to synchronize ``VecNormalize`` stats when observation normalization was disabled
+- Added a check for unbounded actions
 
 Deprecations:
 ^^^^^^^^^^^^^
@@ -42,6 +43,7 @@ Documentation:
 - Added link to PPO ICLR blog post
 - Added remark about breaking Markov assumption and timeout handling
 - Added doc about MLFlow integration via custom logger (@git-thor)
+- Updated Huggingface integration doc
 
 
 Release 1.5.0 (2022-03-25)

stable_baselines3/common/base_class.py

@@ -185,6 +185,11 @@ class BaseAlgorithm(ABC):
             if self.use_sde and not isinstance(self.action_space, gym.spaces.Box):
                 raise ValueError("generalized State-Dependent Exploration (gSDE) can only be used with continuous actions.")
 
+            if isinstance(self.action_space, gym.spaces.Box):
+                assert np.all(
+                    np.isfinite(np.array([self.action_space.low, self.action_space.high]))
+                ), "Continuous action space must have a finite lower and upper bound"
+
     @staticmethod
     def _wrap_env(env: GymEnv, verbose: int = 0, monitor_wrapper: bool = True) -> VecEnv:
         """ "

stable_baselines3/common/env_checker.py

@@ -274,6 +274,11 @@ def check_env(env: gym.Env, warn: bool = True, skip_render_check: bool = True) -
                 "cf https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html"
             )
 
+        if isinstance(action_space, spaces.Box):
+            assert np.all(
+                np.isfinite(np.array([action_space.low, action_space.high]))
+            ), "Continuous action space must have a finite lower and upper bound"
+
         if isinstance(action_space, spaces.Box) and action_space.dtype != np.dtype(np.float32):
             warnings.warn(
                 f"Your action space has dtype {action_space.dtype}, we recommend using np.float32 to avoid cast errors."

tests/test_envs.py

@@ -141,6 +141,8 @@ def test_non_default_spaces(new_obs_space):
         spaces.Box(low=1, high=-1, shape=(2,), dtype=np.float32),
         # Same boundaries
         spaces.Box(low=1, high=1, shape=(2,), dtype=np.float32),
+        # Unbounded action space
+        spaces.Box(low=-np.inf, high=1, shape=(2,), dtype=np.float32),
         # Almost good, except for one dim
         spaces.Box(low=np.array([-1, -1, -1]), high=np.array([1, 1, 0.99]), dtype=np.float32),
     ],
@@ -156,8 +158,14 @@ def test_non_default_action_spaces(new_action_space):
     # Change the action space
     env.action_space = new_action_space
 
-    with pytest.warns(UserWarning):
+    # Unbounded action space throws an error,
-        check_env(env)
+    # the rest only warning
+    if not np.all(np.isfinite(env.action_space.low)):
+        with pytest.raises(AssertionError), pytest.warns(UserWarning):
+            check_env(env)
+    else:
+        with pytest.warns(UserWarning):
+            check_env(env)
 
 
 def check_reset_assert_error(env, new_reset_return):