Raise error when same env object instance is passed in vectorized environment (#1154)

* Raise error when same env object instance is passed in vectorized environment

* At to changelog

* Add raises to docstring

* Add test

* Also test make_vec_env

* Fix test

* Try to enable color for MyPy

* Update version and ignore lint warnings

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>
Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>

.github/workflows/ci.yml

@@ -11,6 +11,9 @@ on:
 
 jobs:
   build:
+    env:
+      TERM: xterm-256color
+      FORCE_COLOR: 1
     # Skip CI if [ci skip] in the commit message
     if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
     runs-on: ubuntu-latest

Makefile

@@ -8,7 +8,7 @@ pytype:
 	pytype -j auto
 
 mypy:
-	MYPY_FORCE_COLOR=1 mypy ${LINT_PATHS}
+	mypy ${LINT_PATHS}
 
 type: pytype mypy
 

docs/misc/changelog.rst

@@ -4,7 +4,7 @@ Changelog
 ==========
 
 
-Release 1.7.0a1 (WIP)
+Release 1.7.0a2 (WIP)
 --------------------------
 
 Breaking Changes:
@@ -26,6 +26,7 @@ Bug Fixes:
 - Fix return type of ``evaluate_actions`` in ``ActorCritcPolicy`` to reflect that entropy is an optional tensor (@Rocamonde)
 - Fix type annotation of ``policy`` in ``BaseAlgorithm`` and ``OffPolicyAlgorithm``
 - Allowed model trained with Python 3.7 to be loaded with Python 3.8+ without the ``custom_objects`` workaround
+- Raise an error when the same gym environment instance is passed as separate environments when creating a vectorized environment with more than one environment. (@Rocamonde)
 - Fix type annotation of ``model`` in ``evaluate_policy``
 - Fixed ``Self`` return type using ``TypeVar``
 

setup.cfg

@@ -94,6 +94,9 @@ per-file-ignores =
 	./stable_baselines3/sac/__init__.py:F401
 	./stable_baselines3/td3/__init__.py:F401
 	./stable_baselines3/common/vec_env/__init__.py:F401
+  # Default implementation in abstract methods
+  ./stable_baselines3/common/callbacks.py:B027
+  ./stable_baselines3/common/noise.py:B027
 exclude =
     # No need to traverse our git directory
     .git,

stable_baselines3/common/vec_env/dummy_vec_env.py

@@ -19,10 +19,21 @@ class DummyVecEnv(VecEnv):
 
     :param env_fns: a list of functions
         that return environments to vectorize
+    :raises ValueError: If the same environment instance is passed as the output of two or more different env_fn.
     """
 
     def __init__(self, env_fns: List[Callable[[], gym.Env]]):
         self.envs = [fn() for fn in env_fns]
+        if len(set([id(env.unwrapped) for env in self.envs])) != len(self.envs):
+            raise ValueError(
+                "You tried to create multiple environments, but the function to create them returned the same instance "
+                "instead of creating different objects. "
+                "You are probably using `make_vec_env(lambda: env)` or `DummyVecEnv([lambda: env] * n_envs)`. "
+                "You should replace `lambda: env` by a `make_env` function that "
+                "creates a new instance of the environment at every call "
+                "(using `gym.make()` for instance). You can take a look at the documentation for an example. "
+                "Please read https://github.com/DLR-RM/stable-baselines3/issues/1151 for more information."
+            )
         env = self.envs[0]
         VecEnv.__init__(self, len(env_fns), env.observation_space, env.action_space)
         obs_space = env.observation_space

stable_baselines3/version.txt

@@ -1 +1 @@
-1.7.0a1
+1.7.0a2

tests/test_utils.py

@@ -45,6 +45,16 @@ def test_make_vec_env(env_id, n_envs, vec_env_cls, wrapper_class):
     env.close()
 
 
+def test_make_vec_env_func_checker():
+    """The functions in ``env_fns'' must return distinct instances since we need distinct environments."""
+    env = gym.make("CartPole-v1")
+
+    with pytest.raises(ValueError):
+        make_vec_env(lambda: env, n_envs=2)
+
+    env.close()
+
+
 @pytest.mark.parametrize("env_id", ["BreakoutNoFrameskip-v4"])
 @pytest.mark.parametrize("n_envs", [1, 2])
 @pytest.mark.parametrize("wrapper_kwargs", [None, dict(clip_reward=False, screen_size=60)])

tests/test_vec_envs.py

@@ -62,6 +62,16 @@ class CustomGymEnv(gym.Env):
         return np.ones((dim_0, dim_1))
 
 
+def test_vecenv_func_checker():
+    """The functions in ``env_fns'' must return distinct instances since we need distinct environments."""
+    env = CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2)))
+
+    with pytest.raises(ValueError):
+        DummyVecEnv([lambda: env for _ in range(N_ENVS)])
+
+    env.close()
+
+
 @pytest.mark.parametrize("vec_env_class", VEC_ENV_CLASSES)
 @pytest.mark.parametrize("vec_env_wrapper", VEC_ENV_WRAPPERS)
 def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper):