From d829a1bb0447cce8fac20961798ff239fcaf77e1 Mon Sep 17 00:00:00 2001
From: Antonin RAFFIN <antonin.raffin@ensta.org>
Date: Mon, 21 Nov 2022 13:15:12 +0100
Subject: [PATCH 01/13] Update README (Acknowledgments)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 26b796c..b4461ef 100644
--- a/README.md
+++ b/README.md
@@ -247,7 +247,7 @@ If you want to contribute, please read [**CONTRIBUTING.md**](./CONTRIBUTING.md)
 
 ## Acknowledgments
 
-The initial work to develop Stable Baselines3 was partially funded by the project *Reduced Complexity Models* from the *Helmholtz-Gemeinschaft Deutscher Forschungszentren*.
+The initial work to develop Stable Baselines3 was partially funded by the project *Reduced Complexity Models* from the *Helmholtz-Gemeinschaft Deutscher Forschungszentren*, and by the EU's Horizon 2020 Research and Innovation Programme under grant number 951992 ([VeriDream](https://www.veridream.eu/)).
 
 The original version, Stable Baselines, was created in the [robotics lab U2IS](http://u2is.ensta-paristech.fr/index.php?lang=en) ([INRIA Flowers](https://flowers.inria.fr/) team) at [ENSTA ParisTech](http://www.ensta-paristech.fr/en).
 

From f3abda5cbc9dab71fbda44d9feca7efd7bb09dfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 22 Nov 2022 13:42:39 +0100
Subject: [PATCH 02/13] Fix `Self` return type (#1167)

* Fix Self annotation

* Update changelog

* Define type var on top

* ClassSelf to SelfClass

* annotate self

* Revert Running meanstd change

* Revert vecnormalize change (static method rejected)

Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
---
 docs/misc/changelog.rst                       |  1 +
 stable_baselines3/a2c/a2c.py                  |  6 ++--
 stable_baselines3/common/base_class.py        | 13 ++++---
 stable_baselines3/common/distributions.py     | 34 ++++++++++++++-----
 .../common/off_policy_algorithm.py            |  6 ++--
 .../common/on_policy_algorithm.py             |  6 ++--
 stable_baselines3/common/policies.py          |  4 +--
 stable_baselines3/ddpg/ddpg.py                |  6 ++--
 stable_baselines3/dqn/dqn.py                  |  6 ++--
 stable_baselines3/ppo/ppo.py                  |  6 ++--
 stable_baselines3/sac/sac.py                  |  6 ++--
 stable_baselines3/td3/td3.py                  |  6 ++--
 12 files changed, 58 insertions(+), 42 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 3ef78d5..99fa5b4 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -27,6 +27,7 @@ Bug Fixes:
 - Fix type annotation of ``policy`` in ``BaseAlgorithm`` and ``OffPolicyAlgorithm``
 - Allowed model trained with Python 3.7 to be loaded with Python 3.8+ without the ``custom_objects`` workaround
 - Fix type annotation of ``model`` in ``evaluate_policy``
+- Fixed ``Self`` return type using ``TypeVar``
 
 Deprecations:
 ^^^^^^^^^^^^^
diff --git a/stable_baselines3/a2c/a2c.py b/stable_baselines3/a2c/a2c.py
index 24d69a6..972e700 100644
--- a/stable_baselines3/a2c/a2c.py
+++ b/stable_baselines3/a2c/a2c.py
@@ -9,7 +9,7 @@ from stable_baselines3.common.policies import ActorCriticCnnPolicy, ActorCriticP
 from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedule
 from stable_baselines3.common.utils import explained_variance
 
-A2CSelf = TypeVar("A2CSelf", bound="A2C")
+SelfA2C = TypeVar("SelfA2C", bound="A2C")
 
 
 class A2C(OnPolicyAlgorithm):
@@ -181,14 +181,14 @@ class A2C(OnPolicyAlgorithm):
             self.logger.record("train/std", th.exp(self.policy.log_std).mean().item())
 
     def learn(
-        self: A2CSelf,
+        self: SelfA2C,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 100,
         tb_log_name: str = "A2C",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> A2CSelf:
+    ) -> SelfA2C:
 
         return super().learn(
             total_timesteps=total_timesteps,
diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py
index ebfb7de..bb14f6a 100644
--- a/stable_baselines3/common/base_class.py
+++ b/stable_baselines3/common/base_class.py
@@ -38,6 +38,8 @@ from stable_baselines3.common.vec_env import (
     unwrap_vec_normalize,
 )
 
+SelfBaseAlgorithm = TypeVar("SelfBaseAlgorithm", bound="BaseAlgorithm")
+
 
 def maybe_make_env(env: Union[GymEnv, str, None], verbose: int) -> Optional[GymEnv]:
     """If env is a string, make the environment; otherwise, return env.
@@ -53,9 +55,6 @@ def maybe_make_env(env: Union[GymEnv, str, None], verbose: int) -> Optional[GymE
     return env
 
 
-BaseAlgorithmSelf = TypeVar("BaseAlgorithmSelf", bound="BaseAlgorithm")
-
-
 class BaseAlgorithm(ABC):
     """
     The base of RL algorithms
@@ -491,14 +490,14 @@ class BaseAlgorithm(ABC):
 
     @abstractmethod
     def learn(
-        self: BaseAlgorithmSelf,
+        self: SelfBaseAlgorithm,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 100,
         tb_log_name: str = "run",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> BaseAlgorithmSelf:
+    ) -> SelfBaseAlgorithm:
         """
         Return a trained model.
 
@@ -617,7 +616,7 @@ class BaseAlgorithm(ABC):
 
     @classmethod
     def load(
-        cls: Type[BaseAlgorithmSelf],
+        cls: Type[SelfBaseAlgorithm],
         path: Union[str, pathlib.Path, io.BufferedIOBase],
         env: Optional[GymEnv] = None,
         device: Union[th.device, str] = "auto",
@@ -625,7 +624,7 @@ class BaseAlgorithm(ABC):
         print_system_info: bool = False,
         force_reset: bool = True,
         **kwargs,
-    ) -> BaseAlgorithmSelf:
+    ) -> SelfBaseAlgorithm:
         """
         Load the model from a zip-file.
         Warning: ``load`` re-creates the model from scratch, it does not update it in-place!
diff --git a/stable_baselines3/common/distributions.py b/stable_baselines3/common/distributions.py
index fc48625..63eb475 100644
--- a/stable_baselines3/common/distributions.py
+++ b/stable_baselines3/common/distributions.py
@@ -1,7 +1,7 @@
 """Probability distributions."""
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union
 
 import gym
 import numpy as np
@@ -12,6 +12,16 @@ from torch.distributions import Bernoulli, Categorical, Normal
 
 from stable_baselines3.common.preprocessing import get_action_dim
 
+SelfDistribution = TypeVar("SelfDistribution", bound="Distribution")
+SelfDiagGaussianDistribution = TypeVar("SelfDiagGaussianDistribution", bound="DiagGaussianDistribution")
+SelfSquashedDiagGaussianDistribution = TypeVar(
+    "SelfSquashedDiagGaussianDistribution", bound="SquashedDiagGaussianDistribution"
+)
+SelfCategoricalDistribution = TypeVar("SelfCategoricalDistribution", bound="CategoricalDistribution")
+SelfMultiCategoricalDistribution = TypeVar("SelfMultiCategoricalDistribution", bound="MultiCategoricalDistribution")
+SelfBernoulliDistribution = TypeVar("SelfBernoulliDistribution", bound="BernoulliDistribution")
+SelfStateDependentNoiseDistribution = TypeVar("SelfStateDependentNoiseDistribution", bound="StateDependentNoiseDistribution")
+
 
 class Distribution(ABC):
     """Abstract base class for distributions."""
@@ -28,7 +38,7 @@ class Distribution(ABC):
         concrete classes."""
 
     @abstractmethod
-    def proba_distribution(self, *args, **kwargs) -> "Distribution":
+    def proba_distribution(self: SelfDistribution, *args, **kwargs) -> SelfDistribution:
         """Set parameters of the distribution.
 
         :return: self
@@ -141,7 +151,9 @@ class DiagGaussianDistribution(Distribution):
         log_std = nn.Parameter(th.ones(self.action_dim) * log_std_init, requires_grad=True)
         return mean_actions, log_std
 
-    def proba_distribution(self, mean_actions: th.Tensor, log_std: th.Tensor) -> "DiagGaussianDistribution":
+    def proba_distribution(
+        self: SelfDiagGaussianDistribution, mean_actions: th.Tensor, log_std: th.Tensor
+    ) -> SelfDiagGaussianDistribution:
         """
         Create the distribution given its parameters (mean, std)
 
@@ -207,7 +219,9 @@ class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
         self.epsilon = epsilon
         self.gaussian_actions = None
 
-    def proba_distribution(self, mean_actions: th.Tensor, log_std: th.Tensor) -> "SquashedDiagGaussianDistribution":
+    def proba_distribution(
+        self: SelfSquashedDiagGaussianDistribution, mean_actions: th.Tensor, log_std: th.Tensor
+    ) -> SelfSquashedDiagGaussianDistribution:
         super().proba_distribution(mean_actions, log_std)
         return self
 
@@ -271,7 +285,7 @@ class CategoricalDistribution(Distribution):
         action_logits = nn.Linear(latent_dim, self.action_dim)
         return action_logits
 
-    def proba_distribution(self, action_logits: th.Tensor) -> "CategoricalDistribution":
+    def proba_distribution(self: SelfCategoricalDistribution, action_logits: th.Tensor) -> SelfCategoricalDistribution:
         self.distribution = Categorical(logits=action_logits)
         return self
 
@@ -323,7 +337,9 @@ class MultiCategoricalDistribution(Distribution):
         action_logits = nn.Linear(latent_dim, sum(self.action_dims))
         return action_logits
 
-    def proba_distribution(self, action_logits: th.Tensor) -> "MultiCategoricalDistribution":
+    def proba_distribution(
+        self: SelfMultiCategoricalDistribution, action_logits: th.Tensor
+    ) -> SelfMultiCategoricalDistribution:
         self.distribution = [Categorical(logits=split) for split in th.split(action_logits, tuple(self.action_dims), dim=1)]
         return self
 
@@ -376,7 +392,7 @@ class BernoulliDistribution(Distribution):
         action_logits = nn.Linear(latent_dim, self.action_dims)
         return action_logits
 
-    def proba_distribution(self, action_logits: th.Tensor) -> "BernoulliDistribution":
+    def proba_distribution(self: SelfBernoulliDistribution, action_logits: th.Tensor) -> SelfBernoulliDistribution:
         self.distribution = Bernoulli(logits=action_logits)
         return self
 
@@ -520,8 +536,8 @@ class StateDependentNoiseDistribution(Distribution):
         return mean_actions_net, log_std
 
     def proba_distribution(
-        self, mean_actions: th.Tensor, log_std: th.Tensor, latent_sde: th.Tensor
-    ) -> "StateDependentNoiseDistribution":
+        self: SelfStateDependentNoiseDistribution, mean_actions: th.Tensor, log_std: th.Tensor, latent_sde: th.Tensor
+    ) -> SelfStateDependentNoiseDistribution:
         """
         Create the distribution given its parameters (mean, std)
 
diff --git a/stable_baselines3/common/off_policy_algorithm.py b/stable_baselines3/common/off_policy_algorithm.py
index 9c2bda2..634d9e9 100644
--- a/stable_baselines3/common/off_policy_algorithm.py
+++ b/stable_baselines3/common/off_policy_algorithm.py
@@ -21,7 +21,7 @@ from stable_baselines3.common.utils import safe_mean, should_collect_more_steps
 from stable_baselines3.common.vec_env import VecEnv
 from stable_baselines3.her.her_replay_buffer import HerReplayBuffer
 
-OffPolicyAlgorithmSelf = TypeVar("OffPolicyAlgorithmSelf", bound="OffPolicyAlgorithm")
+SelfOffPolicyAlgorithm = TypeVar("SelfOffPolicyAlgorithm", bound="OffPolicyAlgorithm")
 
 
 class OffPolicyAlgorithm(BaseAlgorithm):
@@ -311,14 +311,14 @@ class OffPolicyAlgorithm(BaseAlgorithm):
         )
 
     def learn(
-        self: OffPolicyAlgorithmSelf,
+        self: SelfOffPolicyAlgorithm,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 4,
         tb_log_name: str = "run",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> OffPolicyAlgorithmSelf:
+    ) -> SelfOffPolicyAlgorithm:
 
         total_timesteps, callback = self._setup_learn(
             total_timesteps,
diff --git a/stable_baselines3/common/on_policy_algorithm.py b/stable_baselines3/common/on_policy_algorithm.py
index a0018b3..35ad2b9 100644
--- a/stable_baselines3/common/on_policy_algorithm.py
+++ b/stable_baselines3/common/on_policy_algorithm.py
@@ -14,7 +14,7 @@ from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedul
 from stable_baselines3.common.utils import obs_as_tensor, safe_mean
 from stable_baselines3.common.vec_env import VecEnv
 
-OnPolicyAlgorithmSelf = TypeVar("OnPolicyAlgorithmSelf", bound="OnPolicyAlgorithm")
+SelfOnPolicyAlgorithm = TypeVar("SelfOnPolicyAlgorithm", bound="OnPolicyAlgorithm")
 
 
 class OnPolicyAlgorithm(BaseAlgorithm):
@@ -223,14 +223,14 @@ class OnPolicyAlgorithm(BaseAlgorithm):
         raise NotImplementedError
 
     def learn(
-        self: OnPolicyAlgorithmSelf,
+        self: SelfOnPolicyAlgorithm,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 1,
         tb_log_name: str = "OnPolicyAlgorithm",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> OnPolicyAlgorithmSelf:
+    ) -> SelfOnPolicyAlgorithm:
         iteration = 0
 
         total_timesteps, callback = self._setup_learn(
diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py
index 5972fa3..c084872 100644
--- a/stable_baselines3/common/policies.py
+++ b/stable_baselines3/common/policies.py
@@ -32,7 +32,7 @@ from stable_baselines3.common.torch_layers import (
 from stable_baselines3.common.type_aliases import Schedule
 from stable_baselines3.common.utils import get_device, is_vectorized_observation, obs_as_tensor
 
-BaseModelSelf = TypeVar("BaseModelSelf", bound="BaseModel")
+SelfBaseModel = TypeVar("SelfBaseModel", bound="BaseModel")
 
 
 class BaseModel(nn.Module):
@@ -159,7 +159,7 @@ class BaseModel(nn.Module):
         th.save({"state_dict": self.state_dict(), "data": self._get_constructor_parameters()}, path)
 
     @classmethod
-    def load(cls: Type[BaseModelSelf], path: str, device: Union[th.device, str] = "auto") -> BaseModelSelf:
+    def load(cls: Type[SelfBaseModel], path: str, device: Union[th.device, str] = "auto") -> SelfBaseModel:
         """
         Load model from path.
 
diff --git a/stable_baselines3/ddpg/ddpg.py b/stable_baselines3/ddpg/ddpg.py
index 993a8c2..40d67b5 100644
--- a/stable_baselines3/ddpg/ddpg.py
+++ b/stable_baselines3/ddpg/ddpg.py
@@ -8,7 +8,7 @@ from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedul
 from stable_baselines3.td3.policies import TD3Policy
 from stable_baselines3.td3.td3 import TD3
 
-DDPGSelf = TypeVar("DDPGSelf", bound="DDPG")
+SelfDDPG = TypeVar("SelfDDPG", bound="DDPG")
 
 
 class DDPG(TD3):
@@ -113,14 +113,14 @@ class DDPG(TD3):
             self._setup_model()
 
     def learn(
-        self: DDPGSelf,
+        self: SelfDDPG,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 4,
         tb_log_name: str = "DDPG",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> DDPGSelf:
+    ) -> SelfDDPG:
 
         return super().learn(
             total_timesteps=total_timesteps,
diff --git a/stable_baselines3/dqn/dqn.py b/stable_baselines3/dqn/dqn.py
index 9e074a9..e8f4945 100644
--- a/stable_baselines3/dqn/dqn.py
+++ b/stable_baselines3/dqn/dqn.py
@@ -14,7 +14,7 @@ from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedul
 from stable_baselines3.common.utils import get_linear_fn, get_parameters_by_name, is_vectorized_observation, polyak_update
 from stable_baselines3.dqn.policies import CnnPolicy, DQNPolicy, MlpPolicy, MultiInputPolicy
 
-DQNSelf = TypeVar("DQNSelf", bound="DQN")
+SelfDQN = TypeVar("SelfDQN", bound="DQN")
 
 
 class DQN(OffPolicyAlgorithm):
@@ -253,14 +253,14 @@ class DQN(OffPolicyAlgorithm):
         return action, state
 
     def learn(
-        self: DQNSelf,
+        self: SelfDQN,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 4,
         tb_log_name: str = "DQN",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> DQNSelf:
+    ) -> SelfDQN:
 
         return super().learn(
             total_timesteps=total_timesteps,
diff --git a/stable_baselines3/ppo/ppo.py b/stable_baselines3/ppo/ppo.py
index 5d30569..bd80736 100644
--- a/stable_baselines3/ppo/ppo.py
+++ b/stable_baselines3/ppo/ppo.py
@@ -11,7 +11,7 @@ from stable_baselines3.common.policies import ActorCriticCnnPolicy, ActorCriticP
 from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedule
 from stable_baselines3.common.utils import explained_variance, get_schedule_fn
 
-PPOSelf = TypeVar("PPOSelf", bound="PPO")
+SelfPPO = TypeVar("SelfPPO", bound="PPO")
 
 
 class PPO(OnPolicyAlgorithm):
@@ -295,14 +295,14 @@ class PPO(OnPolicyAlgorithm):
             self.logger.record("train/clip_range_vf", clip_range_vf)
 
     def learn(
-        self: PPOSelf,
+        self: SelfPPO,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 1,
         tb_log_name: str = "PPO",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> PPOSelf:
+    ) -> SelfPPO:
 
         return super().learn(
             total_timesteps=total_timesteps,
diff --git a/stable_baselines3/sac/sac.py b/stable_baselines3/sac/sac.py
index 85bdf78..1eafec0 100644
--- a/stable_baselines3/sac/sac.py
+++ b/stable_baselines3/sac/sac.py
@@ -13,7 +13,7 @@ from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedul
 from stable_baselines3.common.utils import get_parameters_by_name, polyak_update
 from stable_baselines3.sac.policies import CnnPolicy, MlpPolicy, MultiInputPolicy, SACPolicy
 
-SACSelf = TypeVar("SACSelf", bound="SAC")
+SelfSAC = TypeVar("SelfSAC", bound="SAC")
 
 
 class SAC(OffPolicyAlgorithm):
@@ -287,14 +287,14 @@ class SAC(OffPolicyAlgorithm):
             self.logger.record("train/ent_coef_loss", np.mean(ent_coef_losses))
 
     def learn(
-        self: SACSelf,
+        self: SelfSAC,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 4,
         tb_log_name: str = "SAC",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> SACSelf:
+    ) -> SelfSAC:
 
         return super().learn(
             total_timesteps=total_timesteps,
diff --git a/stable_baselines3/td3/td3.py b/stable_baselines3/td3/td3.py
index c611481..a8bc3ef 100644
--- a/stable_baselines3/td3/td3.py
+++ b/stable_baselines3/td3/td3.py
@@ -13,7 +13,7 @@ from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback, Schedul
 from stable_baselines3.common.utils import get_parameters_by_name, polyak_update
 from stable_baselines3.td3.policies import CnnPolicy, MlpPolicy, MultiInputPolicy, TD3Policy
 
-TD3Self = TypeVar("TD3Self", bound="TD3")
+SelfTD3 = TypeVar("SelfTD3", bound="TD3")
 
 
 class TD3(OffPolicyAlgorithm):
@@ -203,14 +203,14 @@ class TD3(OffPolicyAlgorithm):
         self.logger.record("train/critic_loss", np.mean(critic_losses))
 
     def learn(
-        self: TD3Self,
+        self: SelfTD3,
         total_timesteps: int,
         callback: MaybeCallback = None,
         log_interval: int = 4,
         tb_log_name: str = "TD3",
         reset_num_timesteps: bool = True,
         progress_bar: bool = False,
-    ) -> TD3Self:
+    ) -> SelfTD3:
 
         return super().learn(
             total_timesteps=total_timesteps,

From 68b190b667fe923889d234aa45864b2faaec651e Mon Sep 17 00:00:00 2001
From: Juan Rocamonde <juancarlosrocamonde@gmail.com>
Date: Tue, 22 Nov 2022 05:28:58 -0800
Subject: [PATCH 03/13] Raise error when same env object instance is passed in
 vectorized environment (#1154)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Raise error when same env object instance is passed in vectorized environment

* At to changelog

* Add raises to docstring

* Add test

* Also test make_vec_env

* Fix test

* Try to enable color for MyPy

* Update version and ignore lint warnings

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>
Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>
---
 .github/workflows/ci.yml                          |  3 +++
 Makefile                                          |  2 +-
 docs/misc/changelog.rst                           |  3 ++-
 setup.cfg                                         |  3 +++
 stable_baselines3/common/vec_env/dummy_vec_env.py | 11 +++++++++++
 stable_baselines3/version.txt                     |  2 +-
 tests/test_utils.py                               | 10 ++++++++++
 tests/test_vec_envs.py                            | 10 ++++++++++
 8 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4bc23e2..b50cc62 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -11,6 +11,9 @@ on:
 
 jobs:
   build:
+    env:
+      TERM: xterm-256color
+      FORCE_COLOR: 1
     # Skip CI if [ci skip] in the commit message
     if: "! contains(toJSON(github.event.commits.*.message), '[ci skip]')"
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index 3e4e734..c806507 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ pytype:
 	pytype -j auto
 
 mypy:
-	MYPY_FORCE_COLOR=1 mypy ${LINT_PATHS}
+	mypy ${LINT_PATHS}
 
 type: pytype mypy
 
diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 99fa5b4..c1d9461 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 ==========
 
 
-Release 1.7.0a1 (WIP)
+Release 1.7.0a2 (WIP)
 --------------------------
 
 Breaking Changes:
@@ -26,6 +26,7 @@ Bug Fixes:
 - Fix return type of ``evaluate_actions`` in ``ActorCritcPolicy`` to reflect that entropy is an optional tensor (@Rocamonde)
 - Fix type annotation of ``policy`` in ``BaseAlgorithm`` and ``OffPolicyAlgorithm``
 - Allowed model trained with Python 3.7 to be loaded with Python 3.8+ without the ``custom_objects`` workaround
+- Raise an error when the same gym environment instance is passed as separate environments when creating a vectorized environment with more than one environment. (@Rocamonde)
 - Fix type annotation of ``model`` in ``evaluate_policy``
 - Fixed ``Self`` return type using ``TypeVar``
 
diff --git a/setup.cfg b/setup.cfg
index 6cf08b3..b79e63a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -94,6 +94,9 @@ per-file-ignores =
 	./stable_baselines3/sac/__init__.py:F401
 	./stable_baselines3/td3/__init__.py:F401
 	./stable_baselines3/common/vec_env/__init__.py:F401
+  # Default implementation in abstract methods
+  ./stable_baselines3/common/callbacks.py:B027
+  ./stable_baselines3/common/noise.py:B027
 exclude =
     # No need to traverse our git directory
     .git,
diff --git a/stable_baselines3/common/vec_env/dummy_vec_env.py b/stable_baselines3/common/vec_env/dummy_vec_env.py
index c0efc8c..72cab7e 100644
--- a/stable_baselines3/common/vec_env/dummy_vec_env.py
+++ b/stable_baselines3/common/vec_env/dummy_vec_env.py
@@ -19,10 +19,21 @@ class DummyVecEnv(VecEnv):
 
     :param env_fns: a list of functions
         that return environments to vectorize
+    :raises ValueError: If the same environment instance is passed as the output of two or more different env_fn.
     """
 
     def __init__(self, env_fns: List[Callable[[], gym.Env]]):
         self.envs = [fn() for fn in env_fns]
+        if len(set([id(env.unwrapped) for env in self.envs])) != len(self.envs):
+            raise ValueError(
+                "You tried to create multiple environments, but the function to create them returned the same instance "
+                "instead of creating different objects. "
+                "You are probably using `make_vec_env(lambda: env)` or `DummyVecEnv([lambda: env] * n_envs)`. "
+                "You should replace `lambda: env` by a `make_env` function that "
+                "creates a new instance of the environment at every call "
+                "(using `gym.make()` for instance). You can take a look at the documentation for an example. "
+                "Please read https://github.com/DLR-RM/stable-baselines3/issues/1151 for more information."
+            )
         env = self.envs[0]
         VecEnv.__init__(self, len(env_fns), env.observation_space, env.action_space)
         obs_space = env.observation_space
diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt
index 12cd5fb..b895285 100644
--- a/stable_baselines3/version.txt
+++ b/stable_baselines3/version.txt
@@ -1 +1 @@
-1.7.0a1
+1.7.0a2
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 7b228f0..e9281c3 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -45,6 +45,16 @@ def test_make_vec_env(env_id, n_envs, vec_env_cls, wrapper_class):
     env.close()
 
 
+def test_make_vec_env_func_checker():
+    """The functions in ``env_fns'' must return distinct instances since we need distinct environments."""
+    env = gym.make("CartPole-v1")
+
+    with pytest.raises(ValueError):
+        make_vec_env(lambda: env, n_envs=2)
+
+    env.close()
+
+
 @pytest.mark.parametrize("env_id", ["BreakoutNoFrameskip-v4"])
 @pytest.mark.parametrize("n_envs", [1, 2])
 @pytest.mark.parametrize("wrapper_kwargs", [None, dict(clip_reward=False, screen_size=60)])
diff --git a/tests/test_vec_envs.py b/tests/test_vec_envs.py
index 93ea348..7a09581 100644
--- a/tests/test_vec_envs.py
+++ b/tests/test_vec_envs.py
@@ -62,6 +62,16 @@ class CustomGymEnv(gym.Env):
         return np.ones((dim_0, dim_1))
 
 
+def test_vecenv_func_checker():
+    """The functions in ``env_fns'' must return distinct instances since we need distinct environments."""
+    env = CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2)))
+
+    with pytest.raises(ValueError):
+        DummyVecEnv([lambda: env for _ in range(N_ENVS)])
+
+    env.close()
+
+
 @pytest.mark.parametrize("vec_env_class", VEC_ENV_CLASSES)
 @pytest.mark.parametrize("vec_env_wrapper", VEC_ENV_WRAPPERS)
 def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper):

From cd630a31213cd3591e09f8974d34241e811fab04 Mon Sep 17 00:00:00 2001
From: Antonin RAFFIN <antonin.raffin@ensta.org>
Date: Fri, 25 Nov 2022 15:14:55 +0100
Subject: [PATCH 04/13] Fixes for flake8 6.0 (#1181)

---
 docs/misc/changelog.rst                | 1 +
 setup.cfg                              | 3 ++-
 stable_baselines3/common/base_class.py | 2 +-
 tests/test_utils.py                    | 4 ++--
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index c1d9461..f5cf0b6 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -36,6 +36,7 @@ Deprecations:
 Others:
 ^^^^^^^
 - Used issue forms instead of issue templates
+- Fixed flake8 config to be compatible with flake8 6+
 
 Documentation:
 ^^^^^^^^^^^^^^
diff --git a/setup.cfg b/setup.cfg
index b79e63a..5e30726 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -80,7 +80,8 @@ exclude = (?x)(
   )
 
 [flake8]
-ignore = W503,W504,E203,E231  # line breaks before and after binary operators
+# line breaks before and after binary operators
+ignore = W503,W504,E203,E231
 # Ignore import not used when aliases are defined
 per-file-ignores =
 	./stable_baselines3/__init__.py:F401
diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py
index bb14f6a..9351bfb 100644
--- a/stable_baselines3/common/base_class.py
+++ b/stable_baselines3/common/base_class.py
@@ -125,7 +125,7 @@ class BaseAlgorithm(ABC):
         # Used for computing fps, it is updated at each call of learn()
         self._num_timesteps_at_start = 0
         self.seed = seed
-        self.action_noise = None  # type: Optional[ActionNoise]
+        self.action_noise: Optional[ActionNoise] = None
         self.start_time = None
         self.policy = None
         self.learning_rate = learning_rate
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e9281c3..e74b1d0 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -298,13 +298,13 @@ def test_evaluate_policy_monitors(vec_env_class):
     episode_rewards, episode_lengths = evaluate_policy(
         model, eval_env, n_eval_episodes, return_episode_rewards=True, warn=False
     )
-    assert all(map(lambda l: l == 1, episode_lengths)), "AlwaysDoneWrapper did not fix episode lengths to one"
+    assert all(map(lambda length: length == 1, episode_lengths)), "AlwaysDoneWrapper did not fix episode lengths to one"
     eval_env.close()
 
     # Should get longer episodes with with Monitor (true episodes)
     eval_env = make_eval_env(with_monitor=True, wrapper_class=AlwaysDoneWrapper)
     episode_rewards, episode_lengths = evaluate_policy(model, eval_env, n_eval_episodes, return_episode_rewards=True)
-    assert all(map(lambda l: l > 1, episode_lengths)), "evaluate_policy did not get episode lengths from Monitor"
+    assert all(map(lambda length: length > 1, episode_lengths)), "evaluate_policy did not get episode lengths from Monitor"
     eval_env.close()
 
 

From e3b24829a57ece38f78436cfea38aaa62d059850 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Mon, 28 Nov 2022 18:22:31 +0100
Subject: [PATCH 05/13] Drop `gym.GoalEnv` and other minor changes initally
 from #780 (#1184)

* Various changes from #780

* Fix env_checker for goal_env detection
---
 README.md                                     | 12 +--
 docs/guide/examples.rst                       |  7 +-
 docs/guide/quickstart.rst                     | 12 +--
 docs/misc/changelog.rst                       |  2 -
 docs/modules/her.rst                          |  6 +-
 setup.py                                      | 13 +--
 stable_baselines3/common/atari_wrappers.py    |  2 +-
 stable_baselines3/common/base_class.py        |  1 +
 stable_baselines3/common/env_checker.py       | 85 +++++++++++++++++--
 .../common/envs/bit_flipping_env.py           |  4 +-
 stable_baselines3/common/noise.py             |  2 +-
 .../common/off_policy_algorithm.py            |  1 -
 stable_baselines3/td3/td3.py                  |  1 -
 tests/test_callbacks.py                       |  2 +-
 tests/test_envs.py                            |  2 +-
 tests/test_predict.py                         |  2 +-
 tests/test_save_load.py                       |  2 +-
 tests/test_utils.py                           |  2 +-
 tests/test_vec_monitor.py                     |  2 +-
 tests/test_vec_normalize.py                   |  2 +-
 20 files changed, 120 insertions(+), 42 deletions(-)

diff --git a/README.md b/README.md
index b4461ef..b487bd6 100644
--- a/README.md
+++ b/README.md
@@ -126,13 +126,15 @@ env = gym.make("CartPole-v1")
 model = PPO("MlpPolicy", env, verbose=1)
 model.learn(total_timesteps=10_000)
 
-obs = env.reset()
+vec_env = model.get_env()
+obs = vec_env.reset()
 for i in range(1000):
     action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = env.step(action)
-    env.render()
-    if done:
-      obs = env.reset()
+    obs, reward, done, info = vec_env.step(action)
+    vec_env.render()
+    # VecEnv resets automatically
+    # if done:
+    #   obs = env.reset()
 
 env.close()
 ```
diff --git a/docs/guide/examples.rst b/docs/guide/examples.rst
index 646eddf..47bdc40 100644
--- a/docs/guide/examples.rst
+++ b/docs/guide/examples.rst
@@ -94,11 +94,12 @@ In the following example, we will train, save and load a DQN model on the Lunar
   mean_reward, std_reward = evaluate_policy(model, model.get_env(), n_eval_episodes=10)
 
   # Enjoy trained agent
-  obs = env.reset()
+  vec_env = model.get_env()
+  obs = vec_env.reset()
   for i in range(1000):
       action, _states = model.predict(obs, deterministic=True)
-      obs, rewards, dones, info = env.step(action)
-      env.render()
+      obs, rewards, dones, info = vec_env.step(action)
+      vec_env.render()
 
 
 Multiprocessing: Unleashing the Power of Vectorized Environments
diff --git a/docs/guide/quickstart.rst b/docs/guide/quickstart.rst
index 7ad9e0e..5d1055a 100644
--- a/docs/guide/quickstart.rst
+++ b/docs/guide/quickstart.rst
@@ -19,13 +19,15 @@ Here is a quick example of how to train and run A2C on a CartPole environment:
   model = A2C("MlpPolicy", env, verbose=1)
   model.learn(total_timesteps=10_000)
 
-  obs = env.reset()
+  vec_env = model.get_env()
+  obs = vec_env.reset()
   for i in range(1000):
       action, _state = model.predict(obs, deterministic=True)
-      obs, reward, done, info = env.step(action)
-      env.render()
-      if done:
-        obs = env.reset()
+      obs, reward, done, info = vec_env.step(action)
+      vec_env.render()
+      # VecEnv resets automatically
+      # if done:
+      #   obs = vec_env.reset()
 
 .. note::
 
diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index f5cf0b6..9ae202b 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -42,7 +42,6 @@ Documentation:
 ^^^^^^^^^^^^^^
 - Updated Hugging Face Integration page (@simoninithomas)
 
-
 Release 1.6.2 (2022-10-10)
 --------------------------
 
@@ -77,7 +76,6 @@ Documentation:
 ^^^^^^^^^^^^^^
 - Extended docstring of the ``wrapper_class`` parameter in ``make_vec_env`` (@AlexPasqua)
 
-
 Release 1.6.1 (2022-09-29)
 ---------------------------
 
diff --git a/docs/modules/her.rst b/docs/modules/her.rst
index 0b73351..817a991 100644
--- a/docs/modules/her.rst
+++ b/docs/modules/her.rst
@@ -19,10 +19,12 @@ It creates "virtual" transitions by relabeling transitions (changing the desired
   but a replay buffer class ``HerReplayBuffer`` that must be passed to an off-policy algorithm
   when using ``MultiInputPolicy`` (to have Dict observation support).
 
-
 .. warning::
 
-    HER requires the environment to inherits from `gym.GoalEnv <https://github.com/openai/gym/blob/3394e245727c1ae6851b504a50ba77c73cd4c65b/gym/core.py#L160>`_
+    HER requires the environment to follow the legacy `gym_robotics.GoalEnv interface <https://github.com/Farama-Foundation/Gymnasium-Robotics/blob/a35b1c1fa669428bf640a2c7101e66eb1627ac3a/gym_robotics/core.py#L8>`_
+    In short, the ``gym.Env`` must have:
+    - a vectorized implementation of ``compute_reward()``
+    - a dictionary observation space with three keys: ``observation``, ``achieved_goal`` and ``desired_goal``
 
 
 .. warning::
diff --git a/setup.py b/setup.py
index 44bcb15..b02e9e4 100644
--- a/setup.py
+++ b/setup.py
@@ -48,13 +48,16 @@ env = gym.make("CartPole-v1")
 model = PPO("MlpPolicy", env, verbose=1)
 model.learn(total_timesteps=10_000)
 
-obs = env.reset()
+vec_env = model.get_env()
+obs = vec_env.reset()
 for i in range(1000):
     action, _states = model.predict(obs, deterministic=True)
-    obs, reward, done, info = env.step(action)
-    env.render()
-    if done:
-        obs = env.reset()
+    obs, reward, done, info = vec_env.step(action)
+    vec_env.render()
+    # VecEnv resets automatically
+    # if done:
+    #   obs = vec_env.reset()
+
 ```
 
 Or just train a model with a one liner if [the environment is registered in Gym](https://www.gymlibrary.ml/content/environment_creation/) and if [the policy is registered](https://stable-baselines3.readthedocs.io/en/master/guide/custom_policy.html):
diff --git a/stable_baselines3/common/atari_wrappers.py b/stable_baselines3/common/atari_wrappers.py
index a9b2eca..62178a1 100644
--- a/stable_baselines3/common/atari_wrappers.py
+++ b/stable_baselines3/common/atari_wrappers.py
@@ -86,7 +86,7 @@ class EpisodicLifeEnv(gym.Wrapper):
         # then update lives to handle bonus lives
         lives = self.env.unwrapped.ale.lives()
         if 0 < lives < self.lives:
-            # for Qbert sometimes we stay in lives == 0 condtion for a few frames
+            # for Qbert sometimes we stay in lives == 0 condition for a few frames
             # so its important to keep lives > 0, so that we only reset once
             # the environment advertises done.
             done = True
diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py
index 9351bfb..1309229 100644
--- a/stable_baselines3/common/base_class.py
+++ b/stable_baselines3/common/base_class.py
@@ -543,6 +543,7 @@ class BaseAlgorithm(ABC):
             return
         set_random_seed(seed, using_cuda=self.device.type == th.device("cuda").type)
         self.action_space.seed(seed)
+        # self.env is always a VecEnv
         if self.env is not None:
             self.env.seed(seed)
 
diff --git a/stable_baselines3/common/env_checker.py b/stable_baselines3/common/env_checker.py
index efc05e3..1b6c3bb 100644
--- a/stable_baselines3/common/env_checker.py
+++ b/stable_baselines3/common/env_checker.py
@@ -1,5 +1,5 @@
 import warnings
-from typing import Union
+from typing import Any, Dict, Union
 
 import gym
 import numpy as np
@@ -93,6 +93,62 @@ def _check_nan(env: gym.Env) -> None:
         _, _, _, _ = vec_env.step(action)
 
 
+def _is_goal_env(env: gym.Env) -> bool:
+    """
+    Check if the env uses the convention for goal-conditioned envs (previously, the gym.GoalEnv interface)
+    """
+    if isinstance(env, gym.Wrapper):  # We need to unwrap the env since gym.Wrapper has the compute_reward method
+        return _is_goal_env(env.unwrapped)
+    return hasattr(env, "compute_reward")
+
+
+def _check_goal_env_obs(obs: dict, observation_space: spaces.Dict, method_name: str) -> None:
+    """
+    Check that an environment implementing the `compute_rewards()` method
+    (previously known as GoalEnv in gym) contains three elements,
+    namely `observation`, `desired_goal`, and `achieved_goal`.
+    """
+    assert len(observation_space.spaces) == 3, (
+        "A goal conditioned env must contain 3 observation keys: `observation`, `desired_goal`, and `achieved_goal`."
+        f"The current observation contains {len(observation_space.spaces)} keys: {list(observation_space.spaces.keys())}"
+    )
+
+    for key in ["observation", "achieved_goal", "desired_goal"]:
+        if key not in observation_space.spaces:
+            raise AssertionError(
+                f"The observation returned by the `{method_name}()` method of a goal-conditioned env requires the '{key}' "
+                "key to be part of the observation dictionary. "
+                f"Current keys are {list(observation_space.spaces.keys())}"
+            )
+
+
+def _check_goal_env_compute_reward(
+    obs: Dict[str, Union[np.ndarray, int]],
+    env: gym.Env,
+    reward: float,
+    info: Dict[str, Any],
+):
+    """
+    Check that reward is computed with `compute_reward`
+    and that the implementation is vectorized.
+    """
+    achieved_goal, desired_goal = obs["achieved_goal"], obs["desired_goal"]
+    assert reward == env.compute_reward(  # type: ignore[attr-defined]
+        achieved_goal, desired_goal, info
+    ), "The reward was not computed with `compute_reward()`"
+
+    achieved_goal, desired_goal = np.array(achieved_goal), np.array(desired_goal)
+    batch_achieved_goals = np.array([achieved_goal, achieved_goal])
+    batch_desired_goals = np.array([desired_goal, desired_goal])
+    if isinstance(achieved_goal, int) or len(achieved_goal.shape) == 0:
+        batch_achieved_goals = batch_achieved_goals.reshape(2, 1)
+        batch_desired_goals = batch_desired_goals.reshape(2, 1)
+    batch_infos = np.array([info, info])
+    rewards = env.compute_reward(batch_achieved_goals, batch_desired_goals, batch_infos)  # type: ignore[attr-defined]
+    assert rewards.shape == (2,), f"Unexpected shape for vectorized computation of reward: {rewards.shape} != (2,)"
+    assert rewards[0] == reward, f"Vectorized computation of reward differs from single computation: {rewards[0]} != {reward}"
+
+
 def _check_obs(obs: Union[tuple, dict, np.ndarray, int], observation_space: spaces.Space, method_name: str) -> None:
     """
     Check that the observation returned by the environment
@@ -141,7 +197,11 @@ def _check_returned_values(env: gym.Env, observation_space: spaces.Space, action
     # because env inherits from gym.Env, we assume that `reset()` and `step()` methods exists
     obs = env.reset()
 
-    if isinstance(observation_space, spaces.Dict):
+    if _is_goal_env(env):
+        # Make mypy happy, already checked
+        assert isinstance(observation_space, spaces.Dict)
+        _check_goal_env_obs(obs, observation_space, "reset")
+    elif isinstance(observation_space, spaces.Dict):
         assert isinstance(obs, dict), "The observation returned by `reset()` must be a dictionary"
 
         if not obs.keys() == observation_space.spaces.keys():
@@ -167,7 +227,12 @@ def _check_returned_values(env: gym.Env, observation_space: spaces.Space, action
     # Unpack
     obs, reward, done, info = data
 
-    if isinstance(observation_space, spaces.Dict):
+    if _is_goal_env(env):
+        # Make mypy happy, already checked
+        assert isinstance(observation_space, spaces.Dict)
+        _check_goal_env_obs(obs, observation_space, "step")
+        _check_goal_env_compute_reward(obs, env, reward, info)
+    elif isinstance(observation_space, spaces.Dict):
         assert isinstance(obs, dict), "The observation returned by `step()` must be a dictionary"
 
         if not obs.keys() == observation_space.spaces.keys():
@@ -190,15 +255,16 @@ def _check_returned_values(env: gym.Env, observation_space: spaces.Space, action
     assert isinstance(done, bool), "The `done` signal must be a boolean"
     assert isinstance(info, dict), "The `info` returned by `step()` must be a python dictionary"
 
-    if isinstance(env, gym.GoalEnv):
-        # For a GoalEnv, the keys are checked at reset
+    # Goal conditioned env
+    if _is_goal_env(env):
         assert reward == env.compute_reward(obs["achieved_goal"], obs["desired_goal"], info)
 
 
 def _check_spaces(env: gym.Env) -> None:
     """
-    Check that the observation and action spaces are defined
-    and inherit from gym.spaces.Space.
+    Check that the observation and action spaces are defined and inherit from gym.spaces.Space. For
+    envs that follow the goal-conditioned standard (previously, the gym.GoalEnv interface) we check
+    the observation space is gym.spaces.Dict
     """
     # Helper to link to the code, because gym has no proper documentation
     gym_spaces = " cf https://github.com/openai/gym/blob/master/gym/spaces/"
@@ -209,6 +275,11 @@ def _check_spaces(env: gym.Env) -> None:
     assert isinstance(env.observation_space, spaces.Space), "The observation space must inherit from gym.spaces" + gym_spaces
     assert isinstance(env.action_space, spaces.Space), "The action space must inherit from gym.spaces" + gym_spaces
 
+    if _is_goal_env(env):
+        assert isinstance(
+            env.observation_space, spaces.Dict
+        ), "Goal conditioned envs (previously gym.GoalEnv) require the observation space to be gym.spaces.Dict"
+
 
 # Check render cannot be covered by CI
 def _check_render(env: gym.Env, warn: bool = True, headless: bool = False) -> None:  # pragma: no cover
diff --git a/stable_baselines3/common/envs/bit_flipping_env.py b/stable_baselines3/common/envs/bit_flipping_env.py
index a881b32..d6724c9 100644
--- a/stable_baselines3/common/envs/bit_flipping_env.py
+++ b/stable_baselines3/common/envs/bit_flipping_env.py
@@ -2,13 +2,13 @@ from collections import OrderedDict
 from typing import Any, Dict, Optional, Union
 
 import numpy as np
-from gym import GoalEnv, spaces
+from gym import Env, spaces
 from gym.envs.registration import EnvSpec
 
 from stable_baselines3.common.type_aliases import GymStepReturn
 
 
-class BitFlippingEnv(GoalEnv):
+class BitFlippingEnv(Env):
     """
     Simple bit flipping env, useful to test HER.
     The goal is to flip all the bits to get a vector of ones.
diff --git a/stable_baselines3/common/noise.py b/stable_baselines3/common/noise.py
index baa72e9..5e8632d 100644
--- a/stable_baselines3/common/noise.py
+++ b/stable_baselines3/common/noise.py
@@ -10,7 +10,7 @@ class ActionNoise(ABC):
     The action noise base class
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
 
     def reset(self) -> None:
diff --git a/stable_baselines3/common/off_policy_algorithm.py b/stable_baselines3/common/off_policy_algorithm.py
index 634d9e9..5e018fc 100644
--- a/stable_baselines3/common/off_policy_algorithm.py
+++ b/stable_baselines3/common/off_policy_algorithm.py
@@ -602,7 +602,6 @@ class OffPolicyAlgorithm(BaseAlgorithm):
                     # Log training infos
                     if log_interval is not None and self._episode_num % log_interval == 0:
                         self._dump_logs()
-
         callback.on_rollout_end()
 
         return RolloutReturn(num_collected_steps * env.num_envs, num_collected_episodes, continue_training)
diff --git a/stable_baselines3/td3/td3.py b/stable_baselines3/td3/td3.py
index a8bc3ef..97812a9 100644
--- a/stable_baselines3/td3/td3.py
+++ b/stable_baselines3/td3/td3.py
@@ -150,7 +150,6 @@ class TD3(OffPolicyAlgorithm):
         self._update_learning_rate([self.actor.optimizer, self.critic.optimizer])
 
         actor_losses, critic_losses = [], []
-
         for _ in range(gradient_steps):
 
             self._n_updates += 1
diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py
index f749d5a..a0e20b7 100644
--- a/tests/test_callbacks.py
+++ b/tests/test_callbacks.py
@@ -102,7 +102,7 @@ def test_callbacks(tmp_path, model_class):
 
 def select_env(model_class) -> str:
     if model_class is DQN:
-        return "CartPole-v0"
+        return "CartPole-v1"
     else:
         return "Pendulum-v1"
 
diff --git a/tests/test_envs.py b/tests/test_envs.py
index 8b8cb8c..1281bb4 100644
--- a/tests/test_envs.py
+++ b/tests/test_envs.py
@@ -28,7 +28,7 @@ ENV_CLASSES = [
 ]
 
 
-@pytest.mark.parametrize("env_id", ["CartPole-v0", "Pendulum-v1"])
+@pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"])
 def test_env(env_id):
     """
     Check that environmnent integrated in Gym pass the test.
diff --git a/tests/test_predict.py b/tests/test_predict.py
index 93bbc9d..22ff4fd 100644
--- a/tests/test_predict.py
+++ b/tests/test_predict.py
@@ -40,7 +40,7 @@ def test_auto_wrap(model_class):
     """Test auto wrapping of env into a VecEnv."""
     # Use different environment for DQN
     if model_class is DQN:
-        env_name = "CartPole-v0"
+        env_name = "CartPole-v1"
     else:
         env_name = "Pendulum-v1"
     env = gym.make(env_name)
diff --git a/tests/test_save_load.py b/tests/test_save_load.py
index 91b0760..f96b69e 100644
--- a/tests/test_save_load.py
+++ b/tests/test_save_load.py
@@ -30,7 +30,7 @@ def select_env(model_class: BaseAlgorithm) -> gym.Env:
     if model_class == DQN:
         return IdentityEnv(10)
     else:
-        return IdentityEnvBox(10)
+        return IdentityEnvBox(-10, 10)
 
 
 @pytest.mark.parametrize("model_class", MODEL_LIST)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e74b1d0..83d695a 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -248,7 +248,7 @@ def test_evaluate_policy_monitors(vec_env_class):
     # Also test VecEnvs
     n_eval_episodes = 3
     n_envs = 2
-    env_id = "CartPole-v0"
+    env_id = "CartPole-v1"
     model = A2C("MlpPolicy", env_id, seed=0)
 
     def make_eval_env(with_monitor, wrapper_class=gym.Wrapper):
diff --git a/tests/test_vec_monitor.py b/tests/test_vec_monitor.py
index 5ccc33e..0a146a0 100644
--- a/tests/test_vec_monitor.py
+++ b/tests/test_vec_monitor.py
@@ -140,7 +140,7 @@ def test_vec_monitor_ppo(recwarn):
 
     # No warnings because using `VecMonitor`
     evaluate_policy(model, monitor_env)
-    assert len(recwarn) == 0
+    assert len(recwarn) == 0, f"{[str(warning) for warning in recwarn]}"
 
 
 def test_vec_monitor_warn():
diff --git a/tests/test_vec_normalize.py b/tests/test_vec_normalize.py
index 07c720f..00af193 100644
--- a/tests/test_vec_normalize.py
+++ b/tests/test_vec_normalize.py
@@ -40,7 +40,7 @@ class DummyRewardEnv(gym.Env):
         return np.array([self.returned_rewards[self.return_reward_idx]])
 
 
-class DummyDictEnv(gym.GoalEnv):
+class DummyDictEnv(gym.Env):
     """
     Dummy gym goal env for testing purposes
     """

From aee0ba03c734cbf5df249ffef72655d8194146dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Mon, 28 Nov 2022 19:36:26 +0100
Subject: [PATCH 06/13] Update changelog for #1184 (#1185)

---
 docs/misc/changelog.rst       | 5 ++++-
 docs/modules/dqn.rst          | 2 +-
 stable_baselines3/version.txt | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 9ae202b..b4355c7 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 ==========
 
 
-Release 1.7.0a2 (WIP)
+Release 1.7.0a3 (WIP)
 --------------------------
 
 Breaking Changes:
@@ -37,10 +37,13 @@ Others:
 ^^^^^^^
 - Used issue forms instead of issue templates
 - Fixed flake8 config to be compatible with flake8 6+
+- Goal-conditioned environments are now characterized by the availability of the ``compute_reward`` method, rather than by their inheritance to ``gym.GoalEnv``
+- Replaced ``CartPole-v0`` by ``CartPole-v1`` is tests
 
 Documentation:
 ^^^^^^^^^^^^^^
 - Updated Hugging Face Integration page (@simoninithomas)
+- Changed ``env`` to ``vec_env`` when environment is vectorized
 
 Release 1.6.2 (2022-10-10)
 --------------------------
diff --git a/docs/modules/dqn.rst b/docs/modules/dqn.rst
index ce43855..8648606 100644
--- a/docs/modules/dqn.rst
+++ b/docs/modules/dqn.rst
@@ -60,7 +60,7 @@ This example is only to demonstrate the use of the library and its functions, an
 
   from stable_baselines3 import DQN
 
-  env = gym.make("CartPole-v0")
+  env = gym.make("CartPole-v1")
 
   model = DQN("MlpPolicy", env, verbose=1)
   model.learn(total_timesteps=10000, log_interval=4)
diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt
index b895285..08b6b37 100644
--- a/stable_baselines3/version.txt
+++ b/stable_baselines3/version.txt
@@ -1 +1 @@
-1.7.0a2
+1.7.0a3

From 0973b01b9dcee853bdd7314db55c4bc524a7f20b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 29 Nov 2022 11:27:59 +0100
Subject: [PATCH 07/13] Fix `tests/test_distributions.py` type hint (#1186)

* Fixed test_distribution type hint

* Impose list[int] for action dim
---
 docs/misc/changelog.rst                   | 1 +
 setup.cfg                                 | 1 -
 stable_baselines3/common/distributions.py | 2 +-
 tests/test_distributions.py               | 6 ++----
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index b4355c7..26b147b 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -39,6 +39,7 @@ Others:
 - Fixed flake8 config to be compatible with flake8 6+
 - Goal-conditioned environments are now characterized by the availability of the ``compute_reward`` method, rather than by their inheritance to ``gym.GoalEnv``
 - Replaced ``CartPole-v0`` by ``CartPole-v1`` is tests
+- Fixed ``tests/test_distributions.py`` type hint
 
 Documentation:
 ^^^^^^^^^^^^^^
diff --git a/setup.cfg b/setup.cfg
index 5e30726..5e41364 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -72,7 +72,6 @@ exclude = (?x)(
     | stable_baselines3/sac/sac.py$
     | stable_baselines3/td3/policies.py$
     | stable_baselines3/td3/td3.py$
-    | tests/test_distributions.py$
     | tests/test_logger.py$
     | tests/test_tensorboard.py$
     | tests/test_train_eval_mode.py$
diff --git a/stable_baselines3/common/distributions.py b/stable_baselines3/common/distributions.py
index 63eb475..b78ef82 100644
--- a/stable_baselines3/common/distributions.py
+++ b/stable_baselines3/common/distributions.py
@@ -679,7 +679,7 @@ def make_proba_distribution(
     elif isinstance(action_space, spaces.Discrete):
         return CategoricalDistribution(action_space.n, **dist_kwargs)
     elif isinstance(action_space, spaces.MultiDiscrete):
-        return MultiCategoricalDistribution(action_space.nvec, **dist_kwargs)
+        return MultiCategoricalDistribution(list(action_space.nvec), **dist_kwargs)
     elif isinstance(action_space, spaces.MultiBinary):
         return BernoulliDistribution(action_space.n, **dist_kwargs)
     else:
diff --git a/tests/test_distributions.py b/tests/test_distributions.py
index 513429b..e782182 100644
--- a/tests/test_distributions.py
+++ b/tests/test_distributions.py
@@ -55,7 +55,7 @@ def test_squashed_gaussian(model_class):
 
 
 @pytest.fixture()
-def dummy_model_distribution_obs_and_actions() -> Tuple[A2C, np.array, np.array]:
+def dummy_model_distribution_obs_and_actions() -> Tuple[A2C, np.ndarray, np.ndarray]:
     """
     Fixture creating a Pendulum-v1 gym env, an A2C model and sampling 10 random observations and actions from the env
     :return: A2C model, random observations, random actions
@@ -165,9 +165,7 @@ def test_categorical(dist, CAT_ACTIONS):
         BernoulliDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS)),
         CategoricalDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS)),
         DiagGaussianDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS), th.rand(N_ACTIONS)),
-        MultiCategoricalDistribution(np.array([N_ACTIONS, N_ACTIONS])).proba_distribution(
-            th.rand(1, sum([N_ACTIONS, N_ACTIONS]))
-        ),
+        MultiCategoricalDistribution([N_ACTIONS, N_ACTIONS]).proba_distribution(th.rand(1, sum([N_ACTIONS, N_ACTIONS]))),
         SquashedDiagGaussianDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS), th.rand(N_ACTIONS)),
         StateDependentNoiseDistribution(N_ACTIONS).proba_distribution(
             th.rand(N_ACTIONS), th.rand([N_ACTIONS, N_ACTIONS]), th.rand([N_ACTIONS, N_ACTIONS])

From 6902fac5e7bacb3efddba94d6cb59167555bd3b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 29 Nov 2022 12:26:16 +0100
Subject: [PATCH 08/13] Fix `stable_baselines3/common/type_aliases.py` type
 hint (#1189)

---
 docs/misc/changelog.rst                  | 1 +
 setup.cfg                                | 1 -
 stable_baselines3/common/type_aliases.py | 4 ++--
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 26b147b..d5968f9 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -40,6 +40,7 @@ Others:
 - Goal-conditioned environments are now characterized by the availability of the ``compute_reward`` method, rather than by their inheritance to ``gym.GoalEnv``
 - Replaced ``CartPole-v0`` by ``CartPole-v1`` is tests
 - Fixed ``tests/test_distributions.py`` type hint
+- Fixed ``stable_baselines3/common/type_aliases.py`` type hint
 
 Documentation:
 ^^^^^^^^^^^^^^
diff --git a/setup.cfg b/setup.cfg
index 5e41364..cbc4402 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -49,7 +49,6 @@ exclude = (?x)(
     | stable_baselines3/common/save_util.py$
     | stable_baselines3/common/sb2_compat/rmsprop_tf_like.py$
     | stable_baselines3/common/torch_layers.py$
-    | stable_baselines3/common/type_aliases.py$
     | stable_baselines3/common/utils.py$
     | stable_baselines3/common/vec_env/__init__.py$
     | stable_baselines3/common/vec_env/base_vec_env.py$
diff --git a/stable_baselines3/common/type_aliases.py b/stable_baselines3/common/type_aliases.py
index 4faad7d..7227667 100644
--- a/stable_baselines3/common/type_aliases.py
+++ b/stable_baselines3/common/type_aliases.py
@@ -36,7 +36,7 @@ class RolloutBufferSamples(NamedTuple):
     returns: th.Tensor
 
 
-class DictRolloutBufferSamples(RolloutBufferSamples):
+class DictRolloutBufferSamples(NamedTuple):
     observations: TensorDict
     actions: th.Tensor
     old_values: th.Tensor
@@ -53,7 +53,7 @@ class ReplayBufferSamples(NamedTuple):
     rewards: th.Tensor
 
 
-class DictReplayBufferSamples(ReplayBufferSamples):
+class DictReplayBufferSamples(NamedTuple):
     observations: TensorDict
     actions: th.Tensor
     next_observations: TensorDict

From 5cd891317eb3ec13b6c8d9881145b5e670162a74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 29 Nov 2022 12:43:16 +0100
Subject: [PATCH 09/13] Add `with_bias` parameter to `create_mlp` (#1188)

* Add with_bias arg

* Update changelog

* move torch_layers to the last position

* Update version

Co-authored-by: Antonin Raffin <antonin.raffin@ensta.org>
---
 docs/misc/changelog.rst                  | 3 ++-
 stable_baselines3/common/torch_layers.py | 8 +++++---
 stable_baselines3/version.txt            | 2 +-
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index d5968f9..6baacb7 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 ==========
 
 
-Release 1.7.0a3 (WIP)
+Release 1.7.0a4 (WIP)
 --------------------------
 
 Breaking Changes:
@@ -17,6 +17,7 @@ Breaking Changes:
 New Features:
 ^^^^^^^^^^^^^
 - Introduced mypy type checking
+- Added ``with_bias`` argument to ``create_mlp``
 
 SB3-Contrib
 ^^^^^^^^^^^
diff --git a/stable_baselines3/common/torch_layers.py b/stable_baselines3/common/torch_layers.py
index f87337c..2ce0cc1 100644
--- a/stable_baselines3/common/torch_layers.py
+++ b/stable_baselines3/common/torch_layers.py
@@ -99,6 +99,7 @@ def create_mlp(
     net_arch: List[int],
     activation_fn: Type[nn.Module] = nn.ReLU,
     squash_output: bool = False,
+    with_bias: bool = True,
 ) -> List[nn.Module]:
     """
     Create a multi layer perceptron (MLP), which is
@@ -113,21 +114,22 @@ def create_mlp(
         to use after each layer.
     :param squash_output: Whether to squash the output using a Tanh
         activation function
+    :param with_bias: If set to False, the layers will not learn an additive bias
     :return:
     """
 
     if len(net_arch) > 0:
-        modules = [nn.Linear(input_dim, net_arch[0]), activation_fn()]
+        modules = [nn.Linear(input_dim, net_arch[0], bias=with_bias), activation_fn()]
     else:
         modules = []
 
     for idx in range(len(net_arch) - 1):
-        modules.append(nn.Linear(net_arch[idx], net_arch[idx + 1]))
+        modules.append(nn.Linear(net_arch[idx], net_arch[idx + 1], bias=with_bias))
         modules.append(activation_fn())
 
     if output_dim > 0:
         last_layer_dim = net_arch[-1] if len(net_arch) > 0 else input_dim
-        modules.append(nn.Linear(last_layer_dim, output_dim))
+        modules.append(nn.Linear(last_layer_dim, output_dim, bias=with_bias))
     if squash_output:
         modules.append(nn.Tanh())
     return modules
diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt
index 08b6b37..0952a4b 100644
--- a/stable_baselines3/version.txt
+++ b/stable_baselines3/version.txt
@@ -1 +1 @@
-1.7.0a3
+1.7.0a4

From b46396a6644d6f262984014c7fe3ecaf713db86d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 29 Nov 2022 15:36:55 +0100
Subject: [PATCH 10/13] Fix `stable_baselines3/common/env_util.py` type hint
 (#1192)

* Remove env_util from mypy exclude

* Fix make_atari_env type hint

* Update changelog
---
 docs/misc/changelog.rst              |  1 +
 setup.cfg                            |  1 -
 stable_baselines3/common/env_util.py | 12 +++---------
 3 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 6baacb7..4885d52 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -42,6 +42,7 @@ Others:
 - Replaced ``CartPole-v0`` by ``CartPole-v1`` is tests
 - Fixed ``tests/test_distributions.py`` type hint
 - Fixed ``stable_baselines3/common/type_aliases.py`` type hint
+- Fixed ``stable_baselines3/common/env_util.py`` type hint
 
 Documentation:
 ^^^^^^^^^^^^^^
diff --git a/setup.cfg b/setup.cfg
index cbc4402..331c8ff 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,7 +36,6 @@ exclude = (?x)(
     | stable_baselines3/common/buffers.py$
     | stable_baselines3/common/callbacks.py$
     | stable_baselines3/common/distributions.py$
-    | stable_baselines3/common/env_util.py$
     | stable_baselines3/common/envs/bit_flipping_env.py$
     | stable_baselines3/common/envs/identity_env.py$
     | stable_baselines3/common/envs/multi_input_envs.py$
diff --git a/stable_baselines3/common/env_util.py b/stable_baselines3/common/env_util.py
index eb893c6..c85d147 100644
--- a/stable_baselines3/common/env_util.py
+++ b/stable_baselines3/common/env_util.py
@@ -116,7 +116,7 @@ def make_atari_env(
     monitor_dir: Optional[str] = None,
     wrapper_kwargs: Optional[Dict[str, Any]] = None,
     env_kwargs: Optional[Dict[str, Any]] = None,
-    vec_env_cls: Optional[Union[DummyVecEnv, SubprocVecEnv]] = None,
+    vec_env_cls: Optional[Union[Type[DummyVecEnv], Type[SubprocVecEnv]]] = None,
     vec_env_kwargs: Optional[Dict[str, Any]] = None,
     monitor_kwargs: Optional[Dict[str, Any]] = None,
 ) -> VecEnv:
@@ -138,22 +138,16 @@ def make_atari_env(
     :param monitor_kwargs: Keyword arguments to pass to the ``Monitor`` class constructor.
     :return: The wrapped environment
     """
-    if wrapper_kwargs is None:
-        wrapper_kwargs = {}
-
-    def atari_wrapper(env: gym.Env) -> gym.Env:
-        env = AtariWrapper(env, **wrapper_kwargs)
-        return env
-
     return make_vec_env(
         env_id,
         n_envs=n_envs,
         seed=seed,
         start_index=start_index,
         monitor_dir=monitor_dir,
-        wrapper_class=atari_wrapper,
+        wrapper_class=AtariWrapper,
         env_kwargs=env_kwargs,
         vec_env_cls=vec_env_cls,
         vec_env_kwargs=vec_env_kwargs,
         monitor_kwargs=monitor_kwargs,
+        wrapper_kwargs=wrapper_kwargs,
     )

From 852d635742e97495d200b104ab8e09f128211e26 Mon Sep 17 00:00:00 2001
From: Zikang Xiong <73256697+ZikangXiong@users.noreply.github.com>
Date: Tue, 29 Nov 2022 17:33:46 -0500
Subject: [PATCH 11/13] Exposed modules in __init__.py with __all__ (#1195)

* Exposed modules in __init__.py with __all__

* Remove flake8 ignore and update root __all__

* Update version

Co-authored-by: Antonin Raffin <antonin.raffin@ensta.org>
---
 docs/misc/changelog.rst                      |  5 +++--
 setup.cfg                                    | 11 ----------
 stable_baselines3/__init__.py                | 12 ++++++++++
 stable_baselines3/a2c/__init__.py            |  2 ++
 stable_baselines3/common/envs/__init__.py    | 11 ++++++++++
 stable_baselines3/common/vec_env/__init__.py | 23 +++++++++++++++++++-
 stable_baselines3/ddpg/__init__.py           |  2 ++
 stable_baselines3/dqn/__init__.py            |  2 ++
 stable_baselines3/her/__init__.py            |  2 ++
 stable_baselines3/ppo/__init__.py            |  2 ++
 stable_baselines3/sac/__init__.py            |  2 ++
 stable_baselines3/td3/__init__.py            |  2 ++
 stable_baselines3/version.txt                |  2 +-
 13 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 4885d52..01fc719 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -4,7 +4,7 @@ Changelog
 ==========
 
 
-Release 1.7.0a4 (WIP)
+Release 1.7.0a5 (WIP)
 --------------------------
 
 Breaking Changes:
@@ -43,6 +43,7 @@ Others:
 - Fixed ``tests/test_distributions.py`` type hint
 - Fixed ``stable_baselines3/common/type_aliases.py`` type hint
 - Fixed ``stable_baselines3/common/env_util.py`` type hint
+- Exposed modules in ``__init__.py`` with the ``__all__`` attribute (@ZikangXiong)
 
 Documentation:
 ^^^^^^^^^^^^^^
@@ -1126,4 +1127,4 @@ And all the contributors:
 @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485
 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede
 @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875
-@anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer
+@anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong
diff --git a/setup.cfg b/setup.cfg
index 331c8ff..733b5c3 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -81,17 +81,6 @@ exclude = (?x)(
 ignore = W503,W504,E203,E231
 # Ignore import not used when aliases are defined
 per-file-ignores =
-	./stable_baselines3/__init__.py:F401
-	./stable_baselines3/common/__init__.py:F401
-	./stable_baselines3/common/envs/__init__.py:F401
-	./stable_baselines3/a2c/__init__.py:F401
-	./stable_baselines3/ddpg/__init__.py:F401
-	./stable_baselines3/dqn/__init__.py:F401
-	./stable_baselines3/her/__init__.py:F401
-	./stable_baselines3/ppo/__init__.py:F401
-	./stable_baselines3/sac/__init__.py:F401
-	./stable_baselines3/td3/__init__.py:F401
-	./stable_baselines3/common/vec_env/__init__.py:F401
   # Default implementation in abstract methods
   ./stable_baselines3/common/callbacks.py:B027
   ./stable_baselines3/common/noise.py:B027
diff --git a/stable_baselines3/__init__.py b/stable_baselines3/__init__.py
index d73f5f0..0775a8e 100644
--- a/stable_baselines3/__init__.py
+++ b/stable_baselines3/__init__.py
@@ -20,3 +20,15 @@ def HER(*args, **kwargs):
         "Since Stable Baselines 2.1.0, `HER` is now a replay buffer class `HerReplayBuffer`.\n "
         "Please check the documentation for more information: https://stable-baselines3.readthedocs.io/"
     )
+
+
+__all__ = [
+    "A2C",
+    "DDPG",
+    "DQN",
+    "PPO",
+    "SAC",
+    "TD3",
+    "HerReplayBuffer",
+    "get_system_info",
+]
diff --git a/stable_baselines3/a2c/__init__.py b/stable_baselines3/a2c/__init__.py
index 7e99964..78fc54f 100644
--- a/stable_baselines3/a2c/__init__.py
+++ b/stable_baselines3/a2c/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.a2c.a2c import A2C
 from stable_baselines3.a2c.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "A2C"]
diff --git a/stable_baselines3/common/envs/__init__.py b/stable_baselines3/common/envs/__init__.py
index 23bd575..3ff0221 100644
--- a/stable_baselines3/common/envs/__init__.py
+++ b/stable_baselines3/common/envs/__init__.py
@@ -7,3 +7,14 @@ from stable_baselines3.common.envs.identity_env import (
     IdentityEnvMultiDiscrete,
 )
 from stable_baselines3.common.envs.multi_input_envs import SimpleMultiObsEnv
+
+__all__ = [
+    "BitFlippingEnv",
+    "FakeImageEnv",
+    "IdentityEnv",
+    "IdentityEnvBox",
+    "IdentityEnvMultiBinary",
+    "IdentityEnvMultiDiscrete",
+    "SimpleMultiObsEnv",
+    "SimpleMultiObsEnv",
+]
diff --git a/stable_baselines3/common/vec_env/__init__.py b/stable_baselines3/common/vec_env/__init__.py
index 3880fbd..33a103a 100644
--- a/stable_baselines3/common/vec_env/__init__.py
+++ b/stable_baselines3/common/vec_env/__init__.py
@@ -1,4 +1,3 @@
-# flake8: noqa F401
 import typing
 from copy import deepcopy
 from typing import Optional, Type, Union
@@ -72,3 +71,25 @@ def sync_envs_normalization(env: "GymEnv", eval_env: "GymEnv") -> None:
             eval_env_tmp.ret_rms = deepcopy(env_tmp.ret_rms)
         env_tmp = env_tmp.venv
         eval_env_tmp = eval_env_tmp.venv
+
+
+__all__ = [
+    "CloudpickleWrapper",
+    "VecEnv",
+    "VecEnvWrapper",
+    "DummyVecEnv",
+    "StackedDictObservations",
+    "StackedObservations",
+    "SubprocVecEnv",
+    "VecCheckNan",
+    "VecExtractDictObs",
+    "VecFrameStack",
+    "VecMonitor",
+    "VecNormalize",
+    "VecTransposeImage",
+    "VecVideoRecorder",
+    "unwrap_vec_wrapper",
+    "unwrap_vec_normalize",
+    "is_vecenv_wrapped",
+    "sync_envs_normalization",
+]
diff --git a/stable_baselines3/ddpg/__init__.py b/stable_baselines3/ddpg/__init__.py
index 262e7f1..257a3e3 100644
--- a/stable_baselines3/ddpg/__init__.py
+++ b/stable_baselines3/ddpg/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.ddpg.ddpg import DDPG
 from stable_baselines3.ddpg.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "DDPG"]
diff --git a/stable_baselines3/dqn/__init__.py b/stable_baselines3/dqn/__init__.py
index f36f96e..2e5e2db 100644
--- a/stable_baselines3/dqn/__init__.py
+++ b/stable_baselines3/dqn/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.dqn.dqn import DQN
 from stable_baselines3.dqn.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "DQN"]
diff --git a/stable_baselines3/her/__init__.py b/stable_baselines3/her/__init__.py
index 1f58921..dc4c8c2 100644
--- a/stable_baselines3/her/__init__.py
+++ b/stable_baselines3/her/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.her.goal_selection_strategy import GoalSelectionStrategy
 from stable_baselines3.her.her_replay_buffer import HerReplayBuffer
+
+__all__ = ["GoalSelectionStrategy", "HerReplayBuffer"]
diff --git a/stable_baselines3/ppo/__init__.py b/stable_baselines3/ppo/__init__.py
index e5c23fc..cd91257 100644
--- a/stable_baselines3/ppo/__init__.py
+++ b/stable_baselines3/ppo/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.ppo.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
 from stable_baselines3.ppo.ppo import PPO
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "PPO"]
diff --git a/stable_baselines3/sac/__init__.py b/stable_baselines3/sac/__init__.py
index 5a84dde..bdf780d 100644
--- a/stable_baselines3/sac/__init__.py
+++ b/stable_baselines3/sac/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.sac.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
 from stable_baselines3.sac.sac import SAC
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "SAC"]
diff --git a/stable_baselines3/td3/__init__.py b/stable_baselines3/td3/__init__.py
index 0b903cd..428141e 100644
--- a/stable_baselines3/td3/__init__.py
+++ b/stable_baselines3/td3/__init__.py
@@ -1,2 +1,4 @@
 from stable_baselines3.td3.policies import CnnPolicy, MlpPolicy, MultiInputPolicy
 from stable_baselines3.td3.td3 import TD3
+
+__all__ = ["CnnPolicy", "MlpPolicy", "MultiInputPolicy", "TD3"]
diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt
index 0952a4b..5d819d4 100644
--- a/stable_baselines3/version.txt
+++ b/stable_baselines3/version.txt
@@ -1 +1 @@
-1.7.0a4
+1.7.0a5

From 002850f8ace0e045f7e9d370149a6fbb6cbcebad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?=
 <45557362+qgallouedec@users.noreply.github.com>
Date: Tue, 29 Nov 2022 23:46:32 +0100
Subject: [PATCH 12/13] Fix `stable_baselines3/common/torch_layers.py` type
 hint (#1191)

* Remove torch layers from mypy exclude

* Make torch layers mypy compliant

* Extra type specification

* Update changelog

Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
---
 docs/misc/changelog.rst                  |  1 +
 setup.cfg                                |  1 -
 stable_baselines3/common/torch_layers.py | 13 ++++++-------
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 01fc719..67b5591 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -42,6 +42,7 @@ Others:
 - Replaced ``CartPole-v0`` by ``CartPole-v1`` is tests
 - Fixed ``tests/test_distributions.py`` type hint
 - Fixed ``stable_baselines3/common/type_aliases.py`` type hint
+- Fixed ``stable_baselines3/common/torch_layers.py`` type hint
 - Fixed ``stable_baselines3/common/env_util.py`` type hint
 - Exposed modules in ``__init__.py`` with the ``__all__`` attribute (@ZikangXiong)
 
diff --git a/setup.cfg b/setup.cfg
index 733b5c3..045638c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -47,7 +47,6 @@ exclude = (?x)(
     | stable_baselines3/common/preprocessing.py$
     | stable_baselines3/common/save_util.py$
     | stable_baselines3/common/sb2_compat/rmsprop_tf_like.py$
-    | stable_baselines3/common/torch_layers.py$
     | stable_baselines3/common/utils.py$
     | stable_baselines3/common/vec_env/__init__.py$
     | stable_baselines3/common/vec_env/base_vec_env.py$
diff --git a/stable_baselines3/common/torch_layers.py b/stable_baselines3/common/torch_layers.py
index 2ce0cc1..5de2af1 100644
--- a/stable_baselines3/common/torch_layers.py
+++ b/stable_baselines3/common/torch_layers.py
@@ -28,9 +28,6 @@ class BaseFeaturesExtractor(nn.Module):
     def features_dim(self) -> int:
         return self._features_dim
 
-    def forward(self, observations: th.Tensor) -> th.Tensor:
-        raise NotImplementedError()
-
 
 class FlattenExtractor(BaseFeaturesExtractor):
     """
@@ -173,9 +170,11 @@ class MlpExtractor(nn.Module):
     ):
         super().__init__()
         device = get_device(device)
-        shared_net, policy_net, value_net = [], [], []
-        policy_only_layers = []  # Layer sizes of the network that only belongs to the policy network
-        value_only_layers = []  # Layer sizes of the network that only belongs to the value network
+        shared_net: List[nn.Module] = []
+        policy_net: List[nn.Module] = []
+        value_net: List[nn.Module] = []
+        policy_only_layers: List[int] = []  # Layer sizes of the network that only belongs to the policy network
+        value_only_layers: List[int] = []  # Layer sizes of the network that only belongs to the value network
         last_layer_dim_shared = feature_dim
 
         # Iterate through the shared layers and build the shared parts of the network
@@ -254,7 +253,7 @@ class CombinedExtractor(BaseFeaturesExtractor):
         # TODO we do not know features-dim here before going over all the items, so put something there. This is dirty!
         super().__init__(observation_space, features_dim=1)
 
-        extractors = {}
+        extractors: Dict[str, nn.Module] = {}
 
         total_concat_size = 0
         for key, subspace in observation_space.spaces.items():

From f7d7ed3fa7095b560e34210e709c1c8b7b7877e5 Mon Sep 17 00:00:00 2001
From: Athanasios Theocharis <athatheoc@gmail.com>
Date: Tue, 6 Dec 2022 17:51:52 +0100
Subject: [PATCH 13/13] Update custom_policy.rst (#1183)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Update custom_policy.rst

* Update changelog

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>
Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>
---
 docs/guide/custom_policy.rst | 6 +++---
 docs/misc/changelog.rst      | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/guide/custom_policy.rst b/docs/guide/custom_policy.rst
index 1a3ae34..4ba3203 100644
--- a/docs/guide/custom_policy.rst
+++ b/docs/guide/custom_policy.rst
@@ -333,11 +333,11 @@ If your task requires even more granular control over the policy/value architect
           :return: (th.Tensor, th.Tensor) latent_policy, latent_value of the specified network.
               If all layers are shared, then ``latent_policy == latent_value``
           """
-          return self.policy_net(features), self.value_net(features)
-          
+          return self.forward_actor(features), self.forward_critic(features)
+
       def forward_actor(self, features: th.Tensor) -> th.Tensor:
           return self.policy_net(features)
-      
+
       def forward_critic(self, features: th.Tensor) -> th.Tensor:
           return self.value_net(features)
 
diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 67b5591..619e1eb 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -50,6 +50,7 @@ Documentation:
 ^^^^^^^^^^^^^^
 - Updated Hugging Face Integration page (@simoninithomas)
 - Changed ``env`` to ``vec_env`` when environment is vectorized
+- Update custom policy documentation (@athatheo)
 
 Release 1.6.2 (2022-10-10)
 --------------------------