From 69b94dd6a8f93cf0b9d2201dcae9c146b8a9c75d Mon Sep 17 00:00:00 2001 From: Vikas Kumar <32395222+theSquaredError@users.noreply.github.com> Date: Sat, 11 Feb 2023 01:45:09 +0530 Subject: [PATCH 1/3] Rename "timesteps" to "episodes" in `log_interval` documentation (#1325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * change timestamp to episode for logging * update changelog * minor format modif * minor format modif --------- Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com> --- docs/misc/changelog.rst | 4 ++-- stable_baselines3/common/base_class.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 6fafd99..f70a81b 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -45,7 +45,7 @@ Documentation: - Renamed ``load_parameters`` to ``set_parameters`` (@DavyMorgan) - Clarified documentation about subproc multiprocessing for A2C (@Bonifatius94) - Fixed typo in ``A2C`` docstring (@AlexPasqua) - +- Renamed timesteps to episodes for ``log_interval`` description (@theSquaredError) Release 1.7.0 (2023-01-10) -------------------------- @@ -1227,4 +1227,4 @@ And all the contributors: @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875 @yuanmingqi @anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong -@DavyMorgan @luizapozzobon @Bonifatius94 +@DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py index b6fba85..17be67a 100644 --- a/stable_baselines3/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -505,7 +505,7 @@ class BaseAlgorithm(ABC): :param total_timesteps: The total number of samples (env steps) to train on :param callback: callback(s) called at every step with state of the algorithm. - :param log_interval: The number of timesteps before logging. + :param log_interval: The number of episodes before logging. :param tb_log_name: the name of the run for TensorBoard logging :param reset_num_timesteps: whether or not to reset the current timestep number (used in logging) :param progress_bar: Display a progress bar using tqdm and rich. From 7a1e429702d7eef17ae026f48013de2884fb77b7 Mon Sep 17 00:00:00 2001 From: harveybellini Date: Wed, 15 Feb 2023 12:14:02 +0000 Subject: [PATCH 2/3] Remove Note from examples - Code works (#1330) * Remove Note Gif creation works with Atari Environments using the script provided below. * Update changelog --------- Co-authored-by: Antonin Raffin --- docs/guide/examples.rst | 4 ---- docs/misc/changelog.rst | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/guide/examples.rst b/docs/guide/examples.rst index 2bdb8d8..61b1fd1 100644 --- a/docs/guide/examples.rst +++ b/docs/guide/examples.rst @@ -769,10 +769,6 @@ Record a mp4 video (here using a random agent). Bonus: Make a GIF of a Trained Agent ------------------------------------ -.. note:: - For Atari games, you need to use a screen recorder such as `Kazam `_. - And then convert the video using `ffmpeg `_ - .. code-block:: python import imageio diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index f70a81b..ddc0ce2 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -46,6 +46,7 @@ Documentation: - Clarified documentation about subproc multiprocessing for A2C (@Bonifatius94) - Fixed typo in ``A2C`` docstring (@AlexPasqua) - Renamed timesteps to episodes for ``log_interval`` description (@theSquaredError) +- Removed note about gif creation for Atari games (@harveybellini) Release 1.7.0 (2023-01-10) -------------------------- @@ -1227,4 +1228,4 @@ And all the contributors: @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875 @yuanmingqi @anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong -@DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError +@DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError @harveybellini From 12e9917c24dc23d7de7694a924f017c6a8e9a6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 15 Feb 2023 14:17:18 +0100 Subject: [PATCH 3/3] Fix image-based normalized env loading (#1321) * Fix * Add test * Update changelog * fix memory error avoidance * Update version * image env test * black * check_shape_equal * check shape equal in vecnormalize * Allow spaces not to be box or dict * rm `test_save_load_vecnormalized_image` in favor of `test_vec_env` * Remove unused imports --------- Co-authored-by: Antonin RAFFIN Co-authored-by: Antonin Raffin --- docs/misc/changelog.rst | 5 +++-- stable_baselines3/common/utils.py | 18 ++++++++++++++++ .../common/vec_env/vec_normalize.py | 9 +++++--- stable_baselines3/version.txt | 2 +- tests/test_utils.py | 21 +++++++++++++++++++ tests/test_vec_normalize.py | 7 ++++++- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index ddc0ce2..c3c6629 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -4,7 +4,7 @@ Changelog ========== -Release 1.8.0a4 (WIP) +Release 1.8.0a5 (WIP) -------------------------- @@ -29,6 +29,7 @@ Bug Fixes: - Fixed Atari wrapper that missed the reset condition (@luizapozzobon) - Added the argument ``dtype`` (default to ``float32``) to the noise for consistency with gym action (@sidney-tio) - Fixed PPO train/n_updates metric not accounting for early stopping (@adamfrly) +- Fixed loading of normalized image-based environments Deprecations: ^^^^^^^^^^^^^ @@ -212,7 +213,7 @@ Bug Fixes: - Fixed missing verbose parameter passing in the ``EvalCallback`` constructor (@burakdmb) - Fixed the issue that when updating the target network in DQN, SAC, TD3, the ``running_mean`` and ``running_var`` properties of batch norm layers are not updated (@honglu2875) - Fixed incorrect type annotation of the replay_buffer_class argument in ``common.OffPolicyAlgorithm`` initializer, where an instance instead of a class was required (@Rocamonde) -- Fixed loading saved model with different number of envrionments +- Fixed loading saved model with different number of environments - Removed ``forward()`` abstract method declaration from ``common.policies.BaseModel`` (already defined in ``torch.nn.Module``) to fix type errors in subclasses (@Rocamonde) - Fixed the return type of ``.load()`` and ``.learn()`` methods in ``BaseAlgorithm`` so that they now use ``TypeVar`` (@Rocamonde) - Fixed an issue where keys with different tags but the same key raised an error in ``common.logger.HumanOutputFormat`` (@Rocamonde and @AdamGleave) diff --git a/stable_baselines3/common/utils.py b/stable_baselines3/common/utils.py index b5cbff4..1234fba 100644 --- a/stable_baselines3/common/utils.py +++ b/stable_baselines3/common/utils.py @@ -230,6 +230,24 @@ def check_for_correct_spaces(env: GymEnv, observation_space: spaces.Space, actio raise ValueError(f"Action spaces do not match: {action_space} != {env.action_space}") +def check_shape_equal(space1: spaces.Space, space2: spaces.Space) -> None: + """ + If the spaces are Box, check that they have the same shape. + + If the spaces are Dict, it recursively checks the subspaces. + + :param space1: Space + :param space2: Other space + """ + if isinstance(space1, spaces.Dict): + assert isinstance(space2, spaces.Dict), "spaces must be of the same type" + assert space1.spaces.keys() == space2.spaces.keys(), "spaces must have the same keys" + for key in space1.spaces.keys(): + check_shape_equal(space1.spaces[key], space2.spaces[key]) + elif isinstance(space1, spaces.Box): + assert space1.shape == space2.shape, "spaces must have the same shape" + + def is_vectorized_box_observation(observation: np.ndarray, observation_space: spaces.Box) -> bool: """ For box observation type, detects and validates the shape, diff --git a/stable_baselines3/common/vec_env/vec_normalize.py b/stable_baselines3/common/vec_env/vec_normalize.py index 8b98413..ffa813f 100644 --- a/stable_baselines3/common/vec_env/vec_normalize.py +++ b/stable_baselines3/common/vec_env/vec_normalize.py @@ -1,3 +1,4 @@ +import inspect import pickle from copy import deepcopy from typing import Any, Dict, List, Optional, Union @@ -159,10 +160,12 @@ class VecNormalize(VecEnvWrapper): """ if self.venv is not None: raise ValueError("Trying to set venv of already initialized VecNormalize wrapper.") - VecEnvWrapper.__init__(self, venv) + self.venv = venv + self.num_envs = venv.num_envs + self.class_attributes = dict(inspect.getmembers(self.__class__)) - # Check only that the observation_space match - utils.check_for_correct_spaces(venv, self.observation_space, venv.action_space) + # Check that the observation_space shape match + utils.check_shape_equal(self.observation_space, venv.observation_space) self.returns = np.zeros(self.num_envs) def step_wait(self) -> VecEnvStepReturn: diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt index e8175d3..3d6a0b8 100644 --- a/stable_baselines3/version.txt +++ b/stable_baselines3/version.txt @@ -1 +1 @@ -1.8.0a4 +1.8.0a5 diff --git a/tests/test_utils.py b/tests/test_utils.py index e5236e8..dff80a0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,6 +15,7 @@ from stable_baselines3.common.evaluation import evaluate_policy from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.noise import ActionNoise, OrnsteinUhlenbeckActionNoise, VectorizedActionNoise from stable_baselines3.common.utils import ( + check_shape_equal, get_parameters_by_name, get_system_info, is_vectorized_observation, @@ -509,3 +510,23 @@ def test_is_vectorized_observation(): discrete_obs = np.ones((1, 1), dtype=np.int8) dict_obs = {"box": box_obs, "discrete": discrete_obs} is_vectorized_observation(dict_obs, dict_space) + + +def test_check_shape_equal(): + space1 = spaces.Box(low=0, high=1, shape=(2, 2)) + space2 = spaces.Box(low=-1, high=1, shape=(2, 2)) + check_shape_equal(space1, space2) + + space1 = spaces.Box(low=0, high=1, shape=(2, 2)) + space2 = spaces.Box(low=-1, high=2, shape=(3, 3)) + with pytest.raises(AssertionError): + check_shape_equal(space1, space2) + + space1 = spaces.Dict({"key1": spaces.Box(low=0, high=1, shape=(2, 2)), "key2": spaces.Box(low=0, high=1, shape=(2, 2))}) + space2 = spaces.Dict({"key1": spaces.Box(low=-1, high=2, shape=(2, 2)), "key2": spaces.Box(low=-1, high=2, shape=(2, 2))}) + check_shape_equal(space1, space2) + + space1 = spaces.Dict({"key1": spaces.Box(low=0, high=1, shape=(2, 2)), "key2": spaces.Box(low=0, high=1, shape=(2, 2))}) + space2 = spaces.Dict({"key1": spaces.Box(low=-1, high=2, shape=(3, 3)), "key2": spaces.Box(low=-1, high=2, shape=(2, 2))}) + with pytest.raises(AssertionError): + check_shape_equal(space1, space2) diff --git a/tests/test_vec_normalize.py b/tests/test_vec_normalize.py index fb37fd3..aca113d 100644 --- a/tests/test_vec_normalize.py +++ b/tests/test_vec_normalize.py @@ -7,6 +7,7 @@ import pytest from gym import spaces from stable_baselines3 import SAC, TD3, HerReplayBuffer +from stable_baselines3.common.envs import FakeImageEnv from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.running_mean_std import RunningMeanStd from stable_baselines3.common.vec_env import ( @@ -118,6 +119,10 @@ def make_dict_env(): return Monitor(DummyDictEnv()) +def make_image_env(): + return Monitor(FakeImageEnv()) + + def check_rms_equal(rmsa, rmsb): if isinstance(rmsa, dict): for key in rmsa.keys(): @@ -244,7 +249,7 @@ def test_obs_rms_vec_normalize(): assert np.allclose(env.ret_rms.mean, 5.688, atol=1e-3) -@pytest.mark.parametrize("make_env", [make_env, make_dict_env]) +@pytest.mark.parametrize("make_env", [make_env, make_dict_env, make_image_env]) def test_vec_env(tmp_path, make_env): """Test VecNormalize Object""" clip_obs = 0.5