From ef10189d80dbb2efb3b5391cba4eb3c2ab5c7aae Mon Sep 17 00:00:00 2001 From: Max Weltevrede <31962715+MWeltevrede@users.noreply.github.com> Date: Mon, 4 Jul 2022 15:08:54 +0200 Subject: [PATCH 01/10] Prohibit simultaneous use of optimize_memory_usage and handle_timeout_termination (#948) * Prohibit simultaneous use of optimize_memory_buffer and handle_timeout_termination * Modify test to avoid unsupported buffer configuration * Change from assertion to raising of ValueError * Update changelog * Update style for consistency * Use handle_timeout_termination when possible Co-authored-by: Anssi Co-authored-by: Antonin Raffin --- docs/misc/changelog.rst | 3 ++- stable_baselines3/common/buffers.py | 7 +++++++ tests/test_save_load.py | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 52bf3e4..5e893d9 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -33,6 +33,7 @@ Bug Fixes: - Added a check for unbounded actions - Fixed issues due to newer version of protobuf (tensorboard) and sphinx - Fix exception causes all over the codebase (@cool-RR) +- Prohibit simultaneous use of optimize_memory_usage and handle_timeout_termination due to a bug (@MWeltevrede) Deprecations: ^^^^^^^^^^^^^ @@ -979,4 +980,4 @@ And all the contributors: @wkirgsn @AechPro @CUN-bjy @batu @IljaAvadiev @timokau @kachayev @cleversonahum @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485 -@Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR +@Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede diff --git a/stable_baselines3/common/buffers.py b/stable_baselines3/common/buffers.py index d7728cb..d36c5f5 100644 --- a/stable_baselines3/common/buffers.py +++ b/stable_baselines3/common/buffers.py @@ -164,6 +164,7 @@ class ReplayBuffer(BaseBuffer): at a cost of more complexity. See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195 and https://github.com/DLR-RM/stable-baselines3/pull/28#issuecomment-637559274 + Cannot be used in combination with handle_timeout_termination. :param handle_timeout_termination: Handle timeout termination (due to timelimit) separately and treat the task as infinite horizon task. https://github.com/DLR-RM/stable-baselines3/issues/284 @@ -188,6 +189,12 @@ class ReplayBuffer(BaseBuffer): if psutil is not None: mem_available = psutil.virtual_memory().available + # there is a bug if both optimize_memory_usage and handle_timeout_termination are true + # see https://github.com/DLR-RM/stable-baselines3/issues/934 + if optimize_memory_usage and handle_timeout_termination: + raise ValueError( + "ReplayBuffer does not support optimize_memory_usage = True and handle_timeout_termination = True simultaneously." + ) self.optimize_memory_usage = optimize_memory_usage self.observations = np.zeros((self.buffer_size, self.n_envs) + self.obs_shape, dtype=observation_space.dtype) diff --git a/tests/test_save_load.py b/tests/test_save_load.py index 2fdebbe..d7a74c5 100644 --- a/tests/test_save_load.py +++ b/tests/test_save_load.py @@ -375,6 +375,9 @@ def test_warn_buffer(recwarn, model_class, optimize_memory_usage): select_env(model_class), buffer_size=100, optimize_memory_usage=optimize_memory_usage, + # we cannot use optimize_memory_usage and handle_timeout_termination + # at the same time + replay_buffer_kwargs={"handle_timeout_termination": not optimize_memory_usage}, policy_kwargs=dict(net_arch=[64]), learning_starts=10, ) From c1f1c3d3d796054f68b3fa741c1a0be4ce2187b5 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 12 Jul 2022 22:50:23 +0200 Subject: [PATCH 02/10] Release v1.6.0 (#958) * Release v1.6.0 + update doc + add copy button * Update read the doc conda env * Update year * Fix bug in kl divergence check * Rephrase requirement for envpool and isaac gym --- docs/conda_env.yml | 6 +++--- docs/conf.py | 13 ++++++++++++- docs/guide/examples.rst | 10 ++++++++++ docs/guide/install.rst | 11 +++++++++++ docs/misc/changelog.rst | 7 ++++++- setup.py | 2 ++ stable_baselines3/common/buffers.py | 3 ++- stable_baselines3/common/distributions.py | 3 ++- stable_baselines3/version.txt | 2 +- tests/test_distributions.py | 4 +++- 10 files changed, 52 insertions(+), 9 deletions(-) diff --git a/docs/conda_env.yml b/docs/conda_env.yml index a01d37b..98a5508 100644 --- a/docs/conda_env.yml +++ b/docs/conda_env.yml @@ -6,9 +6,9 @@ dependencies: - cpuonly=1.0=0 - pip=21.1 - python=3.7 - - pytorch=1.8.1=py3.7_cpu_0 + - pytorch=1.11=py3.7_cpu_0 - pip: - - gym>=0.17.2 + - gym==0.21 - cloudpickle - opencv-python-headless - pandas @@ -16,5 +16,5 @@ dependencies: - matplotlib - sphinx_autodoc_typehints - sphinx>=4.2 - # See https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 - sphinx_rtd_theme>=1.0 + - sphinx_copybutton diff --git a/docs/conf.py b/docs/conf.py index 18898d5..b44be6f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,6 +24,14 @@ try: except ImportError: enable_spell_check = False +# Try to enable copy button +try: + import sphinx_copybutton # noqa: F401 + + enable_copy_button = True +except ImportError: + enable_copy_button = False + # source code directory, relative to this file, for sphinx-autobuild sys.path.insert(0, os.path.abspath("..")) @@ -51,7 +59,7 @@ with open(version_file) as file_handler: # -- Project information ----------------------------------------------------- project = "Stable Baselines3" -copyright = "2020, Stable Baselines3" +copyright = "2022, Stable Baselines3" author = "Stable Baselines3 Contributors" # The short X.Y version @@ -83,6 +91,9 @@ extensions = [ if enable_spell_check: extensions.append("sphinxcontrib.spelling") +if enable_copy_button: + extensions.append("sphinx_copybutton") + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/guide/examples.rst b/docs/guide/examples.rst index a5b56b2..0d7e7c0 100644 --- a/docs/guide/examples.rst +++ b/docs/guide/examples.rst @@ -729,6 +729,16 @@ to keep track of the agent progress. model.learn(10_000) +SB3 with EnvPool or Isaac Gym +----------------------------- + +Just like Procgen (see above), `EnvPool `_ and `Isaac Gym `_ accelerate the environment by +already providing a vectorized implementation. + +To use SB3 with those tools, you must wrap the env with tool's specific ``VecEnvWrapper`` that will pre-process the data for SB3, +you can find links to those wrappers in `issue #772 `_. + + Record a Video -------------- diff --git a/docs/guide/install.rst b/docs/guide/install.rst index 3b26927..a9bb761 100644 --- a/docs/guide/install.rst +++ b/docs/guide/install.rst @@ -54,6 +54,17 @@ Bleeding-edge version pip install git+https://github.com/DLR-RM/stable-baselines3 +.. note:: + + If you want to use latest gym version (0.24+), you have to use + + .. code-block:: bash + + pip install git+https://github.com/carlosluis/stable-baselines3/tree/fix_tests + + See `PR #780 `_ for more information. + + Development version ------------------- diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 5e893d9..62f2ddb 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -4,9 +4,11 @@ Changelog ========== -Release 1.5.1a9 (WIP) +Release 1.6.0 (2022-07-11) --------------------------- +**Recurrent PPO (PPO LSTM), better defaults for learning from pixels with SAC/TD3** + Breaking Changes: ^^^^^^^^^^^^^^^^^ - Changed the way policy "aliases" are handled ("MlpPolicy", "CnnPolicy", ...), removing the former @@ -34,6 +36,7 @@ Bug Fixes: - Fixed issues due to newer version of protobuf (tensorboard) and sphinx - Fix exception causes all over the codebase (@cool-RR) - Prohibit simultaneous use of optimize_memory_usage and handle_timeout_termination due to a bug (@MWeltevrede) +- Fixed a bug in ``kl_divergence`` check that would fail when using numpy arrays with MultiCategorical distribution Deprecations: ^^^^^^^^^^^^^ @@ -51,6 +54,8 @@ Documentation: - Added remark about breaking Markov assumption and timeout handling - Added doc about MLFlow integration via custom logger (@git-thor) - Updated Huggingface integration doc +- Added copy button for code snippets +- Added doc about EnvPool and Isaac Gym support Release 1.5.0 (2022-03-25) diff --git a/setup.py b/setup.py index 05745e9..2816316 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,8 @@ setup( "sphinxcontrib.spelling", # Type hints support "sphinx-autodoc-typehints", + # Copy button for code snippets + "sphinx_copybutton", ], "extra": [ # For render diff --git a/stable_baselines3/common/buffers.py b/stable_baselines3/common/buffers.py index d36c5f5..5ed9b4c 100644 --- a/stable_baselines3/common/buffers.py +++ b/stable_baselines3/common/buffers.py @@ -193,7 +193,8 @@ class ReplayBuffer(BaseBuffer): # see https://github.com/DLR-RM/stable-baselines3/issues/934 if optimize_memory_usage and handle_timeout_termination: raise ValueError( - "ReplayBuffer does not support optimize_memory_usage = True and handle_timeout_termination = True simultaneously." + "ReplayBuffer does not support optimize_memory_usage = True " + "and handle_timeout_termination = True simultaneously." ) self.optimize_memory_usage = optimize_memory_usage diff --git a/stable_baselines3/common/distributions.py b/stable_baselines3/common/distributions.py index 3d1ff5a..7096d01 100644 --- a/stable_baselines3/common/distributions.py +++ b/stable_baselines3/common/distributions.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Tuple, Union import gym +import numpy as np import torch as th from gym import spaces from torch import nn @@ -688,7 +689,7 @@ def kl_divergence(dist_true: Distribution, dist_pred: Distribution) -> th.Tensor # MultiCategoricalDistribution is not a PyTorch Distribution subclass # so we need to implement it ourselves! if isinstance(dist_pred, MultiCategoricalDistribution): - assert dist_pred.action_dims == dist_true.action_dims, "Error: distributions must have the same input space" + assert np.allclose(dist_pred.action_dims, dist_true.action_dims), "Error: distributions must have the same input space" return th.stack( [th.distributions.kl_divergence(p, q) for p, q in zip(dist_true.distribution, dist_pred.distribution)], dim=1, diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt index 125ec27..dc1e644 100644 --- a/stable_baselines3/version.txt +++ b/stable_baselines3/version.txt @@ -1 +1 @@ -1.5.1a9 +1.6.0 diff --git a/tests/test_distributions.py b/tests/test_distributions.py index 3652b18..07920db 100644 --- a/tests/test_distributions.py +++ b/tests/test_distributions.py @@ -163,7 +163,9 @@ def test_categorical(dist, CAT_ACTIONS): BernoulliDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS)), CategoricalDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS)), DiagGaussianDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS), th.rand(N_ACTIONS)), - MultiCategoricalDistribution([N_ACTIONS, N_ACTIONS]).proba_distribution(th.rand(1, sum([N_ACTIONS, N_ACTIONS]))), + MultiCategoricalDistribution(np.array([N_ACTIONS, N_ACTIONS])).proba_distribution( + th.rand(1, sum([N_ACTIONS, N_ACTIONS])) + ), SquashedDiagGaussianDistribution(N_ACTIONS).proba_distribution(th.rand(N_ACTIONS), th.rand(N_ACTIONS)), StateDependentNoiseDistribution(N_ACTIONS).proba_distribution( th.rand(N_ACTIONS), th.rand([N_ACTIONS, N_ACTIONS]), th.rand([N_ACTIONS, N_ACTIONS]) From 38706f12f34b94236f3bf45b9aaef724569ea997 Mon Sep 17 00:00:00 2001 From: Antonin Raffin Date: Tue, 12 Jul 2022 23:47:53 +0200 Subject: [PATCH 03/10] Use ICRL url for PPO blog post --- docs/modules/ppo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ppo.rst b/docs/modules/ppo.rst index a562aa2..829aa33 100644 --- a/docs/modules/ppo.rst +++ b/docs/modules/ppo.rst @@ -25,7 +25,7 @@ Notes - Clear explanation of PPO on Arxiv Insights channel: https://www.youtube.com/watch?v=5P7I-xPq8u8 - OpenAI blog post: https://blog.openai.com/openai-baselines-ppo/ - Spinning Up guide: https://spinningup.openai.com/en/latest/algorithms/ppo.html -- 37 implementation details blog: https://ppo-details.cleanrl.dev//2021/11/05/ppo-implementation-details/ +- 37 implementation details blog: https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/ Can I use? From a18b91e01a3fa0b49512c3a5701c517643ce1e64 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Fri, 15 Jul 2022 22:48:27 +0200 Subject: [PATCH 04/10] Replace "nature" with "Nature" (magazine) to reduce confusion (#965) * Replace "nature" with "Nature" (magazine) to reduce confusion * Replace "nature" with "Nature" (magazine) to reduce confusion * Update changelog Co-authored-by: mel --- docs/guide/migration.rst | 2 +- docs/misc/changelog.rst | 26 ++++++++++++++++++++++++ stable_baselines3/common/torch_layers.py | 2 +- stable_baselines3/dqn/dqn.py | 2 +- stable_baselines3/version.txt | 2 +- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/guide/migration.rst b/docs/guide/migration.rst index 879a5fb..ef26870 100644 --- a/docs/guide/migration.rst +++ b/docs/guide/migration.rst @@ -141,7 +141,7 @@ DQN ^^^ Only the vanilla DQN is implemented right now but extensions will follow. -Default hyperparameters are taken from the nature paper, except for the optimizer and learning rate that were taken from Stable Baselines defaults. +Default hyperparameters are taken from the Nature paper, except for the optimizer and learning rate that were taken from Stable Baselines defaults. DDPG ^^^^ diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 62f2ddb..81ab0ff 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -3,6 +3,31 @@ Changelog ========== +Release 1.6.1a0 (WIP) +--------------------------- + +Breaking Changes: +^^^^^^^^^^^^^^^^^ + +New Features: +^^^^^^^^^^^^^ + +SB3-Contrib +^^^^^^^^^^^ + +Bug Fixes: +^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Others: +^^^^^^^ + +Documentation: +^^^^^^^^^^^^^^ +- Fix typo in docstring "nature" -> "Nature" (@Melanol) + Release 1.6.0 (2022-07-11) --------------------------- @@ -986,3 +1011,4 @@ And all the contributors: @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede +@Melanol diff --git a/stable_baselines3/common/torch_layers.py b/stable_baselines3/common/torch_layers.py index 8fd2237..f87337c 100644 --- a/stable_baselines3/common/torch_layers.py +++ b/stable_baselines3/common/torch_layers.py @@ -50,7 +50,7 @@ class FlattenExtractor(BaseFeaturesExtractor): class NatureCNN(BaseFeaturesExtractor): """ - CNN from DQN nature paper: + CNN from DQN Nature paper: Mnih, Volodymyr, et al. "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533. diff --git a/stable_baselines3/dqn/dqn.py b/stable_baselines3/dqn/dqn.py index fe8f398..0cd6dfb 100644 --- a/stable_baselines3/dqn/dqn.py +++ b/stable_baselines3/dqn/dqn.py @@ -20,7 +20,7 @@ class DQN(OffPolicyAlgorithm): Deep Q-Network (DQN) Paper: https://arxiv.org/abs/1312.5602, https://www.nature.com/articles/nature14236 - Default hyperparameters are taken from the nature paper, + Default hyperparameters are taken from the Nature paper, except for the optimizer and learning rate that were taken from Stable Baselines defaults. :param policy: The policy model to use (MlpPolicy, CnnPolicy, ...) diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt index dc1e644..035e3b6 100644 --- a/stable_baselines3/version.txt +++ b/stable_baselines3/version.txt @@ -1 +1 @@ -1.6.0 +1.6.1a0 From fda3d4d748439a6755260896e4350e0383a9c6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Mon, 18 Jul 2022 11:22:19 +0200 Subject: [PATCH 05/10] Fix returned type in predict (#964) * `arr[0]` to `arr.squeeze(0)` * `squeeze(axis=0)` to `squeeze(0)` * Type testing * Add type test for unvectorized observation * `squeeze(0)` to `squeeze(axis=0)` * Treatment of the laziness symptoms * Update changelog * Udate changelog Co-authored-by: Antonin RAFFIN --- docs/misc/changelog.rst | 3 ++- stable_baselines3/common/distributions.py | 4 ++-- stable_baselines3/common/policies.py | 2 +- tests/test_predict.py | 2 ++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 81ab0ff..b258547 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -17,6 +17,7 @@ SB3-Contrib Bug Fixes: ^^^^^^^^^^ +- Fixed the issue that ``predict`` does not always return action as ``np.ndarray`` (@qgallouedec) Deprecations: ^^^^^^^^^^^^^ @@ -1011,4 +1012,4 @@ And all the contributors: @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede -@Melanol +@Melanol @qgallouedec diff --git a/stable_baselines3/common/distributions.py b/stable_baselines3/common/distributions.py index 7096d01..5247751 100644 --- a/stable_baselines3/common/distributions.py +++ b/stable_baselines3/common/distributions.py @@ -578,10 +578,10 @@ class StateDependentNoiseDistribution(Distribution): return th.mm(latent_sde, self.exploration_mat) # Use batch matrix multiplication for efficient computation # (batch_size, n_features) -> (batch_size, 1, n_features) - latent_sde = latent_sde.unsqueeze(1) + latent_sde = latent_sde.unsqueeze(dim=1) # (batch_size, 1, n_actions) noise = th.bmm(latent_sde, self.exploration_matrices) - return noise.squeeze(1) + return noise.squeeze(dim=1) def actions_from_params( self, mean_actions: th.Tensor, log_std: th.Tensor, latent_sde: th.Tensor, deterministic: bool = False diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py index 51a3d37..a88fad6 100644 --- a/stable_baselines3/common/policies.py +++ b/stable_baselines3/common/policies.py @@ -350,7 +350,7 @@ class BasePolicy(BaseModel): # Remove batch dimension if needed if not vectorized_env: - actions = actions[0] + actions = actions.squeeze(axis=0) return actions, state diff --git a/tests/test_predict.py b/tests/test_predict.py index 853f4d1..89cdb09 100644 --- a/tests/test_predict.py +++ b/tests/test_predict.py @@ -73,11 +73,13 @@ def test_predict(model_class, env_id, device): obs = env.reset() action, _ = model.predict(obs) + assert isinstance(action, np.ndarray) assert action.shape == env.action_space.shape assert env.action_space.contains(action) vec_env_obs = vec_env.reset() action, _ = model.predict(vec_env_obs) + assert isinstance(action, np.ndarray) assert action.shape[0] == vec_env_obs.shape[0] # Special case for DQN to check the epsilon greedy exploration From b1cc15970a40c86b26a247fab5783e025bfe3da1 Mon Sep 17 00:00:00 2001 From: Adam Gleave Date: Mon, 25 Jul 2022 14:02:53 -0700 Subject: [PATCH 06/10] Use higher resolution time_ns() and avoid division by zero (#979) * Use higher resolution time and round up to eps * Update changelog * Add test case * Fix formatting, time()->time_ns * Bugfix: ns is integer not float * Move test to better place * Divide by 1e9 earlier --- docs/misc/changelog.rst | 1 + stable_baselines3/common/base_class.py | 2 +- stable_baselines3/common/off_policy_algorithm.py | 5 +++-- stable_baselines3/common/on_policy_algorithm.py | 6 ++++-- tests/test_logger.py | 14 ++++++++++++++ tests/test_run.py | 5 ++++- 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index b258547..b1ed3b1 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -18,6 +18,7 @@ SB3-Contrib Bug Fixes: ^^^^^^^^^^ - Fixed the issue that ``predict`` does not always return action as ``np.ndarray`` (@qgallouedec) +- Fixed division by zero error when computing FPS when a small number of time has elapsed in operating systems with low-precision timers. Deprecations: ^^^^^^^^^^^^^ diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py index 36a73b8..e8032e7 100644 --- a/stable_baselines3/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -422,7 +422,7 @@ class BaseAlgorithm(ABC): :param tb_log_name: the name of the run for tensorboard log :return: """ - self.start_time = time.time() + self.start_time = time.time_ns() if self.ep_info_buffer is None or reset_num_timesteps: # Initialize buffers if they don't exist, or reinitialize if resetting counters diff --git a/stable_baselines3/common/off_policy_algorithm.py b/stable_baselines3/common/off_policy_algorithm.py index 99a02ff..b841eb0 100644 --- a/stable_baselines3/common/off_policy_algorithm.py +++ b/stable_baselines3/common/off_policy_algorithm.py @@ -1,5 +1,6 @@ import io import pathlib +import sys import time import warnings from copy import deepcopy @@ -427,8 +428,8 @@ class OffPolicyAlgorithm(BaseAlgorithm): """ Write log. """ - time_elapsed = time.time() - self.start_time - fps = int((self.num_timesteps - self._num_timesteps_at_start) / (time_elapsed + 1e-8)) + time_elapsed = max((time.time_ns() - self.start_time) / 1e9, sys.float_info.epsilon) + fps = int((self.num_timesteps - self._num_timesteps_at_start) / time_elapsed) self.logger.record("time/episodes", self._episode_num, exclude="tensorboard") if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0: self.logger.record("rollout/ep_rew_mean", safe_mean([ep_info["r"] for ep_info in self.ep_info_buffer])) diff --git a/stable_baselines3/common/on_policy_algorithm.py b/stable_baselines3/common/on_policy_algorithm.py index 763c108..84c89d9 100644 --- a/stable_baselines3/common/on_policy_algorithm.py +++ b/stable_baselines3/common/on_policy_algorithm.py @@ -1,3 +1,4 @@ +import sys import time from typing import Any, Dict, List, Optional, Tuple, Type, Union @@ -254,13 +255,14 @@ class OnPolicyAlgorithm(BaseAlgorithm): # Display training infos if log_interval is not None and iteration % log_interval == 0: - fps = int((self.num_timesteps - self._num_timesteps_at_start) / (time.time() - self.start_time)) + time_elapsed = max((time.time_ns() - self.start_time) / 1e9, sys.float_info.epsilon) + fps = int((self.num_timesteps - self._num_timesteps_at_start) / time_elapsed) self.logger.record("time/iterations", iteration, exclude="tensorboard") if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0: self.logger.record("rollout/ep_rew_mean", safe_mean([ep_info["r"] for ep_info in self.ep_info_buffer])) self.logger.record("rollout/ep_len_mean", safe_mean([ep_info["l"] for ep_info in self.ep_info_buffer])) self.logger.record("time/fps", fps) - self.logger.record("time/time_elapsed", int(time.time() - self.start_time), exclude="tensorboard") + self.logger.record("time/time_elapsed", int(time_elapsed), exclude="tensorboard") self.logger.record("time/total_timesteps", self.num_timesteps, exclude="tensorboard") self.logger.dump(step=self.num_timesteps) diff --git a/tests/test_logger.py b/tests/test_logger.py index 6fe536f..a55f88a 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,6 +1,7 @@ import os import time from typing import Sequence +from unittest import mock import gym import numpy as np @@ -381,3 +382,16 @@ def test_fps_logger(tmp_path, algo): # third time, FPS should be the same model.learn(100, log_interval=1, reset_num_timesteps=False) assert max_fps / 10 <= logger.name_to_value["time/fps"] <= max_fps + + +@pytest.mark.parametrize("algo", [A2C, DQN]) +def test_fps_no_div_zero(algo): + """Set time to constant and train algorithm to check no division by zero error. + + Time can appear to be constant during short runs on platforms with low-precision + timers. We should avoid division by zero errors e.g. when computing FPS in + this situation.""" + with mock.patch("time.time", lambda: 42.0): + with mock.patch("time.time_ns", lambda: 42.0): + model = algo("MlpPolicy", "CartPole-v1") + model.learn(total_timesteps=100) diff --git a/tests/test_run.py b/tests/test_run.py index e4e8a2e..b0a9a11 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -10,7 +10,10 @@ normal_action_noise = NormalActionNoise(np.zeros(1), 0.1 * np.ones(1)) @pytest.mark.parametrize("model_class", [TD3, DDPG]) -@pytest.mark.parametrize("action_noise", [normal_action_noise, OrnsteinUhlenbeckActionNoise(np.zeros(1), 0.1 * np.ones(1))]) +@pytest.mark.parametrize( + "action_noise", + [normal_action_noise, OrnsteinUhlenbeckActionNoise(np.zeros(1), 0.1 * np.ones(1))], +) def test_deterministic_pg(model_class, action_noise): """ Test for DDPG and variants (TD3). From d532362e94064f7059166a8e53aa8efb1bf253c0 Mon Sep 17 00:00:00 2001 From: Marsel Khisamutdinov Date: Sat, 30 Jul 2022 15:44:25 +0500 Subject: [PATCH 07/10] Adds info on split tensorboard graphs (#989) * Add info on split tensorboard graphs. * Change wording to make it look better. * Update changelog.rst * Rephrase and add link to issue Co-authored-by: Antonin Raffin --- docs/_static/img/split_graph.png | Bin 0 -> 21464 bytes docs/guide/tensorboard.rst | 10 ++++++++++ docs/misc/changelog.rst | 1 + 3 files changed, 11 insertions(+) create mode 100644 docs/_static/img/split_graph.png diff --git a/docs/_static/img/split_graph.png b/docs/_static/img/split_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..c966c5565ac2c8fe2427952aeb0bb970520448af GIT binary patch literal 21464 zcmeFXV|1lWw>G+xj%{>0M#px?wv&!++wR!5ZQHi(j*X7h>Hb#wdHL-<&KY~0v;V!x zSoc_~YSvXXXU&?bJ5)|a6dvXy3;+Ot7Z(#!002NFfZyk!Ab?M5i_mNU0LQ~!N!3w7 z&xOF&-p1I>(uly(&DMy($kogk0B~I^OgC|+YOgE!ZGgrN=F0SGrEv?+{pk%nY_4d6 z=P_$}E{;%52*iwt7$s-*_oGkLGp}djN?B;>!iIg5=O9M<)lbEX+r=vzlf5^uZhfD( zqb?qsB%jXFTcvFqSG=X4Kb`QWySIK^FPrt;vC3suV-782s(VK2bi?1Z$4bp!<<{E?aXejTY2DaCr+0pIb@mpG{lxHsK8E}15OWnxyc9uoZTQ0*5|g^yt7IC? z-@5}k(+8&8%T)7NVE;E3O1GA2b*gU6_w&2wL8n90vg_sWF4u$GZkw%BI+N6^t*~EC z;!no=Kc63$k0K?zJr2@-aeJSh@&3MlX1adNoMk&ZJIYX=)?z9@W-vJYslG6ba^<5r zOd#J}6;x`Z@gxo>f9bNyneU?V#<8_4n?1=_0H*)~+AXkMqH= zhVX-Iy#0H;U7OdW4K)IT)u_#oS@+ovc8^VyG7eWNt+)1?r?}rZ1_!&}SbsmjACaSH z#M8X)=^Y+)MocpAbG#iga_vrymz;cj+IxLX{`v8TZd_s_5qy7?Nl5GlJ_*$FtRF0@ zPqw3-Q71%?&;}<1Hs%k2wgX}7x#e{idoUpt!l~3|@G&HwEyeR7i5DE+h{S@~XVwbJW)||&3Q7k(WrW^A7DApGePvZ=; zLp9?}tA~THs@ub3hM4Hj&sb3sCKHzwsV3k!NnI?(%Y|7vY zg4}cF!F}BMvWFi%=I~p`n~Bu>I{o)|_o)%5-*XPUZx89qNnQu$Kdg7SX^)i0Njwa$~{Q$+3B!;}x#DrQVPLmhic`B9r$FU8E^Z8&Ft`0;G7 zQVTd~6~~NpvbtE1k#ILZPumz4Wv3VlGq8&XuiIqbfy1h08>9AkMZW#HGR!_61cs`}u?kq^bA zK{jNbleb@qhYVT>QJqaLvmt&)Ka}eRM_Z#_4mFFX;zuwTHe5pX?vB#u=^ivVPZfAL z>XkR4%4!`ae2b>Px7JA!`5Y@#E1YARKt4i!P){Z0U7*tYrp zRF4+JfsB6Fbcknp!j4j+GgB4Ux;`>m7&FbYEk~q+Sfe$c6lrrtl0u*2_m&q$JgGB82z{ zmcr2D^FEm8CT+qEmPO$rq8Hw~soa@ScTs5?$VhRK(>b(63Y@=9>N%m-hr6>-8aee6 zvNL4qAZxc@;l>%`ia6g+Qp1*EYgOrQM^c)t@fB((R+#0Si?H|)_;cV=dP#(vsXWM) zH`tXn@jwaR9?31{Gdh?LR%uMF(?7t!!W_ufB$K<{uxvSO%v8vIR{QE9@0&IaHCey5 zrT{)x3weyMc}dT%fZcKpwlYxYEpXT&0%-0pLHC(Zd}OQ{P2}I#1G&3o4JcFuyB170 zLas;kRWTdTq{uZOXGJSPD3E^3EFJE9GM33%g0+@cH_C`CA?-y4y(EMR1(1K}<$<{d zu!R>YJf%S*m9DOvOXCH=L8?H&gH#A>P?*|b*z(Hs`TZ1RQu)HPjlaCBbsJD2;X;A& z6z_um^P{!UPH?F&v=S}*=UYF6>I&FLf&>H^1MMt6d@h|(evsP&733P~GbynFKU0*{ z4`g`XdOT~n2yj|*LWu5pia{GY;^onou5L{@+03pSebvYuE9-B(6bU87TbSY7>|huE)V#2)-&No`;*gS|k#G3P3w@pP+acR;-N}bD6;FRal0Sc%jG=MU zv%_fU9~`aispNI|)u7vAfvN3Nq0K6F+N%pjOoV#vK-l=pog>gogoM zwdB%A@VG5}(bq0cP|%~Vpv~iFJ(c!X>QQi)MR3fMqcKoIkjzo(AyNd^eR)uh+k+k~ zEkUSH4V`{IC}8mWS`>zuY5pXv;G)hsC{YUAx*BtkU{Yp_Zs#R)JY~kHdE%oYtRN=c zD|5ON!_Cp9hUBw$TX=DuBq`+lp}D-e+9cH@aT@|(Jl6UV|Rml91ySyR%TCYwWqAx*(ju!QCAf;AYNJWOk{Jse#2QAod zd5(NRN`oNdtmm|NognDZpZ2R)(Wf>LX}o*|Xt}AV?KyZzIZg{>3@~L9Gm0G?=;T5) zlWzoh3L6Lrzx)MEI7mc=N)wUG9B|M(Rl?okbfl>=#xA|#!0K#-m>d&4=%EX|!^{#MuVzdQdoX%*Dc~s__ttrmTGfh-;=ebr@By z@8bY6-$#f-5PJO=raN?nt7dM5-Z`5PEPOM3@>!SKlqy0FQIr8Qz2xBUFTFxmtEz{3s2!hU4%@b3awnfhgac(vlqU_j$o8N>PDKP`6Onp>hfnHq@ip&xQ))798{s$X8!|?lGVI$-OsFkQix8TI~r2 zSS|q(a_bZ)9;CHs)multkX4&EfO&lj>mi}h^blB83Qf`{+_Y44CB7;qa8@=(7J1ms zgV1qdCtAX=?PC3Q$%Su|2m^oyG8S|r#IkA2$TiX$4Qf0_uCTt#T!(}Y)nq!?P*gDR z_8-*SwpJ(?uWppBL`xZbF-Cu3j1=)~)$Zho9+s0-W9p+M2dGm+=~))Ii&KA(;-dlA zC0&wUl`WGD|V0O==ZaDz1#w4rfm6jc?R{eh$LdPdwT z%v%0=*$sJg-JVLQwEj@9+~z9$x}!~A;c$~DlNSGX-G+KCoM}#*h#&W$`L@4Eo;Jg6 z*VVy3E+u)H@D1fUuheq{*F-QbDg+=$1UkntA*VQ~M0wIktFoE0=}4E;E(gh&P{@{@ z!AnGgMJ`GOPfII{gGfQa3u=7bMT2~hFpmTr%qRCW5%dCBGlFxxgNeilkGa`xT!n z%6GG{+%RVVvB0Kvv!fPG_sin()&`fLY@RI=SfZZIo*p-@aySOb5TK52kEh8gS_FVH z87w9@J^So~esKD;=_QPY;kTEEuBwP>qYuLUw>a;h#r5+E6DjD;B@llhcRsuo`)}%W zJL+C?w z7G4xljIy6~R;i|to`PPWb1EL3sXe0jC9O_#`4wf*eLA4=6FIcq;TnGuipGG)C=P5zf8j*k>S>teW&pSC}{!0)Z+WA8yp2!pdoT9VLMl-Vv5K|I~#Jz_+X zDB;A6$snw(3&$IG1kAn;>l3)KXh=`koOKejBSxZxY-vaY%ib&MLAn3-YHmbhl#mcQ z#(|A?(G|kPiUYOI`>a0VnoV(EB*9h!=MBdaU~41!%{&T>dQ+B;E8{{6ZGSs0qm^B| z3H+#YNkSM3;afUv;e4z`!Uh49F^rt?oNT6eHMQ>-?C^+*%V?TfIjk6t6>NXHAp2JW zp{prUv}jZ8H5$_cXK{aD%rNj^xT#|8GyQ75z?H{qi?cGnh48?8P(toX8k==u`j4?DDJ6Pr+TmO>mpL)Zm@nx0zH zP`P!{g(dda@LM=-U;#};{oSK%x3PLUx6)4!j7e`P_=UlCa)AfUJuE!k!d30Duq-aG zk`fEwjgz@Zsj|iyAmt>@@ycNMT)SwhIF9f`0}WnSdSOoc5s@R)``*BZ&=H=~tGfP&j8c@Ax*7%Q!zZvHv;e`Ass}ki+PJ4t%SqPARIcuI8I48DPci$qP}RV!CsRQ=%dX^@6v+uNM2@`*>Q zP9uQZe?4Z2Gz<(>4Ga^SH8WrhVRhN0il89h`&Uds*c+Pa-H8xaMX_r5(NJzuP~JkA zT6fE>v~(aA*$1}<7R+oSdSSO6!QJI-4+DIIG>y2ONY2@N{HwR*DKlZVRM8&d@$i}T zEqI<4k{vkuzQWRl`Bhq1G}6yQ{76q?0-XvOeuFX~(c@yRMT%dAKUTTe{&zM_>C-&`r62vQpVi+XKk*{O_ZZVu$rj3Xq>D9z7Ak8_O`^JjF z-cSMVUm8(r{BYD`0tiJj{X_Anreg{&q!PJM0 zz7m5h5;ggAvMm}lgn$VaNTRRuQLs%k1BigM^o8J^d6Gy_dV$E%c)~7ni0xMn+ zVr?LE)uz80HHu8S7^bSv_=6O`7G@V-FQ*F0|2od}X)Lb&eqr3*$kg2hjttbpg>qtUhz!h4 z9hrZcev`V|2&b-BQnDZy@^#^hFfDSCAIX?)!aw0M$QJn z+6A&RI&sCh%7a-1XL4^bVc)L9!YDP`5dVj7Bm*#}@;{Xz%`D2JXXGvfadi6%6gAi2 zgD8^n6b%UYuk%(<rk z>gYX@H|MLq&f`54{UdHiIO{kW2bZQ~vc+(cgSi5_*k_y>)mO%{XC|{NmZeSf)?k3x zV?p5Xt#{0@ce=Q1LORh<`7KBM4FXA}9@(aGil0|SXkz%@hOTX43rTiB#8JvB! z%6>m~Uv$&Qpta_j1+ZNGj8pjAASk@^jcUD;s{J$^Q|1{eMH{FdqN-ShDf5e7&zzLf z=o;=%dpGc(K1}p{*%n74%lwYri^DAWd4$5P6QxQoST2|MpyP$_M{pe%6f0V~bR3dK z#cnA25nwEh31oiq!uX*$eTx^rKtOdR#`>#3H{A%=1T-l+k9IqapE5_%-*C0K?aQ+Xk$l83^R~8ta4M|Nb7JV24q+Z=BVDXpG)CiX(?DD9pu+;xdS;Pw)! zpLkW)B+Hh1@O)9@H(u+NpJb+65wD}>6CNc(QBaNpEg)@5P71eAju17t{0fnFVRhrm zxWYSnnFH)?$mVcQ8TKA5ZHvT9)lx_(L=mcPHX|@j-J>CTHkO5wpL)0h?o5AsEz$3OU5m*@u{g@vz%yPN zF3iaY*Py{Ynp_j5co{(oj9k-el-7Uz#jMI2A1Fvb9i%ab4T&E-E9;Pu4NrWqiYjDM zEFs?>eAL*C30d1J-*U=EW!7Hig(OJESK{hMz(qWmZGkR0U_1{Q47#ltTD$&f?1Kp3 z$p#T$091rkBTN5BYvv1SsmzI$bOCmlfR4|7b#7zyjYv-UWQKv2j>ai|NCMsp&KBBR zmJ3fg9V(Qa2pgiAx*N*FCkslrBxwk%AST2jm9ChXxBSB9uV`L}_G>XV>;);%N?)?l z8lMY_(s@2JJuZFb-0l$L4ue&{z~e@2204F}XD-cJ)(c~IBR@vDvXvV* zAg=3-<~kdP3&ah z0aWl_DbykIdweOe>c#IW5t&`ra(rD4rJS>nLgizDT2MmT+D&9bVSMy!^^~R@o8%r_ z?0!ZMU)ts<=;&glGp;>|WXom9k5I%$$l+@k&+#xfUHD1i%o7)v1E~j8E1it>(HwsH z;GEH}-@tY~E)J|OP~zq+#!MEFpHnOBRlLq}rc3-Vi)#7GQGZw?Z;lEW9gydl^^R?? zvX?3?-vp;ih+-)uh_n@(s}XRtWbg-^pvf5$`YK$&?m%kZPBFqvw2OFw8uM?6FLbCP zB%pldFq?aeb!)-jG8&f~?mbbA-jK(ieii%TFCIxkrFY`kLa;IIC*!dS&FL=pM&=A_ z44(Hj!EgYz=)gywBHYXKHNwwMXA3by>p0!&{?8C0k7xY%z)Q$ zs?t)N1~yi-dWJUoMzpS0w!mvT0Dz0v)mG2I!pM<8-^j$wnw#jVy_<-@%#fStD~mL} zw5_0#shOC&y^*53jFN%7g#m{l5ibu6mn$a_z{&Yo6PY?X+H%s- zxwyE{x-ik&*qhKXaBy(Y(KFI9GSUDkXdK+E9ravktR0BoA^yS;GIB7mH?ws#v#}<4 z$JEoeadPA)A_Ddk{C#~^w$jr7fVXz|n-qZZpmWu;rDLF_r?ax6`_~Z;j>67BkiRYH zza8PA1ia0pQ!sL{ak4is5_UGSb|n5+2t$K^jJI{NxBSD7p#hzdrI8ho)B!jv!@o@_ zDlRSej}h-8Ffp^T{WA(E?0=(lG&BCEu>Ngr?>&Fm`PYR2hyMfj->Co2`yXN;m9#Xc zkd1-U`|`wvxQX89=QOl2Ff-)*>noccyOF*T8x1|H5ep5o0Xsd7J_9oc4Lc(PD=UY- zp^=e3{l7qoTRS-FSsNI=Ljl2Q&44&;#(GRl`g(>mEJkdGG|YyE^fW+B1{zi-eHMCV zMtXV{4x@j8khM1hYNej#zpm;X$`A;pr)SJy%mUO4;N0{y%na-XG#tikj5LPq9PG?2 z`o;{5hI)UX3=KF%Z0xP{fZJ(irDtM9XKQWpr{i7Vocwa)+(eADe=Yn^i=3sNqcLy* zH<6T?wUg^VNtMj3j1(R9-lfUF%ECd<&c?yO!NNq(&cyIfA{8Th2cRatV=~awGP3{a zdEXXJATvP0>b>hJ5a3TckQYutdm}wZ8+#=i8%u7Y_XQBVkNl(F1YCb@ikO)LkizX< zdfam*f3ibDP zv;StZ4D}5d*?~Jo!=c9t+;2U0V;T+yLsl9+HV$KbMkacCV`j#GM|ZF>c68CRH{v$| zath=Is6c51t6_gU2E zFkoR|<6xyR*4Ja8VP?`ZqS4pqFrqPFGBPyKV`5`7V*N{r{vp8sPb_jMF)(v7vv4wS z{5y+Wbnk}i@0-d+_g}XCFM@xubU@Mk)dsYnK!Z&84~zUanZ0Yu|Ki`@)a`$91|al* zCix%9_dnwLkGTFv68Ikx|3|z2Bd-6E1pY_F|Ix1h-^2y;&z#Q48W`xg05djUx1M$2 z3z~suf`W45f`Wg49}@r|kM)Y>7VGExG^jH#p-hS3j3hrImq{HG#%_)tCP%mcQ^}d4 zxu$M97gNx}4pRjV$%|2rKs(jO*#hIg4H=RjADYj1F zVg@Vcq$yQ~Y{1f57_OTrg3ysLDrSP(KqXG$2d>T|)Q@}GsrsMOZyfFm9fQ8UVqE=j z^JLnev(_5mr(Or-n+VGXjt26{v|^rO&%%anGQw$LxtPq2=Nty)^=ns#Byj5C6^XM* zqDs?bo&}Z+5^pg~Dm9HJJ#H{XZeDIsmdy}|Lcys8{Gq^=6Xb%Vy%KZc^$De4q49tF zo+XbI#*~AJM7b&D@H@Y4`}}CvD5qz*Bo#*C8xG&|g!{0gl_T$vab+-UmI3HB*_N<* zrgGAKiN8~q495lOT<;lll2m-~9mmc2JsKmun&{6QC;;Nhn{P7);?>OeShlR;fM1Nr z7t|)P34jdl5sR%`_T1WB-CFN9Nqc((sJqx_E@OVA1*YE+`jVnTfZy-`Ii1Dvz!qp* zF*OGO02cNA0|H3PzydZxI*LmRLmoonLero&ezQ`4V12qFs`fe zN0AKZpAkNd2&WL~UV*1!2&5IYX7SDyz|%ZxZ#7LH(QUEWqy-;&-Avy8q@6AwKS{+A zMgrq!r*yaTBY^UUD*S(V2z(u!*}`|7zmJcP3Wmp(G7=`O?s)J?vmGgbHo*3IAosp~ zQkdK*0C|I5xu4XYT$wL7LEK*;onGuj8ON^Ojw)XK6$ucL+QYkUr3+sepd6uqdQJEj#m{k6DM$E)&bM4usl z`GhoxQx8u3ga*o=K!D=^^${Wm<4^G4Jsdx-b$ZMcgnpOi#A9`&H!?b4KWCL@A24JD zF&)D5RVr6jd?+X^Qox7ORZ<47T~hJ`0T!HjA71^cl)AZjaX6uJaevim4bI(DpFBb! z(No{7HLBq7Y1P7|0EgxA_Mr$CoGLYjG`kebfMy=L2S7nVYOQ`a!(XO&K7_u|^fW6~ zQEIKjWVq(D;$p0sDK-7PuO$Ii#rj6~1y zSj}*WX|xZt@qQP+HSt zxog93>yPp(%N=l9Iu`~`sloO0G;ocnEnUgn;`BD_s%yKiLge85GmdWXIj8p}?zqn) zEBVU~B5AR!Z;YcXY)MUtoYB8dsyI*O`i%{^6r}y^iOELF`=#= z2;*N}*n6Yrq6N_MN$y2^Rfq(wPyVbS;J$XEvj_}LX;H;cRW|qk(c9H;SysB2tGBwJ zro5E#b>=|A4q1Z#cpns7XZJ(x>X}2PiucTC?vu7RLe=NW_C<-kBK^|S`<4%c#-7?D zJ8G?=3O|2cBv#fXhy^Gh1t_A3=I8a79ixDrFk)k+>&c*uxIR3Iv_g7hgE}|dJn3mt z<1np4{cbIdkgT6?Dfb~mL>xb1+)H|A6PjtbgPzr~2EHWjBuvhd*HCB-vS;bG2T zsU^URU7_uziVLL1$4|B%MHuu7hKC3dMc}c-r59@sGfh_#e4z?w5{=r<6^;g>w>pi? z;IIcvZ?-fVYAEe!uxA-0EnKl=J)BCN^bd>{TY&8ohccWWO(_}}yngM(W!|*IC2e}R zj^V{ zYr4>M8#Dq$aKC=3c>a+;(0En({AF3&ixfEfV61gNYc2!dEl zQJG$Q0ng1Sw^bV6xD?7*<$9NtyWPMKJ7@BketD?Ce(|ev zb62Zs;PLvTNCH`uiZUEVJN68ZN+^9YQum@Xz{rDhDTiJW7PV%Zz?AlrS3_SaDD|_8 zOWa&Lg|P$7S8Y4&ZnrxZ09GXBjvUFgJNYLOma-C&L^1ZbyDeMx0CUS{SxMfpc+nSm zy!5HH=(^@rU*YP0%uq%yWh3wIA&TU%7kV4~5`+5BJ!G;3&U2;97#lo+OZcZ3vOqil?Hb%V(*#`D$GCKfwiZkpO;tNZ<09yX56d5EoG4 zg5=UNAmz|L`0?coA|`kze2%`$=gOjY>hexVEsKKthL;Y~b;ZcQ$b#Xb(LsnubFYXU zLeiCo5uU3!B!`}p*IyCLr!>6mqz93fh*ABP{&_C<@hag({@0HOjc479;Dp4)&x!Lg zU&%be!vG`pt+wqJP=Nv^qJ|i&w3y15rx?6)FY;MjbI~{K2il*{M9o-^#hbB4>r$T< zea{PS+A4=m8ju7X5U|u3iZIoR3|CuyJV>t`?+h14AM!g+J(3kwv`35Rnn*Ig|9B=d z-tdOS;r$NQQ<$td@-oc|kh`cVBH+#L6c%2&aGp?M(yoq>g%PL^6E4n=9vpe5rQ*HJ zPEBd;vNAAB{@soF3@WuQH}*^md-O12C>oh8`{=bVaqvhc4ha+}BA2?nWVw#`GHHnGC9rLCD@QC^Z|wk#QjlbXIkAb@mKDP!UOjA8Ad5#D{eRT?4iB8>3E* zBufNRE*jWQLu#CfY{))pd)wDi8leB~gT59|jBchvsIR-rF^mOTF9N`EWmLf)k&f?c zlSDig(1;=d=nw7&Q?f#}p+QJ@k=|Q&kKrgqOzi_*~|1GRf%GVM>7f9MUGjG`-bLD+mi2*e;Ofg^W9hfy@P1B*MPNw8 zZaI=uuvbBi>D`SVK3D-AJn)6kY!EM`c?88HO2i<&@u6`vWfR9MEVQOM$o9V0qy_!T z58*y_U?y}h#yH@n2&YOMLn1jI_e(d?_{l;*KlS0t+Wu)O}Mk_4Thw1GL&ahoEeQoQzMkY_KthqPueEFsyrB>BN#<+I68MXj`0HO;HM6a z1jba5cfxlJ3xmnkAB1lp%m#?QgXT*K+`dIG@=GuJMm8RI`AWEISPD!ti$S^0`1j}@ z^q6oaJx;e89b4@V-0evdaagA$38mp2`!glO3$~w&6A)6`QcJ05os{bh?me| zU+CQJ1NqyLM_p81_xV4`swuM~c`l5g`TOb_tQ75~Xswdg||z$tm|G^TrDAE4ah@hZqWwFp+J4qaEGt`fn1^3xv>%{n^{Guv{@MxQJ85k2fIB~Xa$i#BXOivXH)n6=Mdz4D=T8g^ zWVlE;wnDpE0l2Wb*pueXzBbtBm(A_@v6hJo{ZXLDfTwO1)#00C%Wh#l>i$nK-dhG;+4(=1h^@#)D_B$!i z=?2B1d;Z|$`4J z@i2FA`u}n;_7QD_fg&+022Dr>oe99Exc^=*rZ`#Ys z(mG9mY8;h2k0J4mf2WacE3aw_v8UYTCw}+O!Re=a59YebG)7#oH>9Oq!NKKXyv8tB3abC8e zp%47sW#IMCbq0iwn6I@yXS0e?ehFx(CkT+7xF~{|EMtCZ#wgWbF_ zZJVw>kw=E@rK|J#Pd32fr8&&atpRrGX4I3qu#0m~l%WDt{9_ z%%%uQd=vRg=f0+c%`QWo@VGaT3<@wnM;155O1@i);dJHVu8O>9TD|`PFCIP#&L5y~ z+(d+ZvxLOe!fS@;GFz8DvL6XF6(M3n!i=^#BxXVF?wb;2%atgi&_E9mV%Rd3Zt6Vi z7^Lx03yLT~ptNL+a5%J<07>Fq7WuCk24RIH5a>90m`!z(dkU4jUf+;{85YJdHf^&l`n`Br zVE*6xfn^8D#2DgTgO2CF$SU&3$0rJl1by)Ff_y|#f;&FvSYfOu_|$j#9r7Dwo{ZNg zuV3VO!IZO|uk4+6f?u|#(c+Al!<%~`GzJApk!(YrzHLSodR8)K; zCk=|4ync}%!z!Gi7`Mu2s;<>i7fwxB7f2^ENDwGS9|a1hZBF&pbU8DSC{7@q5W8@u z)RZ#rKGB1o(_^RY7Fz0UI?gH1-!Q>nckCpcym^l(mDd5f=$8p{87*ck8c9By5Xl~q z+SeDGh$(G2ts87W?x{T`ew$_n3D8o8h>3gy0UDJ81iWk{@vEtOqPX{p0nw@w$s~9` z>Kn-c2OF?q=7OP3AC>K0vbiHzg*hCK5a=x2kb>r%0HAZJ$@X%~JF&iROPt#lI9AP- z`Y$%Oc6;$8#e2PS|6zjgW2Az1ZFJw!SHORlX>?F!B=HRi&pAC$41DVeOL`N+`%d84 zAt%#)fwAOOmz9^dfAKY*mOc7gFap`Y^@Eur$7{u`m@6lxOL(OdN`00aTf>dURUTP07Wk zSRTnFtWq10ITPhr3vIGJD=?@$x@h_=We8ECNILNlg5@F^Wl1tp#8@|@Qg|K2^A@UH zFYG9Fd^9si9eHvc+0Qp_-f-LuuYEos0fTEa(r3vWEGRHgnl7F9I9IDOIcLh0KY5iZ zNgU4^_5MY^>IFUsx)#H8RxmMO>n9ENpvDNh-z!j`|I}r6@J~cn9~c(a8q9em^2-=a z9;mZ4U!(+}1apzf%E+{m|7IX<#CXRP^ULp0qzvP-^!-^9!4NM$`yB2WN~PG3sTp=8 zZ`Xo$a>sfnSm6j-@t5D{FPu33bbXN0b~j?VBo?Y!(~{=*<^jg&HR~5CmBt;~*x6`N z+Sb7xEXQy|LWtrqYF4?h30s&tt|0wEa6=^@QIEKSJGol#%wWEDjASyI_{s#2ni{I@ z*k4z~{1mj1iytBh6~X@T9zt-P5K=g5=V~^%iqBoYS}fiCN^7LiKD%~J)1rft7Op$F zZOYH(Q&5CWUBqaJI={zQ(HR7fF#zDg$Gbk zDyCAcK|w_EUT}(wOEb0BJMl4&6O%Z?&#fn1HwkEhrWVBjoa|MRorS6SI{rd_M`B2q zk``qsj&r!*TOOxA{9x^3K-Y`VG`Xl7oCJ(W7Z(+kh!`CE7#4b4 zq6|daVGn+NP@bF$B-ZoixB9szOjLXH+Y}gEvxqK*ai5TZ2czV2VJ$>R+kI&;h~`&( zl(Jx0Siq|soSBWGiRcp#HOyKJQC!SDb=pPBT_=d==z|X5SCfIoJX*s_H9F*F)U2R- z{4tjG(Jt!UaoELbGStfMZa}3+p1H3d8!10;HofhVS z%6ep|HYopY?z~3p2eN87h;EOgwKF}?kkP@AC345-*(L+Zj>QzwL-5|(;?{iGC4m~! zhQDh3q12q3Ys_mHLjbtHK%eMbPzJMls5;ttWa}#j#<;t^T>bqfj9tU~R+rI|FW}c68ZC*N10Ja++$SD`$aS3sMmC)BXY+^j{vD zbW@{E3JN%49S{uH9Y^&Fc4-6x*e2V)-ghIThs%h(Fg=a&)rND=N43Tm!4oO0iODKe zMZoGs`sC0CN4;9j59Dj>&&1Q_B%1|*nrhk*`D$5|Q3jvqFs|}g?N;s;g*F|g^=UAE zra~8QZM^lSE3r>DYd^ugZahA0i#025)Ko9-U2oq%*Qgtfika{)o*kzfON8ua_4{E6 zGisZEeInjv<`1YNZMs+)ePS>%kki*G5LD>l#r#&2_qTS1UT+U%O zOtnKBKCz{@ z)vw=veHiYN`YYmG)ldr9v>vu(*}HG*xU_za^7)L5h{Fg4Om(q?2VdvR%tV&2s%=<2 z>LwxT;3>U*xVLlzbVy3ux`?xyMrS&B-ry zyCi~)GP$6By>9~bR6zdwZ3gY7s5 zDjfd=Rl|kSTR2nm*7nX@(C)Cxw&Vw0&zz%E4 zrqviz(;aw`6^E8Ahq?++yUFbVmN;P;pb1D=W4>JUzVGV_Wz?t><44LpZ(_c9nx8Y2 z7Ps>K`mi8|0n9QJHkraF7|KJ9F<9uF>l(suD@|UI+#dU;eT1d( z(KV(v9@Jq7;e@Q0ri*O9aC#E6L?n6!?>u4iack$=H}H*Dwef9KUiiiHO`;o|>lY6u zgyioQ%nFiqMFe@rh}Yko&<=rx2EQN~CH0=v)ZEmo6cvLKq~)pJd9w7#QT zU~bkc2bHsVE`QGnQ$bk;9qLiU;=zn1n1(#oi0>8mep2;oQ#f2l@}Ywl zmxsM~*7~!a_9MPyZrk=}S^BQYC8&L4Y^}9%7YxN!l&g#o|X+#n9 z(?yD(Ey4BxbTl@m)rpq>DS4=GS+`WvlEq2meC4x*v#%7Lj>mCURKP(xf);=z5Llf8 zP1RF~OL|Ip@^}5Vg8|OibzwWQ?FWpEft~jp_MfG>!YArK#y5RRT7Y;0dXr(VPu~cD zJFlfe(%B0|+6dwwOAd4;@pyv-pntR^PN}hWVatKtmkrB8bGBh2Nt6M2F=y0_G3r`; z?d8-PAjW!uO!37W3}`>s^G=k=ib7oC!Z+5==0#zEx%)f8gkh z^SR7jOTW@EluoQo9yt~*;8NZFx9-{`*X?0JOU)|z#y#~bZnUy+mka{8h0mt6Z&EnVZa#{}bC!X|J z0`dt34q^HPNTzNll3)LnfM?5tRDX}iCFmed=a_``K{RKt^QGmX*Ck1LF!y#`A{k2W>aDU#QWLxBX+T~uCLr8dS)e6`Q_nA>95Kj1R*FXMqSG0!& zKj+u`s7YI$X5(n)79_Aya&XfPFNen&>Z+0YP_cJ|Xj>rmP>k~lX{d61=N>bxm&h=dcsLO%I9U9Id@-j^0-Ol4cZ3f2LG_{YmzG~TM;lyCM zIK=f^{sd6N{K@l7Y-?HnytR%XB@Mjz(#R058S_27BWu;OExiIf5J21Gir@jfbe4zXuDw&UO5Wn>)?wZ0r?*1SIV2pOQ(zW+}?ML)L z%zzx#dvwbHtbYR?#@`+Rtc9BMXNq%12de?B@&pEJWQcYSA*<+oeka3CiLQ<%U}0*T+3M> z+29F-G|M+6(555^5KjYr36Y?cylw`484Iy{fOj19ZZvCE5U)>y`fOM!qsRR!CuFIC z0a;$Dcs?z6++dsO7@Vw8tC$5l8yiTJ&L`@tsQWO<#v4W_r_2x@V%5nK$qOyBGZ>1> z6-ww>wt_Nyq2)A9JS@=k^hO}D>BzKzmc078xwsB^-^}zBQf+qnadY^5aPvcY0;U?o zOkMwB%-U1OV@S5A?$U48o!BlzfM4}F zv+Hn2k2Q*A_z48MiFPRBbrr;JZ@lBp(wi4sTSQkasZFi>B$8bLWQ!iQko(hXX(o`M zcc@hNYHT(nK5&sUww>Y-Kf(_z-J7C*-}j967auLXz%TYoOjABETVBM1>@A9zd7VRS zVCZS_3DYaeOcC4UrRxC~S%$Y_Kxs^NB7m2k`$TKri$;Taocx_rLY(#=OY*wjg7}X5 z=p196g&Qpi=I#I9Fs_e@p@DYi=Z#hP5Kt3e_b0f3ayIlq4eG-ZRWem&ebKiVKY<)R zxOx<9{7M~{GgSPl%YVC@r9b$#{l&q$-Mg-o-4V+80~F-)3U!>w$Tnm)xtc-`laa6{ zhb2`syd@9uDwk^(#ljbp!3Dw-9);&meaKT12*47}Y^bJqdLUbK%=5D5T(K0pnl}5w zR<+`US{TPev|U$p?(!2zFom^dv9>lHWG{t=t5$n>=V^5DD7as=2Rs!tB;6Fsq$JjQ z;*r|48IzKwdx5#zpH^}5&nV}lMCR1;<)Da#NdZ&leU_sqM%dWf=fgj7Ou+;9_Ax$jGooSgegvEaHTJ*ugO&favq3~s)Fiw;MoLaYN0!*r$oGQS&)*&{}#j1RPJw?+@N!^J#?92jm{p{;T6t>1maO$PRd z=segOBG(t+K$tF`&^uNTZx_W(O zm^8$8-T{zm(3vxYlY2-1U|gy~fvIW|9Kb|I)q~ym0P=Y^?~`FqfcgWS)<&g1tg9(3 zpFi{^w^(l`_{7P`fX+)>4w<^>R0kHNC?LMy!Ia8EE7+lQ;;Y$EVso;YuYcgIP41d- zxSySTy47%J${O)QAS)wHuMK_v30j?Kxtj;z99X2n&?+N)NL%H^MosIcoI6)QAD&9H zP_1_kEs;J=G z3Qi0Vb-G@0jCnUu>f&3=LA%l$(g+L%YgOI%e>gshmDBUP-`33w0Up###k)0XXP%&A zcWhG%>7R-EE1vsbxAvCl-}XJ+&2WrCYXtIY>Hyg<{Hv8YOWi!bFS#%D)gB=V8$j(3 lY^wZb59+^`_). + And, if you still managed to get your graphs split by other means, just put tensorboard log files into the same folder. + + .. image:: ../_static/img/split_graph.png + :width: 330 + :alt: split_graph + Once the learn function is called, you can monitor the RL agent during or after the training, with the following bash command: .. code-block:: bash diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index b1ed3b1..6a42744 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -29,6 +29,7 @@ Others: Documentation: ^^^^^^^^^^^^^^ - Fix typo in docstring "nature" -> "Nature" (@Melanol) +- Add info on split tensorboard logs into (@Melanol) Release 1.6.0 (2022-07-11) From 646d6d38b6ba9aac612d4431176493a465ac4758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francesco=20Lucian=C3=B2?= <56108851+francescoluciano@users.noreply.github.com> Date: Sat, 30 Jul 2022 12:52:35 +0200 Subject: [PATCH 08/10] Fixed typo in PPO doc (#983) * Fixed typo Fixed typo * Update changelog Co-authored-by: Antonin Raffin --- docs/misc/changelog.rst | 7 ++++--- docs/modules/ppo.rst | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 6a42744..3acdbcd 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -28,8 +28,9 @@ Others: Documentation: ^^^^^^^^^^^^^^ -- Fix typo in docstring "nature" -> "Nature" (@Melanol) -- Add info on split tensorboard logs into (@Melanol) +- Fixed typo in docstring "nature" -> "Nature" (@Melanol) +- Added info on split tensorboard logs into (@Melanol) +- Fixed typo in ppo doc (@francescoluciano) Release 1.6.0 (2022-07-11) @@ -1014,4 +1015,4 @@ And all the contributors: @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede -@Melanol @qgallouedec +@Melanol @qgallouedec @francescoluciano diff --git a/docs/modules/ppo.rst b/docs/modules/ppo.rst index 829aa33..d0c425f 100644 --- a/docs/modules/ppo.rst +++ b/docs/modules/ppo.rst @@ -8,7 +8,7 @@ PPO The `Proximal Policy Optimization `_ algorithm combines ideas from A2C (having multiple workers) and TRPO (it uses a trust region to improve the actor). -The main idea is that after an update, the new policy should be not too far form the old policy. +The main idea is that after an update, the new policy should be not too far from the old policy. For that, ppo uses clipping to avoid too large update. From 6ce33f5bd2dabe389509845ce8789d30de53f298 Mon Sep 17 00:00:00 2001 From: jlp-ue <54306210+jlp-ue@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:54:48 +0200 Subject: [PATCH 09/10] Fix url in docs (#1000) * fixed URL in docs * Update changelog.rst --- docs/guide/install.rst | 2 +- docs/misc/changelog.rst | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/guide/install.rst b/docs/guide/install.rst index a9bb761..0169495 100644 --- a/docs/guide/install.rst +++ b/docs/guide/install.rst @@ -60,7 +60,7 @@ Bleeding-edge version .. code-block:: bash - pip install git+https://github.com/carlosluis/stable-baselines3/tree/fix_tests + pip install git+https://github.com/carlosluis/stable-baselines3@fix_tests See `PR #780 `_ for more information. diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 3acdbcd..b7f27ab 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -31,6 +31,7 @@ Documentation: - Fixed typo in docstring "nature" -> "Nature" (@Melanol) - Added info on split tensorboard logs into (@Melanol) - Fixed typo in ppo doc (@francescoluciano) +- Fixed typo in install doc(@jlp-ue) Release 1.6.0 (2022-07-11) @@ -1015,4 +1016,4 @@ And all the contributors: @eleurent @ac-93 @cove9988 @theDebugger811 @hsuehch @Demetrio92 @thomasgubler @IperGiove @ScheiklP @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede -@Melanol @qgallouedec @francescoluciano +@Melanol @qgallouedec @francescoluciano @jlp-ue From c4f54fcf047d7bf425fb6b88a3c8ed23fe375f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Sat, 6 Aug 2022 14:19:20 +0200 Subject: [PATCH 10/10] Handling multi-dimensional action spaces (#971) * Handle non 1D action shape * Revert changes of observation (out of the scope of this PR) * Apply changes to DictReplayBuffer * Update tests * Rollout buffer n-D actions space handling * Remove error when non 1D action space * ActorCriticPolicy return action with the proper shape * remove useless reshape * Update changelog * Add tests Co-authored-by: Antonin RAFFIN --- docs/misc/changelog.rst | 1 + stable_baselines3/common/buffers.py | 9 +++--- stable_baselines3/common/distributions.py | 1 - stable_baselines3/common/policies.py | 5 ++-- tests/test_spaces.py | 36 +++++++++++++++++++++-- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index b7f27ab..e9d4b78 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -19,6 +19,7 @@ Bug Fixes: ^^^^^^^^^^ - Fixed the issue that ``predict`` does not always return action as ``np.ndarray`` (@qgallouedec) - Fixed division by zero error when computing FPS when a small number of time has elapsed in operating systems with low-precision timers. +- Added multidimensional action space support (@qgallouedec) Deprecations: ^^^^^^^^^^^^^ diff --git a/stable_baselines3/common/buffers.py b/stable_baselines3/common/buffers.py index 5ed9b4c..0eb2651 100644 --- a/stable_baselines3/common/buffers.py +++ b/stable_baselines3/common/buffers.py @@ -247,8 +247,7 @@ class ReplayBuffer(BaseBuffer): next_obs = next_obs.reshape((self.n_envs,) + self.obs_shape) # Same, for actions - if isinstance(self.action_space, spaces.Discrete): - action = action.reshape((self.n_envs, self.action_dim)) + action = action.reshape((self.n_envs, self.action_dim)) # Copy to avoid modification by reference self.observations[self.pos] = np.array(obs).copy() @@ -433,6 +432,9 @@ class RolloutBuffer(BaseBuffer): if isinstance(self.observation_space, spaces.Discrete): obs = obs.reshape((self.n_envs,) + self.obs_shape) + # Same reshape, for actions + action = action.reshape((self.n_envs, self.action_dim)) + self.observations[self.pos] = np.array(obs).copy() self.actions[self.pos] = np.array(action).copy() self.rewards[self.pos] = np.array(reward).copy() @@ -586,8 +588,7 @@ class DictReplayBuffer(ReplayBuffer): self.next_observations[key][self.pos] = np.array(next_obs[key]).copy() # Same reshape, for actions - if isinstance(self.action_space, spaces.Discrete): - action = action.reshape((self.n_envs, self.action_dim)) + action = action.reshape((self.n_envs, self.action_dim)) self.actions[self.pos] = np.array(action).copy() self.rewards[self.pos] = np.array(reward).copy() diff --git a/stable_baselines3/common/distributions.py b/stable_baselines3/common/distributions.py index 5247751..fc48625 100644 --- a/stable_baselines3/common/distributions.py +++ b/stable_baselines3/common/distributions.py @@ -658,7 +658,6 @@ def make_proba_distribution( dist_kwargs = {} if isinstance(action_space, spaces.Box): - assert len(action_space.shape) == 1, "Error: the action space must be a vector" cls = StateDependentNoiseDistribution if use_sde else DiagGaussianDistribution return cls(get_action_dim(action_space), **dist_kwargs) elif isinstance(action_space, spaces.Discrete): diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py index a88fad6..3809b61 100644 --- a/stable_baselines3/common/policies.py +++ b/stable_baselines3/common/policies.py @@ -336,8 +336,8 @@ class BasePolicy(BaseModel): with th.no_grad(): actions = self._predict(observation, deterministic=deterministic) - # Convert to numpy - actions = actions.cpu().numpy() + # Convert to numpy, and reshape to the original action shape + actions = actions.cpu().numpy().reshape((-1,) + self.action_space.shape) if isinstance(self.action_space, gym.spaces.Box): if self.squash_output: @@ -592,6 +592,7 @@ class ActorCriticPolicy(BasePolicy): distribution = self._get_action_dist_from_latent(latent_pi) actions = distribution.get_actions(deterministic=deterministic) log_prob = distribution.log_prob(actions) + actions = actions.reshape((-1,) + self.action_space.shape) return actions, values, log_prob def _get_action_dist_from_latent(self, latent_pi: th.Tensor) -> Distribution: diff --git a/tests/test_spaces.py b/tests/test_spaces.py index b754042..0696492 100644 --- a/tests/test_spaces.py +++ b/tests/test_spaces.py @@ -33,6 +33,19 @@ class DummyMultiBinary(gym.Env): return self.observation_space.sample(), 0.0, False, {} +class DummyMultidimensionalAction(gym.Env): + def __init__(self): + super().__init__() + self.observation_space = gym.spaces.Box(low=-1, high=1, shape=(2,), dtype=np.float32) + self.action_space = gym.spaces.Box(low=-1, high=1, shape=(2, 2), dtype=np.float32) + + def reset(self): + return self.observation_space.sample() + + def step(self, action): + return self.observation_space.sample(), 0.0, False, {} + + @pytest.mark.parametrize("model_class", [SAC, TD3, DQN]) @pytest.mark.parametrize("env", [DummyMultiDiscreteSpace([4, 3]), DummyMultiBinary(8)]) def test_identity_spaces(model_class, env): @@ -53,22 +66,39 @@ def test_identity_spaces(model_class, env): @pytest.mark.parametrize("model_class", [A2C, DDPG, DQN, PPO, SAC, TD3]) -@pytest.mark.parametrize("env", ["Pendulum-v1", "CartPole-v1"]) +@pytest.mark.parametrize("env", ["Pendulum-v1", "CartPole-v1", DummyMultidimensionalAction()]) def test_action_spaces(model_class, env): + kwargs = {} if model_class in [SAC, DDPG, TD3]: - supported_action_space = env == "Pendulum-v1" + supported_action_space = env == "Pendulum-v1" or isinstance(env, DummyMultidimensionalAction) + kwargs["learning_starts"] = 2 + kwargs["train_freq"] = 32 elif model_class == DQN: supported_action_space = env == "CartPole-v1" elif model_class in [A2C, PPO]: supported_action_space = True + kwargs["n_steps"] = 64 if supported_action_space: - model_class("MlpPolicy", env) + model = model_class("MlpPolicy", env, **kwargs) + if isinstance(env, DummyMultidimensionalAction): + model.learn(64) else: with pytest.raises(AssertionError): model_class("MlpPolicy", env) +def test_sde_multi_dim(): + SAC( + "MlpPolicy", + DummyMultidimensionalAction(), + learning_starts=10, + use_sde=True, + sde_sample_freq=2, + use_sde_at_warmup=True, + ).learn(20) + + @pytest.mark.parametrize("model_class", [A2C, PPO, DQN]) @pytest.mark.parametrize("env", ["Taxi-v3"]) def test_discrete_obs_space(model_class, env):