From d542732c8da89568c121a3afe404153b19e08897 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 15:02:35 +0200 Subject: [PATCH 1/9] Rename to stable-baselines3 --- .coveragerc | 2 +- .github/ISSUE_TEMPLATE/issue-template.md | 4 ++-- NOTICE | 2 +- README.md | 4 ++-- docs/conf.py | 20 ++++++++--------- docs/guide/quickstart.rst | 8 +++---- docs/guide/vec_envs.rst | 2 +- docs/index.rst | 8 +++---- docs/misc/changelog.rst | 4 ++-- docs/modules/a2c.rst | 8 +++---- docs/modules/base.rst | 2 +- docs/modules/ppo.rst | 8 +++---- docs/modules/sac.rst | 10 ++++----- docs/modules/td3.rst | 10 ++++----- setup.cfg | 2 +- setup.py | 6 ++--- .../__init__.py | 8 +++---- stable_baselines3/a2c/__init__.py | 2 ++ .../a2c/a2c.py | 10 ++++----- .../common/__init__.py | 0 .../common/base_class.py | 22 +++++++++---------- .../common/buffers.py | 6 ++--- .../common/callbacks.py | 8 +++---- .../common/distributions.py | 2 +- .../common/evaluation.py | 2 +- .../common/identity_env.py | 2 +- .../common/logger.py | 0 .../common/monitor.py | 0 .../common/noise.py | 0 .../common/policies.py | 6 ++--- .../common/preprocessing.py | 0 .../common/results_plotter.py | 2 +- .../common/running_mean_std.py | 0 .../common/save_util.py | 0 .../common/type_aliases.py | 4 ++-- .../common/utils.py | 0 .../common/vec_env/__init__.py | 14 ++++++------ .../common/vec_env/base_vec_env.py | 0 .../common/vec_env/dummy_vec_env.py | 4 ++-- .../common/vec_env/subproc_vec_env.py | 2 +- .../common/vec_env/util.py | 0 .../common/vec_env/vec_frame_stack.py | 2 +- .../common/vec_env/vec_normalize.py | 4 ++-- .../common/vec_env/vec_transpose.py | 6 ++--- stable_baselines3/ppo/__init__.py | 2 ++ .../ppo/policies.py | 4 ++-- .../ppo/ppo.py | 16 +++++++------- .../py.typed | 0 stable_baselines3/sac/__init__.py | 2 ++ .../sac/policies.py | 6 ++--- .../sac/sac.py | 10 ++++----- stable_baselines3/td3/__init__.py | 2 ++ .../td3/policies.py | 6 ++--- .../td3/td3.py | 10 ++++----- .../version.txt | 0 tests/test_callbacks.py | 4 ++-- tests/test_cnn.py | 4 ++-- tests/test_custom_policy.py | 2 +- tests/test_distributions.py | 6 ++--- tests/test_identity.py | 8 +++---- tests/test_logger.py | 4 ++-- tests/test_monitor.py | 2 +- tests/test_predict.py | 4 ++-- tests/test_run.py | 4 ++-- tests/test_save_load.py | 8 +++---- tests/test_sde.py | 2 +- tests/test_vec_envs.py | 2 +- tests/test_vec_normalize.py | 6 ++--- torchy_baselines/a2c/__init__.py | 2 -- torchy_baselines/ppo/__init__.py | 2 -- torchy_baselines/sac/__init__.py | 2 -- torchy_baselines/td3/__init__.py | 2 -- 72 files changed, 164 insertions(+), 164 deletions(-) rename {torchy_baselines => stable_baselines3}/__init__.py (55%) create mode 100644 stable_baselines3/a2c/__init__.py rename {torchy_baselines => stable_baselines3}/a2c/a2c.py (96%) rename {torchy_baselines => stable_baselines3}/common/__init__.py (100%) rename {torchy_baselines => stable_baselines3}/common/base_class.py (97%) rename {torchy_baselines => stable_baselines3}/common/buffers.py (98%) rename {torchy_baselines => stable_baselines3}/common/callbacks.py (97%) rename {torchy_baselines => stable_baselines3}/common/distributions.py (99%) rename {torchy_baselines => stable_baselines3}/common/evaluation.py (97%) rename {torchy_baselines => stable_baselines3}/common/identity_env.py (98%) rename {torchy_baselines => stable_baselines3}/common/logger.py (100%) rename {torchy_baselines => stable_baselines3}/common/monitor.py (100%) rename {torchy_baselines => stable_baselines3}/common/noise.py (100%) rename {torchy_baselines => stable_baselines3}/common/policies.py (99%) rename {torchy_baselines => stable_baselines3}/common/preprocessing.py (100%) rename {torchy_baselines => stable_baselines3}/common/results_plotter.py (98%) rename {torchy_baselines => stable_baselines3}/common/running_mean_std.py (100%) rename {torchy_baselines => stable_baselines3}/common/save_util.py (100%) rename {torchy_baselines => stable_baselines3}/common/type_aliases.py (89%) rename {torchy_baselines => stable_baselines3}/common/utils.py (100%) rename {torchy_baselines => stable_baselines3}/common/vec_env/__init__.py (68%) rename {torchy_baselines => stable_baselines3}/common/vec_env/base_vec_env.py (100%) rename {torchy_baselines => stable_baselines3}/common/vec_env/dummy_vec_env.py (96%) rename {torchy_baselines => stable_baselines3}/common/vec_env/subproc_vec_env.py (99%) rename {torchy_baselines => stable_baselines3}/common/vec_env/util.py (100%) rename {torchy_baselines => stable_baselines3}/common/vec_env/vec_frame_stack.py (96%) rename {torchy_baselines => stable_baselines3}/common/vec_env/vec_normalize.py (97%) rename {torchy_baselines => stable_baselines3}/common/vec_env/vec_transpose.py (89%) create mode 100644 stable_baselines3/ppo/__init__.py rename {torchy_baselines => stable_baselines3}/ppo/policies.py (99%) rename {torchy_baselines => stable_baselines3}/ppo/ppo.py (97%) rename {torchy_baselines => stable_baselines3}/py.typed (100%) create mode 100644 stable_baselines3/sac/__init__.py rename {torchy_baselines => stable_baselines3}/sac/policies.py (98%) rename {torchy_baselines => stable_baselines3}/sac/sac.py (98%) create mode 100644 stable_baselines3/td3/__init__.py rename {torchy_baselines => stable_baselines3}/td3/policies.py (99%) rename {torchy_baselines => stable_baselines3}/td3/td3.py (97%) rename {torchy_baselines => stable_baselines3}/version.txt (100%) delete mode 100644 torchy_baselines/a2c/__init__.py delete mode 100644 torchy_baselines/ppo/__init__.py delete mode 100644 torchy_baselines/sac/__init__.py delete mode 100644 torchy_baselines/td3/__init__.py diff --git a/.coveragerc b/.coveragerc index 511f20d..9020116 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ omit = tests/* setup.py # Require graphical interface - torchy_baselines/common/results_plotter.py + stable_baselines3/common/results_plotter.py [report] exclude_lines = diff --git a/.github/ISSUE_TEMPLATE/issue-template.md b/.github/ISSUE_TEMPLATE/issue-template.md index 2e2e61b..f9ff962 100644 --- a/.github/ISSUE_TEMPLATE/issue-template.md +++ b/.github/ISSUE_TEMPLATE/issue-template.md @@ -13,7 +13,7 @@ If you are submitting a bug report, please fill in the following details. If your issue is related to a custom gym environment, please check it first using: ```python -from torchy_baselines.common.env_checker import check_env +from stable_baselines3.common.env_checker import check_env env = CustomEnv(arg1, ...) # It will check your custom environment and output additional warnings if needed @@ -30,7 +30,7 @@ Please use the [markdown code blocks](https://help.github.com/en/articles/creati for both code and stack traces. ```python -from torchy_baselines import ... +from stable_baselines3 import ... ``` diff --git a/NOTICE b/NOTICE index 9fc6700..6dbbda6 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -Large portion of the code of Torchy-Baselines (in `common/`) were ported from Stable-Baselines, a fork of OpenAI Baselines, +Large portion of the code of Stable-Baselines3 (in `common/`) were ported from Stable-Baselines, a fork of OpenAI Baselines, both licensed under the MIT License: before the fork (June 2018): diff --git a/README.md b/README.md index 843d55e..5f35c82 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.com/hill-a/stable-baselines.svg?branch=master)](https://travis-ci.com/hill-a/stable-baselines) [![Documentation Status](https://readthedocs.org/projects/stable-baselines/badge/?version=master)](https://stable-baselines.readthedocs.io/en/master/?badge=master) -# Torchy Baselines +# Stable Baselines3 PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines), a set of improved implementations of reinforcement learning algorithms. @@ -58,7 +58,7 @@ To cite this repository in publications: ``` @misc{torchy-baselines, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah}, - title = {Torchy Baselines}, + title = {Stable Baselines3}, year = {2019}, publisher = {GitHub}, journal = {GitHub repository}, diff --git a/docs/conf.py b/docs/conf.py index e71d166..a3d83bc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,19 +44,19 @@ MOCK_MODULES = [] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) -import torchy_baselines +import stable_baselines3 # -- Project information ----------------------------------------------------- -project = 'Torchy Baselines' -copyright = '2020, Torchy Baselines' -author = 'Torchy Baselines Contributors' +project = 'Stable Baselines3' +copyright = '2020, Stable Baselines3' +author = 'Stable Baselines3 Contributors' # The short X.Y version -version = 'master (' + torchy_baselines.__version__ + ' )' +version = 'master (' + stable_baselines3.__version__ + ' )' # The full version, including alpha/beta/rc tags -release = torchy_baselines.__version__ +release = stable_baselines3.__version__ # -- General configuration --------------------------------------------------- @@ -179,8 +179,8 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'TorchyBaselines.tex', 'Torchy Baselines Documentation', - 'Torchy Baselines Contributors', 'manual'), + (master_doc, 'TorchyBaselines.tex', 'Stable Baselines3 Documentation', + 'Stable Baselines3 Contributors', 'manual'), ] @@ -189,7 +189,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'torchybaselines', 'Torchy Baselines Documentation', + (master_doc, 'torchybaselines', 'Stable Baselines3 Documentation', [author], 1) ] @@ -200,7 +200,7 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'TorchyBaselines', 'Torchy Baselines Documentation', + (master_doc, 'TorchyBaselines', 'Stable Baselines3 Documentation', author, 'TorchyBaselines', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/guide/quickstart.rst b/docs/guide/quickstart.rst index 58fca6d..e20f36a 100644 --- a/docs/guide/quickstart.rst +++ b/docs/guide/quickstart.rst @@ -12,9 +12,9 @@ Here is a quick example of how to train and run SAC on a Pendulum environment: import gym - from torchy_baselines.sac.policies import MlpPolicy - from torchy_baselines.common.vec_env import DummyVecEnv - from torchy_baselines import SAC + from stable_baselines3.sac.policies import MlpPolicy + from stable_baselines3.common.vec_env import DummyVecEnv + from stable_baselines3 import SAC env = gym.make('Pendulum-v0') @@ -34,6 +34,6 @@ the policy is registered: .. code-block:: python - from torchy_baselines import SAC + from stable_baselines3 import SAC model = SAC('MlpPolicy', 'Pendulum-v0').learn(10000) diff --git a/docs/guide/vec_envs.rst b/docs/guide/vec_envs.rst index e0e930c..2bce8d1 100644 --- a/docs/guide/vec_envs.rst +++ b/docs/guide/vec_envs.rst @@ -1,6 +1,6 @@ .. _vec_env: -.. automodule:: torchy_baselines.common.vec_env +.. automodule:: stable_baselines3.common.vec_env Vectorized Environments ======================= diff --git a/docs/index.rst b/docs/index.rst index c46e317..c97722b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,10 +3,10 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Torchy Baselines docs! - Pytorch RL Baselines +Welcome to Stable Baselines3 docs! - Pytorch RL Baselines ======================================================== -`Torchy Baselines `_ is the PyTorch version of `Stable Baselines `_, +`Stable Baselines3 `_ is the PyTorch version of `Stable Baselines `_, a set of improved implementations of reinforcement learning algorithms. RL Baselines Zoo (collection of pre-trained agents): https://github.com/araffin/rl-baselines-zoo @@ -41,7 +41,7 @@ RL Baselines zoo also offers a simple interface to train, evaluate agents and do misc/changelog -Citing Torchy Baselines +Citing Stable Baselines3 ----------------------- To cite this project in publications: @@ -49,7 +49,7 @@ To cite this project in publications: @misc{torchy-baselines, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah}, - title = {Torchy Baselines}, + title = {Stable Baselines3}, year = {2019}, publisher = {GitHub}, journal = {GitHub repository}, diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index a4fd097..3767b7c 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -111,7 +111,7 @@ Pre-Release 0.2.0 (2020-02-14) Breaking Changes: ^^^^^^^^^^^^^^^^^ -- Python 2 support was dropped, Torchy Baselines now requires Python 3.6 or above +- Python 2 support was dropped, Stable Baselines3 now requires Python 3.6 or above - Return type of ``evaluation.evaluate_policy()`` has been changed - Refactored the replay buffer to avoid transformation between PyTorch and NumPy - Created `OffPolicyRLModel` base class @@ -160,7 +160,7 @@ New Features: Maintainers ----------- -Torchy-Baselines is currently maintained by `Antonin Raffin`_ (aka `@araffin`_). +Stable-Baselines3 is currently maintained by `Antonin Raffin`_ (aka `@araffin`_). .. _Antonin Raffin: https://araffin.github.io/ .. _@araffin: https://github.com/araffin diff --git a/docs/modules/a2c.rst b/docs/modules/a2c.rst index d8f5cf2..0ee3aa7 100644 --- a/docs/modules/a2c.rst +++ b/docs/modules/a2c.rst @@ -1,6 +1,6 @@ .. _a2c: -.. automodule:: torchy_baselines.a2c +.. automodule:: stable_baselines3.a2c A2C @@ -44,9 +44,9 @@ Train a A2C agent on `CartPole-v1` using 4 processes. import gym - from torchy_baselines.common.policies import MlpPolicy - from torchy_baselines.common import make_vec_env - from torchy_baselines import A2C + from stable_baselines3.common.policies import MlpPolicy + from stable_baselines3.common import make_vec_env + from stable_baselines3 import A2C # Parallel environments env = make_vec_env('CartPole-v1', n_envs=4) diff --git a/docs/modules/base.rst b/docs/modules/base.rst index d32268d..7fa2a59 100644 --- a/docs/modules/base.rst +++ b/docs/modules/base.rst @@ -1,6 +1,6 @@ .. _base_algo: -.. automodule:: torchy_baselines.common.base_class +.. automodule:: stable_baselines3.common.base_class Base RL Class diff --git a/docs/modules/ppo.rst b/docs/modules/ppo.rst index 9c9887d..477c13f 100644 --- a/docs/modules/ppo.rst +++ b/docs/modules/ppo.rst @@ -1,6 +1,6 @@ .. _ppo2: -.. automodule:: torchy_baselines.ppo +.. automodule:: stable_baselines3.ppo PPO === @@ -53,9 +53,9 @@ Train a PPO agent on `Pendulum-v0` using 4 processes. import gym - from torchy_baselines.ppo.policies import MlpPolicy - from torchy_baselines.common.vec_env import SubprocVecEnv - from torchy_baselines import PPO + from stable_baselines3.ppo.policies import MlpPolicy + from stable_baselines3.common.vec_env import SubprocVecEnv + from stable_baselines3 import PPO # multiprocess environment n_cpu = 4 diff --git a/docs/modules/sac.rst b/docs/modules/sac.rst index 7bd949d..c067bcf 100644 --- a/docs/modules/sac.rst +++ b/docs/modules/sac.rst @@ -1,6 +1,6 @@ .. _sac: -.. automodule:: torchy_baselines.sac +.. automodule:: stable_baselines3.sac SAC @@ -14,7 +14,7 @@ A key feature of SAC, and a major difference with common RL algorithms, is that .. warning:: - The SAC model does not support ``torchy_baselines.common.policies`` because it uses double q-values + The SAC model does not support ``stable_baselines3.common.policies`` because it uses double q-values and value estimation, as a result it must use its own policy models (see :ref:`sac_policies`). @@ -72,9 +72,9 @@ Example import gym import numpy as np - from torchy_baselines.sac.policies import MlpPolicy - from torchy_baselines.common.vec_env import DummyVecEnv - from torchy_baselines import SAC + from stable_baselines3.sac.policies import MlpPolicy + from stable_baselines3.common.vec_env import DummyVecEnv + from stable_baselines3 import SAC env = gym.make('Pendulum-v0') env = DummyVecEnv([lambda: env]) diff --git a/docs/modules/td3.rst b/docs/modules/td3.rst index 9fd6806..338b9da 100644 --- a/docs/modules/td3.rst +++ b/docs/modules/td3.rst @@ -1,6 +1,6 @@ .. _td3: -.. automodule:: torchy_baselines.td3 +.. automodule:: stable_baselines3.td3 TD3 @@ -14,7 +14,7 @@ We recommend reading `OpenAI Spinning guide on TD3 =0.11', 'numpy', diff --git a/torchy_baselines/__init__.py b/stable_baselines3/__init__.py similarity index 55% rename from torchy_baselines/__init__.py rename to stable_baselines3/__init__.py index 28742f5..562ca36 100644 --- a/torchy_baselines/__init__.py +++ b/stable_baselines3/__init__.py @@ -1,9 +1,9 @@ import os -from torchy_baselines.a2c import A2C -from torchy_baselines.ppo import PPO -from torchy_baselines.sac import SAC -from torchy_baselines.td3 import TD3 +from stable_baselines3.a2c import A2C +from stable_baselines3.ppo import PPO +from stable_baselines3.sac import SAC +from stable_baselines3.td3 import TD3 # Read version from file version_file = os.path.join(os.path.dirname(__file__), 'version.txt') diff --git a/stable_baselines3/a2c/__init__.py b/stable_baselines3/a2c/__init__.py new file mode 100644 index 0000000..7dba39a --- /dev/null +++ b/stable_baselines3/a2c/__init__.py @@ -0,0 +1,2 @@ +from stable_baselines3.a2c.a2c import A2C +from stable_baselines3.ppo.policies import MlpPolicy diff --git a/torchy_baselines/a2c/a2c.py b/stable_baselines3/a2c/a2c.py similarity index 96% rename from torchy_baselines/a2c/a2c.py rename to stable_baselines3/a2c/a2c.py index ff10bf4..3c5ead9 100644 --- a/torchy_baselines/a2c/a2c.py +++ b/stable_baselines3/a2c/a2c.py @@ -3,11 +3,11 @@ import torch.nn.functional as F from gym import spaces from typing import Type, Union, Callable, Optional, Dict, Any -from torchy_baselines.common import logger -from torchy_baselines.common.type_aliases import GymEnv, MaybeCallback -from torchy_baselines.common.utils import explained_variance -from torchy_baselines.ppo.policies import PPOPolicy -from torchy_baselines.ppo.ppo import PPO +from stable_baselines3.common import logger +from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback +from stable_baselines3.common.utils import explained_variance +from stable_baselines3.ppo.policies import PPOPolicy +from stable_baselines3.ppo.ppo import PPO class A2C(PPO): diff --git a/torchy_baselines/common/__init__.py b/stable_baselines3/common/__init__.py similarity index 100% rename from torchy_baselines/common/__init__.py rename to stable_baselines3/common/__init__.py diff --git a/torchy_baselines/common/base_class.py b/stable_baselines3/common/base_class.py similarity index 97% rename from torchy_baselines/common/base_class.py rename to stable_baselines3/common/base_class.py index bb1404f..1aee061 100644 --- a/torchy_baselines/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -11,17 +11,17 @@ import gym import torch as th import numpy as np -from torchy_baselines.common import logger -from torchy_baselines.common.policies import BasePolicy, get_policy_from_name -from torchy_baselines.common.utils import set_random_seed, get_schedule_fn, update_learning_rate, get_device -from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize, VecNormalize, VecTransposeImage -from torchy_baselines.common.preprocessing import is_image_space -from torchy_baselines.common.save_util import data_to_json, json_to_data, recursive_getattr, recursive_setattr -from torchy_baselines.common.type_aliases import GymEnv, TensorDict, RolloutReturn, MaybeCallback -from torchy_baselines.common.callbacks import BaseCallback, CallbackList, ConvertCallback, EvalCallback -from torchy_baselines.common.monitor import Monitor -from torchy_baselines.common.noise import ActionNoise -from torchy_baselines.common.buffers import ReplayBuffer +from stable_baselines3.common import logger +from stable_baselines3.common.policies import BasePolicy, get_policy_from_name +from stable_baselines3.common.utils import set_random_seed, get_schedule_fn, update_learning_rate, get_device +from stable_baselines3.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize, VecNormalize, VecTransposeImage +from stable_baselines3.common.preprocessing import is_image_space +from stable_baselines3.common.save_util import data_to_json, json_to_data, recursive_getattr, recursive_setattr +from stable_baselines3.common.type_aliases import GymEnv, TensorDict, RolloutReturn, MaybeCallback +from stable_baselines3.common.callbacks import BaseCallback, CallbackList, ConvertCallback, EvalCallback +from stable_baselines3.common.monitor import Monitor +from stable_baselines3.common.noise import ActionNoise +from stable_baselines3.common.buffers import ReplayBuffer class BaseRLModel(ABC): diff --git a/torchy_baselines/common/buffers.py b/stable_baselines3/common/buffers.py similarity index 98% rename from torchy_baselines/common/buffers.py rename to stable_baselines3/common/buffers.py index fd78d3e..4fb4422 100644 --- a/torchy_baselines/common/buffers.py +++ b/stable_baselines3/common/buffers.py @@ -4,9 +4,9 @@ import numpy as np import torch as th from gym import spaces -from torchy_baselines.common.vec_env import VecNormalize -from torchy_baselines.common.type_aliases import RolloutBufferSamples, ReplayBufferSamples -from torchy_baselines.common.preprocessing import get_action_dim, get_obs_shape +from stable_baselines3.common.vec_env import VecNormalize +from stable_baselines3.common.type_aliases import RolloutBufferSamples, ReplayBufferSamples +from stable_baselines3.common.preprocessing import get_action_dim, get_obs_shape class BaseBuffer(object): diff --git a/torchy_baselines/common/callbacks.py b/stable_baselines3/common/callbacks.py similarity index 97% rename from torchy_baselines/common/callbacks.py rename to stable_baselines3/common/callbacks.py index b5a015d..16143a7 100644 --- a/torchy_baselines/common/callbacks.py +++ b/stable_baselines3/common/callbacks.py @@ -7,12 +7,12 @@ from typing import Union, List, Dict, Any, Optional import gym import numpy as np -from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, sync_envs_normalization -from torchy_baselines.common.evaluation import evaluate_policy -from torchy_baselines.common.logger import Logger +from stable_baselines3.common.vec_env import DummyVecEnv, VecEnv, sync_envs_normalization +from stable_baselines3.common.evaluation import evaluate_policy +from stable_baselines3.common.logger import Logger if typing.TYPE_CHECKING: - from torchy_baselines.common.base_class import BaseRLModel # pytype: disable=pyi-error + from stable_baselines3.common.base_class import BaseRLModel # pytype: disable=pyi-error class BaseCallback(ABC): diff --git a/torchy_baselines/common/distributions.py b/stable_baselines3/common/distributions.py similarity index 99% rename from torchy_baselines/common/distributions.py rename to stable_baselines3/common/distributions.py index 9a67b93..e3ff2a7 100644 --- a/torchy_baselines/common/distributions.py +++ b/stable_baselines3/common/distributions.py @@ -6,7 +6,7 @@ import torch.nn as nn from torch.distributions import Normal, Categorical from gym import spaces -from torchy_baselines.common.preprocessing import get_action_dim +from stable_baselines3.common.preprocessing import get_action_dim class Distribution(object): diff --git a/torchy_baselines/common/evaluation.py b/stable_baselines3/common/evaluation.py similarity index 97% rename from torchy_baselines/common/evaluation.py rename to stable_baselines3/common/evaluation.py index b5017c9..c8d0b97 100644 --- a/torchy_baselines/common/evaluation.py +++ b/stable_baselines3/common/evaluation.py @@ -1,7 +1,7 @@ # Copied from stable_baselines import numpy as np -from torchy_baselines.common.vec_env import VecEnv +from stable_baselines3.common.vec_env import VecEnv def evaluate_policy(model, env, n_eval_episodes=10, deterministic=True, diff --git a/torchy_baselines/common/identity_env.py b/stable_baselines3/common/identity_env.py similarity index 98% rename from torchy_baselines/common/identity_env.py rename to stable_baselines3/common/identity_env.py index 8a76d3b..0a3daad 100644 --- a/torchy_baselines/common/identity_env.py +++ b/stable_baselines3/common/identity_env.py @@ -5,7 +5,7 @@ from gym import Env from gym.spaces import Discrete, MultiDiscrete, MultiBinary, Box -from torchy_baselines.common.type_aliases import GymStepReturn, GymObs +from stable_baselines3.common.type_aliases import GymStepReturn, GymObs class IdentityEnv(Env): diff --git a/torchy_baselines/common/logger.py b/stable_baselines3/common/logger.py similarity index 100% rename from torchy_baselines/common/logger.py rename to stable_baselines3/common/logger.py diff --git a/torchy_baselines/common/monitor.py b/stable_baselines3/common/monitor.py similarity index 100% rename from torchy_baselines/common/monitor.py rename to stable_baselines3/common/monitor.py diff --git a/torchy_baselines/common/noise.py b/stable_baselines3/common/noise.py similarity index 100% rename from torchy_baselines/common/noise.py rename to stable_baselines3/common/noise.py diff --git a/torchy_baselines/common/policies.py b/stable_baselines3/common/policies.py similarity index 99% rename from torchy_baselines/common/policies.py rename to stable_baselines3/common/policies.py index f6c6644..22b51df 100644 --- a/torchy_baselines/common/policies.py +++ b/stable_baselines3/common/policies.py @@ -7,9 +7,9 @@ import torch as th import torch.nn as nn import numpy as np -from torchy_baselines.common.preprocessing import preprocess_obs, get_flattened_obs_dim, is_image_space -from torchy_baselines.common.utils import get_device -from torchy_baselines.common.vec_env import VecTransposeImage +from stable_baselines3.common.preprocessing import preprocess_obs, get_flattened_obs_dim, is_image_space +from stable_baselines3.common.utils import get_device +from stable_baselines3.common.vec_env import VecTransposeImage class BaseFeaturesExtractor(nn.Module): diff --git a/torchy_baselines/common/preprocessing.py b/stable_baselines3/common/preprocessing.py similarity index 100% rename from torchy_baselines/common/preprocessing.py rename to stable_baselines3/common/preprocessing.py diff --git a/torchy_baselines/common/results_plotter.py b/stable_baselines3/common/results_plotter.py similarity index 98% rename from torchy_baselines/common/results_plotter.py rename to stable_baselines3/common/results_plotter.py index d447344..4f879be 100644 --- a/torchy_baselines/common/results_plotter.py +++ b/stable_baselines3/common/results_plotter.py @@ -6,7 +6,7 @@ import pandas as pd # matplotlib.use('TkAgg') # Can change to 'Agg' for non-interactive mode import matplotlib.pyplot as plt -from torchy_baselines.common.monitor import load_results +from stable_baselines3.common.monitor import load_results X_TIMESTEPS = 'timesteps' diff --git a/torchy_baselines/common/running_mean_std.py b/stable_baselines3/common/running_mean_std.py similarity index 100% rename from torchy_baselines/common/running_mean_std.py rename to stable_baselines3/common/running_mean_std.py diff --git a/torchy_baselines/common/save_util.py b/stable_baselines3/common/save_util.py similarity index 100% rename from torchy_baselines/common/save_util.py rename to stable_baselines3/common/save_util.py diff --git a/torchy_baselines/common/type_aliases.py b/stable_baselines3/common/type_aliases.py similarity index 89% rename from torchy_baselines/common/type_aliases.py rename to stable_baselines3/common/type_aliases.py index 70b63a8..139cd60 100644 --- a/torchy_baselines/common/type_aliases.py +++ b/stable_baselines3/common/type_aliases.py @@ -7,8 +7,8 @@ import numpy as np import torch as th import gym -from torchy_baselines.common.vec_env import VecEnv -from torchy_baselines.common.callbacks import BaseCallback +from stable_baselines3.common.vec_env import VecEnv +from stable_baselines3.common.callbacks import BaseCallback GymEnv = Union[gym.Env, VecEnv] diff --git a/torchy_baselines/common/utils.py b/stable_baselines3/common/utils.py similarity index 100% rename from torchy_baselines/common/utils.py rename to stable_baselines3/common/utils.py diff --git a/torchy_baselines/common/vec_env/__init__.py b/stable_baselines3/common/vec_env/__init__.py similarity index 68% rename from torchy_baselines/common/vec_env/__init__.py rename to stable_baselines3/common/vec_env/__init__.py index b119a9f..2c89181 100644 --- a/torchy_baselines/common/vec_env/__init__.py +++ b/stable_baselines3/common/vec_env/__init__.py @@ -3,17 +3,17 @@ import typing from typing import Optional, Union from copy import deepcopy -from torchy_baselines.common.vec_env.base_vec_env import (AlreadySteppingError, NotSteppingError, +from stable_baselines3.common.vec_env.base_vec_env import (AlreadySteppingError, NotSteppingError, VecEnv, VecEnvWrapper, CloudpickleWrapper) -from torchy_baselines.common.vec_env.dummy_vec_env import DummyVecEnv -from torchy_baselines.common.vec_env.subproc_vec_env import SubprocVecEnv -from torchy_baselines.common.vec_env.vec_frame_stack import VecFrameStack -from torchy_baselines.common.vec_env.vec_normalize import VecNormalize -from torchy_baselines.common.vec_env.vec_transpose import VecTransposeImage +from stable_baselines3.common.vec_env.dummy_vec_env import DummyVecEnv +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from stable_baselines3.common.vec_env.vec_frame_stack import VecFrameStack +from stable_baselines3.common.vec_env.vec_normalize import VecNormalize +from stable_baselines3.common.vec_env.vec_transpose import VecTransposeImage # Avoid circular import if typing.TYPE_CHECKING: - from torchy_baselines.common.type_aliases import GymEnv + from stable_baselines3.common.type_aliases import GymEnv def unwrap_vec_normalize(env: Union['GymEnv', VecEnv]) -> Optional[VecNormalize]: diff --git a/torchy_baselines/common/vec_env/base_vec_env.py b/stable_baselines3/common/vec_env/base_vec_env.py similarity index 100% rename from torchy_baselines/common/vec_env/base_vec_env.py rename to stable_baselines3/common/vec_env/base_vec_env.py diff --git a/torchy_baselines/common/vec_env/dummy_vec_env.py b/stable_baselines3/common/vec_env/dummy_vec_env.py similarity index 96% rename from torchy_baselines/common/vec_env/dummy_vec_env.py rename to stable_baselines3/common/vec_env/dummy_vec_env.py index 2d0211e..669fede 100644 --- a/torchy_baselines/common/vec_env/dummy_vec_env.py +++ b/stable_baselines3/common/vec_env/dummy_vec_env.py @@ -3,8 +3,8 @@ from copy import deepcopy import numpy as np -from torchy_baselines.common.vec_env.base_vec_env import VecEnv -from torchy_baselines.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info +from stable_baselines3.common.vec_env.base_vec_env import VecEnv +from stable_baselines3.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info class DummyVecEnv(VecEnv): diff --git a/torchy_baselines/common/vec_env/subproc_vec_env.py b/stable_baselines3/common/vec_env/subproc_vec_env.py similarity index 99% rename from torchy_baselines/common/vec_env/subproc_vec_env.py rename to stable_baselines3/common/vec_env/subproc_vec_env.py index 5e6ee85..128cdca 100644 --- a/torchy_baselines/common/vec_env/subproc_vec_env.py +++ b/stable_baselines3/common/vec_env/subproc_vec_env.py @@ -4,7 +4,7 @@ from collections import OrderedDict import gym import numpy as np -from torchy_baselines.common.vec_env.base_vec_env import VecEnv, CloudpickleWrapper +from stable_baselines3.common.vec_env.base_vec_env import VecEnv, CloudpickleWrapper def _worker(remote, parent_remote, env_fn_wrapper): diff --git a/torchy_baselines/common/vec_env/util.py b/stable_baselines3/common/vec_env/util.py similarity index 100% rename from torchy_baselines/common/vec_env/util.py rename to stable_baselines3/common/vec_env/util.py diff --git a/torchy_baselines/common/vec_env/vec_frame_stack.py b/stable_baselines3/common/vec_env/vec_frame_stack.py similarity index 96% rename from torchy_baselines/common/vec_env/vec_frame_stack.py rename to stable_baselines3/common/vec_env/vec_frame_stack.py index 6676162..b32ddeb 100644 --- a/torchy_baselines/common/vec_env/vec_frame_stack.py +++ b/stable_baselines3/common/vec_env/vec_frame_stack.py @@ -3,7 +3,7 @@ import warnings import numpy as np from gym import spaces -from torchy_baselines.common.vec_env.base_vec_env import VecEnv, VecEnvWrapper +from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvWrapper class VecFrameStack(VecEnvWrapper): diff --git a/torchy_baselines/common/vec_env/vec_normalize.py b/stable_baselines3/common/vec_env/vec_normalize.py similarity index 97% rename from torchy_baselines/common/vec_env/vec_normalize.py rename to stable_baselines3/common/vec_env/vec_normalize.py index 87fb70a..94cf74b 100644 --- a/torchy_baselines/common/vec_env/vec_normalize.py +++ b/stable_baselines3/common/vec_env/vec_normalize.py @@ -2,8 +2,8 @@ import pickle import numpy as np -from torchy_baselines.common.vec_env.base_vec_env import VecEnvWrapper -from torchy_baselines.common.running_mean_std import RunningMeanStd +from stable_baselines3.common.vec_env.base_vec_env import VecEnvWrapper +from stable_baselines3.common.running_mean_std import RunningMeanStd class VecNormalize(VecEnvWrapper): diff --git a/torchy_baselines/common/vec_env/vec_transpose.py b/stable_baselines3/common/vec_env/vec_transpose.py similarity index 89% rename from torchy_baselines/common/vec_env/vec_transpose.py rename to stable_baselines3/common/vec_env/vec_transpose.py index 3a514e9..e4901b4 100644 --- a/torchy_baselines/common/vec_env/vec_transpose.py +++ b/stable_baselines3/common/vec_env/vec_transpose.py @@ -2,11 +2,11 @@ import typing import numpy as np from gym import spaces -from torchy_baselines.common.vec_env.base_vec_env import VecEnv, VecEnvWrapper -from torchy_baselines.common.preprocessing import is_image_space +from stable_baselines3.common.vec_env.base_vec_env import VecEnv, VecEnvWrapper +from stable_baselines3.common.preprocessing import is_image_space if typing.TYPE_CHECKING: - from torchy_baselines.common.type_aliases import GymStepReturn + from stable_baselines3.common.type_aliases import GymStepReturn class VecTransposeImage(VecEnvWrapper): diff --git a/stable_baselines3/ppo/__init__.py b/stable_baselines3/ppo/__init__.py new file mode 100644 index 0000000..8c9ed8e --- /dev/null +++ b/stable_baselines3/ppo/__init__.py @@ -0,0 +1,2 @@ +from stable_baselines3.ppo.ppo import PPO +from stable_baselines3.ppo.policies import MlpPolicy diff --git a/torchy_baselines/ppo/policies.py b/stable_baselines3/ppo/policies.py similarity index 99% rename from torchy_baselines/ppo/policies.py rename to stable_baselines3/ppo/policies.py index 2b9164a..e997769 100644 --- a/torchy_baselines/ppo/policies.py +++ b/stable_baselines3/ppo/policies.py @@ -6,10 +6,10 @@ import torch as th import torch.nn as nn import numpy as np -from torchy_baselines.common.policies import (BasePolicy, register_policy, MlpExtractor, +from stable_baselines3.common.policies import (BasePolicy, register_policy, MlpExtractor, create_sde_features_extractor, NatureCNN, BaseFeaturesExtractor, FlattenExtractor) -from torchy_baselines.common.distributions import (make_proba_distribution, Distribution, +from stable_baselines3.common.distributions import (make_proba_distribution, Distribution, DiagGaussianDistribution, CategoricalDistribution, StateDependentNoiseDistribution) diff --git a/torchy_baselines/ppo/ppo.py b/stable_baselines3/ppo/ppo.py similarity index 97% rename from torchy_baselines/ppo/ppo.py rename to stable_baselines3/ppo/ppo.py index d9496f0..3e81fdb 100644 --- a/torchy_baselines/ppo/ppo.py +++ b/stable_baselines3/ppo/ppo.py @@ -14,14 +14,14 @@ import torch.nn.functional as F # SummaryWriter = None import numpy as np -from torchy_baselines.common import logger -from torchy_baselines.common.base_class import BaseRLModel -from torchy_baselines.common.type_aliases import GymEnv, MaybeCallback -from torchy_baselines.common.buffers import RolloutBuffer -from torchy_baselines.common.utils import explained_variance, get_schedule_fn -from torchy_baselines.common.vec_env import VecEnv -from torchy_baselines.common.callbacks import BaseCallback -from torchy_baselines.ppo.policies import PPOPolicy +from stable_baselines3.common import logger +from stable_baselines3.common.base_class import BaseRLModel +from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback +from stable_baselines3.common.buffers import RolloutBuffer +from stable_baselines3.common.utils import explained_variance, get_schedule_fn +from stable_baselines3.common.vec_env import VecEnv +from stable_baselines3.common.callbacks import BaseCallback +from stable_baselines3.ppo.policies import PPOPolicy class PPO(BaseRLModel): diff --git a/torchy_baselines/py.typed b/stable_baselines3/py.typed similarity index 100% rename from torchy_baselines/py.typed rename to stable_baselines3/py.typed diff --git a/stable_baselines3/sac/__init__.py b/stable_baselines3/sac/__init__.py new file mode 100644 index 0000000..8c89378 --- /dev/null +++ b/stable_baselines3/sac/__init__.py @@ -0,0 +1,2 @@ +from stable_baselines3.sac.sac import SAC +from stable_baselines3.sac.policies import MlpPolicy diff --git a/torchy_baselines/sac/policies.py b/stable_baselines3/sac/policies.py similarity index 98% rename from torchy_baselines/sac/policies.py rename to stable_baselines3/sac/policies.py index ecd6faf..868fde6 100644 --- a/torchy_baselines/sac/policies.py +++ b/stable_baselines3/sac/policies.py @@ -4,11 +4,11 @@ import gym import torch as th import torch.nn as nn -from torchy_baselines.common.preprocessing import get_action_dim -from torchy_baselines.common.policies import (BasePolicy, register_policy, create_mlp, +from stable_baselines3.common.preprocessing import get_action_dim +from stable_baselines3.common.policies import (BasePolicy, register_policy, create_mlp, create_sde_features_extractor, NatureCNN, BaseFeaturesExtractor, FlattenExtractor) -from torchy_baselines.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution +from stable_baselines3.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution # CAP the standard deviation of the actor LOG_STD_MAX = 2 diff --git a/torchy_baselines/sac/sac.py b/stable_baselines3/sac/sac.py similarity index 98% rename from torchy_baselines/sac/sac.py rename to stable_baselines3/sac/sac.py index c1cbc64..85aba9e 100644 --- a/torchy_baselines/sac/sac.py +++ b/stable_baselines3/sac/sac.py @@ -4,11 +4,11 @@ import torch as th import torch.nn.functional as F import numpy as np -from torchy_baselines.common import logger -from torchy_baselines.common.base_class import OffPolicyRLModel -from torchy_baselines.common.type_aliases import GymEnv, MaybeCallback -from torchy_baselines.common.noise import ActionNoise -from torchy_baselines.sac.policies import SACPolicy +from stable_baselines3.common import logger +from stable_baselines3.common.base_class import OffPolicyRLModel +from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback +from stable_baselines3.common.noise import ActionNoise +from stable_baselines3.sac.policies import SACPolicy class SAC(OffPolicyRLModel): diff --git a/stable_baselines3/td3/__init__.py b/stable_baselines3/td3/__init__.py new file mode 100644 index 0000000..96cecdf --- /dev/null +++ b/stable_baselines3/td3/__init__.py @@ -0,0 +1,2 @@ +from stable_baselines3.td3.td3 import TD3 +from stable_baselines3.td3.policies import MlpPolicy diff --git a/torchy_baselines/td3/policies.py b/stable_baselines3/td3/policies.py similarity index 99% rename from torchy_baselines/td3/policies.py rename to stable_baselines3/td3/policies.py index b43adaf..5965334 100644 --- a/torchy_baselines/td3/policies.py +++ b/stable_baselines3/td3/policies.py @@ -4,11 +4,11 @@ import gym import torch as th import torch.nn as nn -from torchy_baselines.common.preprocessing import get_action_dim -from torchy_baselines.common.policies import (BasePolicy, register_policy, create_mlp, +from stable_baselines3.common.preprocessing import get_action_dim +from stable_baselines3.common.policies import (BasePolicy, register_policy, create_mlp, create_sde_features_extractor, NatureCNN, BaseFeaturesExtractor, FlattenExtractor) -from torchy_baselines.common.distributions import StateDependentNoiseDistribution +from stable_baselines3.common.distributions import StateDependentNoiseDistribution class Actor(BasePolicy): diff --git a/torchy_baselines/td3/td3.py b/stable_baselines3/td3/td3.py similarity index 97% rename from torchy_baselines/td3/td3.py rename to stable_baselines3/td3/td3.py index 09742df..18361f2 100644 --- a/torchy_baselines/td3/td3.py +++ b/stable_baselines3/td3/td3.py @@ -2,11 +2,11 @@ import torch as th import torch.nn.functional as F from typing import List, Tuple, Type, Union, Callable, Optional, Dict, Any -from torchy_baselines.common import logger -from torchy_baselines.common.base_class import OffPolicyRLModel -from torchy_baselines.common.noise import ActionNoise -from torchy_baselines.common.type_aliases import GymEnv, MaybeCallback -from torchy_baselines.td3.policies import TD3Policy +from stable_baselines3.common import logger +from stable_baselines3.common.base_class import OffPolicyRLModel +from stable_baselines3.common.noise import ActionNoise +from stable_baselines3.common.type_aliases import GymEnv, MaybeCallback +from stable_baselines3.td3.policies import TD3Policy class TD3(OffPolicyRLModel): diff --git a/torchy_baselines/version.txt b/stable_baselines3/version.txt similarity index 100% rename from torchy_baselines/version.txt rename to stable_baselines3/version.txt diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 5f0fc07..ff8958f 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -4,8 +4,8 @@ import shutil import pytest import gym -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.callbacks import (CallbackList, CheckpointCallback, EvalCallback, +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.callbacks import (CallbackList, CheckpointCallback, EvalCallback, EveryNTimesteps, StopTrainingOnRewardThreshold) diff --git a/tests/test_cnn.py b/tests/test_cnn.py index 1aec9d4..9dc342d 100644 --- a/tests/test_cnn.py +++ b/tests/test_cnn.py @@ -3,8 +3,8 @@ import os import numpy as np import pytest -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.identity_env import FakeImageEnv +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.identity_env import FakeImageEnv SAVE_PATH = './cnn_model.zip' diff --git a/tests/test_custom_policy.py b/tests/test_custom_policy.py index dac69dc..9637f4e 100644 --- a/tests/test_custom_policy.py +++ b/tests/test_custom_policy.py @@ -1,7 +1,7 @@ import pytest import torch as th -from torchy_baselines import A2C, PPO, SAC, TD3 +from stable_baselines3 import A2C, PPO, SAC, TD3 @pytest.mark.parametrize('net_arch', [ diff --git a/tests/test_distributions.py b/tests/test_distributions.py index de97b13..1f340bb 100644 --- a/tests/test_distributions.py +++ b/tests/test_distributions.py @@ -1,11 +1,11 @@ import pytest import torch as th -from torchy_baselines import A2C, PPO -from torchy_baselines.common.distributions import (DiagGaussianDistribution, TanhBijector, +from stable_baselines3 import A2C, PPO +from stable_baselines3.common.distributions import (DiagGaussianDistribution, TanhBijector, StateDependentNoiseDistribution, CategoricalDistribution, SquashedDiagGaussianDistribution) -from torchy_baselines.common.utils import set_random_seed +from stable_baselines3.common.utils import set_random_seed N_ACTIONS = 2 diff --git a/tests/test_identity.py b/tests/test_identity.py index b717c0a..d937c7e 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -1,10 +1,10 @@ import numpy as np import pytest -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.identity_env import IdentityEnvBox, IdentityEnv -from torchy_baselines.common.evaluation import evaluate_policy -from torchy_baselines.common.noise import NormalActionNoise +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.identity_env import IdentityEnvBox, IdentityEnv +from stable_baselines3.common.evaluation import evaluate_policy +from stable_baselines3.common.noise import NormalActionNoise @pytest.mark.parametrize("model_class", [A2C, PPO]) diff --git a/tests/test_logger.py b/tests/test_logger.py index df6d059..f648091 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -4,7 +4,7 @@ import shutil import pytest import numpy as np -from torchy_baselines.common.logger import (make_output_format, read_csv, read_json, DEBUG, ScopedConfigure, +from stable_baselines3.common.logger import (make_output_format, read_csv, read_json, DEBUG, ScopedConfigure, info, debug, set_level, configure, logkv, logkvs, dumpkvs, logkv_mean, warn, error, reset) @@ -18,7 +18,7 @@ KEY_VALUES = { "g": np.array([[[1]]]), } -LOG_DIR = '/tmp/torchy_baselines/' +LOG_DIR = '/tmp/stable_baselines3/' def test_main(): diff --git a/tests/test_monitor.py b/tests/test_monitor.py index b21c33b..00a7802 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -5,7 +5,7 @@ import os import pandas import gym -from torchy_baselines.common.monitor import Monitor, get_monitor_files, load_results +from stable_baselines3.common.monitor import Monitor, get_monitor_files, load_results def test_monitor(tmp_path): diff --git a/tests/test_predict.py b/tests/test_predict.py index 5fd5064..fad35c7 100644 --- a/tests/test_predict.py +++ b/tests/test_predict.py @@ -1,8 +1,8 @@ import gym import pytest -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.vec_env import DummyVecEnv +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.vec_env import DummyVecEnv MODEL_LIST = [ PPO, diff --git a/tests/test_run.py b/tests/test_run.py index 0c6ea81..0c7174c 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,8 +1,8 @@ import numpy as np import pytest -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise normal_action_noise = NormalActionNoise(np.zeros(1), 0.1 * np.ones(1)) diff --git a/tests/test_save_load.py b/tests/test_save_load.py index 5e637bc..b7c0924 100644 --- a/tests/test_save_load.py +++ b/tests/test_save_load.py @@ -5,10 +5,10 @@ import pytest import numpy as np import torch as th -from torchy_baselines import A2C, PPO, SAC, TD3 -from torchy_baselines.common.identity_env import IdentityEnvBox -from torchy_baselines.common.vec_env import DummyVecEnv -from torchy_baselines.common.identity_env import FakeImageEnv +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.identity_env import IdentityEnvBox +from stable_baselines3.common.vec_env import DummyVecEnv +from stable_baselines3.common.identity_env import FakeImageEnv MODEL_LIST = [ diff --git a/tests/test_sde.py b/tests/test_sde.py index cf3bbdb..eed012f 100644 --- a/tests/test_sde.py +++ b/tests/test_sde.py @@ -2,7 +2,7 @@ import pytest import torch as th from torch.distributions import Normal -from torchy_baselines import A2C, TD3, SAC, PPO +from stable_baselines3 import A2C, TD3, SAC, PPO def test_state_dependent_exploration_grad(): diff --git a/tests/test_vec_envs.py b/tests/test_vec_envs.py index f2dd1c2..c6ab43b 100644 --- a/tests/test_vec_envs.py +++ b/tests/test_vec_envs.py @@ -7,7 +7,7 @@ import pytest import gym import numpy as np -from torchy_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv, VecNormalize, VecFrameStack +from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv, VecNormalize, VecFrameStack N_ENVS = 3 VEC_ENV_CLASSES = [DummyVecEnv, SubprocVecEnv] diff --git a/tests/test_vec_normalize.py b/tests/test_vec_normalize.py index 73c0824..5ec80c7 100644 --- a/tests/test_vec_normalize.py +++ b/tests/test_vec_normalize.py @@ -2,9 +2,9 @@ import gym import pytest import numpy as np -from torchy_baselines.common.running_mean_std import RunningMeanStd -from torchy_baselines.common.vec_env import DummyVecEnv, VecNormalize, VecFrameStack, sync_envs_normalization, unwrap_vec_normalize -from torchy_baselines import SAC, TD3 +from stable_baselines3.common.running_mean_std import RunningMeanStd +from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize, VecFrameStack, sync_envs_normalization, unwrap_vec_normalize +from stable_baselines3 import SAC, TD3 ENV_ID = 'Pendulum-v0' diff --git a/torchy_baselines/a2c/__init__.py b/torchy_baselines/a2c/__init__.py deleted file mode 100644 index 0cc4be0..0000000 --- a/torchy_baselines/a2c/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from torchy_baselines.a2c.a2c import A2C -from torchy_baselines.ppo.policies import MlpPolicy diff --git a/torchy_baselines/ppo/__init__.py b/torchy_baselines/ppo/__init__.py deleted file mode 100644 index 72a5560..0000000 --- a/torchy_baselines/ppo/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from torchy_baselines.ppo.ppo import PPO -from torchy_baselines.ppo.policies import MlpPolicy diff --git a/torchy_baselines/sac/__init__.py b/torchy_baselines/sac/__init__.py deleted file mode 100644 index 1132a37..0000000 --- a/torchy_baselines/sac/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from torchy_baselines.sac.sac import SAC -from torchy_baselines.sac.policies import MlpPolicy diff --git a/torchy_baselines/td3/__init__.py b/torchy_baselines/td3/__init__.py deleted file mode 100644 index 148be49..0000000 --- a/torchy_baselines/td3/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from torchy_baselines.td3.td3 import TD3 -from torchy_baselines.td3.policies import MlpPolicy From 2c34a4d6946d0f64a9d12026cd359b1e52a316e0 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 16:28:38 +0200 Subject: [PATCH 2/9] Sync with Stable-Baselines --- .coveragerc | 2 + Makefile | 23 ++ README.md | 4 +- docs/index.rst | 10 +- docs/misc/changelog.rst | 35 ++- setup.py | 2 +- stable_baselines3/common/bit_flipping_env.py | 121 ++++++++++ stable_baselines3/common/env_checker.py | 222 ++++++++++++++++++ stable_baselines3/common/vec_env/__init__.py | 2 + .../common/vec_env/vec_check_nan.py | 86 +++++++ .../common/vec_env/vec_video_recorder.py | 112 +++++++++ stable_baselines3/version.txt | 2 +- tests/test_envs.py | 149 ++++++++++++ tests/test_vec_check_nan.py | 72 ++++++ 14 files changed, 832 insertions(+), 10 deletions(-) create mode 100644 stable_baselines3/common/bit_flipping_env.py create mode 100644 stable_baselines3/common/env_checker.py create mode 100644 stable_baselines3/common/vec_env/vec_check_nan.py create mode 100644 stable_baselines3/common/vec_env/vec_video_recorder.py create mode 100644 tests/test_envs.py create mode 100644 tests/test_vec_check_nan.py diff --git a/.coveragerc b/.coveragerc index 9020116..8ee0fb9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,8 @@ omit = setup.py # Require graphical interface stable_baselines3/common/results_plotter.py + # Require ffmpeg + stable_baselines3/common/vec_env/vec_video_recorder.py [report] exclude_lines = diff --git a/Makefile b/Makefile index 5a31e2d..4d95b7d 100644 --- a/Makefile +++ b/Makefile @@ -16,3 +16,26 @@ clean: cd docs && make clean .PHONY: clean spelling doc + +# TODO: create Dockerfile +# # Build docker images +# # If you do export RELEASE=True, it will also push them +# docker: docker-cpu docker-gpu +# +# docker-cpu: +# ./scripts/build_docker.sh +# +# docker-gpu: +# USE_GPU=True ./scripts/build_docker.sh + +# PyPi package release +release: + python setup.py sdist + python setup.py bdist_wheel + twine upload dist/* + +# Test PyPi package release +test-release: + python setup.py sdist + python setup.py bdist_wheel + twine upload --repository-url https://test.pypi.org/legacy/ dist/* diff --git a/README.md b/README.md index 5f35c82..bc0efa7 100644 --- a/README.md +++ b/README.md @@ -56,12 +56,12 @@ make spelling To cite this repository in publications: ``` -@misc{torchy-baselines, +@misc{stable-baselines3, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah}, title = {Stable Baselines3}, year = {2019}, publisher = {GitHub}, journal = {GitHub repository}, - howpublished = {\url{https://github.com/araffin/torchy-baselines}}, + howpublished = {\url{https://github.com/DLR-RM/stable-baselines3}}, } ``` diff --git a/docs/index.rst b/docs/index.rst index c97722b..9e74bd7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,8 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to Stable Baselines3 docs! - Pytorch RL Baselines -======================================================== +Welcome to Stable Baselines3 docs! +================================== `Stable Baselines3 `_ is the PyTorch version of `Stable Baselines `_, a set of improved implementations of reinforcement learning algorithms. @@ -42,18 +42,18 @@ RL Baselines zoo also offers a simple interface to train, evaluate agents and do Citing Stable Baselines3 ------------------------ +------------------------ To cite this project in publications: .. code-block:: bibtex - @misc{torchy-baselines, + @misc{stable-baselines3, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah}, title = {Stable Baselines3}, year = {2019}, publisher = {GitHub}, journal = {GitHub repository}, - howpublished = {\url{https://github.com/araffin/torchy-baselines}}, + howpublished = {\url{https://github.com/DLR-RM/stable-baselines3}}, } Indices and tables diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 3767b7c..86b7246 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -3,6 +3,32 @@ Changelog ========== +Pre-Release 0.6.0a (WIP) +------------------------------ + + +Breaking Changes: +^^^^^^^^^^^^^^^^^ + +New Features: +^^^^^^^^^^^^^ +- Added env checker (Sync with Stable Baselines) +- Added ``VecCheckNan`` and ``VecVideoRecorder`` (Sync with Stable Baselines) + +Bug Fixes: +^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Others: +^^^^^^^ +- Renamed to Stable-Baseline3 + +Documentation: +^^^^^^^^^^^^^^ + + Pre-Release 0.5.0 (2020-05-05) ------------------------------ @@ -160,10 +186,17 @@ New Features: Maintainers ----------- -Stable-Baselines3 is currently maintained by `Antonin Raffin`_ (aka `@araffin`_). +Stable-Baselines3 is currently maintained by `Antonin Raffin`_ (aka `@araffin`_), `Ashley Hill`_ (aka @hill-a), +`Maximilian Ernestus`_ (aka @erniejunior), `Adam Gleave`_ (`@AdamGleave`_) and `Anssi Kanervisto`_ (aka `@Miffyli`_). +.. _Ashley Hill: https://github.com/hill-a .. _Antonin Raffin: https://araffin.github.io/ +.. _Maximilian Ernestus: https://github.com/erniejunior +.. _Adam Gleave: https://gleave.me/ .. _@araffin: https://github.com/araffin +.. _@AdamGleave: https://github.com/adamgleave +.. _Anssi Kanervisto: https://github.com/Miffyli +.. _@Miffyli: https://github.com/Miffyli diff --git a/setup.py b/setup.py index 780f504..40aec1f 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup(name='stable_baselines3', }, description='Pytorch version of Stable Baselines, implementations of reinforcement learning algorithms.', author='Antonin Raffin', - url='', + url='https://github.com/DLR-RM/stable-baselines3', author_email='antonin.raffin@dlr.de', keywords="reinforcement-learning-algorithms reinforcement-learning machine-learning " "gym openai stable baselines toolbox python data-science", diff --git a/stable_baselines3/common/bit_flipping_env.py b/stable_baselines3/common/bit_flipping_env.py new file mode 100644 index 0000000..1dd8815 --- /dev/null +++ b/stable_baselines3/common/bit_flipping_env.py @@ -0,0 +1,121 @@ +from collections import OrderedDict +from typing import Optional, Union + +import numpy as np +from gym import GoalEnv, spaces + +from stable_baselines3.common.type_aliases import GymStepReturn + + +class BitFlippingEnv(GoalEnv): + """ + Simple bit flipping env, useful to test HER. + The goal is to flip all the bits to get a vector of ones. + In the continuous variant, if the ith action component has a value > 0, + then the ith bit will be flipped. + + :param n_bits: (int) Number of bits to flip + :param continuous: (bool) Whether to use the continuous actions version or not, + by default, it uses the discrete one + :param max_steps: (Optional[int]) Max number of steps, by default, equal to n_bits + :param discrete_obs_space: (bool) Whether to use the discrete observation + version or not, by default, it uses the MultiBinary one + """ + def __init__(self, n_bits: int = 10, + continuous: bool = False, + max_steps: Optional[int] = None, + discrete_obs_space: bool = False): + super(BitFlippingEnv, self).__init__() + # The achieved goal is determined by the current state + # here, it is a special where they are equal + if discrete_obs_space: + # In the discrete case, the agent act on the binary + # representation of the observation + self.observation_space = spaces.Dict({ + 'observation': spaces.Discrete(2 ** n_bits - 1), + 'achieved_goal': spaces.Discrete(2 ** n_bits - 1), + 'desired_goal': spaces.Discrete(2 ** n_bits - 1) + }) + else: + self.observation_space = spaces.Dict({ + 'observation': spaces.MultiBinary(n_bits), + 'achieved_goal': spaces.MultiBinary(n_bits), + 'desired_goal': spaces.MultiBinary(n_bits) + }) + + self.obs_space = spaces.MultiBinary(n_bits) + + if continuous: + self.action_space = spaces.Box(-1, 1, shape=(n_bits,), dtype=np.float32) + else: + self.action_space = spaces.Discrete(n_bits) + self.continuous = continuous + self.discrete_obs_space = discrete_obs_space + self.state = None + self.desired_goal = np.ones((n_bits,)) + if max_steps is None: + max_steps = n_bits + self.max_steps = max_steps + self.current_step = 0 + self.reset() + + def convert_if_needed(self, state: np.ndarray) -> Union[int, np.ndarray]: + """ + Convert to discrete space if needed. + + :param state: (np.ndarray) + :return: (np.ndarray or int) + """ + if self.discrete_obs_space: + # The internal state is the binary representation of the + # observed one + return int(sum([state[i] * 2**i for i in range(len(state))])) + return state + + def _get_obs(self) -> OrderedDict: + """ + Helper to create the observation. + + :return: (OrderedDict) + """ + return OrderedDict([ + ('observation', self.convert_if_needed(self.state.copy())), + ('achieved_goal', self.convert_if_needed(self.state.copy())), + ('desired_goal', self.convert_if_needed(self.desired_goal.copy())) + ]) + + def reset(self) -> OrderedDict: + self.current_step = 0 + self.state = self.obs_space.sample() + return self._get_obs() + + def step(self, action: Union[np.ndarray, int]) -> GymStepReturn: + if self.continuous: + self.state[action > 0] = 1 - self.state[action > 0] + else: + self.state[action] = 1 - self.state[action] + obs = self._get_obs() + reward = self.compute_reward(obs['achieved_goal'], obs['desired_goal'], None) + done = reward == 0 + self.current_step += 1 + # Episode terminate when we reached the goal or the max number of steps + info = {'is_success': done} + done = done or self.current_step >= self.max_steps + return obs, reward, done, info + + def compute_reward(self, + achieved_goal: np.ndarray, + desired_goal: np.ndarray, + _info) -> float: + # Deceptive reward: it is positive only when the goal is achieved + if self.discrete_obs_space: + return 0.0 if achieved_goal == desired_goal else -1.0 + return 0.0 if (achieved_goal == desired_goal).all() else -1.0 + + def render(self, mode: str = 'human') -> Optional[np.ndarray]: + if mode == 'rgb_array': + return self.state.copy() + print(self.state) + + def close(self) -> None: + pass diff --git a/stable_baselines3/common/env_checker.py b/stable_baselines3/common/env_checker.py new file mode 100644 index 0000000..751b204 --- /dev/null +++ b/stable_baselines3/common/env_checker.py @@ -0,0 +1,222 @@ +import warnings +from typing import Union + +import gym +from gym import spaces +import numpy as np + +from stable_baselines.common.vec_env import DummyVecEnv, VecCheckNan + + +def _enforce_array_obs(observation_space: spaces.Space) -> bool: + """ + Whether to check that the returned observation is a numpy array + it is not mandatory for `Dict` and `Tuple` spaces. + """ + return not isinstance(observation_space, (spaces.Dict, spaces.Tuple)) + + +def _check_image_input(observation_space: spaces.Box) -> None: + """ + Check that the input will be compatible with Stable-Baselines + when the observation is apparently an image. + """ + if observation_space.dtype != np.uint8: + warnings.warn("It seems that your observation is an image but the `dtype` " + "of your observation_space is not `np.uint8`. " + "If your observation is not an image, we recommend you to flatten the observation " + "to have only a 1D vector") + + if np.any(observation_space.low != 0) or np.any(observation_space.high != 255): + warnings.warn("It seems that your observation space is an image but the " + "upper and lower bounds are not in [0, 255]. " + "Because the CNN policy normalize automatically the observation " + "you may encounter issue if the values are not in that range." + ) + + if observation_space.shape[0] < 36 or observation_space.shape[1] < 36: + warnings.warn("The minimal resolution for an image is 36x36 for the default CnnPolicy. " + "You might need to use a custom `cnn_extractor` " + "cf https://stable-baselines.readthedocs.io/en/master/guide/custom_policy.html") + + +def _check_unsupported_obs_spaces(env: gym.Env, observation_space: spaces.Space) -> None: + """Emit warnings when the observation space used is not supported by Stable-Baselines.""" + + if isinstance(observation_space, spaces.Dict) and not isinstance(env, gym.GoalEnv): + warnings.warn("The observation space is a Dict but the environment is not a gym.GoalEnv " + "(cf https://github.com/openai/gym/blob/master/gym/core.py), " + "this is currently not supported by Stable Baselines " + "(cf https://github.com/hill-a/stable-baselines/issues/133), " + "you will need to use a custom policy. " + ) + + if isinstance(observation_space, spaces.Tuple): + warnings.warn("The observation space is a Tuple," + "this is currently not supported by Stable Baselines " + "(cf https://github.com/hill-a/stable-baselines/issues/133), " + "you will need to flatten the observation and maybe use a custom policy. " + ) + + +def _check_nan(env: gym.Env) -> None: + """Check for Inf and NaN using the VecWrapper.""" + vec_env = VecCheckNan(DummyVecEnv([lambda: env])) + for _ in range(10): + action = [env.action_space.sample()] + _, _, _, _ = vec_env.step(action) + + +def _check_obs(obs: Union[tuple, dict, np.ndarray, int], + observation_space: spaces.Space, + method_name: str) -> None: + """ + Check that the observation returned by the environment + correspond to the declared one. + """ + if not isinstance(observation_space, spaces.Tuple): + assert not isinstance(obs, tuple), ("The observation returned by the `{}()` " + "method should be a single value, not a tuple".format(method_name)) + + # The check for a GoalEnv is done by the base class + if isinstance(observation_space, spaces.Discrete): + assert isinstance(obs, int), "The observation returned by `{}()` method must be an int".format(method_name) + elif _enforce_array_obs(observation_space): + assert isinstance(obs, np.ndarray), ("The observation returned by `{}()` " + "method must be a numpy array".format(method_name)) + + assert observation_space.contains(obs), ("The observation returned by the `{}()` " + "method does not match the given observation space".format(method_name)) + + +def _check_returned_values(env: gym.Env, observation_space: spaces.Space, action_space: spaces.Space) -> None: + """ + Check the returned values by the env when calling `.reset()` or `.step()` methods. + """ + # because env inherits from gym.Env, we assume that `reset()` and `step()` methods exists + obs = env.reset() + + _check_obs(obs, observation_space, 'reset') + + # Sample a random action + action = action_space.sample() + data = env.step(action) + + assert len(data) == 4, "The `step()` method must return four values: obs, reward, done, info" + + # Unpack + obs, reward, done, info = data + + _check_obs(obs, observation_space, 'step') + + # We also allow int because the reward will be cast to float + assert isinstance(reward, (float, int)), "The reward returned by `step()` must be a float" + assert isinstance(done, bool), "The `done` signal must be a boolean" + assert isinstance(info, dict), "The `info` returned by `step()` must be a python dictionary" + + if isinstance(env, gym.GoalEnv): + # For a GoalEnv, the keys are checked at reset + assert reward == env.compute_reward(obs['achieved_goal'], obs['desired_goal'], info) + + +def _check_spaces(env: gym.Env) -> None: + """ + Check that the observation and action spaces are defined + and inherit from gym.spaces.Space. + """ + # Helper to link to the code, because gym has no proper documentation + gym_spaces = " cf https://github.com/openai/gym/blob/master/gym/spaces/" + + assert hasattr(env, 'observation_space'), "You must specify an observation space (cf gym.spaces)" + gym_spaces + assert hasattr(env, 'action_space'), "You must specify an action space (cf gym.spaces)" + gym_spaces + + assert isinstance(env.observation_space, + spaces.Space), "The observation space must inherit from gym.spaces" + gym_spaces + assert isinstance(env.action_space, spaces.Space), "The action space must inherit from gym.spaces" + gym_spaces + + +def _check_render(env: gym.Env, warn: bool = True, headless: bool = False) -> None: + """ + Check the declared render modes and the `render()`/`close()` + method of the environment. + + :param env: (gym.Env) The environment to check + :param warn: (bool) Whether to output additional warnings + :param headless: (bool) Whether to disable render modes + that require a graphical interface. False by default. + """ + render_modes = env.metadata.get('render.modes') + if render_modes is None: + if warn: + warnings.warn("No render modes was declared in the environment " + " (env.metadata['render.modes'] is None or not defined), " + "you may have trouble when calling `.render()`") + + else: + # Don't check render mode that require a + # graphical interface (useful for CI) + if headless and 'human' in render_modes: + render_modes.remove('human') + # Check all declared render modes + for render_mode in render_modes: + env.render(mode=render_mode) + env.close() + + +def check_env(env: gym.Env, warn: bool = True, skip_render_check: bool = True) -> None: + """ + Check that an environment follows Gym API. + This is particularly useful when using a custom environment. + Please take a look at https://github.com/openai/gym/blob/master/gym/core.py + for more information about the API. + + It also optionally check that the environment is compatible with Stable-Baselines. + + :param env: (gym.Env) The Gym environment that will be checked + :param warn: (bool) Whether to output additional warnings + mainly related to the interaction with Stable Baselines + :param skip_render_check: (bool) Whether to skip the checks for the render method. + True by default (useful for the CI) + """ + assert isinstance(env, gym.Env), ("You environment must inherit from gym.Env class " + " cf https://github.com/openai/gym/blob/master/gym/core.py") + + # ============= Check the spaces (observation and action) ================ + _check_spaces(env) + + # Define aliases for convenience + observation_space = env.observation_space + action_space = env.action_space + + # Warn the user if needed. + # A warning means that the environment may run but not work properly with Stable Baselines algorithms + if warn: + _check_unsupported_obs_spaces(env, observation_space) + + # If image, check the low and high values, the type and the number of channels + # and the shape (minimal value) + if isinstance(observation_space, spaces.Box) and len(observation_space.shape) == 3: + _check_image_input(observation_space) + + if isinstance(observation_space, spaces.Box) and len(observation_space.shape) not in [1, 3]: + warnings.warn("Your observation has an unconventional shape (neither an image, nor a 1D vector). " + "We recommend you to flatten the observation " + "to have only a 1D vector") + + # Check for the action space, it may lead to hard-to-debug issues + if (isinstance(action_space, spaces.Box) and + (np.any(np.abs(action_space.low) != np.abs(action_space.high)) + or np.any(np.abs(action_space.low) > 1) or np.any(np.abs(action_space.high) > 1))): + warnings.warn("We recommend you to use a symmetric and normalized Box action space (range=[-1, 1]) " + "cf https://stable-baselines.readthedocs.io/en/master/guide/rl_tips.html") + + # ============ Check the returned values =============== + _check_returned_values(env, observation_space, action_space) + + # ==== Check the render method and the declared render modes ==== + if not skip_render_check: + _check_render(env, warn=warn) + + # The check only works with numpy arrays + if _enforce_array_obs(observation_space): + _check_nan(env) diff --git a/stable_baselines3/common/vec_env/__init__.py b/stable_baselines3/common/vec_env/__init__.py index 2c89181..8f7719a 100644 --- a/stable_baselines3/common/vec_env/__init__.py +++ b/stable_baselines3/common/vec_env/__init__.py @@ -10,6 +10,8 @@ from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv from stable_baselines3.common.vec_env.vec_frame_stack import VecFrameStack from stable_baselines3.common.vec_env.vec_normalize import VecNormalize from stable_baselines3.common.vec_env.vec_transpose import VecTransposeImage +from stable_baselines3.common.vec_env.vec_video_recorder import VecVideoRecorder +from stable_baselines3.common.vec_env.vec_check_nan import VecCheckNan # Avoid circular import if typing.TYPE_CHECKING: diff --git a/stable_baselines3/common/vec_env/vec_check_nan.py b/stable_baselines3/common/vec_env/vec_check_nan.py new file mode 100644 index 0000000..613be0d --- /dev/null +++ b/stable_baselines3/common/vec_env/vec_check_nan.py @@ -0,0 +1,86 @@ +import warnings + +import numpy as np + +from stable_baselines3.common.vec_env.base_vec_env import VecEnvWrapper + + +class VecCheckNan(VecEnvWrapper): + """ + NaN and inf checking wrapper for vectorized environment, will raise a warning by default, + allowing you to know from what the NaN of inf originated from. + + :param venv: (VecEnv) the vectorized environment to wrap + :param raise_exception: (bool) Whether or not to raise a ValueError, instead of a UserWarning + :param warn_once: (bool) Whether or not to only warn once. + :param check_inf: (bool) Whether or not to check for +inf or -inf as well + """ + + def __init__(self, venv, raise_exception=False, warn_once=True, check_inf=True): + VecEnvWrapper.__init__(self, venv) + self.raise_exception = raise_exception + self.warn_once = warn_once + self.check_inf = check_inf + self._actions = None + self._observations = None + self._user_warned = False + + def step_async(self, actions): + self._check_val(async_step=True, actions=actions) + + self._actions = actions + self.venv.step_async(actions) + + def step_wait(self): + observations, rewards, news, infos = self.venv.step_wait() + + self._check_val(async_step=False, observations=observations, rewards=rewards, news=news) + + self._observations = observations + return observations, rewards, news, infos + + def reset(self): + observations = self.venv.reset() + self._actions = None + + self._check_val(async_step=False, observations=observations) + + self._observations = observations + return observations + + def _check_val(self, *, async_step, **kwargs): + # if warn and warn once and have warned once: then stop checking + if not self.raise_exception and self.warn_once and self._user_warned: + return + + found = [] + for name, val in kwargs.items(): + has_nan = np.any(np.isnan(val)) + has_inf = self.check_inf and np.any(np.isinf(val)) + if has_inf: + found.append((name, "inf")) + if has_nan: + found.append((name, "nan")) + + if found: + self._user_warned = True + msg = "" + for i, (name, type_val) in enumerate(found): + msg += "found {} in {}".format(type_val, name) + if i != len(found) - 1: + msg += ", " + + msg += ".\r\nOriginated from the " + + if not async_step: + if self._actions is None: + msg += "environment observation (at reset)" + else: + msg += "environment, Last given value was: \r\n\taction={}".format(self._actions) + else: + msg += "RL model, Last given value was: \r\n\tobservations={}".format(self._observations) + + if self.raise_exception: + raise ValueError(msg) + else: + warnings.warn(msg, UserWarning) diff --git a/stable_baselines3/common/vec_env/vec_video_recorder.py b/stable_baselines3/common/vec_env/vec_video_recorder.py new file mode 100644 index 0000000..35409b0 --- /dev/null +++ b/stable_baselines3/common/vec_env/vec_video_recorder.py @@ -0,0 +1,112 @@ +import os + +from gym.wrappers.monitoring import video_recorder + +from stable_baselines3.common import logger +from stable_baselines3.common.vec_env.base_vec_env import VecEnvWrapper +from stable_baselines3.common.vec_env.dummy_vec_env import DummyVecEnv +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from stable_baselines3.common.vec_env.vec_frame_stack import VecFrameStack +from stable_baselines3.common.vec_env.vec_normalize import VecNormalize + + +class VecVideoRecorder(VecEnvWrapper): + """ + Wraps a VecEnv or VecEnvWrapper object to record rendered image as mp4 video. + It requires ffmpeg or avconv to be installed on the machine. + + :param venv: (VecEnv or VecEnvWrapper) + :param video_folder: (str) Where to save videos + :param record_video_trigger: (func) Function that defines when to start recording. + The function takes the current number of step, + and returns whether we should start recording or not. + :param video_length: (int) Length of recorded videos + :param name_prefix: (str) Prefix to the video name + """ + + def __init__(self, venv, video_folder, record_video_trigger, + video_length=200, name_prefix='rl-video'): + + VecEnvWrapper.__init__(self, venv) + + self.env = venv + # Temp variable to retrieve metadata + temp_env = venv + + # Unwrap to retrieve metadata dict + # that will be used by gym recorder + while isinstance(temp_env, VecNormalize) or isinstance(temp_env, VecFrameStack): + temp_env = temp_env.venv + + if isinstance(temp_env, DummyVecEnv) or isinstance(temp_env, SubprocVecEnv): + metadata = temp_env.get_attr('metadata')[0] + else: + metadata = temp_env.metadata + + self.env.metadata = metadata + + self.record_video_trigger = record_video_trigger + self.video_recorder = None + + self.video_folder = os.path.abspath(video_folder) + # Create output folder if needed + os.makedirs(self.video_folder, exist_ok=True) + + self.name_prefix = name_prefix + self.step_id = 0 + self.video_length = video_length + + self.recording = False + self.recorded_frames = 0 + + def reset(self): + obs = self.venv.reset() + self.start_video_recorder() + return obs + + def start_video_recorder(self): + self.close_video_recorder() + + video_name = '{}-step-{}-to-step-{}'.format(self.name_prefix, self.step_id, + self.step_id + self.video_length) + base_path = os.path.join(self.video_folder, video_name) + self.video_recorder = video_recorder.VideoRecorder( + env=self.env, + base_path=base_path, + metadata={'step_id': self.step_id} + ) + + self.video_recorder.capture_frame() + self.recorded_frames = 1 + self.recording = True + + def _video_enabled(self): + return self.record_video_trigger(self.step_id) + + def step_wait(self): + obs, rews, dones, infos = self.venv.step_wait() + + self.step_id += 1 + if self.recording: + self.video_recorder.capture_frame() + self.recorded_frames += 1 + if self.recorded_frames > self.video_length: + logger.info("Saving video to ", self.video_recorder.path) + self.close_video_recorder() + elif self._video_enabled(): + self.start_video_recorder() + + return obs, rews, dones, infos + + def close_video_recorder(self): + if self.recording: + self.video_recorder.close() + self.recording = False + self.recorded_frames = 1 + + def close(self): + VecEnvWrapper.close(self) + self.close_video_recorder() + + def __del__(self): + self.close() diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt index 8f0916f..b7c431c 100644 --- a/stable_baselines3/version.txt +++ b/stable_baselines3/version.txt @@ -1 +1 @@ -0.5.0 +0.6.0a0 diff --git a/tests/test_envs.py b/tests/test_envs.py new file mode 100644 index 0000000..9c11d71 --- /dev/null +++ b/tests/test_envs.py @@ -0,0 +1,149 @@ +import pytest +import gym +from gym import spaces +import numpy as np + +from stable_baselines3.common.env_checker import check_env +from stable_baselines3.common.bit_flipping_env import BitFlippingEnv +from stable_baselines3.common.identity_env import (IdentityEnv, IdentityEnvBox, + IdentityEnvMultiBinary, IdentityEnvMultiDiscrete,) + +ENV_CLASSES = [BitFlippingEnv, IdentityEnv, IdentityEnvBox, IdentityEnvMultiBinary, + IdentityEnvMultiDiscrete] + + +@pytest.mark.parametrize("env_id", ['CartPole-v0', 'Pendulum-v0']) +def test_env(env_id): + """ + Check that environmnent integrated in Gym pass the test. + + :param env_id: (str) + """ + env = gym.make(env_id) + with pytest.warns(None) as record: + check_env(env) + + # Pendulum-v0 will produce a warning because the action space is + # in [-2, 2] and not [-1, 1] + if env_id == 'Pendulum-v0': + assert len(record) == 1 + else: + # The other environments must pass without warning + assert len(record) == 0 + + +@pytest.mark.parametrize("env_class", ENV_CLASSES) +def test_custom_envs(env_class): + env = env_class() + check_env(env) + + +def test_high_dimension_action_space(): + """ + Test for continuous action space + with more than one action. + """ + env = gym.make('Pendulum-v0') + # Patch the action space + env.action_space = spaces.Box(low=-1, high=1, shape=(20,), dtype=np.float32) + # Patch to avoid error + def patched_step(_action): + return env.observation_space.sample(), 0.0, False, {} + env.step = patched_step + check_env(env) + + +@pytest.mark.parametrize("new_obs_space", [ + # Small image + spaces.Box(low=0, high=255, shape=(32, 32, 3), dtype=np.uint8), + # Range not in [0, 255] + spaces.Box(low=0, high=1, shape=(64, 64, 3), dtype=np.uint8), + # Wrong dtype + spaces.Box(low=0, high=255, shape=(64, 64, 3), dtype=np.float32), + # Not an image, it should be a 1D vector + spaces.Box(low=-1, high=1, shape=(64, 3), dtype=np.float32), + # Tuple space is not supported by SB + spaces.Tuple([spaces.Discrete(5), spaces.Discrete(10)]), + # Dict space is not supported by SB when env is not a GoalEnv + spaces.Dict({"position": spaces.Discrete(5)}), +]) +def test_non_default_spaces(new_obs_space): + env = gym.make('BreakoutNoFrameskip-v4') + env.observation_space = new_obs_space + # Patch methods to avoid errors + env.reset = new_obs_space.sample + + def patched_step(_action): + return new_obs_space.sample(), 0.0, False, {} + + env.step = patched_step + with pytest.warns(UserWarning): + check_env(env) + + +def check_reset_assert_error(env, new_reset_return): + """ + Helper to check that the error is caught. + :param env: (gym.Env) + :param new_reset_return: (Any) + """ + + def wrong_reset(): + return new_reset_return + + # Patch the reset method with a wrong one + env.reset = wrong_reset + with pytest.raises(AssertionError): + check_env(env) + + +def test_common_failures_reset(): + """ + Test that common failure cases of the `reset_method` are caught + """ + env = IdentityEnvBox() + # Return an observation that does not match the observation_space + check_reset_assert_error(env, np.ones((3,))) + # The observation is not a numpy array + check_reset_assert_error(env, 1) + + # Return not only the observation + check_reset_assert_error(env, (env.observation_space.sample(), False)) + + +def check_step_assert_error(env, new_step_return=()): + """ + Helper to check that the error is caught. + :param env: (gym.Env) + :param new_step_return: (tuple) + """ + + def wrong_step(_action): + return new_step_return + + # Patch the step method with a wrong one + env.step = wrong_step + with pytest.raises(AssertionError): + check_env(env) + + +def test_common_failures_step(): + """ + Test that common failure cases of the `step` method are caught + """ + env = IdentityEnvBox() + + # Wrong shape for the observation + check_step_assert_error(env, (np.ones((4,)), 1.0, False, {})) + # Obs is not a numpy array + check_step_assert_error(env, (1, 1.0, False, {})) + + # Return a wrong reward + check_step_assert_error(env, (env.observation_space.sample(), np.ones(1), False, {})) + + # Info dict is not returned + check_step_assert_error(env, (env.observation_space.sample(), 0.0, False)) + + # Done is not a boolean + check_step_assert_error(env, (env.observation_space.sample(), 0.0, 3.0, {})) + check_step_assert_error(env, (env.observation_space.sample(), 0.0, 1, {})) diff --git a/tests/test_vec_check_nan.py b/tests/test_vec_check_nan.py new file mode 100644 index 0000000..85dd884 --- /dev/null +++ b/tests/test_vec_check_nan.py @@ -0,0 +1,72 @@ +import gym +from gym import spaces +import numpy as np + +from stable_baselines.common.vec_env import DummyVecEnv, VecCheckNan + + +class NanAndInfEnv(gym.Env): + """Custom Environment that raised NaNs and Infs""" + metadata = {'render.modes': ['human']} + + def __init__(self): + super(NanAndInfEnv, self).__init__() + self.action_space = spaces.Box(low=-np.inf, high=np.inf, shape=(1,), dtype=np.float64) + self.observation_space = spaces.Box(low=-np.inf, high=np.inf, shape=(1,), dtype=np.float64) + + @staticmethod + def step(action): + if np.all(np.array(action) > 0): + obs = float('NaN') + elif np.all(np.array(action) < 0): + obs = float('inf') + else: + obs = 0 + return [obs], 0.0, False, {} + + @staticmethod + def reset(): + return [0.0] + + def render(self, mode='human', close=False): + pass + + +def test_check_nan(): + """Test VecCheckNan Object""" + + env = DummyVecEnv([NanAndInfEnv]) + env = VecCheckNan(env, raise_exception=True) + + env.step([[0]]) + + try: + env.step([[float('NaN')]]) + except ValueError: + pass + else: + assert False + + try: + env.step([[float('inf')]]) + except ValueError: + pass + else: + assert False + + try: + env.step([[-1]]) + except ValueError: + pass + else: + assert False + + try: + env.step([[1]]) + except ValueError: + pass + else: + assert False + + + env.step(np.array([[0, 1], [0, 1]])) From 62bde9a970d0ef1213de6fb77cf3c0105acf88cf Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 16:30:26 +0200 Subject: [PATCH 3/9] Fix import --- stable_baselines3/common/env_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stable_baselines3/common/env_checker.py b/stable_baselines3/common/env_checker.py index 751b204..6a7f781 100644 --- a/stable_baselines3/common/env_checker.py +++ b/stable_baselines3/common/env_checker.py @@ -5,7 +5,7 @@ import gym from gym import spaces import numpy as np -from stable_baselines.common.vec_env import DummyVecEnv, VecCheckNan +from stable_baselines3.common.vec_env import DummyVecEnv, VecCheckNan def _enforce_array_obs(observation_space: spaces.Space) -> bool: From 04d85ac2e2d9999d60df3fd2de2ec4a419c68452 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 16:32:08 +0200 Subject: [PATCH 4/9] Fix import in tests --- tests/test_vec_check_nan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_vec_check_nan.py b/tests/test_vec_check_nan.py index 85dd884..a04410a 100644 --- a/tests/test_vec_check_nan.py +++ b/tests/test_vec_check_nan.py @@ -2,7 +2,7 @@ import gym from gym import spaces import numpy as np -from stable_baselines.common.vec_env import DummyVecEnv, VecCheckNan +from stable_baselines3.common.vec_env import DummyVecEnv, VecCheckNan class NanAndInfEnv(gym.Env): From cf1ae840c81acc34629172418e6d461c42e434d0 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 16:52:22 +0200 Subject: [PATCH 5/9] Sync identity envs --- stable_baselines3/common/identity_env.py | 66 ++++++++++++------------ stable_baselines3/common/utils.py | 7 ++- tests/test_deterministic.py | 36 +++++++++++++ tests/test_envs.py | 8 +-- 4 files changed, 78 insertions(+), 39 deletions(-) create mode 100644 tests/test_deterministic.py diff --git a/stable_baselines3/common/identity_env.py b/stable_baselines3/common/identity_env.py index 0a3daad..e1bb9c2 100644 --- a/stable_baselines3/common/identity_env.py +++ b/stable_baselines3/common/identity_env.py @@ -1,7 +1,7 @@ -from typing import List, Union +from typing import List, Union, Optional import numpy as np -from gym import Env +from gym import Env, Space from gym.spaces import Discrete, MultiDiscrete, MultiBinary, Box @@ -9,22 +9,36 @@ from stable_baselines3.common.type_aliases import GymStepReturn, GymObs class IdentityEnv(Env): - def __init__(self, dim, ep_length=100): + def __init__(self, + dim: Optional[int] = None, + space: Optional[Space] = None, + ep_length: int = 100): """ Identity environment for testing purposes - :param dim: (int) the size of the dimensions you want to learn - :param ep_length: (int) the length of each episodes in timesteps + :param dim: the size of the action and observation dimension you want + to learn. Provide at most one of ``dim`` and ``space``. If both are + None, then initialization proceeds with ``dim=1`` and ``space=None``. + :param space: the action and observation space. Provide at most one of + ``dim`` and ``space``. + :param ep_length: the length of each episode in timesteps """ - self.action_space = Discrete(dim) - self.observation_space = self.action_space + if space is None: + if dim is None: + dim = 1 + space = Discrete(dim) + else: + assert dim is None, "arguments for both 'dim' and 'space' provided: at most one allowed" + + self.action_space = self.observation_space = space self.ep_length = ep_length self.current_step = 0 - self.dim = dim + self.num_resets = -1 # Becomes 0 after __init__ exits. self.reset() def reset(self) -> GymObs: self.current_step = 0 + self.num_resets += 1 self._choose_next_state() return self.state @@ -55,18 +69,11 @@ class IdentityEnvBox(IdentityEnv): :param low: (float) the lower bound of the box dim :param high: (float) the upper bound of the box dim :param eps: (float) the epsilon bound for correct value - :param ep_length: (int) the length of each episodes in timesteps + :param ep_length: (int) the length of each episode in timesteps """ - super(IdentityEnvBox, self).__init__(1, ep_length) - self.action_space = Box(low=low, high=high, shape=(1,), dtype=np.float32) - self.observation_space = self.action_space + space = Box(low=low, high=high, shape=(1,), dtype=np.float32) + super().__init__(ep_length=ep_length, space=space) self.eps = eps - self.reset() - - def reset(self) -> np.ndarray: - self.current_step = 0 - self._choose_next_state() - return self.state def step(self, action: np.ndarray) -> GymStepReturn: reward = self._get_reward(action) @@ -75,39 +82,32 @@ class IdentityEnvBox(IdentityEnv): done = self.current_step >= self.ep_length return self.state, reward, done, {} - def _choose_next_state(self) -> None: - self.state = self.observation_space.sample() - def _get_reward(self, action: np.ndarray) -> float: return 1.0 if (self.state - self.eps) <= action <= (self.state + self.eps) else 0.0 class IdentityEnvMultiDiscrete(IdentityEnv): - def __init__(self, dim: int, ep_length: int = 100): + def __init__(self, dim: int = 1, ep_length: int = 100): """ Identity environment for testing purposes :param dim: (int) the size of the dimensions you want to learn - :param ep_length: (int) the length of each episodes in timesteps + :param ep_length: (int) the length of each episode in timesteps """ - super(IdentityEnvMultiDiscrete, self).__init__(dim, ep_length) - self.action_space = MultiDiscrete([dim, dim]) - self.observation_space = self.action_space - self.reset() + space = MultiDiscrete([dim, dim]) + super().__init__(ep_length=ep_length, space=space) class IdentityEnvMultiBinary(IdentityEnv): - def __init__(self, dim: int, ep_length: int = 100): + def __init__(self, dim: int = 1, ep_length: int = 100): """ Identity environment for testing purposes :param dim: (int) the size of the dimensions you want to learn - :param ep_length: (int) the length of each episodes in timesteps + :param ep_length: (int) the length of each episode in timesteps """ - super(IdentityEnvMultiBinary, self).__init__(dim, ep_length) - self.action_space = MultiBinary(dim) - self.observation_space = self.action_space - self.reset() + space = MultiBinary(dim) + super().__init__(ep_length=ep_length, space=space) class FakeImageEnv(Env): diff --git a/stable_baselines3/common/utils.py b/stable_baselines3/common/utils.py index b6fbb64..7a5b676 100644 --- a/stable_baselines3/common/utils.py +++ b/stable_baselines3/common/utils.py @@ -11,14 +11,17 @@ def set_random_seed(seed: int, using_cuda: bool = False) -> None: :param seed: (int) :param using_cuda: (bool) """ + # Seed python RNG random.seed(seed) + # Seed numpy RNG np.random.seed(seed) + # seed the RNG for all devices (both CPU and CUDA) th.manual_seed(seed) if using_cuda: - # Make CuDNN Determinist + # Deterministic operations for CuDNN, it may impact performances th.backends.cudnn.deterministic = True - th.cuda.manual_seed(seed) + th.backends.cudnn.benchmark = False # From stable baselines diff --git a/tests/test_deterministic.py b/tests/test_deterministic.py new file mode 100644 index 0000000..bf86de9 --- /dev/null +++ b/tests/test_deterministic.py @@ -0,0 +1,36 @@ +import pytest + +from stable_baselines3 import A2C, PPO, SAC, TD3 +from stable_baselines3.common.noise import NormalActionNoise + +N_STEPS_TRAINING = 3000 +SEED = 0 + + +@pytest.mark.parametrize("algo", [A2C, PPO, SAC, TD3]) +def test_deterministic_training_common(algo): + results = [[], []] + rewards = [[], []] + # Smaller network + kwargs = {'policy_kwargs': dict(net_arch=[64])} + if algo in [TD3, SAC]: + env_id = 'Pendulum-v0' + kwargs.update({'action_noise': NormalActionNoise(0.0, 0.1), + 'learning_starts': 100}) + else: + env_id = 'CartPole-v1' + # if algo == DQN: + # kwargs.update({'learning_starts': 100}) + + for i in range(2): + model = algo('MlpPolicy', env_id, seed=SEED, **kwargs) + model.learn(N_STEPS_TRAINING) + env = model.get_env() + obs = env.reset() + for _ in range(100): + action, _ = model.predict(obs, deterministic=False) + obs, reward, _, _ = env.step(action) + results[i].append(action) + rewards[i].append(reward) + assert sum(results[0]) == sum(results[1]), results + assert sum(rewards[0]) == sum(rewards[1]), rewards diff --git a/tests/test_envs.py b/tests/test_envs.py index 9c11d71..8d57c2f 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -5,11 +5,11 @@ import numpy as np from stable_baselines3.common.env_checker import check_env from stable_baselines3.common.bit_flipping_env import BitFlippingEnv -from stable_baselines3.common.identity_env import (IdentityEnv, IdentityEnvBox, +from stable_baselines3.common.identity_env import (IdentityEnv, IdentityEnvBox, FakeImageEnv, IdentityEnvMultiBinary, IdentityEnvMultiDiscrete,) ENV_CLASSES = [BitFlippingEnv, IdentityEnv, IdentityEnvBox, IdentityEnvMultiBinary, - IdentityEnvMultiDiscrete] + IdentityEnvMultiDiscrete, FakeImageEnv] @pytest.mark.parametrize("env_id", ['CartPole-v0', 'Pendulum-v0']) @@ -43,7 +43,7 @@ def test_high_dimension_action_space(): Test for continuous action space with more than one action. """ - env = gym.make('Pendulum-v0') + env = FakeImageEnv() # Patch the action space env.action_space = spaces.Box(low=-1, high=1, shape=(20,), dtype=np.float32) # Patch to avoid error @@ -68,7 +68,7 @@ def test_high_dimension_action_space(): spaces.Dict({"position": spaces.Discrete(5)}), ]) def test_non_default_spaces(new_obs_space): - env = gym.make('BreakoutNoFrameskip-v4') + env = FakeImageEnv() env.observation_space = new_obs_space # Patch methods to avoid errors env.reset = new_obs_space.sample From 0481fbe72722d36c761211a006335c12dc4c22ea Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 16:54:33 +0200 Subject: [PATCH 6/9] Update changelog --- docs/misc/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index 86b7246..d12616a 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -14,6 +14,7 @@ New Features: ^^^^^^^^^^^^^ - Added env checker (Sync with Stable Baselines) - Added ``VecCheckNan`` and ``VecVideoRecorder`` (Sync with Stable Baselines) +- Added determinism tests Bug Fixes: ^^^^^^^^^^ From 4a4da90671f0aa49cc640cfe93b168bf7cbe7428 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 17:19:21 +0200 Subject: [PATCH 7/9] Remove saved device + update doc --- stable_baselines3/common/base_class.py | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py index 1aee061..12320a8 100644 --- a/stable_baselines3/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -219,9 +219,9 @@ class BaseRLModel(ABC): def get_vec_normalize_env(self) -> Optional[VecNormalize]: """ - Return the `VecNormalize` wrapper of the training env + Return the ``VecNormalize`` wrapper of the training env if it exists. - :return: Optional[VecNormalize] The `VecNormalize` env. + :return: Optional[VecNormalize] The ``VecNormalize`` env. """ return self._vec_normalize_env @@ -267,7 +267,7 @@ class BaseRLModel(ABC): def get_torch_variables(self) -> Tuple[List[str], List[str]]: """ Get the name of the torch variable that will be saved. - `th.save` and `th.load` will be used with the right device + ``th.save`` and ``th.load`` will be used with the right device instead of the default pickling strategy. :return: (Tuple[List[str], List[str]]) @@ -297,7 +297,7 @@ class BaseRLModel(ABC): :param tb_log_name: (str) the name of the run for tensorboard log :param reset_num_timesteps: (bool) whether or not to reset the current timestep number (used in logging) :param eval_env: (gym.Env) Environment that will be used to evaluate the agent - :param eval_freq: (int) Evaluate the agent every `eval_freq` timesteps (this may vary a little) + :param eval_freq: (int) Evaluate the agent every ``eval_freq`` timesteps (this may vary a little) :param n_eval_episodes: (int) Number of episode to evaluate the agent :param eval_log_path: (Optional[str]) Path to a folder where the evaluations will be saved :param reset_num_timesteps: (bool) @@ -333,6 +333,11 @@ class BaseRLModel(ABC): """ data, params, tensors = cls._load_from_file(load_path) + if 'policy_kwargs' in data: + for arg_to_remove in ['device']: + if arg_to_remove in data['policy_kwargs']: + del data['policy_kwargs'][arg_to_remove] + if 'policy_kwargs' in kwargs and kwargs['policy_kwargs'] != data['policy_kwargs']: raise ValueError(f"The specified policy kwargs do not equal the stored policy kwargs." f"Stored kwargs: {data['policy_kwargs']}, specified kwargs: {kwargs['policy_kwargs']}") @@ -354,7 +359,7 @@ class BaseRLModel(ABC): model.__dict__.update(data) model.__dict__.update(kwargs) if not hasattr(model, "_setup_model") and len(params) > 0: - raise NotImplementedError(f"{cls} has no `_setup_model()` method") + raise NotImplementedError(f"{cls} has no ``_setup_model()`` method") model._setup_model() # put state_dicts back in place @@ -417,7 +422,7 @@ class BaseRLModel(ABC): file_content.write(tensor_file.read()) # go to start of file file_content.seek(0) - # load the parameters with the right `map_location` + # load the parameters with the right ``map_location`` tensors = th.load(file_content, map_location=device) # check for all other .pth files @@ -434,7 +439,7 @@ class BaseRLModel(ABC): file_content.write(opt_param_file.read()) # go to start of file file_content.seek(0) - # load the parameters with the right `map_location` + # load the parameters with the right ``map_location`` params[os.path.splitext(file_path)[0]] = th.load(file_content, map_location=device) except zipfile.BadZipFile: @@ -502,7 +507,7 @@ class BaseRLModel(ABC): :param eval_freq: (int) :param n_eval_episodes: (int) :param log_path (Optional[str]): Path to a log folder - :param reset_num_timesteps: (bool) Whether to reset or not the `num_timesteps` attribute + :param reset_num_timesteps: (bool) Whether to reset or not the ``num_timesteps`` attribute :return: (BaseCallback) """ self.start_time = time.time() @@ -516,7 +521,7 @@ class BaseRLModel(ABC): self.num_timesteps = 0 self._episode_num = 0 - # Avoid resetting the environment when calling `.learn()` consecutive times + # Avoid resetting the environment when calling ``.learn()`` consecutive times if reset_num_timesteps or self._last_obs is None: self._last_obs = self.env.reset() # Retrieve unnormalized observation for saving into the buffer @@ -762,9 +767,9 @@ class OffPolicyRLModel(BaseRLModel): :param env: (VecEnv) The training environment :param n_episodes: (int) Number of episodes to use to collect rollout data - You can also specify a `n_steps` instead + You can also specify a ``n_steps`` instead :param n_steps: (int) Number of steps to use to collect rollout data - You can also specify a `n_episodes` instead. + You can also specify a ``n_episodes`` instead. :param action_noise: (Optional[ActionNoise]) Action noise that will be used for exploration Required for deterministic policy (e.g. TD3). This can also be used in addition to the stochastic policy for SAC. @@ -772,7 +777,7 @@ class OffPolicyRLModel(BaseRLModel): (and at the beginning and end of the rollout) :param learning_starts: (int) Number of steps before learning for the warm-up phase. :param replay_buffer: (ReplayBuffer) - :param log_interval: (int) Log data every `log_interval` episodes + :param log_interval: (int) Log data every ``log_interval`` episodes :return: (RolloutReturn) """ episode_rewards, total_timesteps = [], [] From 580317158b5609c9958328dbf3a3cb1899712ae1 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 17:21:56 +0200 Subject: [PATCH 8/9] Update changelog --- docs/misc/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index d12616a..401a64a 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -18,6 +18,7 @@ New Features: Bug Fixes: ^^^^^^^^^^ +- Fixed a bug that prevented model trained on cpu to be loaded on gpu Deprecations: ^^^^^^^^^^^^^ From a3f9efe04a479c348cc57c1e039ebdf8bce1d9f6 Mon Sep 17 00:00:00 2001 From: Antonin RAFFIN Date: Tue, 5 May 2020 17:41:57 +0200 Subject: [PATCH 9/9] Update doc --- docs/conf.py | 14 +++++++++----- docs/index.rst | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a3d83bc..ab0e388 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -152,7 +152,7 @@ html_static_path = ['_static'] # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'TorchyBaselinesdoc' +htmlhelp_basename = 'StableBaselines3doc' # -- Options for LaTeX output ------------------------------------------------ @@ -179,7 +179,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'TorchyBaselines.tex', 'Stable Baselines3 Documentation', + (master_doc, 'StableBaselines3.tex', 'Stable Baselines3 Documentation', 'Stable Baselines3 Contributors', 'manual'), ] @@ -189,7 +189,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'torchybaselines', 'Stable Baselines3 Documentation', + (master_doc, 'stablebaselines3', 'Stable Baselines3 Documentation', [author], 1) ] @@ -200,8 +200,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'TorchyBaselines', 'Stable Baselines3 Documentation', - author, 'TorchyBaselines', 'One line description of project.', + (master_doc, 'StableBaselines3', 'Stable Baselines3 Documentation', + author, 'StableBaselines3', 'One line description of project.', 'Miscellaneous'), ] @@ -214,3 +214,7 @@ texinfo_documents = [ # 'numpy': ('http://docs.scipy.org/doc/numpy/', None), # 'torch': ('http://pytorch.org/docs/master/', None), # } + +# kornia's hack to get rtd builder to install latest pytorch +# if 'READTHEDOCS' in os.environ: +# os.system('pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html') diff --git a/docs/index.rst b/docs/index.rst index 9e74bd7..9cffb4d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,12 +6,12 @@ Welcome to Stable Baselines3 docs! ================================== -`Stable Baselines3 `_ is the PyTorch version of `Stable Baselines `_, +`Stable Baselines3 `_ is the next major version (PyTorch edition) of `Stable Baselines `_, a set of improved implementations of reinforcement learning algorithms. -RL Baselines Zoo (collection of pre-trained agents): https://github.com/araffin/rl-baselines-zoo +RL Baselines3 Zoo (collection of pre-trained agents): https://github.com/DLR-RM/rl-baselines3-zoo -RL Baselines zoo also offers a simple interface to train, evaluate agents and do hyperparameter tuning. +RL Baselines3 Zoo also offers a simple interface to train, evaluate agents and do hyperparameter tuning.