diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 7cbc198..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: "\U0001F41B Bug Report" -about: Submit a bug report to help us improve Stable-Baselines3 -labels: bug -title: "[Bug] bug title" ---- - -**Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. -Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. - - -If your issue is related to a **custom gym environment**, please use the custom gym env template. - -### 🐛 Bug - -A clear and concise description of what the bug is. - - -### To Reproduce - -Steps to reproduce the behavior. - -Please try to provide a minimal example to reproduce the bug. Error messages and stack traces are also helpful. - -Please use the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) -for both code and stack traces. - -```python -from stable_baselines3 import ... - -``` - -```bash -Traceback (most recent call last): File ... - -``` - -### Expected behavior - -A clear and concise description of what you expected to happen. - - -### System Info - -Describe the characteristic of your environment: - * Describe how the library was installed (pip, docker, source, ...) - * GPU models and configuration - * Python version - * PyTorch version - * Gym version - * Versions of any other relevant libraries - -You can use `sb3.get_system_info()` to print relevant packages info: -```python -import stable_baselines3 as sb3 -sb3.get_system_info() -``` - -### Additional context -Add any other context about the problem here. - -### Checklist - -- [ ] I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo (**required**) -- [ ] I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) (**required**) -- [ ] I have provided a minimal working example to reproduce the bug (**required**) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..8defe9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,71 @@ +name: "\U0001F41B Bug Report" +description: Submit a bug report to help us improve Stable-Baselines3 +title: "[Bug]: bug title" +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + **Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. + Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. + + If your issue is related to a **custom gym environment**, please use the custom gym env template. + - type: textarea + id: description + attributes: + label: 🐛 Bug + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: To Reproduce + description: | + Steps to reproduce the behavior. Please try to provide a minimal example to reproduce the bug. Error messages and stack traces are also helpful. + Please use the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) for both code and stack traces. + value: | + ```python + from stable_baselines3 import ... + + ``` + + - type: textarea + id: traceback + attributes: + label: Relevant log output / Error message + description: Please copy and paste any relevant log output / error message. This will be automatically formatted into code, so no need for backticks. + placeholder: "Traceback (most recent call last): File ..." + render: shell + + - type: textarea + id: system-info + attributes: + label: System Info + description: | + Describe the characteristic of your environment: + * Describe how the library was installed (pip, docker, source, ...) + * GPU models and configuration + * Python version + * PyTorch version + * Gym version + * Versions of any other relevant libraries + + You can use `sb3.get_system_info()` to print relevant packages info: + ```python + import stable_baselines3 as sb3 + sb3.get_system_info() + ``` + - type: checkboxes + id: terms + attributes: + label: Checklist + options: + - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo + required: true + - label: I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) + required: true + - label: I have provided a minimal working example to reproduce the bug + required: true + - label: I've used the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) for both code and stack traces. + required: true diff --git a/.github/ISSUE_TEMPLATE/custom_env.md b/.github/ISSUE_TEMPLATE/custom_env.md deleted file mode 100644 index 0a12a68..0000000 --- a/.github/ISSUE_TEMPLATE/custom_env.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -name: "\U0001F916 Custom Gym Environment Issue" -about: How to report an issue when using a custom Gym environment -labels: question, custom gym env ---- - -**Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. -Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. - -### 🤖 Custom Gym Environment - -**Please check your environment first using**: - -```python -from stable_baselines3.common.env_checker import check_env - -env = CustomEnv(arg1, ...) -# It will check your custom environment and output additional warnings if needed -check_env(env) -``` - -### Describe the bug - -A clear and concise description of what the bug is. - -### Code example - -Please try to provide a minimal example to reproduce the bug. -For a custom environment, you need to give at least the observation space, action space, `reset()` and `step()` methods -(see working example below). -Error messages and stack traces are also helpful. - -Please use the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) -for both code and stack traces. - -```python -import gym -import numpy as np - -from stable_baselines3 import A2C -from stable_baselines3.common.env_checker import check_env - - -class CustomEnv(gym.Env): - - def __init__(self): - super(CustomEnv, self).__init__() - self.observation_space = gym.spaces.Box(low=-np.inf, high=np.inf, shape=(14,)) - self.action_space = gym.spaces.Box(low=-1, high=1, shape=(6,)) - - def reset(self): - return self.observation_space.sample() - - def step(self, action): - obs = self.observation_space.sample() - reward = 1.0 - done = False - info = {} - return obs, reward, done, info - -env = CustomEnv() -check_env(env) - -model = A2C("MlpPolicy", env, verbose=1).learn(1000) -``` - -```bash -Traceback (most recent call last): File ... - -``` - -### System Info -Describe the characteristic of your environment: - * Describe how the library was installed (pip, docker, source, ...) - * GPU models and configuration - * Python version - * PyTorch version - * Gym version - * Versions of any other relevant libraries - -You can use `sb3.get_system_info()` to print relevant packages info: -```python -import stable_baselines3 as sb3 -sb3.get_system_info() -``` - -### Additional context -Add any other context about the problem here. - -### Checklist - -- [ ] I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) (**required**) -- [ ] I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo (**required**) -- [ ] I have checked my env using the env checker (**required**) -- [ ] I have provided a minimal working example to reproduce the bug (**required**) diff --git a/.github/ISSUE_TEMPLATE/custom_env.yml b/.github/ISSUE_TEMPLATE/custom_env.yml new file mode 100644 index 0000000..7887ef6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/custom_env.yml @@ -0,0 +1,107 @@ +name: "\U0001F916 Custom Gym Environment Issue" +description: How to report an issue when using a custom Gym environment +labels: ["question", "custom gym env"] +body: + - type: markdown + attributes: + value: | + **Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. + Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. + + **Please check your environment first using**: + ```python + from stable_baselines3.common.env_checker import check_env + + env = CustomEnv(arg1, ...) + # It will check your custom environment and output additional warnings if needed + check_env(env) + ``` + - type: textarea + id: description + attributes: + label: 🐛 Bug + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: code-example + attributes: + label: Code example + description: | + Please try to provide a minimal example to reproduce the bug. + For a custom environment, you need to give at least the observation space, action space, `reset()` and `step()` methods (see working example below). + Error messages and stack traces are also helpful. + Please use the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) for both code and stack traces. + value: | + ```python + import gym + import numpy as np + + from stable_baselines3 import A2C + from stable_baselines3.common.env_checker import check_env + + + class CustomEnv(gym.Env): + + def __init__(self): + super().__init__() + self.observation_space = gym.spaces.Box(low=-np.inf, high=np.inf, shape=(14,)) + self.action_space = gym.spaces.Box(low=-1, high=1, shape=(6,)) + + def reset(self): + return self.observation_space.sample() + + def step(self, action): + obs = self.observation_space.sample() + reward = 1.0 + done = False + info = {} + return obs, reward, done, info + + env = CustomEnv() + check_env(env) + + model = A2C("MlpPolicy", env, verbose=1).learn(1000) + ``` + + - type: textarea + id: traceback + attributes: + label: Relevant log output / Error message + description: Please copy and paste any relevant log output / error message. This will be automatically formatted into code, so no need for backticks. + placeholder: "Traceback (most recent call last): File ..." + render: shell + + - type: textarea + id: system-info + attributes: + label: System Info + description: | + Describe the characteristic of your environment: + * Describe how the library was installed (pip, docker, source, ...) + * GPU models and configuration + * Python version + * PyTorch version + * Gym version + * Versions of any other relevant libraries + + You can use `sb3.get_system_info()` to print relevant packages info: + ```python + import stable_baselines3 as sb3 + sb3.get_system_info() + ``` + - type: checkboxes + id: terms + attributes: + label: Checklist + options: + - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo + required: true + - label: I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) + required: true + - label: I have provided a minimal working example to reproduce the bug + required: true + - label: I have checked my env using the env checker + required: true + - label: I've used the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) for both code and stack traces. + required: true diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md deleted file mode 100644 index 59e5da5..0000000 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: "\U0001F4DA Documentation" -about: Report an issue related to Stable-Baselines3 documentation -labels: documentation ---- - -**Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. -Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. - -### 📚 Documentation - -A clear and concise description of what should be improved in the documentation. - -### Checklist - -- [ ] I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) (**required**) -- [ ] I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo (**required**) - - - - diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 0000000..025e2d3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,25 @@ +name: "\U0001F4DA Documentation" +description: Report an issue related to Stable-Baselines3 documentation +labels: ["documentation"] +body: + - type: markdown + attributes: + value: | + **Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. + Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. + - type: textarea + id: description + attributes: + label: 📚 Documentation + description: A clear and concise description of what should be improved in the documentation. + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Checklist + options: + - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo + required: true + - label: I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index a650863..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -name: "\U0001F680Feature Request" -about: How to create an issue for requesting a feature -labels: enhancement -title: "[Feature Request] request title" ---- - -**Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. -Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. - - -### 🚀 Feature - -A clear and concise description of the feature proposal. - -### Motivation - -Please outline the motivation for the proposal. -Is your feature request related to a problem? e.g.,"I'm always frustrated when [...]". -If this is related to another GitHub issue, please link here too. - -### Pitch - -A clear and concise description of what you want to happen. - -### Alternatives - -A clear and concise description of any alternative solutions or features you've considered, if any. - -### Additional context - -Add any other context or screenshots about the feature request here. - -### Checklist - -- [ ] I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo (**required**) - - - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..1d598a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,44 @@ +name: "\U0001F680 Feature Request" +description: How to create an issue for requesting a feature +title: "[Feature Request] request title" +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + **Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. + Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. + - type: textarea + id: description + attributes: + label: 🚀 Feature + description: A clear and concise description of the feature proposal. + validations: + required: true + - type: textarea + id: motivation + attributes: + label: Motivation + description: Please outline the motivation for the proposal. Is your feature request related to a problem? e.g.,"I'm always frustrated when [...]". If this is related to another GitHub issue, please link here too. + - type: textarea + id: pitch + attributes: + label: Pitch + description: A clear and concise description of what you want to happen. + - type: textarea + id: alternatives + attributes: + label: Alternatives + description: A clear and concise description of any alternative solutions or features you've considered, if any. + - type: textarea + id: additional-context + attributes: + label: Additional context + description: Add any other context or screenshots about the feature request here. + - type: checkboxes + id: terms + attributes: + label: Checklist + options: + - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo + required: true diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md deleted file mode 100644 index b3288d8..0000000 --- a/.github/ISSUE_TEMPLATE/question.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: ❓Question -about: How to ask a question regarding Stable-Baselines3 -labels: question -title: "[Question] question title" ---- - -**Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. -Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. - - -### Question - -Your question. This can be e.g. questions regarding confusing or unclear behaviour of functions or a question if X can be done using stable-baselines3. Make sure to check out the documentation first. - -### Additional context - -Add any other context about the question here. - - -### Checklist - -- [ ] I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) (**required**) -- [ ] I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo (**required**) - - diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..b2fb2f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,30 @@ +name: "❓ Question" +description: How to ask a question regarding Stable-Baselines3 +title: "[Question] question title" +labels: ["question"] +body: + - type: markdown + attributes: + value: | + **Important Note: We do not do technical support, nor consulting** and don't answer personal questions per email. + Please post your question on the [RL Discord](https://discord.com/invite/xhfNqQv), [Reddit](https://www.reddit.com/r/reinforcementlearning/) or [Stack Overflow](https://stackoverflow.com/) in that case. + - type: textarea + id: question + attributes: + label: ❓ Question + description: Your question. This can be e.g. questions regarding confusing or unclear behaviour of functions or a question if X can be done using stable-baselines3. Make sure to check out the documentation first. + validations: + required: true + - type: checkboxes + id: terms + attributes: + label: Checklist + options: + - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo + required: true + - label: I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/) + required: true + - label: If code there is, it is minimal and working + required: true + - label: If code there is, it is formatted using the [markdown code blocks](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) for both code and stack traces. + required: true diff --git a/docs/guide/imitation.rst b/docs/guide/imitation.rst index df7895c..c4a0726 100644 --- a/docs/guide/imitation.rst +++ b/docs/guide/imitation.rst @@ -10,46 +10,10 @@ imitation learning algorithms on top of Stable-Baselines3, including: - `DAgger `_ with synthetic examples - `Adversarial Inverse Reinforcement Learning `_ (AIRL) - `Generative Adversarial Imitation Learning `_ (GAIL) + - `Deep RL from Human Preferences `_ (DRLHP) - -It also provides `CLI scripts <#cli-quickstart>`_ for training and saving -demonstrations from RL experts, and for training imitation learners on these demonstrations. - - -Installation ------------- - -Installation requires Python 3.7+: - -:: - - pip install imitation - - -CLI Quickstart ---------------------- - -:: - - # Train PPO agent on cartpole and collect expert demonstrations - python -m imitation.scripts.expert_demos with fast cartpole log_dir=quickstart - - # Train GAIL from demonstrations - python -m imitation.scripts.train_adversarial with fast gail cartpole rollout_path=quickstart/rollouts/final.pkl - - # Train AIRL from demonstrations - python -m imitation.scripts.train_adversarial with fast airl cartpole rollout_path=quickstart/rollouts/final.pkl - - -.. note:: - - You can remove the ``fast`` option to run training to completion. For more CLI options - and information on reading Tensorboard plots, see the - `README `_. - - -Python Interface Quickstart ---------------------------- - -This `example script `_ -uses the Python API to train BC, GAIL, and AIRL models on CartPole data. +You can install imitation with ``pip install imitation``. The `imitation +documentation `_ has more details +on how to use the library, including `a quick start guide +`_ +for the impatient. diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst index cccaddc..6db4fa9 100644 --- a/docs/misc/changelog.rst +++ b/docs/misc/changelog.rst @@ -3,6 +3,38 @@ Changelog ========== + +Release 1.7.0a0 (WIP) +-------------------------- + +Breaking Changes: +^^^^^^^^^^^^^^^^^ +- Removed deprecated ``create_eval_env``, ``eval_env``, ``eval_log_path``, ``n_eval_episodes`` and ``eval_freq`` parameters, + please use an ``EvalCallback`` instead +- Removed deprecated ``sde_net_arch`` parameter +- Removed ``ret`` attributes in ``VecNormalize``, please use ``returns`` instead + +New Features: +^^^^^^^^^^^^^ + +SB3-Contrib +^^^^^^^^^^^ + +Bug Fixes: +^^^^^^^^^^ + +Deprecations: +^^^^^^^^^^^^^ + +Others: +^^^^^^^ +- Used issue forms instead of issue templates + +Documentation: +^^^^^^^^^^^^^^ + + + Release 1.6.2 (2022-10-10) -------------------------- diff --git a/setup.cfg b/setup.cfg index eac74ea..eb3b04c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,9 @@ [metadata] # This includes the license file in the wheel. license_files = LICENSE +project_urls = + Code = https://github.com/DLR-RM/stable-baselines3 + Documentation = https://stable-baselines3.readthedocs.io/ [tool:pytest] # Deterministic ordering for tests; useful for pytest-xdist. diff --git a/stable_baselines3/a2c/a2c.py b/stable_baselines3/a2c/a2c.py index d59eebb..24d69a6 100644 --- a/stable_baselines3/a2c/a2c.py +++ b/stable_baselines3/a2c/a2c.py @@ -43,10 +43,6 @@ class A2C(OnPolicyAlgorithm): Default: -1 (only sample at the beginning of the rollout) :param normalize_advantage: Whether to normalize or not the advantage :param tensorboard_log: the log location for tensorboard (if None, no logging) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -79,7 +75,6 @@ class A2C(OnPolicyAlgorithm): sde_sample_freq: int = -1, normalize_advantage: bool = False, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -103,7 +98,6 @@ class A2C(OnPolicyAlgorithm): policy_kwargs=policy_kwargs, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, _init_setup_model=False, supported_action_spaces=( @@ -191,11 +185,7 @@ class A2C(OnPolicyAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 100, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "A2C", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> A2CSelf: @@ -204,11 +194,7 @@ class A2C(OnPolicyAlgorithm): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/common/base_class.py b/stable_baselines3/common/base_class.py index 42dca11..05a8cf4 100644 --- a/stable_baselines3/common/base_class.py +++ b/stable_baselines3/common/base_class.py @@ -3,7 +3,6 @@ import io import pathlib import time -import warnings from abc import ABC, abstractmethod from collections import deque from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union @@ -13,7 +12,7 @@ import numpy as np import torch as th from stable_baselines3.common import utils -from stable_baselines3.common.callbacks import BaseCallback, CallbackList, ConvertCallback, EvalCallback, ProgressBarCallback +from stable_baselines3.common.callbacks import BaseCallback, CallbackList, ConvertCallback, ProgressBarCallback from stable_baselines3.common.env_util import is_wrapped from stable_baselines3.common.logger import Logger from stable_baselines3.common.monitor import Monitor @@ -75,10 +74,6 @@ class BaseAlgorithm(ABC): if it is not possible. :param support_multi_env: Whether the algorithm supports training with multiple environments (as in A2C) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param monitor_wrapper: When creating an environment, whether to wrap it or not in a Monitor wrapper. :param seed: Seed for the pseudo random generators @@ -102,7 +97,6 @@ class BaseAlgorithm(ABC): verbose: int = 0, device: Union[th.device, str] = "auto", support_multi_env: bool = False, - create_eval_env: bool = False, monitor_wrapper: bool = True, seed: Optional[int] = None, use_sde: bool = False, @@ -131,7 +125,6 @@ class BaseAlgorithm(ABC): self._total_timesteps = 0 # Used for computing fps, it is updated at each call of learn() self._num_timesteps_at_start = 0 - self.eval_env = None self.seed = seed self.action_noise = None # type: Optional[ActionNoise] self.start_time = None @@ -162,19 +155,6 @@ class BaseAlgorithm(ABC): # Create and wrap the env if needed if env is not None: - if isinstance(env, str): - if create_eval_env: - warnings.warn( - "The parameter `create_eval_env` is deprecated and will be removed in the future. " - "Please use `EvalCallback` or a custom Callback instead.", - DeprecationWarning, - # By setting the `stacklevel` we refer to the initial caller of the deprecated feature. - # This causes the the `DepricationWarning` to not be ignored and to be shown to the user. See - # https://github.com/DLR-RM/stable-baselines3/pull/1082#discussion_r989842855 for more details. - stacklevel=4, - ) - self.eval_env = maybe_make_env(env, self.verbose) - env = maybe_make_env(env, self.verbose) env = self._wrap_env(env, self.verbose, monitor_wrapper) @@ -275,21 +255,6 @@ class BaseAlgorithm(ABC): """Getter for the logger object.""" return self._logger - def _get_eval_env(self, eval_env: Optional[GymEnv]) -> Optional[GymEnv]: - """ - Return the environment that will be used for evaluation. - - :param eval_env:) - :return: - """ - if eval_env is None: - eval_env = self.eval_env - - if eval_env is not None: - eval_env = self._wrap_env(eval_env, self.verbose) - assert eval_env.num_envs == 1 - return eval_env - def _setup_lr_schedule(self) -> None: """Transform to callable if needed.""" self.lr_schedule = get_schedule_fn(self.learning_rate) @@ -332,7 +297,6 @@ class BaseAlgorithm(ABC): "policy", "device", "env", - "eval_env", "replay_buffer", "rollout_buffer", "_vec_normalize_env", @@ -379,20 +343,10 @@ class BaseAlgorithm(ABC): def _init_callback( self, callback: MaybeCallback, - eval_env: Optional[VecEnv] = None, - eval_freq: int = 10000, - n_eval_episodes: int = 5, - log_path: Optional[str] = None, progress_bar: bool = False, ) -> BaseCallback: """ :param callback: Callback(s) called at every step with state of the algorithm. - :param eval_freq: How many steps between evaluations; if None, do not evaluate. - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. - :param n_eval_episodes: How many episodes to play per evaluation - :param n_eval_episodes: Number of episodes to rollout during evaluation. - :param log_path: Path to a folder where the evaluations will be saved :param progress_bar: Display a progress bar using tqdm and rich. :return: A hybrid callback calling `callback` and performing evaluation. """ @@ -408,29 +362,13 @@ class BaseAlgorithm(ABC): if progress_bar: callback = CallbackList([callback, ProgressBarCallback()]) - # Create eval callback in charge of the evaluation - if eval_env is not None: - eval_callback = EvalCallback( - eval_env, - best_model_save_path=log_path, - log_path=log_path, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, - verbose=self.verbose, - ) - callback = CallbackList([callback, eval_callback]) - callback.init_callback(self) return callback def _setup_learn( self, total_timesteps: int, - eval_env: Optional[GymEnv], callback: MaybeCallback = None, - eval_freq: int = 10000, - n_eval_episodes: int = 5, - log_path: Optional[str] = None, reset_num_timesteps: bool = True, tb_log_name: str = "run", progress_bar: bool = False, @@ -439,32 +377,12 @@ class BaseAlgorithm(ABC): Initialize different variables needed for training. :param total_timesteps: The total number of samples (env steps) to train on - :param eval_env: Environment to use for evaluation. - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param callback: Callback(s) called at every step with state of the algorithm. - :param eval_freq: How many steps between evaluations - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. - :param n_eval_episodes: How many episodes to play per evaluation - :param log_path: Path to a folder where the evaluations will be saved :param reset_num_timesteps: Whether to reset or not the ``num_timesteps`` attribute :param tb_log_name: the name of the run for tensorboard log :param progress_bar: Display a progress bar using tqdm and rich. :return: Total timesteps and callback(s) """ - - if eval_env is not None or eval_freq != -1: - warnings.warn( - "Parameters `eval_env` and `eval_freq` are deprecated and will be removed in the future. " - "Please use `EvalCallback` or a custom Callback instead.", - DeprecationWarning, - # By setting the `stacklevel` we refer to the initial caller of the deprecated feature. - # This causes the the `DepricationWarning` to not be ignored and to be shown to the user. See - # https://github.com/DLR-RM/stable-baselines3/pull/1082#discussion_r989842855 for more details. - stacklevel=4, - ) - self.start_time = time.time_ns() if self.ep_info_buffer is None or reset_num_timesteps: @@ -492,17 +410,12 @@ class BaseAlgorithm(ABC): if self._vec_normalize_env is not None: self._last_original_obs = self._vec_normalize_env.get_original_obs() - if eval_env is not None and self.seed is not None: - eval_env.seed(self.seed) - - eval_env = self._get_eval_env(eval_env) - # Configure logger's outputs if no logger was passed if not self._custom_logger: self._logger = utils.configure_logger(self.verbose, self.tensorboard_log, tb_log_name, reset_num_timesteps) # Create eval callback if needed - callback = self._init_callback(callback, eval_env, eval_freq, n_eval_episodes, log_path, progress_bar) + callback = self._init_callback(callback, progress_bar) return total_timesteps, callback @@ -583,10 +496,6 @@ class BaseAlgorithm(ABC): callback: MaybeCallback = None, log_interval: int = 100, tb_log_name: str = "run", - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> BaseAlgorithmSelf: @@ -597,13 +506,6 @@ class BaseAlgorithm(ABC): :param callback: callback(s) called at every step with state of the algorithm. :param log_interval: The number of timesteps before logging. :param tb_log_name: the name of the run for TensorBoard logging - :param eval_env: Environment that will be used to evaluate the agent. Caution, this parameter - is deprecated and will be removed in the future. Please use ``EvalCallback`` instead. - :param eval_freq: Evaluate the agent every ``eval_freq`` timesteps (this may vary a little). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. - :param n_eval_episodes: Number of episode to evaluate the agent - :param eval_log_path: Path to a folder where the evaluations will be saved :param reset_num_timesteps: whether or not to reset the current timestep number (used in logging) :param progress_bar: Display a progress bar using tqdm and rich. :return: the trained model @@ -644,8 +546,6 @@ class BaseAlgorithm(ABC): self.action_space.seed(seed) if self.env is not None: self.env.seed(seed) - if self.eval_env is not None: - self.eval_env.seed(seed) def set_parameters( self, diff --git a/stable_baselines3/common/off_policy_algorithm.py b/stable_baselines3/common/off_policy_algorithm.py index d64d4e8..d309322 100644 --- a/stable_baselines3/common/off_policy_algorithm.py +++ b/stable_baselines3/common/off_policy_algorithm.py @@ -60,10 +60,6 @@ class OffPolicyAlgorithm(BaseAlgorithm): if it is not possible. :param support_multi_env: Whether the algorithm supports training with multiple environments (as in A2C) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically. (Only available when passing string for the environment) - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param monitor_wrapper: When creating an environment, whether to wrap it or not in a Monitor wrapper. :param seed: Seed for the pseudo random generators @@ -98,7 +94,6 @@ class OffPolicyAlgorithm(BaseAlgorithm): verbose: int = 0, device: Union[th.device, str] = "auto", support_multi_env: bool = False, - create_eval_env: bool = False, monitor_wrapper: bool = True, seed: Optional[int] = None, use_sde: bool = False, @@ -117,7 +112,6 @@ class OffPolicyAlgorithm(BaseAlgorithm): verbose=verbose, device=device, support_multi_env=support_multi_env, - create_eval_env=create_eval_env, monitor_wrapper=monitor_wrapper, seed=seed, use_sde=use_sde, @@ -271,11 +265,7 @@ class OffPolicyAlgorithm(BaseAlgorithm): def _setup_learn( self, total_timesteps: int, - eval_env: Optional[GymEnv], callback: MaybeCallback = None, - eval_freq: int = 10000, - n_eval_episodes: int = 5, - log_path: Optional[str] = None, reset_num_timesteps: bool = True, tb_log_name: str = "run", progress_bar: bool = False, @@ -314,11 +304,7 @@ class OffPolicyAlgorithm(BaseAlgorithm): return super()._setup_learn( total_timesteps, - eval_env, callback, - eval_freq, - n_eval_episodes, - log_path, reset_num_timesteps, tb_log_name, progress_bar, @@ -329,22 +315,14 @@ class OffPolicyAlgorithm(BaseAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 4, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "run", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> OffPolicyAlgorithmSelf: total_timesteps, callback = self._setup_learn( total_timesteps, - eval_env, callback, - eval_freq, - n_eval_episodes, - eval_log_path, reset_num_timesteps, tb_log_name, progress_bar, diff --git a/stable_baselines3/common/on_policy_algorithm.py b/stable_baselines3/common/on_policy_algorithm.py index d19640c..a0018b3 100644 --- a/stable_baselines3/common/on_policy_algorithm.py +++ b/stable_baselines3/common/on_policy_algorithm.py @@ -38,10 +38,6 @@ class OnPolicyAlgorithm(BaseAlgorithm): :param sde_sample_freq: Sample a new noise matrix every n steps when using gSDE Default: -1 (only sample at the beginning of the rollout) :param tensorboard_log: the log location for tensorboard (if None, no logging) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically. (Only available when passing string for the environment) - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param monitor_wrapper: When creating an environment, whether to wrap it or not in a Monitor wrapper. :param policy_kwargs: additional arguments to be passed to the policy on creation @@ -68,7 +64,6 @@ class OnPolicyAlgorithm(BaseAlgorithm): use_sde: bool, sde_sample_freq: int, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, monitor_wrapper: bool = True, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, @@ -87,7 +82,6 @@ class OnPolicyAlgorithm(BaseAlgorithm): device=device, use_sde=use_sde, sde_sample_freq=sde_sample_freq, - create_eval_env=create_eval_env, support_multi_env=True, seed=seed, tensorboard_log=tensorboard_log, @@ -233,11 +227,7 @@ class OnPolicyAlgorithm(BaseAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 1, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "OnPolicyAlgorithm", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> OnPolicyAlgorithmSelf: @@ -245,11 +235,7 @@ class OnPolicyAlgorithm(BaseAlgorithm): total_timesteps, callback = self._setup_learn( total_timesteps, - eval_env, callback, - eval_freq, - n_eval_episodes, - eval_log_path, reset_num_timesteps, tb_log_name, progress_bar, diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py index 4632e48..18f90c0 100644 --- a/stable_baselines3/common/policies.py +++ b/stable_baselines3/common/policies.py @@ -171,14 +171,6 @@ class BaseModel(nn.Module): device = get_device(device) saved_variables = th.load(path, map_location=device) - # Allow to load policy saved with older version of SB3 - if "sde_net_arch" in saved_variables["data"]: - warnings.warn( - "sde_net_arch is deprecated, please downgrade to SB3 v1.2.0 if you need such parameter.", - DeprecationWarning, - ) - del saved_variables["data"]["sde_net_arch"] - # Create policy object model = cls(**saved_variables["data"]) # pytype: disable=not-instantiable # Load weights @@ -389,9 +381,6 @@ class ActorCriticPolicy(BasePolicy): :param log_std_init: Initial value for the log standard deviation :param full_std: Whether to use (n_features x n_actions) parameters for the std instead of only (n_features,) when using gSDE - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -419,7 +408,6 @@ class ActorCriticPolicy(BasePolicy): use_sde: bool = False, log_std_init: float = 0.0, full_std: bool = True, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, squash_output: bool = False, features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor, @@ -471,9 +459,6 @@ class ActorCriticPolicy(BasePolicy): "learn_features": False, } - if sde_net_arch is not None: - warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning) - self.use_sde = use_sde self.dist_kwargs = dist_kwargs @@ -684,9 +669,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy): :param log_std_init: Initial value for the log standard deviation :param full_std: Whether to use (n_features x n_actions) parameters for the std instead of only (n_features,) when using gSDE - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -714,7 +696,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy): use_sde: bool = False, log_std_init: float = 0.0, full_std: bool = True, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, squash_output: bool = False, features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN, @@ -733,7 +714,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy): use_sde, log_std_init, full_std, - sde_net_arch, use_expln, squash_output, features_extractor_class, @@ -759,9 +739,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy): :param log_std_init: Initial value for the log standard deviation :param full_std: Whether to use (n_features x n_actions) parameters for the std instead of only (n_features,) when using gSDE - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -789,7 +766,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy): use_sde: bool = False, log_std_init: float = 0.0, full_std: bool = True, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, squash_output: bool = False, features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor, @@ -808,7 +784,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy): use_sde, log_std_init, full_std, - sde_net_arch, use_expln, squash_output, features_extractor_class, diff --git a/stable_baselines3/common/vec_env/vec_normalize.py b/stable_baselines3/common/vec_env/vec_normalize.py index f3ee588..53e94af 100644 --- a/stable_baselines3/common/vec_env/vec_normalize.py +++ b/stable_baselines3/common/vec_env/vec_normalize.py @@ -289,8 +289,3 @@ class VecNormalize(VecEnvWrapper): """ with open(save_path, "wb") as file_handler: pickle.dump(self, file_handler) - - @property - def ret(self) -> np.ndarray: - warnings.warn("`VecNormalize` `ret` attribute is deprecated. Please use `returns` instead.", DeprecationWarning) - return self.returns diff --git a/stable_baselines3/ddpg/ddpg.py b/stable_baselines3/ddpg/ddpg.py index 26e0745..993a8c2 100644 --- a/stable_baselines3/ddpg/ddpg.py +++ b/stable_baselines3/ddpg/ddpg.py @@ -44,10 +44,6 @@ class DDPG(TD3): :param optimize_memory_usage: Enable a memory efficient variant of the replay buffer at a cost of more complexity. See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195 - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -74,7 +70,6 @@ class DDPG(TD3): replay_buffer_kwargs: Optional[Dict[str, Any]] = None, optimize_memory_usage: bool = False, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -100,7 +95,6 @@ class DDPG(TD3): tensorboard_log=tensorboard_log, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, optimize_memory_usage=optimize_memory_usage, # Remove all tricks from TD3 to obtain DDPG: @@ -123,11 +117,7 @@ class DDPG(TD3): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 4, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "DDPG", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> DDPGSelf: @@ -136,11 +126,7 @@ class DDPG(TD3): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/dqn/dqn.py b/stable_baselines3/dqn/dqn.py index cc13ecf..8c67838 100644 --- a/stable_baselines3/dqn/dqn.py +++ b/stable_baselines3/dqn/dqn.py @@ -52,10 +52,6 @@ class DQN(OffPolicyAlgorithm): :param exploration_final_eps: final value of random action probability :param max_grad_norm: The maximum value for the gradient clipping :param tensorboard_log: the log location for tensorboard (if None, no logging) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -92,7 +88,6 @@ class DQN(OffPolicyAlgorithm): exploration_final_eps: float = 0.05, max_grad_norm: float = 10, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -118,7 +113,6 @@ class DQN(OffPolicyAlgorithm): tensorboard_log=tensorboard_log, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, sde_support=False, optimize_memory_usage=optimize_memory_usage, @@ -263,11 +257,7 @@ class DQN(OffPolicyAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 4, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "DQN", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> DQNSelf: @@ -276,11 +266,7 @@ class DQN(OffPolicyAlgorithm): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/ppo/ppo.py b/stable_baselines3/ppo/ppo.py index b187ec8..5d30569 100644 --- a/stable_baselines3/ppo/ppo.py +++ b/stable_baselines3/ppo/ppo.py @@ -57,10 +57,6 @@ class PPO(OnPolicyAlgorithm): see issue #213 (cf https://github.com/hill-a/stable-baselines/issues/213) By default, there is no limit on the kl div. :param tensorboard_log: the log location for tensorboard (if None, no logging) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -96,7 +92,6 @@ class PPO(OnPolicyAlgorithm): sde_sample_freq: int = -1, target_kl: Optional[float] = None, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -120,7 +115,6 @@ class PPO(OnPolicyAlgorithm): policy_kwargs=policy_kwargs, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, _init_setup_model=False, supported_action_spaces=( @@ -305,11 +299,7 @@ class PPO(OnPolicyAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 1, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "PPO", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> PPOSelf: @@ -318,11 +308,7 @@ class PPO(OnPolicyAlgorithm): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/sac/policies.py b/stable_baselines3/sac/policies.py index 255bd75..368cd67 100644 --- a/stable_baselines3/sac/policies.py +++ b/stable_baselines3/sac/policies.py @@ -38,9 +38,6 @@ class Actor(BasePolicy): :param log_std_init: Initial value for the log standard deviation :param full_std: Whether to use (n_features x n_actions) parameters for the std instead of only (n_features,) when using gSDE. - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -60,7 +57,6 @@ class Actor(BasePolicy): use_sde: bool = False, log_std_init: float = -3, full_std: bool = True, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, clip_mean: float = 2.0, normalize_images: bool = True, @@ -80,14 +76,10 @@ class Actor(BasePolicy): self.features_dim = features_dim self.activation_fn = activation_fn self.log_std_init = log_std_init - self.sde_net_arch = sde_net_arch self.use_expln = use_expln self.full_std = full_std self.clip_mean = clip_mean - if sde_net_arch is not None: - warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning) - action_dim = get_action_dim(self.action_space) latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn) self.latent_pi = nn.Sequential(*latent_pi_net) @@ -196,9 +188,6 @@ class SACPolicy(BasePolicy): :param activation_fn: Activation function :param use_sde: Whether to use State Dependent Exploration or not :param log_std_init: Initial value for the log standard deviation - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -226,7 +215,6 @@ class SACPolicy(BasePolicy): activation_fn: Type[nn.Module] = nn.ReLU, use_sde: bool = False, log_std_init: float = -3, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, clip_mean: float = 2.0, features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor, @@ -263,9 +251,6 @@ class SACPolicy(BasePolicy): } self.actor_kwargs = self.net_args.copy() - if sde_net_arch is not None: - warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning) - sde_kwargs = { "use_sde": use_sde, "log_std_init": log_std_init, @@ -382,9 +367,6 @@ class CnnPolicy(SACPolicy): :param activation_fn: Activation function :param use_sde: Whether to use State Dependent Exploration or not :param log_std_init: Initial value for the log standard deviation - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -410,7 +392,6 @@ class CnnPolicy(SACPolicy): activation_fn: Type[nn.Module] = nn.ReLU, use_sde: bool = False, log_std_init: float = -3, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, clip_mean: float = 2.0, features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN, @@ -429,7 +410,6 @@ class CnnPolicy(SACPolicy): activation_fn, use_sde, log_std_init, - sde_net_arch, use_expln, clip_mean, features_extractor_class, @@ -453,9 +433,6 @@ class MultiInputPolicy(SACPolicy): :param activation_fn: Activation function :param use_sde: Whether to use State Dependent Exploration or not :param log_std_init: Initial value for the log standard deviation - :param sde_net_arch: Network architecture for extracting features - when using gSDE. If None, the latent features from the policy will be used. - Pass an empty list to use the states as features. :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure a positive standard deviation (cf paper). It allows to keep variance above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. @@ -481,7 +458,6 @@ class MultiInputPolicy(SACPolicy): activation_fn: Type[nn.Module] = nn.ReLU, use_sde: bool = False, log_std_init: float = -3, - sde_net_arch: Optional[List[int]] = None, use_expln: bool = False, clip_mean: float = 2.0, features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor, @@ -500,7 +476,6 @@ class MultiInputPolicy(SACPolicy): activation_fn, use_sde, log_std_init, - sde_net_arch, use_expln, clip_mean, features_extractor_class, diff --git a/stable_baselines3/sac/sac.py b/stable_baselines3/sac/sac.py index 6d7ad29..85bdf78 100644 --- a/stable_baselines3/sac/sac.py +++ b/stable_baselines3/sac/sac.py @@ -65,10 +65,6 @@ class SAC(OffPolicyAlgorithm): Default: -1 (only sample at the beginning of the rollout) :param use_sde_at_warmup: Whether to use gSDE instead of uniform sampling during the warm up phase (before learning starts) - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -107,7 +103,6 @@ class SAC(OffPolicyAlgorithm): sde_sample_freq: int = -1, use_sde_at_warmup: bool = False, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -133,7 +128,6 @@ class SAC(OffPolicyAlgorithm): tensorboard_log=tensorboard_log, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, use_sde=use_sde, sde_sample_freq=sde_sample_freq, @@ -297,11 +291,7 @@ class SAC(OffPolicyAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 4, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "SAC", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> SACSelf: @@ -310,11 +300,7 @@ class SAC(OffPolicyAlgorithm): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/td3/td3.py b/stable_baselines3/td3/td3.py index f7dd08b..c611481 100644 --- a/stable_baselines3/td3/td3.py +++ b/stable_baselines3/td3/td3.py @@ -53,10 +53,6 @@ class TD3(OffPolicyAlgorithm): :param target_policy_noise: Standard deviation of Gaussian noise added to target policy (smoothing noise) :param target_noise_clip: Limit for absolute value of target policy smoothing noise. - :param create_eval_env: Whether to create a second environment that will be - used for evaluating the agent periodically (Only available when passing string for the environment). - Caution, this parameter is deprecated and will be removed in the future. - Please use `EvalCallback` or a custom Callback instead. :param policy_kwargs: additional arguments to be passed to the policy on creation :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for debug messages @@ -92,7 +88,6 @@ class TD3(OffPolicyAlgorithm): target_policy_noise: float = 0.2, target_noise_clip: float = 0.5, tensorboard_log: Optional[str] = None, - create_eval_env: bool = False, policy_kwargs: Optional[Dict[str, Any]] = None, verbose: int = 0, seed: Optional[int] = None, @@ -118,7 +113,6 @@ class TD3(OffPolicyAlgorithm): tensorboard_log=tensorboard_log, verbose=verbose, device=device, - create_eval_env=create_eval_env, seed=seed, sde_support=False, optimize_memory_usage=optimize_memory_usage, @@ -213,11 +207,7 @@ class TD3(OffPolicyAlgorithm): total_timesteps: int, callback: MaybeCallback = None, log_interval: int = 4, - eval_env: Optional[GymEnv] = None, - eval_freq: int = -1, - n_eval_episodes: int = 5, tb_log_name: str = "TD3", - eval_log_path: Optional[str] = None, reset_num_timesteps: bool = True, progress_bar: bool = False, ) -> TD3Self: @@ -226,11 +216,7 @@ class TD3(OffPolicyAlgorithm): total_timesteps=total_timesteps, callback=callback, log_interval=log_interval, - eval_env=eval_env, - eval_freq=eval_freq, - n_eval_episodes=n_eval_episodes, tb_log_name=tb_log_name, - eval_log_path=eval_log_path, reset_num_timesteps=reset_num_timesteps, progress_bar=progress_bar, ) diff --git a/stable_baselines3/version.txt b/stable_baselines3/version.txt index fdd3be6..56fee06 100644 --- a/stable_baselines3/version.txt +++ b/stable_baselines3/version.txt @@ -1 +1 @@ -1.6.2 +1.7.0a0 diff --git a/tests/test_predict.py b/tests/test_predict.py index abbf254..93bbc9d 100644 --- a/tests/test_predict.py +++ b/tests/test_predict.py @@ -37,20 +37,15 @@ class CustomSubClassedSpaceEnv(gym.Env): @pytest.mark.parametrize("model_class", MODEL_LIST) def test_auto_wrap(model_class): - # test auto wrapping of env into a VecEnv - + """Test auto wrapping of env into a VecEnv.""" # Use different environment for DQN if model_class is DQN: env_name = "CartPole-v0" else: env_name = "Pendulum-v1" env = gym.make(env_name) - eval_env = gym.make(env_name) model = model_class("MlpPolicy", env) - - # Catch DeprecationWarnings - with pytest.warns(DeprecationWarning): # `eval_env` is deprecated - model.learn(100, eval_env=eval_env) + model.learn(100) @pytest.mark.parametrize("model_class", MODEL_LIST) diff --git a/tests/test_run.py b/tests/test_run.py index 66b0ff8..71236a3 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -18,25 +18,22 @@ def test_deterministic_pg(model_class, action_noise): """ Test for DDPG and variants (TD3). """ - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - model = model_class( - "MlpPolicy", - "Pendulum-v1", - policy_kwargs=dict(net_arch=[64, 64]), - learning_starts=100, - verbose=1, - create_eval_env=True, - buffer_size=250, - action_noise=action_noise, - ) - model.learn(total_timesteps=300, eval_freq=250) + model = model_class( + "MlpPolicy", + "Pendulum-v1", + policy_kwargs=dict(net_arch=[64, 64]), + learning_starts=100, + verbose=1, + buffer_size=250, + action_noise=action_noise, + ) + model.learn(total_timesteps=200) @pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"]) def test_a2c(env_id): - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - model = A2C("MlpPolicy", env_id, seed=0, policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True) - model.learn(total_timesteps=1000, eval_freq=500) + model = A2C("MlpPolicy", env_id, seed=0, policy_kwargs=dict(net_arch=[16]), verbose=1) + model.learn(total_timesteps=64) @pytest.mark.parametrize("model_class", [A2C, PPO]) @@ -49,48 +46,44 @@ def test_advantage_normalization(model_class, normalize_advantage): @pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"]) @pytest.mark.parametrize("clip_range_vf", [None, 0.2, -0.2]) def test_ppo(env_id, clip_range_vf): - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - if clip_range_vf is not None and clip_range_vf < 0: - # Should throw an error - with pytest.raises(AssertionError): - model = PPO( - "MlpPolicy", - env_id, - seed=0, - policy_kwargs=dict(net_arch=[16]), - verbose=1, - create_eval_env=True, - clip_range_vf=clip_range_vf, - ) - else: + if clip_range_vf is not None and clip_range_vf < 0: + # Should throw an error + with pytest.raises(AssertionError): model = PPO( "MlpPolicy", env_id, - n_steps=512, seed=0, policy_kwargs=dict(net_arch=[16]), verbose=1, - create_eval_env=True, clip_range_vf=clip_range_vf, ) - model.learn(total_timesteps=1000, eval_freq=500) + else: + model = PPO( + "MlpPolicy", + env_id, + n_steps=512, + seed=0, + policy_kwargs=dict(net_arch=[16]), + verbose=1, + clip_range_vf=clip_range_vf, + n_epochs=2, + ) + model.learn(total_timesteps=1000) @pytest.mark.parametrize("ent_coef", ["auto", 0.01, "auto_0.01"]) def test_sac(ent_coef): - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - model = SAC( - "MlpPolicy", - "Pendulum-v1", - policy_kwargs=dict(net_arch=[64, 64]), - learning_starts=100, - verbose=1, - create_eval_env=True, - buffer_size=250, - ent_coef=ent_coef, - action_noise=NormalActionNoise(np.zeros(1), np.zeros(1)), - ) - model.learn(total_timesteps=300, eval_freq=250) + model = SAC( + "MlpPolicy", + "Pendulum-v1", + policy_kwargs=dict(net_arch=[64, 64]), + learning_starts=100, + verbose=1, + buffer_size=250, + ent_coef=ent_coef, + action_noise=NormalActionNoise(np.zeros(1), np.zeros(1)), + ) + model.learn(total_timesteps=200) @pytest.mark.parametrize("n_critics", [1, 3]) @@ -104,22 +97,20 @@ def test_n_critics(n_critics): buffer_size=10000, verbose=1, ) - model.learn(total_timesteps=300) + model.learn(total_timesteps=200) def test_dqn(): - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - model = DQN( - "MlpPolicy", - "CartPole-v1", - policy_kwargs=dict(net_arch=[64, 64]), - learning_starts=100, - buffer_size=500, - learning_rate=3e-4, - verbose=1, - create_eval_env=True, - ) - model.learn(total_timesteps=500, eval_freq=250) + model = DQN( + "MlpPolicy", + "CartPole-v1", + policy_kwargs=dict(net_arch=[64, 64]), + learning_starts=100, + buffer_size=500, + learning_rate=3e-4, + verbose=1, + ) + model.learn(total_timesteps=200) @pytest.mark.parametrize("train_freq", [4, (4, "step"), (1, "episode")]) diff --git a/tests/test_sde.py b/tests/test_sde.py index 4348207..4fc16fc 100644 --- a/tests/test_sde.py +++ b/tests/test_sde.py @@ -63,18 +63,16 @@ def test_sde_check(): @pytest.mark.parametrize("use_expln", [False, True]) def test_state_dependent_noise(model_class, use_expln): kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64} - with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated - model = model_class( - "MlpPolicy", - "Pendulum-v1", - use_sde=True, - seed=None, - create_eval_env=True, - verbose=1, - policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]), - **kwargs, - ) - model.learn(total_timesteps=255, eval_freq=250) + model = model_class( + "MlpPolicy", + "Pendulum-v1", + use_sde=True, + seed=None, + verbose=1, + policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]), + **kwargs, + ) + model.learn(total_timesteps=255) model.policy.reset_noise() if model_class == SAC: model.policy.actor.get_std() diff --git a/tests/test_train_eval_mode.py b/tests/test_train_eval_mode.py index a1a63c0..f3a012f 100644 --- a/tests/test_train_eval_mode.py +++ b/tests/test_train_eval_mode.py @@ -332,7 +332,7 @@ def test_a2c_ppo_collect_rollouts_with_batch_norm(model_class, env_id): bias_before, running_mean_before = clone_on_policy_batch_norm(model) - total_timesteps, callback = model._setup_learn(total_timesteps=2 * 64, eval_env=model.get_env()) + total_timesteps, callback = model._setup_learn(total_timesteps=2 * 64) for _ in range(2): model.collect_rollouts(model.get_env(), callback, model.rollout_buffer, n_rollout_steps=model.n_steps) diff --git a/tests/test_vec_normalize.py b/tests/test_vec_normalize.py index 0fef682..1fbf5b7 100644 --- a/tests/test_vec_normalize.py +++ b/tests/test_vec_normalize.py @@ -118,15 +118,6 @@ def make_dict_env(): return Monitor(DummyDictEnv()) -def test_deprecation(): - venv = DummyVecEnv([lambda: gym.make("CartPole-v1")]) - venv = VecNormalize(venv) - with warnings.catch_warnings(record=True) as record: - assert np.allclose(venv.ret, venv.returns) - # Deprecation warning when using .ret - assert len(record) == 1 - - def check_rms_equal(rmsa, rmsb): if isinstance(rmsa, dict): for key in rmsa.keys(): @@ -380,8 +371,7 @@ def test_offpolicy_normalization(model_class, online_sampling): assert model.get_vec_normalize_env() is eval_env model.learn(total_timesteps=10) model.set_env(env) - with pytest.warns(DeprecationWarning): # `eval_env` and `eval_freq` are deprecated - model.learn(total_timesteps=150, eval_env=eval_env, eval_freq=75) + model.learn(total_timesteps=150) # Check getter assert isinstance(model.get_vec_normalize_env(), VecNormalize)