saymrwulf/stable-baselines3
1
0
Fork 0
mirror of https://github.com/saymrwulf/stable-baselines3.git synced 2026-09-15 22:10:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove deprecated features and attributes (#1104)

Browse source 
* Remove deprecated eval env

* Remove deprecated ret attribute

* Remove sde net arch

* Remove unused code

* Update test comment

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>
This commit is contained in:
Antonin RAFFIN 2022-10-11 10:55:16 +02:00 committed by GitHub
parent 5e8f06b3cb
commit 508f8ffd59
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 97 additions and 367 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
docs/misc/changelog.rst
View file

@ -3,6 +3,37 @@
Changelog Changelog
========== ==========
Release 1.7.0a0 (WIP)
--------------------------
Breaking Changes:
^^^^^^^^^^^^^^^^^
- Removed deprecated ``create_eval_env``, ``eval_env``, ``eval_log_path``, ``n_eval_episodes`` and ``eval_freq`` parameters,
please use an ``EvalCallback`` instead
- Removed deprecated ``sde_net_arch`` parameter
- Removed ``ret`` attributes in ``VecNormalize``, please use ``returns`` instead
New Features:
^^^^^^^^^^^^^
SB3-Contrib
^^^^^^^^^^^
Bug Fixes:
^^^^^^^^^^
Deprecations:
^^^^^^^^^^^^^
Others:
^^^^^^^
Documentation:
^^^^^^^^^^^^^^
Release 1.6.2 (2022-10-10) Release 1.6.2 (2022-10-10)
-------------------------- --------------------------

14
stable_baselines3/a2c/a2c.py
View file

@ -43,10 +43,6 @@ class A2C(OnPolicyAlgorithm):
Default: -1 (only sample at the beginning of the rollout) Default: -1 (only sample at the beginning of the rollout)
:param normalize_advantage: Whether to normalize or not the advantage :param normalize_advantage: Whether to normalize or not the advantage
:param tensorboard_log: the log location for tensorboard (if None, no logging) :param tensorboard_log: the log location for tensorboard (if None, no logging)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -79,7 +75,6 @@ class A2C(OnPolicyAlgorithm):
sde_sample_freq: int = -1, sde_sample_freq: int = -1,
normalize_advantage: bool = False, normalize_advantage: bool = False,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -103,7 +98,6 @@ class A2C(OnPolicyAlgorithm):
policy_kwargs=policy_kwargs, policy_kwargs=policy_kwargs,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
_init_setup_model=False, _init_setup_model=False,
supported_action_spaces=( supported_action_spaces=(
@ -191,11 +185,7 @@ class A2C(OnPolicyAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 100, log_interval: int = 100,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "A2C", tb_log_name: str = "A2C",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> A2CSelf: ) -> A2CSelf:
@ -204,11 +194,7 @@ class A2C(OnPolicyAlgorithm):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

104
stable_baselines3/common/base_class.py
View file

@ -3,7 +3,6 @@
import io import io
import pathlib import pathlib
import time import time
import warnings
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections import deque from collections import deque
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union
@ -13,7 +12,7 @@ import numpy as np
import torch as th import torch as th
from stable_baselines3.common import utils from stable_baselines3.common import utils
from stable_baselines3.common.callbacks import BaseCallback, CallbackList, ConvertCallback, EvalCallback, ProgressBarCallback from stable_baselines3.common.callbacks import BaseCallback, CallbackList, ConvertCallback, ProgressBarCallback
from stable_baselines3.common.env_util import is_wrapped from stable_baselines3.common.env_util import is_wrapped
from stable_baselines3.common.logger import Logger from stable_baselines3.common.logger import Logger
from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.monitor import Monitor
@ -75,10 +74,6 @@ class BaseAlgorithm(ABC):
if it is not possible. if it is not possible.
:param support_multi_env: Whether the algorithm supports training :param support_multi_env: Whether the algorithm supports training
with multiple environments (as in A2C) with multiple environments (as in A2C)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param monitor_wrapper: When creating an environment, whether to wrap it :param monitor_wrapper: When creating an environment, whether to wrap it
or not in a Monitor wrapper. or not in a Monitor wrapper.
:param seed: Seed for the pseudo random generators :param seed: Seed for the pseudo random generators
@ -102,7 +97,6 @@ class BaseAlgorithm(ABC):
verbose: int = 0, verbose: int = 0,
device: Union[th.device, str] = "auto", device: Union[th.device, str] = "auto",
support_multi_env: bool = False, support_multi_env: bool = False,
create_eval_env: bool = False,
monitor_wrapper: bool = True, monitor_wrapper: bool = True,
seed: Optional[int] = None, seed: Optional[int] = None,
use_sde: bool = False, use_sde: bool = False,
@ -131,7 +125,6 @@ class BaseAlgorithm(ABC):
self._total_timesteps = 0 self._total_timesteps = 0
# Used for computing fps, it is updated at each call of learn() # Used for computing fps, it is updated at each call of learn()
self._num_timesteps_at_start = 0 self._num_timesteps_at_start = 0
self.eval_env = None
self.seed = seed self.seed = seed
self.action_noise = None # type: Optional[ActionNoise] self.action_noise = None # type: Optional[ActionNoise]
self.start_time = None self.start_time = None
@ -162,19 +155,6 @@ class BaseAlgorithm(ABC):
# Create and wrap the env if needed # Create and wrap the env if needed
if env is not None: if env is not None:
if isinstance(env, str):
if create_eval_env:
warnings.warn(
"The parameter `create_eval_env` is deprecated and will be removed in the future. "
"Please use `EvalCallback` or a custom Callback instead.",
DeprecationWarning,
# By setting the `stacklevel` we refer to the initial caller of the deprecated feature.
# This causes the the `DepricationWarning` to not be ignored and to be shown to the user. See
# https://github.com/DLR-RM/stable-baselines3/pull/1082#discussion_r989842855 for more details.
stacklevel=4,
)
self.eval_env = maybe_make_env(env, self.verbose)
env = maybe_make_env(env, self.verbose) env = maybe_make_env(env, self.verbose)
env = self._wrap_env(env, self.verbose, monitor_wrapper) env = self._wrap_env(env, self.verbose, monitor_wrapper)
@ -275,21 +255,6 @@ class BaseAlgorithm(ABC):
"""Getter for the logger object.""" """Getter for the logger object."""
return self._logger return self._logger
def _get_eval_env(self, eval_env: Optional[GymEnv]) -> Optional[GymEnv]:
"""
Return the environment that will be used for evaluation.
:param eval_env:)
:return:
"""
if eval_env is None:
eval_env = self.eval_env
if eval_env is not None:
eval_env = self._wrap_env(eval_env, self.verbose)
assert eval_env.num_envs == 1
return eval_env
def _setup_lr_schedule(self) -> None: def _setup_lr_schedule(self) -> None:
"""Transform to callable if needed.""" """Transform to callable if needed."""
self.lr_schedule = get_schedule_fn(self.learning_rate) self.lr_schedule = get_schedule_fn(self.learning_rate)
@ -332,7 +297,6 @@ class BaseAlgorithm(ABC):
"policy", "policy",
"device", "device",
"env", "env",
"eval_env",
"replay_buffer", "replay_buffer",
"rollout_buffer", "rollout_buffer",
"_vec_normalize_env", "_vec_normalize_env",
@ -379,20 +343,10 @@ class BaseAlgorithm(ABC):
def _init_callback( def _init_callback(
self, self,
callback: MaybeCallback, callback: MaybeCallback,
eval_env: Optional[VecEnv] = None,
eval_freq: int = 10000,
n_eval_episodes: int = 5,
log_path: Optional[str] = None,
progress_bar: bool = False, progress_bar: bool = False,
) -> BaseCallback: ) -> BaseCallback:
""" """
:param callback: Callback(s) called at every step with state of the algorithm. :param callback: Callback(s) called at every step with state of the algorithm.
:param eval_freq: How many steps between evaluations; if None, do not evaluate.
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param n_eval_episodes: How many episodes to play per evaluation
:param n_eval_episodes: Number of episodes to rollout during evaluation.
:param log_path: Path to a folder where the evaluations will be saved
:param progress_bar: Display a progress bar using tqdm and rich. :param progress_bar: Display a progress bar using tqdm and rich.
:return: A hybrid callback calling `callback` and performing evaluation. :return: A hybrid callback calling `callback` and performing evaluation.
""" """
@ -408,29 +362,13 @@ class BaseAlgorithm(ABC):
if progress_bar: if progress_bar:
callback = CallbackList([callback, ProgressBarCallback()]) callback = CallbackList([callback, ProgressBarCallback()])
# Create eval callback in charge of the evaluation
if eval_env is not None:
eval_callback = EvalCallback(
eval_env,
best_model_save_path=log_path,
log_path=log_path,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
verbose=self.verbose,
)
callback = CallbackList([callback, eval_callback])
callback.init_callback(self) callback.init_callback(self)
return callback return callback
def _setup_learn( def _setup_learn(
self, self,
total_timesteps: int, total_timesteps: int,
eval_env: Optional[GymEnv],
callback: MaybeCallback = None, callback: MaybeCallback = None,
eval_freq: int = 10000,
n_eval_episodes: int = 5,
log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
tb_log_name: str = "run", tb_log_name: str = "run",
progress_bar: bool = False, progress_bar: bool = False,
@ -439,32 +377,12 @@ class BaseAlgorithm(ABC):
Initialize different variables needed for training. Initialize different variables needed for training.
:param total_timesteps: The total number of samples (env steps) to train on :param total_timesteps: The total number of samples (env steps) to train on
:param eval_env: Environment to use for evaluation.
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param callback: Callback(s) called at every step with state of the algorithm. :param callback: Callback(s) called at every step with state of the algorithm.
:param eval_freq: How many steps between evaluations
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param n_eval_episodes: How many episodes to play per evaluation
:param log_path: Path to a folder where the evaluations will be saved
:param reset_num_timesteps: Whether to reset or not the ``num_timesteps`` attribute :param reset_num_timesteps: Whether to reset or not the ``num_timesteps`` attribute
:param tb_log_name: the name of the run for tensorboard log :param tb_log_name: the name of the run for tensorboard log
:param progress_bar: Display a progress bar using tqdm and rich. :param progress_bar: Display a progress bar using tqdm and rich.
:return: Total timesteps and callback(s) :return: Total timesteps and callback(s)
""" """
if eval_env is not None or eval_freq != -1:
warnings.warn(
"Parameters `eval_env` and `eval_freq` are deprecated and will be removed in the future. "
"Please use `EvalCallback` or a custom Callback instead.",
DeprecationWarning,
# By setting the `stacklevel` we refer to the initial caller of the deprecated feature.
# This causes the the `DepricationWarning` to not be ignored and to be shown to the user. See
# https://github.com/DLR-RM/stable-baselines3/pull/1082#discussion_r989842855 for more details.
stacklevel=4,
)
self.start_time = time.time_ns() self.start_time = time.time_ns()
if self.ep_info_buffer is None or reset_num_timesteps: if self.ep_info_buffer is None or reset_num_timesteps:
@ -492,17 +410,12 @@ class BaseAlgorithm(ABC):
if self._vec_normalize_env is not None: if self._vec_normalize_env is not None:
self._last_original_obs = self._vec_normalize_env.get_original_obs() self._last_original_obs = self._vec_normalize_env.get_original_obs()
if eval_env is not None and self.seed is not None:
eval_env.seed(self.seed)
eval_env = self._get_eval_env(eval_env)
# Configure logger's outputs if no logger was passed # Configure logger's outputs if no logger was passed
if not self._custom_logger: if not self._custom_logger:
self._logger = utils.configure_logger(self.verbose, self.tensorboard_log, tb_log_name, reset_num_timesteps) self._logger = utils.configure_logger(self.verbose, self.tensorboard_log, tb_log_name, reset_num_timesteps)
# Create eval callback if needed # Create eval callback if needed
callback = self._init_callback(callback, eval_env, eval_freq, n_eval_episodes, log_path, progress_bar) callback = self._init_callback(callback, progress_bar)
return total_timesteps, callback return total_timesteps, callback
@ -583,10 +496,6 @@ class BaseAlgorithm(ABC):
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 100, log_interval: int = 100,
tb_log_name: str = "run", tb_log_name: str = "run",
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> BaseAlgorithmSelf: ) -> BaseAlgorithmSelf:
@ -597,13 +506,6 @@ class BaseAlgorithm(ABC):
:param callback: callback(s) called at every step with state of the algorithm. :param callback: callback(s) called at every step with state of the algorithm.
:param log_interval: The number of timesteps before logging. :param log_interval: The number of timesteps before logging.
:param tb_log_name: the name of the run for TensorBoard logging :param tb_log_name: the name of the run for TensorBoard logging
:param eval_env: Environment that will be used to evaluate the agent. Caution, this parameter
is deprecated and will be removed in the future. Please use ``EvalCallback`` instead.
:param eval_freq: Evaluate the agent every ``eval_freq`` timesteps (this may vary a little).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param n_eval_episodes: Number of episode to evaluate the agent
:param eval_log_path: Path to a folder where the evaluations will be saved
:param reset_num_timesteps: whether or not to reset the current timestep number (used in logging) :param reset_num_timesteps: whether or not to reset the current timestep number (used in logging)
:param progress_bar: Display a progress bar using tqdm and rich. :param progress_bar: Display a progress bar using tqdm and rich.
:return: the trained model :return: the trained model
@ -644,8 +546,6 @@ class BaseAlgorithm(ABC):
self.action_space.seed(seed) self.action_space.seed(seed)
if self.env is not None: if self.env is not None:
self.env.seed(seed) self.env.seed(seed)
if self.eval_env is not None:
self.eval_env.seed(seed)
def set_parameters( def set_parameters(
self, self,

22
stable_baselines3/common/off_policy_algorithm.py
View file

@ -60,10 +60,6 @@ class OffPolicyAlgorithm(BaseAlgorithm):
if it is not possible. if it is not possible.
:param support_multi_env: Whether the algorithm supports training :param support_multi_env: Whether the algorithm supports training
with multiple environments (as in A2C) with multiple environments (as in A2C)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically. (Only available when passing string for the environment)
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param monitor_wrapper: When creating an environment, whether to wrap it :param monitor_wrapper: When creating an environment, whether to wrap it
or not in a Monitor wrapper. or not in a Monitor wrapper.
:param seed: Seed for the pseudo random generators :param seed: Seed for the pseudo random generators
@ -98,7 +94,6 @@ class OffPolicyAlgorithm(BaseAlgorithm):
verbose: int = 0, verbose: int = 0,
device: Union[th.device, str] = "auto", device: Union[th.device, str] = "auto",
support_multi_env: bool = False, support_multi_env: bool = False,
create_eval_env: bool = False,
monitor_wrapper: bool = True, monitor_wrapper: bool = True,
seed: Optional[int] = None, seed: Optional[int] = None,
use_sde: bool = False, use_sde: bool = False,
@ -117,7 +112,6 @@ class OffPolicyAlgorithm(BaseAlgorithm):
verbose=verbose, verbose=verbose,
device=device, device=device,
support_multi_env=support_multi_env, support_multi_env=support_multi_env,
create_eval_env=create_eval_env,
monitor_wrapper=monitor_wrapper, monitor_wrapper=monitor_wrapper,
seed=seed, seed=seed,
use_sde=use_sde, use_sde=use_sde,
@ -271,11 +265,7 @@ class OffPolicyAlgorithm(BaseAlgorithm):
def _setup_learn( def _setup_learn(
self, self,
total_timesteps: int, total_timesteps: int,
eval_env: Optional[GymEnv],
callback: MaybeCallback = None, callback: MaybeCallback = None,
eval_freq: int = 10000,
n_eval_episodes: int = 5,
log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
tb_log_name: str = "run", tb_log_name: str = "run",
progress_bar: bool = False, progress_bar: bool = False,
@ -314,11 +304,7 @@ class OffPolicyAlgorithm(BaseAlgorithm):
return super()._setup_learn( return super()._setup_learn(
total_timesteps, total_timesteps,
eval_env,
callback, callback,
eval_freq,
n_eval_episodes,
log_path,
reset_num_timesteps, reset_num_timesteps,
tb_log_name, tb_log_name,
progress_bar, progress_bar,
@ -329,22 +315,14 @@ class OffPolicyAlgorithm(BaseAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 4, log_interval: int = 4,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "run", tb_log_name: str = "run",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> OffPolicyAlgorithmSelf: ) -> OffPolicyAlgorithmSelf:
total_timesteps, callback = self._setup_learn( total_timesteps, callback = self._setup_learn(
total_timesteps, total_timesteps,
eval_env,
callback, callback,
eval_freq,
n_eval_episodes,
eval_log_path,
reset_num_timesteps, reset_num_timesteps,
tb_log_name, tb_log_name,
progress_bar, progress_bar,

14
stable_baselines3/common/on_policy_algorithm.py
View file

@ -38,10 +38,6 @@ class OnPolicyAlgorithm(BaseAlgorithm):
:param sde_sample_freq: Sample a new noise matrix every n steps when using gSDE :param sde_sample_freq: Sample a new noise matrix every n steps when using gSDE
Default: -1 (only sample at the beginning of the rollout) Default: -1 (only sample at the beginning of the rollout)
:param tensorboard_log: the log location for tensorboard (if None, no logging) :param tensorboard_log: the log location for tensorboard (if None, no logging)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically. (Only available when passing string for the environment)
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param monitor_wrapper: When creating an environment, whether to wrap it :param monitor_wrapper: When creating an environment, whether to wrap it
or not in a Monitor wrapper. or not in a Monitor wrapper.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
@ -68,7 +64,6 @@ class OnPolicyAlgorithm(BaseAlgorithm):
use_sde: bool, use_sde: bool,
sde_sample_freq: int, sde_sample_freq: int,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
monitor_wrapper: bool = True, monitor_wrapper: bool = True,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
@ -87,7 +82,6 @@ class OnPolicyAlgorithm(BaseAlgorithm):
device=device, device=device,
use_sde=use_sde, use_sde=use_sde,
sde_sample_freq=sde_sample_freq, sde_sample_freq=sde_sample_freq,
create_eval_env=create_eval_env,
support_multi_env=True, support_multi_env=True,
seed=seed, seed=seed,
tensorboard_log=tensorboard_log, tensorboard_log=tensorboard_log,
@ -233,11 +227,7 @@ class OnPolicyAlgorithm(BaseAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 1, log_interval: int = 1,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "OnPolicyAlgorithm", tb_log_name: str = "OnPolicyAlgorithm",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> OnPolicyAlgorithmSelf: ) -> OnPolicyAlgorithmSelf:
@ -245,11 +235,7 @@ class OnPolicyAlgorithm(BaseAlgorithm):
total_timesteps, callback = self._setup_learn( total_timesteps, callback = self._setup_learn(
total_timesteps, total_timesteps,
eval_env,
callback, callback,
eval_freq,
n_eval_episodes,
eval_log_path,
reset_num_timesteps, reset_num_timesteps,
tb_log_name, tb_log_name,
progress_bar, progress_bar,

25
stable_baselines3/common/policies.py
View file

@ -171,14 +171,6 @@ class BaseModel(nn.Module):
device = get_device(device) device = get_device(device)
saved_variables = th.load(path, map_location=device) saved_variables = th.load(path, map_location=device)
# Allow to load policy saved with older version of SB3
if "sde_net_arch" in saved_variables["data"]:
warnings.warn(
"sde_net_arch is deprecated, please downgrade to SB3 v1.2.0 if you need such parameter.",
DeprecationWarning,
)
del saved_variables["data"]["sde_net_arch"]
# Create policy object # Create policy object
model = cls(**saved_variables["data"]) # pytype: disable=not-instantiable model = cls(**saved_variables["data"]) # pytype: disable=not-instantiable
# Load weights # Load weights
@ -389,9 +381,6 @@ class ActorCriticPolicy(BasePolicy):
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param full_std: Whether to use (n_features x n_actions) parameters :param full_std: Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using gSDE for the std instead of only (n_features,) when using gSDE
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -419,7 +408,6 @@ class ActorCriticPolicy(BasePolicy):
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = 0.0, log_std_init: float = 0.0,
full_std: bool = True, full_std: bool = True,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
squash_output: bool = False, squash_output: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor, features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor,
@ -471,9 +459,6 @@ class ActorCriticPolicy(BasePolicy):
"learn_features": False, "learn_features": False,
} }
if sde_net_arch is not None:
warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
self.use_sde = use_sde self.use_sde = use_sde
self.dist_kwargs = dist_kwargs self.dist_kwargs = dist_kwargs
@ -684,9 +669,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy):
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param full_std: Whether to use (n_features x n_actions) parameters :param full_std: Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using gSDE for the std instead of only (n_features,) when using gSDE
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -714,7 +696,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy):
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = 0.0, log_std_init: float = 0.0,
full_std: bool = True, full_std: bool = True,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
squash_output: bool = False, squash_output: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN, features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN,
@ -733,7 +714,6 @@ class ActorCriticCnnPolicy(ActorCriticPolicy):
use_sde, use_sde,
log_std_init, log_std_init,
full_std, full_std,
sde_net_arch,
use_expln, use_expln,
squash_output, squash_output,
features_extractor_class, features_extractor_class,
@ -759,9 +739,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy):
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param full_std: Whether to use (n_features x n_actions) parameters :param full_std: Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using gSDE for the std instead of only (n_features,) when using gSDE
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -789,7 +766,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy):
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = 0.0, log_std_init: float = 0.0,
full_std: bool = True, full_std: bool = True,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
squash_output: bool = False, squash_output: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor, features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor,
@ -808,7 +784,6 @@ class MultiInputActorCriticPolicy(ActorCriticPolicy):
use_sde, use_sde,
log_std_init, log_std_init,
full_std, full_std,
sde_net_arch,
use_expln, use_expln,
squash_output, squash_output,
features_extractor_class, features_extractor_class,

5
stable_baselines3/common/vec_env/vec_normalize.py
View file

@ -289,8 +289,3 @@ class VecNormalize(VecEnvWrapper):
""" """
with open(save_path, "wb") as file_handler: with open(save_path, "wb") as file_handler:
pickle.dump(self, file_handler) pickle.dump(self, file_handler)
@property
def ret(self) -> np.ndarray:
warnings.warn("`VecNormalize` `ret` attribute is deprecated. Please use `returns` instead.", DeprecationWarning)
return self.returns

14
stable_baselines3/ddpg/ddpg.py
View file

@ -44,10 +44,6 @@ class DDPG(TD3):
:param optimize_memory_usage: Enable a memory efficient variant of the replay buffer :param optimize_memory_usage: Enable a memory efficient variant of the replay buffer
at a cost of more complexity. at a cost of more complexity.
See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195 See https://github.com/DLR-RM/stable-baselines3/issues/37#issuecomment-637501195
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -74,7 +70,6 @@ class DDPG(TD3):
replay_buffer_kwargs: Optional[Dict[str, Any]] = None, replay_buffer_kwargs: Optional[Dict[str, Any]] = None,
optimize_memory_usage: bool = False, optimize_memory_usage: bool = False,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -100,7 +95,6 @@ class DDPG(TD3):
tensorboard_log=tensorboard_log, tensorboard_log=tensorboard_log,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
optimize_memory_usage=optimize_memory_usage, optimize_memory_usage=optimize_memory_usage,
# Remove all tricks from TD3 to obtain DDPG: # Remove all tricks from TD3 to obtain DDPG:
@ -123,11 +117,7 @@ class DDPG(TD3):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 4, log_interval: int = 4,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "DDPG", tb_log_name: str = "DDPG",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> DDPGSelf: ) -> DDPGSelf:
@ -136,11 +126,7 @@ class DDPG(TD3):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

14
stable_baselines3/dqn/dqn.py
View file

@ -52,10 +52,6 @@ class DQN(OffPolicyAlgorithm):
:param exploration_final_eps: final value of random action probability :param exploration_final_eps: final value of random action probability
:param max_grad_norm: The maximum value for the gradient clipping :param max_grad_norm: The maximum value for the gradient clipping
:param tensorboard_log: the log location for tensorboard (if None, no logging) :param tensorboard_log: the log location for tensorboard (if None, no logging)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -92,7 +88,6 @@ class DQN(OffPolicyAlgorithm):
exploration_final_eps: float = 0.05, exploration_final_eps: float = 0.05,
max_grad_norm: float = 10, max_grad_norm: float = 10,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -118,7 +113,6 @@ class DQN(OffPolicyAlgorithm):
tensorboard_log=tensorboard_log, tensorboard_log=tensorboard_log,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
sde_support=False, sde_support=False,
optimize_memory_usage=optimize_memory_usage, optimize_memory_usage=optimize_memory_usage,
@ -263,11 +257,7 @@ class DQN(OffPolicyAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 4, log_interval: int = 4,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "DQN", tb_log_name: str = "DQN",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> DQNSelf: ) -> DQNSelf:
@ -276,11 +266,7 @@ class DQN(OffPolicyAlgorithm):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

14
stable_baselines3/ppo/ppo.py
View file

@ -57,10 +57,6 @@ class PPO(OnPolicyAlgorithm):
see issue #213 (cf https://github.com/hill-a/stable-baselines/issues/213) see issue #213 (cf https://github.com/hill-a/stable-baselines/issues/213)
By default, there is no limit on the kl div. By default, there is no limit on the kl div.
:param tensorboard_log: the log location for tensorboard (if None, no logging) :param tensorboard_log: the log location for tensorboard (if None, no logging)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -96,7 +92,6 @@ class PPO(OnPolicyAlgorithm):
sde_sample_freq: int = -1, sde_sample_freq: int = -1,
target_kl: Optional[float] = None, target_kl: Optional[float] = None,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -120,7 +115,6 @@ class PPO(OnPolicyAlgorithm):
policy_kwargs=policy_kwargs, policy_kwargs=policy_kwargs,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
_init_setup_model=False, _init_setup_model=False,
supported_action_spaces=( supported_action_spaces=(
@ -305,11 +299,7 @@ class PPO(OnPolicyAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 1, log_interval: int = 1,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "PPO", tb_log_name: str = "PPO",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> PPOSelf: ) -> PPOSelf:
@ -318,11 +308,7 @@ class PPO(OnPolicyAlgorithm):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

25
stable_baselines3/sac/policies.py
View file

@ -38,9 +38,6 @@ class Actor(BasePolicy):
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param full_std: Whether to use (n_features x n_actions) parameters :param full_std: Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using gSDE. for the std instead of only (n_features,) when using gSDE.
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -60,7 +57,6 @@ class Actor(BasePolicy):
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = -3, log_std_init: float = -3,
full_std: bool = True, full_std: bool = True,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
clip_mean: float = 2.0, clip_mean: float = 2.0,
normalize_images: bool = True, normalize_images: bool = True,
@ -80,14 +76,10 @@ class Actor(BasePolicy):
self.features_dim = features_dim self.features_dim = features_dim
self.activation_fn = activation_fn self.activation_fn = activation_fn
self.log_std_init = log_std_init self.log_std_init = log_std_init
self.sde_net_arch = sde_net_arch
self.use_expln = use_expln self.use_expln = use_expln
self.full_std = full_std self.full_std = full_std
self.clip_mean = clip_mean self.clip_mean = clip_mean
if sde_net_arch is not None:
warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
action_dim = get_action_dim(self.action_space) action_dim = get_action_dim(self.action_space)
latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn) latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn)
self.latent_pi = nn.Sequential(*latent_pi_net) self.latent_pi = nn.Sequential(*latent_pi_net)
@ -196,9 +188,6 @@ class SACPolicy(BasePolicy):
:param activation_fn: Activation function :param activation_fn: Activation function
:param use_sde: Whether to use State Dependent Exploration or not :param use_sde: Whether to use State Dependent Exploration or not
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -226,7 +215,6 @@ class SACPolicy(BasePolicy):
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = -3, log_std_init: float = -3,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
clip_mean: float = 2.0, clip_mean: float = 2.0,
features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor, features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor,
@ -263,9 +251,6 @@ class SACPolicy(BasePolicy):
} }
self.actor_kwargs = self.net_args.copy() self.actor_kwargs = self.net_args.copy()
if sde_net_arch is not None:
warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
sde_kwargs = { sde_kwargs = {
"use_sde": use_sde, "use_sde": use_sde,
"log_std_init": log_std_init, "log_std_init": log_std_init,
@ -382,9 +367,6 @@ class CnnPolicy(SACPolicy):
:param activation_fn: Activation function :param activation_fn: Activation function
:param use_sde: Whether to use State Dependent Exploration or not :param use_sde: Whether to use State Dependent Exploration or not
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -410,7 +392,6 @@ class CnnPolicy(SACPolicy):
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = -3, log_std_init: float = -3,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
clip_mean: float = 2.0, clip_mean: float = 2.0,
features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN, features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN,
@ -429,7 +410,6 @@ class CnnPolicy(SACPolicy):
activation_fn, activation_fn,
use_sde, use_sde,
log_std_init, log_std_init,
sde_net_arch,
use_expln, use_expln,
clip_mean, clip_mean,
features_extractor_class, features_extractor_class,
@ -453,9 +433,6 @@ class MultiInputPolicy(SACPolicy):
:param activation_fn: Activation function :param activation_fn: Activation function
:param use_sde: Whether to use State Dependent Exploration or not :param use_sde: Whether to use State Dependent Exploration or not
:param log_std_init: Initial value for the log standard deviation :param log_std_init: Initial value for the log standard deviation
:param sde_net_arch: Network architecture for extracting features
when using gSDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure :param use_expln: Use ``expln()`` function instead of ``exp()`` when using gSDE to ensure
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
@ -481,7 +458,6 @@ class MultiInputPolicy(SACPolicy):
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False, use_sde: bool = False,
log_std_init: float = -3, log_std_init: float = -3,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
clip_mean: float = 2.0, clip_mean: float = 2.0,
features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor, features_extractor_class: Type[BaseFeaturesExtractor] = CombinedExtractor,
@ -500,7 +476,6 @@ class MultiInputPolicy(SACPolicy):
activation_fn, activation_fn,
use_sde, use_sde,
log_std_init, log_std_init,
sde_net_arch,
use_expln, use_expln,
clip_mean, clip_mean,
features_extractor_class, features_extractor_class,

14
stable_baselines3/sac/sac.py
View file

@ -65,10 +65,6 @@ class SAC(OffPolicyAlgorithm):
Default: -1 (only sample at the beginning of the rollout) Default: -1 (only sample at the beginning of the rollout)
:param use_sde_at_warmup: Whether to use gSDE instead of uniform sampling :param use_sde_at_warmup: Whether to use gSDE instead of uniform sampling
during the warm up phase (before learning starts) during the warm up phase (before learning starts)
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -107,7 +103,6 @@ class SAC(OffPolicyAlgorithm):
sde_sample_freq: int = -1, sde_sample_freq: int = -1,
use_sde_at_warmup: bool = False, use_sde_at_warmup: bool = False,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -133,7 +128,6 @@ class SAC(OffPolicyAlgorithm):
tensorboard_log=tensorboard_log, tensorboard_log=tensorboard_log,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
use_sde=use_sde, use_sde=use_sde,
sde_sample_freq=sde_sample_freq, sde_sample_freq=sde_sample_freq,
@ -297,11 +291,7 @@ class SAC(OffPolicyAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 4, log_interval: int = 4,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "SAC", tb_log_name: str = "SAC",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> SACSelf: ) -> SACSelf:
@ -310,11 +300,7 @@ class SAC(OffPolicyAlgorithm):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

14
stable_baselines3/td3/td3.py
View file

@ -53,10 +53,6 @@ class TD3(OffPolicyAlgorithm):
:param target_policy_noise: Standard deviation of Gaussian noise added to target policy :param target_policy_noise: Standard deviation of Gaussian noise added to target policy
(smoothing noise) (smoothing noise)
:param target_noise_clip: Limit for absolute value of target policy smoothing noise. :param target_noise_clip: Limit for absolute value of target policy smoothing noise.
:param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically (Only available when passing string for the environment).
Caution, this parameter is deprecated and will be removed in the future.
Please use `EvalCallback` or a custom Callback instead.
:param policy_kwargs: additional arguments to be passed to the policy on creation :param policy_kwargs: additional arguments to be passed to the policy on creation
:param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for :param verbose: Verbosity level: 0 for no output, 1 for info messages (such as device or wrappers used), 2 for
debug messages debug messages
@ -92,7 +88,6 @@ class TD3(OffPolicyAlgorithm):
target_policy_noise: float = 0.2, target_policy_noise: float = 0.2,
target_noise_clip: float = 0.5, target_noise_clip: float = 0.5,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False,
policy_kwargs: Optional[Dict[str, Any]] = None, policy_kwargs: Optional[Dict[str, Any]] = None,
verbose: int = 0, verbose: int = 0,
seed: Optional[int] = None, seed: Optional[int] = None,
@ -118,7 +113,6 @@ class TD3(OffPolicyAlgorithm):
tensorboard_log=tensorboard_log, tensorboard_log=tensorboard_log,
verbose=verbose, verbose=verbose,
device=device, device=device,
create_eval_env=create_eval_env,
seed=seed, seed=seed,
sde_support=False, sde_support=False,
optimize_memory_usage=optimize_memory_usage, optimize_memory_usage=optimize_memory_usage,
@ -213,11 +207,7 @@ class TD3(OffPolicyAlgorithm):
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
log_interval: int = 4, log_interval: int = 4,
eval_env: Optional[GymEnv] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
tb_log_name: str = "TD3", tb_log_name: str = "TD3",
eval_log_path: Optional[str] = None,
reset_num_timesteps: bool = True, reset_num_timesteps: bool = True,
progress_bar: bool = False, progress_bar: bool = False,
) -> TD3Self: ) -> TD3Self:
@ -226,11 +216,7 @@ class TD3(OffPolicyAlgorithm):
total_timesteps=total_timesteps, total_timesteps=total_timesteps,
callback=callback, callback=callback,
log_interval=log_interval, log_interval=log_interval,
eval_env=eval_env,
eval_freq=eval_freq,
n_eval_episodes=n_eval_episodes,
tb_log_name=tb_log_name, tb_log_name=tb_log_name,
eval_log_path=eval_log_path,
reset_num_timesteps=reset_num_timesteps, reset_num_timesteps=reset_num_timesteps,
progress_bar=progress_bar, progress_bar=progress_bar,
) )

2
stable_baselines3/version.txt
View file

@ -1 +1 @@
1.6.2 1.7.0a0

9
tests/test_predict.py
View file

@ -37,20 +37,15 @@ class CustomSubClassedSpaceEnv(gym.Env):
@pytest.mark.parametrize("model_class", MODEL_LIST) @pytest.mark.parametrize("model_class", MODEL_LIST)
def test_auto_wrap(model_class): def test_auto_wrap(model_class):
# test auto wrapping of env into a VecEnv """Test auto wrapping of env into a VecEnv."""
# Use different environment for DQN # Use different environment for DQN
if model_class is DQN: if model_class is DQN:
env_name = "CartPole-v0" env_name = "CartPole-v0"
else: else:
env_name = "Pendulum-v1" env_name = "Pendulum-v1"
env = gym.make(env_name) env = gym.make(env_name)
eval_env = gym.make(env_name)
model = model_class("MlpPolicy", env) model = model_class("MlpPolicy", env)
model.learn(100)
# Catch DeprecationWarnings
with pytest.warns(DeprecationWarning): # `eval_env` is deprecated
model.learn(100, eval_env=eval_env)
@pytest.mark.parametrize("model_class", MODEL_LIST) @pytest.mark.parametrize("model_class", MODEL_LIST)

25
tests/test_run.py
View file

@ -18,25 +18,22 @@ def test_deterministic_pg(model_class, action_noise):
""" """
Test for DDPG and variants (TD3). Test for DDPG and variants (TD3).
""" """
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated
model = model_class( model = model_class(
"MlpPolicy", "MlpPolicy",
"Pendulum-v1", "Pendulum-v1",
policy_kwargs=dict(net_arch=[64, 64]), policy_kwargs=dict(net_arch=[64, 64]),
learning_starts=100, learning_starts=100,
verbose=1, verbose=1,
create_eval_env=True,
buffer_size=250, buffer_size=250,
action_noise=action_noise, action_noise=action_noise,
) )
model.learn(total_timesteps=300, eval_freq=250) model.learn(total_timesteps=200)
@pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"]) @pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"])
def test_a2c(env_id): def test_a2c(env_id):
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated model = A2C("MlpPolicy", env_id, seed=0, policy_kwargs=dict(net_arch=[16]), verbose=1)
model = A2C("MlpPolicy", env_id, seed=0, policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True) model.learn(total_timesteps=64)
model.learn(total_timesteps=1000, eval_freq=500)
@pytest.mark.parametrize("model_class", [A2C, PPO]) @pytest.mark.parametrize("model_class", [A2C, PPO])
@ -49,7 +46,6 @@ def test_advantage_normalization(model_class, normalize_advantage):
@pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"]) @pytest.mark.parametrize("env_id", ["CartPole-v1", "Pendulum-v1"])
@pytest.mark.parametrize("clip_range_vf", [None, 0.2, -0.2]) @pytest.mark.parametrize("clip_range_vf", [None, 0.2, -0.2])
def test_ppo(env_id, clip_range_vf): def test_ppo(env_id, clip_range_vf):
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated
if clip_range_vf is not None and clip_range_vf < 0: if clip_range_vf is not None and clip_range_vf < 0:
# Should throw an error # Should throw an error
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
@ -59,7 +55,6 @@ def test_ppo(env_id, clip_range_vf):
seed=0, seed=0,
policy_kwargs=dict(net_arch=[16]), policy_kwargs=dict(net_arch=[16]),
verbose=1, verbose=1,
create_eval_env=True,
clip_range_vf=clip_range_vf, clip_range_vf=clip_range_vf,
) )
else: else:
@ -70,27 +65,25 @@ def test_ppo(env_id, clip_range_vf):
seed=0, seed=0,
policy_kwargs=dict(net_arch=[16]), policy_kwargs=dict(net_arch=[16]),
verbose=1, verbose=1,
create_eval_env=True,
clip_range_vf=clip_range_vf, clip_range_vf=clip_range_vf,
n_epochs=2,
) )
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000)
@pytest.mark.parametrize("ent_coef", ["auto", 0.01, "auto_0.01"]) @pytest.mark.parametrize("ent_coef", ["auto", 0.01, "auto_0.01"])
def test_sac(ent_coef): def test_sac(ent_coef):
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated
model = SAC( model = SAC(
"MlpPolicy", "MlpPolicy",
"Pendulum-v1", "Pendulum-v1",
policy_kwargs=dict(net_arch=[64, 64]), policy_kwargs=dict(net_arch=[64, 64]),
learning_starts=100, learning_starts=100,
verbose=1, verbose=1,
create_eval_env=True,
buffer_size=250, buffer_size=250,
ent_coef=ent_coef, ent_coef=ent_coef,
action_noise=NormalActionNoise(np.zeros(1), np.zeros(1)), action_noise=NormalActionNoise(np.zeros(1), np.zeros(1)),
) )
model.learn(total_timesteps=300, eval_freq=250) model.learn(total_timesteps=200)
@pytest.mark.parametrize("n_critics", [1, 3]) @pytest.mark.parametrize("n_critics", [1, 3])
@ -104,11 +97,10 @@ def test_n_critics(n_critics):
buffer_size=10000, buffer_size=10000,
verbose=1, verbose=1,
) )
model.learn(total_timesteps=300) model.learn(total_timesteps=200)
def test_dqn(): def test_dqn():
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated
model = DQN( model = DQN(
"MlpPolicy", "MlpPolicy",
"CartPole-v1", "CartPole-v1",
@ -117,9 +109,8 @@ def test_dqn():
buffer_size=500, buffer_size=500,
learning_rate=3e-4, learning_rate=3e-4,
verbose=1, verbose=1,
create_eval_env=True,
) )
model.learn(total_timesteps=500, eval_freq=250) model.learn(total_timesteps=200)
@pytest.mark.parametrize("train_freq", [4, (4, "step"), (1, "episode")]) @pytest.mark.parametrize("train_freq", [4, (4, "step"), (1, "episode")])

4
tests/test_sde.py
View file

@ -63,18 +63,16 @@ def test_sde_check():
@pytest.mark.parametrize("use_expln", [False, True]) @pytest.mark.parametrize("use_expln", [False, True])
def test_state_dependent_noise(model_class, use_expln): def test_state_dependent_noise(model_class, use_expln):
kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64} kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64}
with pytest.warns(DeprecationWarning): # `create_eval_env` and `eval_freq` are deprecated
model = model_class( model = model_class(
"MlpPolicy", "MlpPolicy",
"Pendulum-v1", "Pendulum-v1",
use_sde=True, use_sde=True,
seed=None, seed=None,
create_eval_env=True,
verbose=1, verbose=1,
policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]), policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]),
**kwargs, **kwargs,
) )
model.learn(total_timesteps=255, eval_freq=250) model.learn(total_timesteps=255)
model.policy.reset_noise() model.policy.reset_noise()
if model_class == SAC: if model_class == SAC:
model.policy.actor.get_std() model.policy.actor.get_std()

2
tests/test_train_eval_mode.py
View file

@ -332,7 +332,7 @@ def test_a2c_ppo_collect_rollouts_with_batch_norm(model_class, env_id):
bias_before, running_mean_before = clone_on_policy_batch_norm(model) bias_before, running_mean_before = clone_on_policy_batch_norm(model)
total_timesteps, callback = model._setup_learn(total_timesteps=2 * 64, eval_env=model.get_env()) total_timesteps, callback = model._setup_learn(total_timesteps=2 * 64)
for _ in range(2): for _ in range(2):
model.collect_rollouts(model.get_env(), callback, model.rollout_buffer, n_rollout_steps=model.n_steps) model.collect_rollouts(model.get_env(), callback, model.rollout_buffer, n_rollout_steps=model.n_steps)

12
tests/test_vec_normalize.py
View file

@ -118,15 +118,6 @@ def make_dict_env():
return Monitor(DummyDictEnv()) return Monitor(DummyDictEnv())
def test_deprecation():
venv = DummyVecEnv([lambda: gym.make("CartPole-v1")])
venv = VecNormalize(venv)
with warnings.catch_warnings(record=True) as record:
assert np.allclose(venv.ret, venv.returns)
# Deprecation warning when using .ret
assert len(record) == 1
def check_rms_equal(rmsa, rmsb): def check_rms_equal(rmsa, rmsb):
if isinstance(rmsa, dict): if isinstance(rmsa, dict):
for key in rmsa.keys(): for key in rmsa.keys():
@ -380,8 +371,7 @@ def test_offpolicy_normalization(model_class, online_sampling):
assert model.get_vec_normalize_env() is eval_env assert model.get_vec_normalize_env() is eval_env
model.learn(total_timesteps=10) model.learn(total_timesteps=10)
model.set_env(env) model.set_env(env)
with pytest.warns(DeprecationWarning): # `eval_env` and `eval_freq` are deprecated model.learn(total_timesteps=150)
model.learn(total_timesteps=150, eval_env=eval_env, eval_freq=75)
# Check getter # Check getter
assert isinstance(model.get_vec_normalize_env(), VecNormalize) assert isinstance(model.get_vec_normalize_env(), VecNormalize)