Merge pull request #89 from DLR-RM/base-class-review

Refactor and clean-up of common code
This commit is contained in:
Adam Gleave 2020-07-07 18:53:51 -07:00 committed by GitHub
commit c39ed397ac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 209 additions and 192 deletions

1
.gitignore vendored
View file

@ -22,6 +22,7 @@ keys/
# Virtualenv # Virtualenv
/env /env
/venv
*.sublime-project *.sublime-project

View file

@ -1,4 +1,5 @@
SHELL=/bin/bash SHELL=/bin/bash
LINT_PATHS=stable_baselines3/ tests/ docs/conf.py setup.py
pytest: pytest:
./scripts/run_tests.sh ./scripts/run_tests.sh
@ -9,9 +10,9 @@ type:
lint: lint:
# stop the build if there are Python syntax errors or undefined names # stop the build if there are Python syntax errors or undefined names
# see https://lintlyci.github.io/Flake8Rules/ # see https://lintlyci.github.io/Flake8Rules/
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics flake8 ${LINT_PATHS} --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. # exit-zero treats all errors as warnings.
flake8 . --count --exit-zero --statistics flake8 ${LINT_PATHS} --count --exit-zero --statistics
doc: doc:
cd docs && make html cd docs && make html

View file

@ -19,7 +19,7 @@ from unittest.mock import MagicMock
# We CANNOT enable 'sphinxcontrib.spelling' because ReadTheDocs.org does not support # We CANNOT enable 'sphinxcontrib.spelling' because ReadTheDocs.org does not support
# PyEnchant. # PyEnchant.
try: try:
import sphinxcontrib.spelling import sphinxcontrib.spelling # noqa: F401
enable_spell_check = True enable_spell_check = True
except ImportError: except ImportError:
enable_spell_check = False enable_spell_check = False
@ -129,6 +129,7 @@ html_logo = '_static/img/logo.png'
def setup(app): def setup(app):
app.add_stylesheet("css/baselines_theme.css") app.add_stylesheet("css/baselines_theme.css")
# Theme options are theme-specific and customize the look and feel of a theme # Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the # further. For a list of options available for each theme, see the
# documentation. # documentation.

View file

@ -1,5 +1,7 @@
"""Abstract base classes for RL algorithms."""
import time import time
from typing import Union, Type, Optional, Dict, Any, List, Tuple, Callable from typing import Union, Type, Optional, Dict, Any, Iterable, List, Tuple, Callable
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections import deque from collections import deque
import pathlib import pathlib
@ -23,12 +25,30 @@ from stable_baselines3.common.monitor import Monitor
from stable_baselines3.common.noise import ActionNoise from stable_baselines3.common.noise import ActionNoise
def maybe_make_env(env: Union[GymEnv, str, None], monitor_wrapper: bool, verbose: int) -> Optional[GymEnv]:
"""If env is a string, make the environment; otherwise, return env.
:param env: (Union[GymEnv, str, None]) The environment to learn from.
:param monitor_wrapper: (bool) Whether to wrap env in a Monitor when creating env.
:param verbose: (int) logging verbosity
:return A Gym (vector) environment.
"""
if isinstance(env, str):
if verbose >= 1:
print(f"Creating environment from the given name '{env}'")
env = gym.make(env)
if monitor_wrapper:
env = Monitor(env, filename=None)
return env
class BaseAlgorithm(ABC): class BaseAlgorithm(ABC):
""" """
The base of RL algorithms The base of RL algorithms
:param policy: (Type[BasePolicy]) Policy object :param policy: (Type[BasePolicy]) Policy object
:param env: (Union[GymEnv, str]) The environment to learn from :param env: (Union[GymEnv, str, None]) The environment to learn from
(if registered in Gym, can be str. Can be None for loading trained models) (if registered in Gym, can be str. Can be None for loading trained models)
:param policy_base: (Type[BasePolicy]) The base policy used by this method :param policy_base: (Type[BasePolicy]) The base policy used by this method
:param learning_rate: (float or callable) learning rate for the optimizer, :param learning_rate: (float or callable) learning rate for the optimizer,
@ -54,7 +74,7 @@ class BaseAlgorithm(ABC):
def __init__(self, def __init__(self,
policy: Type[BasePolicy], policy: Type[BasePolicy],
env: Union[GymEnv, str], env: Union[GymEnv, str, None],
policy_base: Type[BasePolicy], policy_base: Type[BasePolicy],
learning_rate: Union[float, Callable], learning_rate: Union[float, Callable],
policy_kwargs: Dict[str, Any] = None, policy_kwargs: Dict[str, Any] = None,
@ -116,18 +136,9 @@ class BaseAlgorithm(ABC):
if env is not None: if env is not None:
if isinstance(env, str): if isinstance(env, str):
if create_eval_env: if create_eval_env:
eval_env = gym.make(env) self.eval_env = maybe_make_env(env, monitor_wrapper, self.verbose)
if monitor_wrapper:
eval_env = Monitor(eval_env, filename=None)
self.eval_env = DummyVecEnv([lambda: eval_env])
if self.verbose >= 1:
print("Creating environment from the given name, wrapped in a DummyVecEnv.")
env = gym.make(env)
if monitor_wrapper:
env = Monitor(env, filename=None)
env = DummyVecEnv([lambda: env])
env = maybe_make_env(env, monitor_wrapper, self.verbose)
env = self._wrap_env(env) env = self._wrap_env(env)
self.observation_space = env.observation_space self.observation_space = env.observation_space
@ -136,8 +147,8 @@ class BaseAlgorithm(ABC):
self.env = env self.env = env
if not support_multi_env and self.n_envs > 1: if not support_multi_env and self.n_envs > 1:
raise ValueError("Error: the model does not support multiple envs requires a single vectorized" raise ValueError("Error: the model does not support multiple envs; it requires "
" environment.") "a single vectorized environment.")
def _wrap_env(self, env: GymEnv) -> VecEnv: def _wrap_env(self, env: GymEnv) -> VecEnv:
if not isinstance(env, VecEnv): if not isinstance(env, VecEnv):
@ -153,10 +164,7 @@ class BaseAlgorithm(ABC):
@abstractmethod @abstractmethod
def _setup_model(self) -> None: def _setup_model(self) -> None:
""" """Create networks, buffer and optimizers."""
Create networks, buffer and optimizers
"""
raise NotImplementedError()
def _get_eval_env(self, eval_env: Optional[GymEnv]) -> Optional[GymEnv]: def _get_eval_env(self, eval_env: Optional[GymEnv]) -> Optional[GymEnv]:
""" """
@ -238,7 +246,7 @@ class BaseAlgorithm(ABC):
def get_torch_variables(self) -> Tuple[List[str], List[str]]: def get_torch_variables(self) -> Tuple[List[str], List[str]]:
""" """
Get the name of the torch variable that will be saved. Get the name of the torch variables that will be saved.
``th.save`` and ``th.load`` will be used with the right device ``th.save`` and ``th.load`` will be used with the right device
instead of the default pickling strategy. instead of the default pickling strategy.
@ -263,10 +271,9 @@ class BaseAlgorithm(ABC):
Return a trained model. Return a trained model.
:param total_timesteps: (int) The total number of samples (env steps) to train on :param total_timesteps: (int) The total number of samples (env steps) to train on
:param callback: (function (dict, dict)) -> boolean function called at every steps with state of the algorithm. :param callback: (MaybeCallback) callback(s) called at every step with state of the algorithm.
It takes the local and global variables. If it returns False, training is aborted.
:param log_interval: (int) The number of timesteps before logging. :param log_interval: (int) The number of timesteps before logging.
:param tb_log_name: (str) the name of the run for tensorboard log :param tb_log_name: (str) the name of the run for TensorBoard logging
:param eval_env: (gym.Env) Environment that will be used to evaluate the agent :param eval_env: (gym.Env) Environment that will be used to evaluate the agent
:param eval_freq: (int) Evaluate the agent every ``eval_freq`` timesteps (this may vary a little) :param eval_freq: (int) Evaluate the agent every ``eval_freq`` timesteps (this may vary a little)
:param n_eval_episodes: (int) Number of episode to evaluate the agent :param n_eval_episodes: (int) Number of episode to evaluate the agent
@ -274,7 +281,6 @@ class BaseAlgorithm(ABC):
:param reset_num_timesteps: (bool) whether or not to reset the current timestep number (used in logging) :param reset_num_timesteps: (bool) whether or not to reset the current timestep number (used in logging)
:return: (BaseAlgorithm) the trained model :return: (BaseAlgorithm) the trained model
""" """
raise NotImplementedError()
def predict(self, observation: np.ndarray, def predict(self, observation: np.ndarray,
state: Optional[np.ndarray] = None, state: Optional[np.ndarray] = None,
@ -329,8 +335,6 @@ class BaseAlgorithm(ABC):
# load parameters # load parameters
model.__dict__.update(data) model.__dict__.update(data)
model.__dict__.update(kwargs) model.__dict__.update(kwargs)
if not hasattr(model, "_setup_model") and len(params) > 0:
raise NotImplementedError(f"{cls} has no ``_setup_model()`` method")
model._setup_model() model._setup_model()
# put state_dicts back in place # put state_dicts back in place
@ -366,14 +370,18 @@ class BaseAlgorithm(ABC):
self.eval_env.seed(seed) self.eval_env.seed(seed)
def _init_callback(self, def _init_callback(self,
callback: Union[None, Callable, List[BaseCallback], BaseCallback], callback: MaybeCallback,
eval_env: Optional[VecEnv] = None, eval_env: Optional[VecEnv] = None,
eval_freq: int = 10000, eval_freq: int = 10000,
n_eval_episodes: int = 5, n_eval_episodes: int = 5,
log_path: Optional[str] = None) -> BaseCallback: log_path: Optional[str] = None) -> BaseCallback:
""" """
:param callback: (Union[callable, [BaseCallback], BaseCallback, None]) :param callback: (MaybeCallback) Callback(s) called at every step with state of the algorithm.
:return: (BaseCallback) :param eval_freq: (Optional[int]) How many steps between evaluations; if None, do not evaluate.
:param n_eval_episodes: (int) How many episodes to play per evaluation
:param n_eval_episodes: (int) Number of episodes to rollout during evaluation.
:param log_path: (Optional[str]) Path to a folder where the evaluations will be saved
:return: (BaseCallback) A hybrid callback calling `callback` and performing evaluation.
""" """
# Convert a list of callbacks into a callback # Convert a list of callbacks into a callback
if isinstance(callback, list): if isinstance(callback, list):
@ -396,7 +404,7 @@ class BaseAlgorithm(ABC):
def _setup_learn(self, def _setup_learn(self,
total_timesteps: int, total_timesteps: int,
eval_env: Optional[GymEnv], eval_env: Optional[GymEnv],
callback: Union[None, Callable, List[BaseCallback], BaseCallback] = None, callback: MaybeCallback = None,
eval_freq: int = 10000, eval_freq: int = 10000,
n_eval_episodes: int = 5, n_eval_episodes: int = 5,
log_path: Optional[str] = None, log_path: Optional[str] = None,
@ -407,11 +415,11 @@ class BaseAlgorithm(ABC):
Initialize different variables needed for training. Initialize different variables needed for training.
:param total_timesteps: (int) The total number of samples (env steps) to train on :param total_timesteps: (int) The total number of samples (env steps) to train on
:param eval_env: (Optional[GymEnv]) :param eval_env: (Optional[VecEnv]) Environment to use for evaluation.
:param callback: (Union[None, BaseCallback, List[BaseCallback, Callable]]) :param callback: (MaybeCallback) Callback(s) called at every step with state of the algorithm.
:param eval_freq: (int) How many steps between evaluations :param eval_freq: (int) How many steps between evaluations
:param n_eval_episodes: (int) How many episodes to play per evaluation :param n_eval_episodes: (int) How many episodes to play per evaluation
:param log_path (Optional[str]): Path to a log folder :param log_path: (Optional[str]) Path to a folder where the evaluations will be saved
:param reset_num_timesteps: (bool) Whether to reset or not the ``num_timesteps`` attribute :param reset_num_timesteps: (bool) Whether to reset or not the ``num_timesteps`` attribute
:param tb_log_name: (str) the name of the run for tensorboard log :param tb_log_name: (str) the name of the run for tensorboard log
:return: (Tuple[int, BaseCallback]) :return: (Tuple[int, BaseCallback])
@ -480,8 +488,8 @@ class BaseAlgorithm(ABC):
def save( def save(
self, self,
path: Union[str, pathlib.Path, io.BufferedIOBase], path: Union[str, pathlib.Path, io.BufferedIOBase],
exclude: Optional[List[str]] = None, exclude: Optional[Iterable[str]] = None,
include: Optional[List[str]] = None, include: Optional[Iterable[str]] = None,
) -> None: ) -> None:
""" """
Save all the attributes of the object and the model parameters in a zip-file. Save all the attributes of the object and the model parameters in a zip-file.
@ -492,16 +500,15 @@ class BaseAlgorithm(ABC):
""" """
# copy parameter list so we don't mutate the original dict # copy parameter list so we don't mutate the original dict
data = self.__dict__.copy() data = self.__dict__.copy()
# use standard list of excluded parameters if none given
if exclude is None:
exclude = self.excluded_save_params()
else:
# append standard exclude params to the given params
exclude.extend([param for param in self.excluded_save_params() if param not in exclude])
# do not exclude params if they are specifically included # Exclude is union of specified parameters (if any) and standard exclusions
if exclude is None:
exclude = []
exclude = set(exclude).union(self.excluded_save_params())
# Do not exclude params if they are specifically included
if include is not None: if include is not None:
exclude = [param_name for param_name in exclude if param_name not in include] exclude = exclude.difference(include)
state_dicts_names, tensors_names = self.get_torch_variables() state_dicts_names, tensors_names = self.get_torch_variables()
# any params that are in the save vars must not be saved by data # any params that are in the save vars must not be saved by data
@ -509,11 +516,10 @@ class BaseAlgorithm(ABC):
for torch_var in torch_variables: for torch_var in torch_variables:
# we need to get only the name of the top most module as we'll remove that # we need to get only the name of the top most module as we'll remove that
var_name = torch_var.split('.')[0] var_name = torch_var.split('.')[0]
exclude.append(var_name) exclude.add(var_name)
# Remove parameter entries of parameters which are to be excluded # Remove parameter entries of parameters which are to be excluded
for param_name in exclude: for param_name in exclude:
if param_name in data:
data.pop(param_name, None) data.pop(param_name, None)
# Build dict of tensor variables # Build dict of tensor variables

View file

@ -1,3 +1,6 @@
"""Probability distributions."""
from abc import ABC, abstractmethod
from typing import Optional, Tuple, Dict, Any, List from typing import Optional, Tuple, Dict, Any, List
import gym import gym
import torch as th import torch as th
@ -8,36 +11,38 @@ from gym import spaces
from stable_baselines3.common.preprocessing import get_action_dim from stable_baselines3.common.preprocessing import get_action_dim
class Distribution(object): class Distribution(ABC):
"""Abstract base class for distributions."""
def __init__(self): def __init__(self):
super(Distribution, self).__init__() super(Distribution, self).__init__()
@abstractmethod
def log_prob(self, x: th.Tensor) -> th.Tensor: def log_prob(self, x: th.Tensor) -> th.Tensor:
""" """
returns the log likelihood Returns the log likelihood
:param x: (th.Tensor) the taken action :param x: (th.Tensor) the taken action
:return: (th.Tensor) The log likelihood of the distribution :return: (th.Tensor) The log likelihood of the distribution
""" """
raise NotImplementedError
@abstractmethod
def entropy(self) -> Optional[th.Tensor]: def entropy(self) -> Optional[th.Tensor]:
""" """
Returns Shannon's entropy of the probability Returns Shannon's entropy of the probability
:return: (Optional[th.Tensor]) the entropy, :return: (Optional[th.Tensor]) the entropy, or None if no analytical form is known
return None if no analytical form is known
""" """
raise NotImplementedError
@abstractmethod
def sample(self) -> th.Tensor: def sample(self) -> th.Tensor:
""" """
Returns a sample from the probability distribution Returns a sample from the probability distribution
:return: (th.Tensor) the stochastic action :return: (th.Tensor) the stochastic action
""" """
raise NotImplementedError
@abstractmethod
def mode(self) -> th.Tensor: def mode(self) -> th.Tensor:
""" """
Returns the most likely action (deterministic output) Returns the most likely action (deterministic output)
@ -45,7 +50,6 @@ class Distribution(object):
:return: (th.Tensor) the stochastic action :return: (th.Tensor) the stochastic action
""" """
raise NotImplementedError
def get_actions(self, deterministic: bool = False) -> th.Tensor: def get_actions(self, deterministic: bool = False) -> th.Tensor:
""" """
@ -58,6 +62,7 @@ class Distribution(object):
return self.mode() return self.mode()
return self.sample() return self.sample()
@abstractmethod
def actions_from_params(self, *args, **kwargs) -> th.Tensor: def actions_from_params(self, *args, **kwargs) -> th.Tensor:
""" """
Returns samples from the probability distribution Returns samples from the probability distribution
@ -65,8 +70,8 @@ class Distribution(object):
:return: (th.Tensor) actions :return: (th.Tensor) actions
""" """
raise NotImplementedError
@abstractmethod
def log_prob_from_params(self, *args, **kwargs) -> Tuple[th.Tensor, th.Tensor]: def log_prob_from_params(self, *args, **kwargs) -> Tuple[th.Tensor, th.Tensor]:
""" """
Returns samples and the associated log probabilities Returns samples and the associated log probabilities
@ -74,14 +79,12 @@ class Distribution(object):
:return: (th.Tuple[th.Tensor, th.Tensor]) actions and log prob :return: (th.Tuple[th.Tensor, th.Tensor]) actions and log prob
""" """
raise NotImplementedError
def sum_independent_dims(tensor: th.Tensor) -> th.Tensor: def sum_independent_dims(tensor: th.Tensor) -> th.Tensor:
""" """
Continuous actions are usually considered to be independent, Continuous actions are usually considered to be independent,
so we can sum the components for the ``log_prob`` so we can sum components of the ``log_prob`` or the entropy.
or the entropy.
:param tensor: (th.Tensor) shape: (n_batch, n_actions) or (n_batch,) :param tensor: (th.Tensor) shape: (n_batch, n_actions) or (n_batch,)
:return: (th.Tensor) shape: (n_batch,) :return: (th.Tensor) shape: (n_batch,)
@ -95,8 +98,7 @@ def sum_independent_dims(tensor: th.Tensor) -> th.Tensor:
class DiagGaussianDistribution(Distribution): class DiagGaussianDistribution(Distribution):
""" """
Gaussian distribution with diagonal covariance matrix, Gaussian distribution with diagonal covariance matrix, for continuous actions.
for continuous actions.
:param action_dim: (int) Dimension of the action space. :param action_dim: (int) Dimension of the action space.
""" """
@ -115,7 +117,7 @@ class DiagGaussianDistribution(Distribution):
one output will be the mean of the Gaussian, the other parameter will be the one output will be the mean of the Gaussian, the other parameter will be the
standard deviation (log std in fact to allow negative values) standard deviation (log std in fact to allow negative values)
:param latent_dim: (int) Dimension og the last layer of the policy (before the action layer) :param latent_dim: (int) Dimension of the last layer of the policy (before the action layer)
:param log_std_init: (float) Initial value for the log standard deviation :param log_std_init: (float) Initial value for the log standard deviation
:return: (nn.Linear, nn.Parameter) :return: (nn.Linear, nn.Parameter)
""" """
@ -137,15 +139,26 @@ class DiagGaussianDistribution(Distribution):
self.distribution = Normal(mean_actions, action_std) self.distribution = Normal(mean_actions, action_std)
return self return self
def mode(self) -> th.Tensor: def log_prob(self, actions: th.Tensor) -> th.Tensor:
return self.distribution.mean """
Get the log probabilities of actions according to the distribution.
Note that you must first call the ``proba_distribution()`` method.
:param actions: (th.Tensor)
:return: (th.Tensor)
"""
log_prob = self.distribution.log_prob(actions)
return sum_independent_dims(log_prob)
def entropy(self) -> th.Tensor:
return sum_independent_dims(self.distribution.entropy())
def sample(self) -> th.Tensor: def sample(self) -> th.Tensor:
# Reparametrization trick to pass gradients # Reparametrization trick to pass gradients
return self.distribution.rsample() return self.distribution.rsample()
def entropy(self) -> th.Tensor: def mode(self) -> th.Tensor:
return sum_independent_dims(self.distribution.entropy()) return self.distribution.mean
def actions_from_params(self, mean_actions: th.Tensor, def actions_from_params(self, mean_actions: th.Tensor,
log_std: th.Tensor, log_std: th.Tensor,
@ -168,22 +181,10 @@ class DiagGaussianDistribution(Distribution):
log_prob = self.log_prob(actions) log_prob = self.log_prob(actions)
return actions, log_prob return actions, log_prob
def log_prob(self, actions: th.Tensor) -> th.Tensor:
"""
Get the log probabilities of actions according to the distribution.
Note that you must call ``proba_distribution()`` method before.
:param actions: (th.Tensor)
:return: (th.Tensor)
"""
log_prob = self.distribution.log_prob(actions)
return sum_independent_dims(log_prob)
class SquashedDiagGaussianDistribution(DiagGaussianDistribution): class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
""" """
Gaussian distribution with diagonal covariance matrix, Gaussian distribution with diagonal covariance matrix, followed by a squashing function (tanh) to ensure bounds.
followed by a squashing function (tanh) to ensure bounds.
:param action_dim: (int) Dimension of the action space. :param action_dim: (int) Dimension of the action space.
:param epsilon: (float) small value to avoid NaN due to numerical imprecision. :param epsilon: (float) small value to avoid NaN due to numerical imprecision.
@ -200,27 +201,6 @@ class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
super(SquashedDiagGaussianDistribution, self).proba_distribution(mean_actions, log_std) super(SquashedDiagGaussianDistribution, self).proba_distribution(mean_actions, log_std)
return self return self
def mode(self) -> th.Tensor:
self.gaussian_actions = self.distribution.mean
# Squash the output
return th.tanh(self.gaussian_actions)
def entropy(self) -> Optional[th.Tensor]:
# No analytical form,
# entropy needs to be estimated using -log_prob.mean()
return None
def sample(self) -> th.Tensor:
# Reparametrization trick to pass gradients
self.gaussian_actions = self.distribution.rsample()
return th.tanh(self.gaussian_actions)
def log_prob_from_params(self, mean_actions: th.Tensor,
log_std: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
action = self.actions_from_params(mean_actions, log_std)
log_prob = self.log_prob(action, self.gaussian_actions)
return action, log_prob
def log_prob(self, actions: th.Tensor, def log_prob(self, actions: th.Tensor,
gaussian_actions: Optional[th.Tensor] = None) -> th.Tensor: gaussian_actions: Optional[th.Tensor] = None) -> th.Tensor:
# Inverse tanh # Inverse tanh
@ -237,6 +217,27 @@ class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
log_prob -= th.sum(th.log(1 - actions ** 2 + self.epsilon), dim=1) log_prob -= th.sum(th.log(1 - actions ** 2 + self.epsilon), dim=1)
return log_prob return log_prob
def entropy(self) -> Optional[th.Tensor]:
# No analytical form,
# entropy needs to be estimated using -log_prob.mean()
return None
def sample(self) -> th.Tensor:
# Reparametrization trick to pass gradients
self.gaussian_actions = super().sample()
return th.tanh(self.gaussian_actions)
def mode(self) -> th.Tensor:
self.gaussian_actions = super().mode()
# Squash the output
return th.tanh(self.gaussian_actions)
def log_prob_from_params(self, mean_actions: th.Tensor,
log_std: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
action = self.actions_from_params(mean_actions, log_std)
log_prob = self.log_prob(action, self.gaussian_actions)
return action, log_prob
class CategoricalDistribution(Distribution): class CategoricalDistribution(Distribution):
""" """
@ -267,14 +268,17 @@ class CategoricalDistribution(Distribution):
self.distribution = Categorical(logits=action_logits) self.distribution = Categorical(logits=action_logits)
return self return self
def mode(self) -> th.Tensor: def log_prob(self, actions: th.Tensor) -> th.Tensor:
return th.argmax(self.distribution.probs, dim=1) return self.distribution.log_prob(actions)
def entropy(self) -> th.Tensor:
return self.distribution.entropy()
def sample(self) -> th.Tensor: def sample(self) -> th.Tensor:
return self.distribution.sample() return self.distribution.sample()
def entropy(self) -> th.Tensor: def mode(self) -> th.Tensor:
return self.distribution.entropy() return th.argmax(self.distribution.probs, dim=1)
def actions_from_params(self, action_logits: th.Tensor, def actions_from_params(self, action_logits: th.Tensor,
deterministic: bool = False) -> th.Tensor: deterministic: bool = False) -> th.Tensor:
@ -287,9 +291,6 @@ class CategoricalDistribution(Distribution):
log_prob = self.log_prob(actions) log_prob = self.log_prob(actions)
return actions, log_prob return actions, log_prob
def log_prob(self, actions: th.Tensor) -> th.Tensor:
return self.distribution.log_prob(actions)
class MultiCategoricalDistribution(Distribution): class MultiCategoricalDistribution(Distribution):
""" """
@ -321,14 +322,19 @@ class MultiCategoricalDistribution(Distribution):
self.distributions = [Categorical(logits=split) for split in th.split(action_logits, tuple(self.action_dims), dim=1)] self.distributions = [Categorical(logits=split) for split in th.split(action_logits, tuple(self.action_dims), dim=1)]
return self return self
def mode(self) -> th.Tensor: def log_prob(self, actions: th.Tensor) -> th.Tensor:
return th.stack([th.argmax(dist.probs, dim=1) for dist in self.distributions], dim=1) # Extract each discrete action and compute log prob for their respective distributions
return th.stack([dist.log_prob(action) for dist, action in zip(self.distributions,
th.unbind(actions, dim=1))], dim=1).sum(dim=1)
def entropy(self) -> th.Tensor:
return th.stack([dist.entropy() for dist in self.distributions], dim=1).sum(dim=1)
def sample(self) -> th.Tensor: def sample(self) -> th.Tensor:
return th.stack([dist.sample() for dist in self.distributions], dim=1) return th.stack([dist.sample() for dist in self.distributions], dim=1)
def entropy(self) -> th.Tensor: def mode(self) -> th.Tensor:
return th.stack([dist.entropy() for dist in self.distributions], dim=1).sum(dim=1) return th.stack([th.argmax(dist.probs, dim=1) for dist in self.distributions], dim=1)
def actions_from_params(self, action_logits: th.Tensor, def actions_from_params(self, action_logits: th.Tensor,
deterministic: bool = False) -> th.Tensor: deterministic: bool = False) -> th.Tensor:
@ -341,11 +347,6 @@ class MultiCategoricalDistribution(Distribution):
log_prob = self.log_prob(actions) log_prob = self.log_prob(actions)
return actions, log_prob return actions, log_prob
def log_prob(self, actions: th.Tensor) -> th.Tensor:
# Extract each discrete action and compute log prob for their respective distributions
return th.stack([dist.log_prob(action) for dist, action in zip(self.distributions,
th.unbind(actions, dim=1))], dim=1).sum(dim=1)
class BernoulliDistribution(Distribution): class BernoulliDistribution(Distribution):
""" """
@ -375,14 +376,17 @@ class BernoulliDistribution(Distribution):
self.distribution = Bernoulli(logits=action_logits) self.distribution = Bernoulli(logits=action_logits)
return self return self
def mode(self) -> th.Tensor: def log_prob(self, actions: th.Tensor) -> th.Tensor:
return th.round(self.distribution.probs) return self.distribution.log_prob(actions).sum(dim=1)
def entropy(self) -> th.Tensor:
return self.distribution.entropy().sum(dim=1)
def sample(self) -> th.Tensor: def sample(self) -> th.Tensor:
return self.distribution.sample() return self.distribution.sample()
def entropy(self) -> th.Tensor: def mode(self) -> th.Tensor:
return self.distribution.entropy().sum(dim=1) return th.round(self.distribution.probs)
def actions_from_params(self, action_logits: th.Tensor, def actions_from_params(self, action_logits: th.Tensor,
deterministic: bool = False) -> th.Tensor: deterministic: bool = False) -> th.Tensor:
@ -395,9 +399,6 @@ class BernoulliDistribution(Distribution):
log_prob = self.log_prob(actions) log_prob = self.log_prob(actions)
return actions, log_prob return actions, log_prob
def log_prob(self, actions: th.Tensor) -> th.Tensor:
return self.distribution.log_prob(actions).sum(dim=1)
class StateDependentNoiseDistribution(Distribution): class StateDependentNoiseDistribution(Distribution):
""" """
@ -414,7 +415,7 @@ class StateDependentNoiseDistribution(Distribution):
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param squash_output: (bool) Whether to squash the output using a tanh function, :param squash_output: (bool) Whether to squash the output using a tanh function,
this allows to ensure boundaries. this ensures bounds are satisfied.
:param learn_features: (bool) Whether to learn features for gSDE or not. :param learn_features: (bool) Whether to learn features for gSDE or not.
This will enable gradients to be backpropagated through the features This will enable gradients to be backpropagated through the features
``latent_sde`` in the code. ``latent_sde`` in the code.
@ -529,6 +530,35 @@ class StateDependentNoiseDistribution(Distribution):
self.distribution = Normal(mean_actions, th.sqrt(variance + self.epsilon)) self.distribution = Normal(mean_actions, th.sqrt(variance + self.epsilon))
return self return self
def log_prob(self, actions: th.Tensor) -> th.Tensor:
if self.bijector is not None:
gaussian_actions = self.bijector.inverse(actions)
else:
gaussian_actions = actions
# log likelihood for a gaussian
log_prob = self.distribution.log_prob(gaussian_actions)
# Sum along action dim
log_prob = sum_independent_dims(log_prob)
if self.bijector is not None:
# Squash correction (from original SAC implementation)
log_prob -= th.sum(self.bijector.log_prob_correction(gaussian_actions), dim=1)
return log_prob
def entropy(self) -> Optional[th.Tensor]:
if self.bijector is not None:
# No analytical form,
# entropy needs to be estimated using -log_prob.mean()
return None
return sum_independent_dims(self.distribution.entropy())
def sample(self) -> th.Tensor:
noise = self.get_noise(self._latent_sde)
actions = self.distribution.mean + noise
if self.bijector is not None:
return self.bijector.forward(actions)
return actions
def mode(self) -> th.Tensor: def mode(self) -> th.Tensor:
actions = self.distribution.mean actions = self.distribution.mean
if self.bijector is not None: if self.bijector is not None:
@ -547,20 +577,6 @@ class StateDependentNoiseDistribution(Distribution):
noise = th.bmm(latent_sde, self.exploration_matrices) noise = th.bmm(latent_sde, self.exploration_matrices)
return noise.squeeze(1) return noise.squeeze(1)
def sample(self) -> th.Tensor:
noise = self.get_noise(self._latent_sde)
actions = self.distribution.mean + noise
if self.bijector is not None:
return self.bijector.forward(actions)
return actions
def entropy(self) -> Optional[th.Tensor]:
# No analytical form,
# entropy needs to be estimated using -log_prob.mean()
if self.bijector is not None:
return None
return sum_independent_dims(self.distribution.entropy())
def actions_from_params(self, mean_actions: th.Tensor, def actions_from_params(self, mean_actions: th.Tensor,
log_std: th.Tensor, log_std: th.Tensor,
latent_sde: th.Tensor, latent_sde: th.Tensor,
@ -576,21 +592,6 @@ class StateDependentNoiseDistribution(Distribution):
log_prob = self.log_prob(actions) log_prob = self.log_prob(actions)
return actions, log_prob return actions, log_prob
def log_prob(self, actions: th.Tensor) -> th.Tensor:
if self.bijector is not None:
gaussian_actions = self.bijector.inverse(actions)
else:
gaussian_actions = actions
# log likelihood for a gaussian
log_prob = self.distribution.log_prob(gaussian_actions)
# Sum along action dim
log_prob = sum_independent_dims(log_prob)
if self.bijector is not None:
# Squash correction (from original SAC implementation)
log_prob -= th.sum(self.bijector.log_prob_correction(gaussian_actions), dim=1)
return log_prob
class TanhBijector(object): class TanhBijector(object):
""" """
@ -653,9 +654,8 @@ def make_proba_distribution(action_space: gym.spaces.Space,
if isinstance(action_space, spaces.Box): if isinstance(action_space, spaces.Box):
assert len(action_space.shape) == 1, "Error: the action space must be a vector" assert len(action_space.shape) == 1, "Error: the action space must be a vector"
if use_sde: cls = StateDependentNoiseDistribution if use_sde else DiagGaussianDistribution
return StateDependentNoiseDistribution(get_action_dim(action_space), **dist_kwargs) return cls(get_action_dim(action_space), **dist_kwargs)
return DiagGaussianDistribution(get_action_dim(action_space), **dist_kwargs)
elif isinstance(action_space, spaces.Discrete): elif isinstance(action_space, spaces.Discrete):
return CategoricalDistribution(action_space.n, **dist_kwargs) return CategoricalDistribution(action_space.n, **dist_kwargs)
elif isinstance(action_space, spaces.MultiDiscrete): elif isinstance(action_space, spaces.MultiDiscrete):

View file

@ -135,7 +135,7 @@ class OffPolicyAlgorithm(BaseAlgorithm):
self.action_space, self.device, self.action_space, self.device,
optimize_memory_usage=self.optimize_memory_usage) optimize_memory_usage=self.optimize_memory_usage)
self.policy = self.policy_class(self.observation_space, self.action_space, self.policy = self.policy_class(self.observation_space, self.action_space,
self.lr_schedule, **self.policy_kwargs) self.lr_schedule, **self.policy_kwargs) # pytype:disable=not-instantiable
self.policy = self.policy.to(self.device) self.policy = self.policy.to(self.device)
def save_replay_buffer(self, path: Union[str, pathlib.Path, io.BufferedIOBase]) -> None: def save_replay_buffer(self, path: Union[str, pathlib.Path, io.BufferedIOBase]) -> None:

View file

@ -96,7 +96,7 @@ class OnPolicyAlgorithm(BaseAlgorithm):
n_envs=self.n_envs) n_envs=self.n_envs)
self.policy = self.policy_class(self.observation_space, self.action_space, self.policy = self.policy_class(self.observation_space, self.action_space,
self.lr_schedule, use_sde=self.use_sde, device=self.device, self.lr_schedule, use_sde=self.use_sde, device=self.device,
**self.policy_kwargs) **self.policy_kwargs) # pytype:disable=not-instantiable
self.policy = self.policy.to(self.device) self.policy = self.policy.to(self.device)
def collect_rollouts(self, def collect_rollouts(self,

View file

@ -1,3 +1,7 @@
"""Policies: abstract base class and concrete implementations."""
from abc import ABC, abstractmethod
import collections
from typing import Union, Type, Dict, List, Tuple, Optional, Any, Callable from typing import Union, Type, Dict, List, Tuple, Optional, Any, Callable
from functools import partial from functools import partial
@ -17,7 +21,7 @@ from stable_baselines3.common.distributions import (make_proba_distribution, Dis
StateDependentNoiseDistribution) StateDependentNoiseDistribution)
class BasePolicy(nn.Module): class BasePolicy(nn.Module, ABC):
""" """
The base policy object The base policy object
@ -98,17 +102,22 @@ class BasePolicy(nn.Module):
module.bias.data.fill_(0.0) module.bias.data.fill_(0.0)
@staticmethod @staticmethod
def _dummy_schedule(_progress_remaining: float) -> float: def _dummy_schedule(progress_remaining: float) -> float:
""" (float) Useful for pickling policy.""" """ (float) Useful for pickling policy."""
del progress_remaining
return 0.0 return 0.0
def forward(self, *_args, **kwargs): @abstractmethod
raise NotImplementedError() def forward(self, *args, **kwargs):
del args, kwargs
def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor: def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor:
""" """
Get the action according to the policy for a given observation. Get the action according to the policy for a given observation.
By default provides a dummy implementation -- not all BasePolicy classes
implement this, e.g. if they are a Critic in an Actor-Critic method.
:param observation: (th.Tensor) :param observation: (th.Tensor)
:param deterministic: (bool) Whether to use stochastic or deterministic actions :param deterministic: (bool) Whether to use stochastic or deterministic actions
:return: (th.Tensor) Taken action according to the policy :return: (th.Tensor) Taken action according to the policy
@ -140,10 +149,8 @@ class BasePolicy(nn.Module):
# Handle the different cases for images # Handle the different cases for images
# as PyTorch use channel first format # as PyTorch use channel first format
if is_image_space(self.observation_space): if is_image_space(self.observation_space):
if (observation.shape == self.observation_space.shape if not (observation.shape == self.observation_space.shape
or observation.shape[1:] == self.observation_space.shape): or observation.shape[1:] == self.observation_space.shape):
pass
else:
# Try to re-order the channels # Try to re-order the channels
transpose_obs = VecTransposeImage.transpose_image(observation) transpose_obs = VecTransposeImage.transpose_image(observation)
if (transpose_obs.shape == self.observation_space.shape if (transpose_obs.shape == self.observation_space.shape
@ -160,21 +167,21 @@ class BasePolicy(nn.Module):
# Convert to numpy # Convert to numpy
actions = actions.cpu().numpy() actions = actions.cpu().numpy()
if isinstance(self.action_space, gym.spaces.Box):
if self.squash_output:
# Rescale to proper domain when using squashing # Rescale to proper domain when using squashing
if isinstance(self.action_space, gym.spaces.Box) and self.squash_output:
actions = self.unscale_action(actions) actions = self.unscale_action(actions)
else:
clipped_actions = actions # Actions could be on arbitrary scale, so clip the actions to avoid
# Clip the actions to avoid out of bound error when using gaussian distribution # out of bound error (e.g. if sampling from a Gaussian distribution)
if isinstance(self.action_space, gym.spaces.Box) and not self.squash_output: actions = np.clip(actions, self.action_space.low, self.action_space.high)
clipped_actions = np.clip(actions, self.action_space.low, self.action_space.high)
if not vectorized_env: if not vectorized_env:
if state is not None: if state is not None:
raise ValueError("Error: The environment must be vectorized when using recurrent policies.") raise ValueError("Error: The environment must be vectorized when using recurrent policies.")
clipped_actions = clipped_actions[0] actions = actions[0]
return clipped_actions, state return actions, state
def scale_action(self, action: np.ndarray) -> np.ndarray: def scale_action(self, action: np.ndarray) -> np.ndarray:
""" """
@ -294,7 +301,7 @@ class ActorCriticPolicy(BasePolicy):
def __init__(self, def __init__(self,
observation_space: gym.spaces.Space, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
lr_schedule: Callable, lr_schedule: Callable[[float], float],
net_arch: Optional[List[Union[int, Dict[str, List[int]]]]] = None, net_arch: Optional[List[Union[int, Dict[str, List[int]]]]] = None,
device: Union[th.device, str] = 'auto', device: Union[th.device, str] = 'auto',
activation_fn: Type[nn.Module] = nn.Tanh, activation_fn: Type[nn.Module] = nn.Tanh,
@ -313,7 +320,7 @@ class ActorCriticPolicy(BasePolicy):
if optimizer_kwargs is None: if optimizer_kwargs is None:
optimizer_kwargs = {} optimizer_kwargs = {}
# Small values to avoid NaN in ADAM optimizer # Small values to avoid NaN in Adam optimizer
if optimizer_class == th.optim.Adam: if optimizer_class == th.optim.Adam:
optimizer_kwargs['eps'] = 1e-5 optimizer_kwargs['eps'] = 1e-5
@ -366,15 +373,17 @@ class ActorCriticPolicy(BasePolicy):
def _get_data(self) -> Dict[str, Any]: def _get_data(self) -> Dict[str, Any]:
data = super()._get_data() data = super()._get_data()
default_none_kwargs = self.dist_kwargs or collections.defaultdict(lambda: None)
data.update(dict( data.update(dict(
net_arch=self.net_arch, net_arch=self.net_arch,
activation_fn=self.activation_fn, activation_fn=self.activation_fn,
use_sde=self.use_sde, use_sde=self.use_sde,
log_std_init=self.log_std_init, log_std_init=self.log_std_init,
squash_output=self.dist_kwargs['squash_output'] if self.dist_kwargs else None, squash_output=default_none_kwargs['squash_output'],
full_std=self.dist_kwargs['full_std'] if self.dist_kwargs else None, full_std=default_none_kwargs['full_std'],
sde_net_arch=self.dist_kwargs['sde_net_arch'] if self.dist_kwargs else None, sde_net_arch=default_none_kwargs['sde_net_arch'],
use_expln=self.dist_kwargs['use_expln'] if self.dist_kwargs else None, use_expln=default_none_kwargs['use_expln'],
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
ortho_init=self.ortho_init, ortho_init=self.ortho_init,
optimizer_class=self.optimizer_class, optimizer_class=self.optimizer_class,
@ -394,7 +403,7 @@ class ActorCriticPolicy(BasePolicy):
StateDependentNoiseDistribution), 'reset_noise() is only available when using gSDE' StateDependentNoiseDistribution), 'reset_noise() is only available when using gSDE'
self.action_dist.sample_weights(self.log_std, batch_size=n_envs) self.action_dist.sample_weights(self.log_std, batch_size=n_envs)
def _build(self, lr_schedule: Callable) -> None: def _build(self, lr_schedule: Callable[[float], float]) -> None:
""" """
Create the networks and the optimizer. Create the networks and the optimizer.
@ -719,9 +728,9 @@ def get_policy_from_name(base_policy_type: Type[BasePolicy], name: str) -> Type[
:return: (Type[BasePolicy]) the policy :return: (Type[BasePolicy]) the policy
""" """
if base_policy_type not in _policy_registry: if base_policy_type not in _policy_registry:
raise ValueError(f"Error: the policy type {base_policy_type} is not registered!") raise KeyError(f"Error: the policy type {base_policy_type} is not registered!")
if name not in _policy_registry[base_policy_type]: if name not in _policy_registry[base_policy_type]:
raise ValueError(f"Error: unknown policy type {name}," raise KeyError(f"Error: unknown policy type {name},"
f"the only registed policy type are: {list(_policy_registry[base_policy_type].keys())}!") f"the only registed policy type are: {list(_policy_registry[base_policy_type].keys())}!")
return _policy_registry[base_policy_type][name] return _policy_registry[base_policy_type][name]

View file

@ -1,6 +1,5 @@
""" """Common aliases for type hints"""
Common aliases for type hint
"""
from typing import Union, Dict, Any, NamedTuple, List, Callable, Tuple from typing import Union, Dict, Any, NamedTuple, List, Callable, Tuple
import numpy as np import numpy as np