Merge pull request #44 from Antonin-Raffin/typing

Add typing and update requirement to python 3.6
This commit is contained in:
Raffin, Antonin 2020-01-27 11:57:48 +01:00 committed by GitHub Enterprise
commit cc3b023533
29 changed files with 251 additions and 184 deletions

1
.gitignore vendored
View file

@ -12,6 +12,7 @@ __pycache__/
_build/ _build/
*.npz *.npz
*.pth *.pth
.pytype/
# Setuptools distribution and build folders. # Setuptools distribution and build folders.
/dist/ /dist/

13
Makefile Normal file
View file

@ -0,0 +1,13 @@
SHELL=/bin/bash
pytest:
./scripts/run_tests.sh
type:
pytype
docs:
cd docs && make html
spelling:
cd docs && make spelling

View file

@ -6,6 +6,8 @@
PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines), a set of improved implementations of reinforcement learning algorithms. PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines), a set of improved implementations of reinforcement learning algorithms.
NOTE: Python 3.6 is required!
## Implemented Algorithms ## Implemented Algorithms
- A2C - A2C
@ -22,6 +24,34 @@ PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines
- cf github Roadmap - cf github Roadmap
## Run the Tests
```
pip install -e .[tests]
make pytest
```
## Type check
```
pip install -e .[tests]
make type
```
## Build the Documentation
```
pip install -e .[docs]
make docs
```
Spell check for the documentation:
```
make spelling
```
## Citing the Project ## Citing the Project
To cite this repository in publications: To cite this repository in publications:

View file

@ -70,6 +70,7 @@ release = torchy_baselines.__version__
# ones. # ones.
extensions = [ extensions = [
'sphinx.ext.autodoc', 'sphinx.ext.autodoc',
'sphinx_autodoc_typehints',
'sphinx.ext.autosummary', 'sphinx.ext.autosummary',
'sphinx.ext.mathjax', 'sphinx.ext.mathjax',
'sphinx.ext.ifconfig', 'sphinx.ext.ifconfig',

View file

@ -8,9 +8,11 @@ Pre-Release 0.2.0a0 (WIP)
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- Python 2 support was dropped, Torchy Baselines now requires Python 3.6 or above
New Features: New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
- Add `seed()` method to `VecEnv` class
Bug Fixes: Bug Fixes:
^^^^^^^^^^ ^^^^^^^^^^
@ -18,9 +20,10 @@ Bug Fixes:
Deprecations: Deprecations:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
Others: Others:
^^^^^^^ ^^^^^^^
- Add type check
- Converted all format string to f-strings
Documentation: Documentation:
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^

View file

@ -24,6 +24,7 @@ Notes
- Original paper: https://arxiv.org/abs/1707.06347 - Original paper: https://arxiv.org/abs/1707.06347
- Clear explanation of PPO on Arxiv Insights channel: https://www.youtube.com/watch?v=5P7I-xPq8u8 - Clear explanation of PPO on Arxiv Insights channel: https://www.youtube.com/watch?v=5P7I-xPq8u8
- OpenAI blog post: https://blog.openai.com/openai-baselines-ppo/ - OpenAI blog post: https://blog.openai.com/openai-baselines-ppo/
- Spinning Up guide: https://spinningup.openai.com/en/latest/algorithms/ppo.html
Can I use? Can I use?

View file

@ -113,3 +113,4 @@ optimizers
Deprecations Deprecations
forkserver forkserver
cuda cuda
Polyak

View file

@ -15,3 +15,6 @@ filterwarnings =
# Gym warnings # Gym warnings
ignore:Parameters to load are deprecated.:DeprecationWarning ignore:Parameters to load are deprecated.:DeprecationWarning
ignore:the imp module is deprecated in favour of importlib:PendingDeprecationWarning ignore:the imp module is deprecated in favour of importlib:PendingDeprecationWarning
[pytype]
inputs = torchy_baselines

View file

@ -18,13 +18,16 @@ setup(name='torchy_baselines',
'pytest-cov', 'pytest-cov',
'pytest-env', 'pytest-env',
'pytest-xdist', 'pytest-xdist',
'pytype',
], ],
'docs': [ 'docs': [
'sphinx', 'sphinx',
'sphinx-autobuild', 'sphinx-autobuild',
'sphinx-rtd-theme', 'sphinx-rtd-theme',
# For spelling # For spelling
'sphinxcontrib.spelling' 'sphinxcontrib.spelling',
# Type hints support
'sphinx-autodoc-typehints'
], ],
'extra': [ 'extra': [
# For render # For render

View file

@ -33,7 +33,7 @@ class CEMRL(TD3):
:param learning_starts: (int) how many steps of the model to collect transitions for before learning starts :param learning_starts: (int) how many steps of the model to collect transitions for before learning starts
:param gamma: (float) the discount factor :param gamma: (float) the discount factor
:param batch_size: (int) Minibatch size for each gradient update :param batch_size: (int) Minibatch size for each gradient update
:param tau: (float) the soft update coefficient ("polyak update" of the target networks, between 0 and 1) :param tau: (float) the soft update coefficient ("Polyak update" of the target networks, between 0 and 1)
:param action_noise: (ActionNoise) the action noise type. Cf common.noise for the different action noise type. :param action_noise: (ActionNoise) the action noise type. Cf common.noise for the different action noise type.
:param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy :param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy
(smoothing noise) (smoothing noise)
@ -103,6 +103,7 @@ class CEMRL(TD3):
eval_env=None, eval_freq=-1, n_eval_episodes=5, tb_log_name="CEMRL", reset_num_timesteps=True): eval_env=None, eval_freq=-1, n_eval_episodes=5, tb_log_name="CEMRL", reset_num_timesteps=True):
timesteps_since_eval, episode_num, evaluations, obs, eval_env = self._setup_learn(eval_env) timesteps_since_eval, episode_num, evaluations, obs, eval_env = self._setup_learn(eval_env)
actor_steps = 0
while self.num_timesteps < total_timesteps: while self.num_timesteps < total_timesteps:

View file

@ -1,54 +1,68 @@
import time import time
from abc import ABCMeta, abstractmethod
from collections import deque
import os import os
import io import io
import zipfile import zipfile
import typing
from typing import Union, Type, Optional, Dict, Any, List, Tuple
from abc import ABC, abstractmethod
from collections import deque
import gym import gym
import torch as th import torch as th
import numpy as np import numpy as np
from torchy_baselines.common import logger from torchy_baselines.common import logger
from torchy_baselines.common.policies import get_policy_from_name from torchy_baselines.common.policies import BasePolicy, get_policy_from_name
from torchy_baselines.common.utils import set_random_seed, get_schedule_fn, update_learning_rate from torchy_baselines.common.utils import set_random_seed, get_schedule_fn, update_learning_rate
from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize, sync_envs_normalization from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize, sync_envs_normalization
from torchy_baselines.common.monitor import Monitor from torchy_baselines.common.monitor import Monitor
from torchy_baselines.common.evaluation import evaluate_policy from torchy_baselines.common.evaluation import evaluate_policy
from torchy_baselines.common.save_util import data_to_json, json_to_data from torchy_baselines.common.save_util import data_to_json, json_to_data
# TODO: define aliases, ex GymEnv = Union[gym.Env, VecEnv]
if typing.TYPE_CHECKING:
from torchy_baselines.common.noise import ActionNoise
class BaseRLModel(object):
class BaseRLModel(ABC):
""" """
The base RL model The base RL model
:param policy: (BasePolicy) Policy object :param policy: Policy object
:param env: (Gym environment) The environment to learn from :param env: The environment to learn from
(if registered in Gym, can be str. Can be None for loading trained models) (if registered in Gym, can be str. Can be None for loading trained models)
:param policy_base: (BasePolicy) the base policy used by this method :param policy_base: The base policy used by this method
:param policy_kwargs: (dict) additional arguments to be passed to the policy on creation :param policy_kwargs: Additional arguments to be passed to the policy on creation
:param verbose: (int) the verbosity level: 0 none, 1 training information, 2 debug :param verbose: The verbosity level: 0 none, 1 training information, 2 debug
:param device: (str or th.device) Device on which the code should run. :param device: Device on which the code should run.
By default, it will try to use a Cuda compatible device and fallback to cpu By default, it will try to use a Cuda compatible device and fallback to cpu
if it is not possible. if it is not possible.
:param support_multi_env: (bool) Whether the algorithm supports training :param support_multi_env: Whether the algorithm supports training
with multiple environments (as in A2C) with multiple environments (as in A2C)
:param create_eval_env: (bool) Whether to create a second environment that will be :param create_eval_env: Whether to create a second environment that will be
used for evaluating the agent periodically. (Only available when passing string for the environment) used for evaluating the agent periodically. (Only available when passing string for the environment)
:param monitor_wrapper: (bool) When creating an environment, whether to wrap it :param monitor_wrapper: When creating an environment, whether to wrap it
or not in a Monitor wrapper. or not in a Monitor wrapper.
:param seed: (int) Seed for the pseudo random generators :param seed: Seed for the pseudo random generators
:param use_sde: (bool) Whether to use State Dependent Exploration (SDE) :param use_sde: Whether to use State Dependent Exploration (SDE)
instead of action noise exploration (default: False) instead of action noise exploration (default: False)
:param sde_sample_freq: (int) Sample a new noise matrix every n steps when using SDE :param sde_sample_freq: Sample a new noise matrix every n steps when using SDE
Default: -1 (only sample at the beginning of the rollout) Default: -1 (only sample at the beginning of the rollout)
""" """
__metaclass__ = ABCMeta def __init__(self,
policy: Type[BasePolicy],
env: Union[gym.Env, VecEnv, str],
policy_base: Type[BasePolicy],
policy_kwargs : Dict[str, Any] = None,
verbose: int = 0,
device: Union[th.device, str] = 'auto',
support_multi_env: bool = False,
create_eval_env: bool = False,
monitor_wrapper: bool = True,
seed: Optional[int] = None,
use_sde: bool = False,
sde_sample_freq: int = -1):
def __init__(self, policy, env, policy_base, policy_kwargs=None,
verbose=0, device='auto', support_multi_env=False,
create_eval_env=False, monitor_wrapper=True, seed=None,
use_sde=False, sde_sample_freq=-1):
if isinstance(policy, str) and policy_base is not None: if isinstance(policy, str) and policy_base is not None:
self.policy_class = get_policy_from_name(policy_base, policy) self.policy_class = get_policy_from_name(policy_base, policy)
else: else:
@ -59,9 +73,9 @@ class BaseRLModel(object):
self.device = th.device(device) self.device = th.device(device)
if verbose > 0: if verbose > 0:
print("Using {} device".format(self.device)) print(f"Using {self.device} device")
self.env = env self.env = None # type: Union[gym.Env, VecEnv]
# get VecNormalize object if needed # get VecNormalize object if needed
self._vec_normalize_env = unwrap_vec_normalize(env) self._vec_normalize_env = unwrap_vec_normalize(env)
self.verbose = verbose self.verbose = verbose
@ -73,7 +87,10 @@ class BaseRLModel(object):
self.eval_env = None self.eval_env = None
self.replay_buffer = None self.replay_buffer = None
self.seed = seed self.seed = seed
self.action_noise = None self.action_noise = None # type: ActionNoise
self.start_time = None
self.policy, self.actor = None, None
self.learning_rate = None
# Used for SDE only # Used for SDE only
self.rollout_data = None self.rollout_data = None
self.on_policy_exploration = False self.on_policy_exploration = False
@ -112,12 +129,12 @@ class BaseRLModel(object):
raise ValueError("Error: the model does not support multiple envs requires a single vectorized" raise ValueError("Error: the model does not support multiple envs requires a single vectorized"
" environment.") " environment.")
def _get_eval_env(self, eval_env): def _get_eval_env(self, eval_env: Union[gym.Env, VecEnv, None]) -> Union[gym.Env, VecEnv, None]:
""" """
Return the environment that will be used for evaluation. Return the environment that will be used for evaluation.
:param eval_env: (gym.Env or VecEnv) :param eval_env:
:return: (VecEnv) :return:
""" """
if eval_env is None: if eval_env is None:
eval_env = self.eval_env eval_env = self.eval_env
@ -128,48 +145,47 @@ class BaseRLModel(object):
assert eval_env.num_envs == 1 assert eval_env.num_envs == 1
return eval_env return eval_env
def scale_action(self, action): def scale_action(self, action: np.ndarray) -> np.ndarray:
""" """
Rescale the action from [low, high] to [-1, 1] Rescale the action from [low, high] to [-1, 1]
(no need for symmetric action space) (no need for symmetric action space)
:param action: (np.ndarray) :param action:
:return: (np.ndarray) :return:
""" """
low, high = self.action_space.low, self.action_space.high low, high = self.action_space.low, self.action_space.high
return 2.0 * ((action - low) / (high - low)) - 1.0 return 2.0 * ((action - low) / (high - low)) - 1.0
def unscale_action(self, scaled_action): def unscale_action(self, scaled_action: np.ndarray) -> np.ndarray:
""" """
Rescale the action from [-1, 1] to [low, high] Rescale the action from [-1, 1] to [low, high]
(no need for symmetric action space) (no need for symmetric action space)
:param scaled_action: (np.ndarray) :param scaled_action:
:return: (np.ndarray) :return:
""" """
low, high = self.action_space.low, self.action_space.high low, high = self.action_space.low, self.action_space.high
return low + (0.5 * (scaled_action + 1.0) * (high - low)) return low + (0.5 * (scaled_action + 1.0) * (high - low))
def _setup_learning_rate(self): def _setup_learning_rate(self) -> None:
"""Transform to callable if needed.""" """Transform to callable if needed."""
self.learning_rate = get_schedule_fn(self.learning_rate) self.learning_rate = get_schedule_fn(self.learning_rate)
def _update_current_progress(self, num_timesteps, total_timesteps): def _update_current_progress(self, num_timesteps: int, total_timesteps: int) -> None:
""" """
Compute current progress (from 1 to 0) Compute current progress (from 1 to 0)
:param num_timesteps: (int) current number of timesteps :param num_timesteps: current number of timesteps
:param total_timesteps: (int) :param total_timesteps:
""" """
self._current_progress = 1.0 - float(num_timesteps) / float(total_timesteps) self._current_progress = 1.0 - float(num_timesteps) / float(total_timesteps)
def _update_learning_rate(self, optimizers): def _update_learning_rate(self, optimizers: Union[List[th.optim.Optimizer], th.optim.Optimizer]) -> None:
""" """
Update the optimizers learning rate using the current learning rate schedule Update the optimizers learning rate using the current learning rate schedule
and the current progress (from 1 to 0). and the current progress (from 1 to 0).
:param optimizers: ([th.optim.Optimizer] or Optimizer) An optimizer :param optimizers: An optimizer or a list of optimizer.
or a list of optimizer.
""" """
# Log the current learning rate # Log the current learning rate
logger.logkv("learning_rate", self.learning_rate(self._current_progress)) logger.logkv("learning_rate", self.learning_rate(self._current_progress))
@ -180,32 +196,32 @@ class BaseRLModel(object):
update_learning_rate(optimizer, self.learning_rate(self._current_progress)) update_learning_rate(optimizer, self.learning_rate(self._current_progress))
@staticmethod @staticmethod
def safe_mean(arr): def safe_mean(arr: Union[np.ndarray, list]) -> np.ndarray:
""" """
Compute the mean of an array if there is at least one element. Compute the mean of an array if there is at least one element.
For empty array, return NaN. It is used for logging only. For empty array, return NaN. It is used for logging only.
:param arr: (np.ndarray) :param arr:
:return: (float) :return:
""" """
return np.nan if len(arr) == 0 else np.mean(arr) return np.nan if len(arr) == 0 else np.mean(arr)
def get_env(self): def get_env(self) -> Union[VecEnv, None]:
""" """
Returns the current environment (can be None if not defined). Returns the current environment (can be None if not defined).
:return: (gym.Env) The current environment :return: The current environment
""" """
return self.env return self.env
@staticmethod @staticmethod
def check_env(env, observation_space, action_space): def check_env(env, observation_space: gym.spaces.Space, action_space: gym.spaces.Space) -> bool:
""" """
Checks the validity of the environment and returns if it is consistent. Checks the validity of the environment and returns if it is consistent.
Checked parameters: Checked parameters:
- observation_space - observation_space
- action_space - action_space
:return: (bool) True if environment seems to be coherent :return: True if environment seems to be coherent
""" """
if observation_space != env.observation_space: if observation_space != env.observation_space:
return False return False
@ -214,7 +230,7 @@ class BaseRLModel(object):
# return true if no check failed # return true if no check failed
return True return True
def set_env(self, env): def set_env(self, env: Union[gym.Env, VecEnv]) -> None:
""" """
Checks the validity of the environment, and if it is coherent, set it as the current environment. Checks the validity of the environment, and if it is coherent, set it as the current environment.
Furthermore wrap any non vectorized env into a vectorized Furthermore wrap any non vectorized env into a vectorized
@ -222,7 +238,7 @@ class BaseRLModel(object):
- observation_space - observation_space
- action_space - action_space
:param env: (gym.Env) The environment for learning a policy :param env: The environment for learning a policy
""" """
if self.check_env(env, self.observation_space, self.action_space) is False: if self.check_env(env, self.observation_space, self.action_space) is False:
raise ValueError("The given environment is not compatible with model: " raise ValueError("The given environment is not compatible with model: "
@ -236,23 +252,24 @@ class BaseRLModel(object):
self.n_envs = env.num_envs self.n_envs = env.num_envs
self.env = env self.env = env
def get_parameters(self): def get_parameters(self) -> Tuple[Dict[str, Any], Dict[str, Any]]:
""" """
Returns policy and optimizer parameters as a tuple Returns policy and optimizer parameters as a tuple
:return: (dict,dict) policy_parameters, opt_parameters
:return: policy_parameters, opt_parameters
""" """
return self.get_policy_parameters(), self.get_opt_parameters() return self.get_policy_parameters(), self.get_opt_parameters()
def get_policy_parameters(self): def get_policy_parameters(self) -> Dict[str, Any]:
""" """
Get current model policy parameters as dictionary of variable name -> tensors. Get current model policy parameters as dictionary of variable name -> tensors.
:return: (dict) Dictionary of variable name -> tensor of model's policy parameters. :return: Dictionary of variable name -> tensor of model's policy parameters.
""" """
return self.policy.state_dict() return self.policy.state_dict()
@abstractmethod @abstractmethod
def get_opt_parameters(self): def get_opt_parameters(self)-> Dict[str, Any]:
""" """
Get current model optimizer parameters as dictionary of variable names -> tensors Get current model optimizer parameters as dictionary of variable names -> tensors
:return: (dict) Dictionary of variable name -> tensor of model's optimizer parameters :return: (dict) Dictionary of variable name -> tensor of model's optimizer parameters
@ -260,8 +277,13 @@ class BaseRLModel(object):
raise NotImplementedError() raise NotImplementedError()
@abstractmethod @abstractmethod
def learn(self, total_timesteps, callback=None, log_interval=100, tb_log_name="run", def learn(self, total_timesteps: int,
eval_env=None, eval_freq=-1, n_eval_episodes=5, reset_num_timesteps=True): callback=None, log_interval: int = 100,
tb_log_name: str = "run",
eval_env: Union[gym.Env, VecEnv, None] = None,
eval_freq: int = -1,
n_eval_episodes: int = 5,
reset_num_timesteps: bool = True):
""" """
Return a trained model. Return a trained model.
@ -279,19 +301,22 @@ class BaseRLModel(object):
raise NotImplementedError() raise NotImplementedError()
@abstractmethod @abstractmethod
def predict(self, observation, state=None, mask=None, deterministic=False): def predict(self, observation: np.ndarray,
state: Optional[np.ndarray] = None,
mask: Optional[np.ndarray] = None,
deterministic: bool = False) -> np.ndarray:
""" """
Get the model's action from an observation Get the model's action from an observation
:param observation: (np.ndarray) the input observation :param observation: the input observation
:param state: (np.ndarray) The last states (can be None, used in recurrent policies) :param state: The last states (can be None, used in recurrent policies)
:param mask: (np.ndarray) The last masks (can be None, used in recurrent policies) :param mask: The last masks (can be None, used in recurrent policies)
:param deterministic: (bool) Whether or not to return deterministic actions. :param deterministic: Whether or not to return deterministic actions.
:return: (np.ndarray, np.ndarray) the model's action and the next state (used in recurrent policies) :return: the model's action and the next state (used in recurrent policies)
""" """
raise NotImplementedError() raise NotImplementedError()
def load_parameters(self, load_dict, opt_params): def load_parameters(self, load_dict: Dict[str, Any], opt_params: Dict[str, Any]) -> None:
""" """
Load model parameters from a dictionary Load model parameters from a dictionary
load_dict should contain all keys from torch.model.state_dict() load_dict should contain all keys from torch.model.state_dict()
@ -299,29 +324,28 @@ class BaseRLModel(object):
but can only be handled in child classes. but can only be handled in child classes.
:param load_dict: (dict) dict of parameters from model.state_dict() :param load_dict: dict of parameters from model.state_dict()
:param opt_params: (dict of dicts) dict of optimizer state_dicts should be handled in child_class :param opt_params: dict of optimizer state_dicts should be handled in child_class
""" """
if opt_params is not None: if opt_params is not None:
raise ValueError("Optimizer Parameters where given but no overloaded load function exists for this class") raise ValueError("Optimizer Parameters where given but no overloaded load function exists for this class")
self.policy.load_state_dict(load_dict) self.policy.load_state_dict(load_dict)
@classmethod @classmethod
def load(cls, load_path, env=None, **kwargs): def load(cls, load_path: str, env: Union[gym.Env, VecEnv, None] = None, **kwargs):
""" """
Load the model from a zip-file Load the model from a zip-file
:param load_path: (str) the location of the saved data :param load_path: the location of the saved data
:param env: (Gym Environment) the new environment to run the loaded model on :param env: the new environment to run the loaded model on
(can be None if you only need prediction from a trained model) has priority over any saved environment (can be None if you only need prediction from a trained model) has priority over any saved environment
:param kwargs: extra arguments to change the model when loading :param kwargs: extra arguments to change the model when loading
""" """
data, params, opt_params = cls._load_from_file(load_path) data, params, opt_params = cls._load_from_file(load_path)
if 'policy_kwargs' in kwargs and kwargs['policy_kwargs'] != data['policy_kwargs']: if 'policy_kwargs' in kwargs and kwargs['policy_kwargs'] != data['policy_kwargs']:
raise ValueError("The specified policy kwargs do not equal the stored policy kwargs." raise ValueError(f"The specified policy kwargs do not equal the stored policy kwargs."
"Stored kwargs: {}, specified kwargs: {}".format(data['policy_kwargs'], "Stored kwargs: {data['policy_kwargs']}, specified kwargs: {kwargs['policy_kwargs']}")
kwargs['policy_kwargs']))
# check if observation space and action space are part of the saved parameters # check if observation space and action space are part of the saved parameters
if ("observation_space" not in data or "action_space" not in data) and "env" not in data: if ("observation_space" not in data or "action_space" not in data) and "env" not in data:
@ -344,11 +368,11 @@ class BaseRLModel(object):
return model return model
@staticmethod @staticmethod
def _load_from_file(load_path, load_data=True): def _load_from_file(load_path: str, load_data: bool = True):
""" Load model data from a .zip archive """ Load model data from a .zip archive
:param load_path: (str) Where to load the model from :param load_path: Where to load the model from
:param load_data: (bool) Whether we should load and return data :param load_data: Whether we should load and return data
(class parameters). Mainly used by 'load_parameters' to only load model parameters (weights) (class parameters). Mainly used by 'load_parameters' to only load model parameters (weights)
:return: (dict),(dict),(dict) Class parameters, model parameters (state_dict) :return: (dict),(dict),(dict) Class parameters, model parameters (state_dict)
and dict of optimizer parameters (dict of state_dict) and dict of optimizer parameters (dict of state_dict)
@ -359,7 +383,7 @@ class BaseRLModel(object):
if os.path.exists(load_path + ".zip"): if os.path.exists(load_path + ".zip"):
load_path += ".zip" load_path += ".zip"
else: else:
raise ValueError("Error: the file {} could not be found".format(load_path)) raise ValueError(f"Error: the file {load_path} could not be found")
# Open the zip archive and load data # Open the zip archive and load data
try: try:
@ -406,11 +430,11 @@ class BaseRLModel(object):
opt_params[os.path.splitext(file_path)[0]] = th.load(file_content) opt_params[os.path.splitext(file_path)[0]] = th.load(file_content)
except zipfile.BadZipFile: except zipfile.BadZipFile:
# load_path wasn't a zip file # load_path wasn't a zip file
raise ValueError("Error: the file {} wasn't a zip-file".format(load_path)) raise ValueError(f"Error: the file {load_path} wasn't a zip-file")
return data, params, opt_params return data, params, opt_params
def set_random_seed(self, seed=None): def set_random_seed(self, seed: Optional[int] = None) -> None:
""" """
Set the seed of the pseudo-random generators Set the seed of the pseudo-random generators
(python, numpy, pytorch, gym, action_space) (python, numpy, pytorch, gym, action_space)
@ -446,7 +470,7 @@ class BaseRLModel(object):
eval_env.seed(self.seed) eval_env.seed(self.seed)
eval_env = self._get_eval_env(eval_env) eval_env = self._get_eval_env(eval_env)
obs = self.env.reset() obs = self.env.reset() # type: Union[gym.Env, VecEnv]
return timesteps_since_eval, episode_num, evaluations, obs, eval_env return timesteps_since_eval, episode_num, evaluations, obs, eval_env
def _update_info_buffer(self, infos): def _update_info_buffer(self, infos):
@ -629,13 +653,15 @@ class BaseRLModel(object):
return mean_reward, total_steps, total_episodes, obs return mean_reward, total_steps, total_episodes, obs
@staticmethod @staticmethod
def _save_to_file_zip(save_path, data=None, params=None, opt_params=None): def _save_to_file_zip(save_path: str, data: Dict[str, Any] = None,
"""Save model to a zip archive params: Dict[str, Any] = None, opt_params: Dict[str, Any] = None) -> None:
"""
Save model to a zip archive.
:param save_path: (str) Where to store the model :param save_path: Where to store the model
:param data: (dict) Class parameters being stored :param data: Class parameters being stored
:param params: (dict) Model parameters being stored expected to be state_dict :param params: Model parameters being stored expected to be state_dict
:param opt_params: (dict) Optimizer parameters being stored expected to contain an entry for every :param opt_params: Optimizer parameters being stored expected to contain an entry for every
optimizer with its name and the state_dict optimizer with its name and the state_dict
""" """
@ -666,7 +692,7 @@ class BaseRLModel(object):
th.save(dict_, opt_param_file) th.save(dict_, opt_param_file)
@staticmethod @staticmethod
def excluded_save_params(): def excluded_save_params() -> List[str]:
""" """
Returns the names of the parameters that should be excluded by default Returns the names of the parameters that should be excluded by default
when saving the model. when saving the model.
@ -675,13 +701,13 @@ class BaseRLModel(object):
""" """
return ["env", "eval_env", "replay_buffer", "rollout_buffer", "_vec_normalize_env"] return ["env", "eval_env", "replay_buffer", "rollout_buffer", "_vec_normalize_env"]
def save(self, path, exclude=None, include=None): def save(self, path: str, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> None:
""" """
Save all the attributes of the object and the model parameters in a zip-file. Save all the attributes of the object and the model parameters in a zip-file.
:param path: (str) path to the file where the rl agent should be saved :param path: path to the file where the rl agent should be saved
:param exclude: ([str]) name of parameters that should be excluded in addition to the default one :param exclude: name of parameters that should be excluded in addition to the default one
:param include: ([str]) name of parameters that might be excluded but should be included anyway :param include: name of parameters that might be excluded but should be included anyway
""" """
# copy parameter list so we don't mutate the original dict # copy parameter list so we don't mutate the original dict
data = self.__dict__.copy() data = self.__dict__.copy()
@ -704,25 +730,26 @@ class BaseRLModel(object):
opt_params_to_save = self.get_opt_parameters() opt_params_to_save = self.get_opt_parameters()
self._save_to_file_zip(path, data=data, params=params_to_save, opt_params=opt_params_to_save) self._save_to_file_zip(path, data=data, params=params_to_save, opt_params=opt_params_to_save)
def _eval_policy(self, eval_freq, eval_env, n_eval_episodes, def _eval_policy(self, eval_freq: int, eval_env: int, n_eval_episodes: int,
timesteps_since_eval, deterministic=True): timesteps_since_eval: int, render: bool = False, deterministic: bool = True) -> int:
""" """
Evaluate the current policy on a test environment. Evaluate the current policy on a test environment.
:param eval_env: (gym.Env) Environment that will be used to evaluate the agent :param eval_freq: Evaluate the agent every `eval_freq` timesteps (this may vary a little)
:param eval_freq: (int) Evaluate the agent every `eval_freq` timesteps (this may vary a little) :param n_eval_episodes: Number of episode to evaluate the agent
:param n_eval_episodes: (int) Number of episode to evaluate the agent :parma timesteps_since_eval: Number of timesteps since last evaluation
:parma timesteps_since_eval: (int) Number of timesteps since last evaluation :param deterministic: Whether to use deterministic or stochastic actions
:param deterministic: (bool) Whether to use deterministic or stochastic actions :param render: Whether to render the eval env or not
:return: (int) Number of timesteps since last evaluation :return: Number of timesteps since last evaluation
""" """
if 0 < eval_freq <= timesteps_since_eval and eval_env is not None: if 0 < eval_freq <= timesteps_since_eval and eval_env is not None:
timesteps_since_eval %= eval_freq timesteps_since_eval %= eval_freq
# Synchronise the normalization stats if needed # Synchronise the normalization stats if needed
sync_envs_normalization(self.env, eval_env) sync_envs_normalization(self.env, eval_env)
mean_reward, std_reward = evaluate_policy(self, eval_env, n_eval_episodes, deterministic=deterministic) mean_reward, std_reward = evaluate_policy(self, eval_env, n_eval_episodes,
render=render, deterministic=deterministic)
if self.verbose > 0: if self.verbose > 0:
print("Eval num_timesteps={}, " print(f"Eval num_timesteps={self.num_timesteps}, "
"episode_reward={:.2f} +/- {:.2f}".format(self.num_timesteps, mean_reward, std_reward)) f"episode_reward={mean_reward:.2f} +/- {std_reward:.2f}")
print("FPS: {:.2f}".format(self.num_timesteps / (time.time() - self.start_time))) print(f"FPS: {self.num_timesteps / (time.time() - self.start_time):.2f}")
return timesteps_since_eval return timesteps_since_eval

View file

@ -467,6 +467,5 @@ def make_proba_distribution(action_space, use_sde=False, dist_kwargs=None):
# elif isinstance(action_space, spaces.MultiBinary): # elif isinstance(action_space, spaces.MultiBinary):
# return BernoulliDistribution(action_space.n, **dist_kwargs) # return BernoulliDistribution(action_space.n, **dist_kwargs)
else: else:
raise NotImplementedError("Error: probability distribution, not implemented for action space of type {}." raise NotImplementedError(f"Error: probability distribution, not implemented for action space of type {type(action_space)}."
.format(type(action_space)) +
" Must be of type Gym Spaces: Box, Discrete, MultiDiscrete or MultiBinary.") " Must be of type Gym Spaces: Box, Discrete, MultiDiscrete or MultiBinary.")

View file

@ -47,8 +47,8 @@ def evaluate_policy(model, env, n_eval_episodes=10, deterministic=True,
mean_reward = np.mean(episode_rewards) mean_reward = np.mean(episode_rewards)
std_reward = np.std(episode_rewards) std_reward = np.std(episode_rewards)
if reward_threshold is not None: if reward_threshold is not None:
assert mean_reward > reward_threshold, 'Mean reward below threshold: '\ assert mean_reward > reward_threshold, (f'Mean reward below threshold: '
'{:.2f} < {:.2f}'.format(mean_reward, reward_threshold) '{mean_reward:.2f} < {reward_threshold:.2f}')
if return_episode_rewards: if return_episode_rewards:
return episode_rewards, n_steps return episode_rewards, n_steps
return mean_reward, std_reward return mean_reward, std_reward

View file

@ -53,7 +53,7 @@ class HumanOutputFormat(KVWriter, SeqWriter):
self.file = open(filename_or_file, 'wt') self.file = open(filename_or_file, 'wt')
self.own_file = True self.own_file = True
else: else:
assert hasattr(filename_or_file, 'write'), 'Expected file or str, got {}'.format(filename_or_file) assert hasattr(filename_or_file, 'write'), f'Expected file or str, got {filename_or_file}'
self.file = filename_or_file self.file = filename_or_file
self.own_file = False self.own_file = False
@ -272,7 +272,7 @@ def getkvs():
return Logger.CURRENT.name2val return Logger.CURRENT.name2val
def log(*args, **kwargs): def log(*args, level=INFO):
""" """
Write the sequence of args, with no separators, Write the sequence of args, with no separators,
to the console and output files (if you've configured an output file). to the console and output files (if you've configured an output file).
@ -283,7 +283,6 @@ def log(*args, **kwargs):
:param args: (list) log the arguments :param args: (list) log the arguments
:param level: (int) the logging level (can be DEBUG=10, INFO=20, WARN=30, ERROR=40, DISABLED=50) :param level: (int) the logging level (can be DEBUG=10, INFO=20, WARN=30, ERROR=40, DISABLED=50)
""" """
level = kwargs.get('level', INFO)
Logger.CURRENT.log(*args, level=level) Logger.CURRENT.log(*args, level=level)
@ -424,7 +423,7 @@ class Logger(object):
self.name2val.clear() self.name2val.clear()
self.name2cnt.clear() self.name2cnt.clear()
def log(self, *args, **kwargs): def log(self, *args, level=INFO):
""" """
Write the sequence of args, with no separators, Write the sequence of args, with no separators,
to the console and output files (if you've configured an output file). to the console and output files (if you've configured an output file).
@ -435,7 +434,6 @@ class Logger(object):
:param args: (list) log the arguments :param args: (list) log the arguments
:param level: (int) the logging level (can be DEBUG=10, INFO=20, WARN=30, ERROR=40, DISABLED=50) :param level: (int) the logging level (can be DEBUG=10, INFO=20, WARN=30, ERROR=40, DISABLED=50)
""" """
level = kwargs.get('level', INFO)
if self.level <= level: if self.level <= level:
self._do_log(args) self._do_log(args)

View file

@ -30,7 +30,7 @@ class NormalActionNoise(ActionNoise):
return np.random.normal(self._mu, self._sigma) return np.random.normal(self._mu, self._sigma)
def __repr__(self): def __repr__(self):
return 'NormalActionNoise(mu={}, sigma={})'.format(self._mu, self._sigma) return f'NormalActionNoise(mu={self._mu}, sigma={self._sigma})'
class OrnsteinUhlenbeckActionNoise(ActionNoise): class OrnsteinUhlenbeckActionNoise(ActionNoise):
@ -68,4 +68,4 @@ class OrnsteinUhlenbeckActionNoise(ActionNoise):
self.noise_prev = self.initial_noise if self.initial_noise is not None else np.zeros_like(self._mu) self.noise_prev = self.initial_noise if self.initial_noise is not None else np.zeros_like(self._mu)
def __repr__(self): def __repr__(self):
return 'OrnsteinUhlenbeckActionNoise(mu={}, sigma={})'.format(self._mu, self._sigma) return f'OrnsteinUhlenbeckActionNoise(mu={self._mu}, sigma={self._sigma})'

View file

@ -150,10 +150,10 @@ def get_policy_from_name(base_policy_type, name):
:return: (base_policy_type) the policy :return: (base_policy_type) the policy
""" """
if base_policy_type not in _policy_registry: if base_policy_type not in _policy_registry:
raise ValueError("Error: the policy type {} is not registered!".format(base_policy_type)) raise ValueError(f"Error: the policy type {base_policy_type} is not registered!")
if name not in _policy_registry[base_policy_type]: if name not in _policy_registry[base_policy_type]:
raise ValueError("Error: unknown policy type {}, the only registed policy type are: {}!" raise ValueError(f"Error: unknown policy type {name},"
.format(name, list(_policy_registry[base_policy_type].keys()))) "the only registed policy type are: {list(_policy_registry[base_policy_type].keys())}!")
return _policy_registry[base_policy_type][name] return _policy_registry[base_policy_type][name]
@ -174,13 +174,12 @@ def register_policy(name, policy):
except AttributeError: except AttributeError:
sub_class = str(th.random.randint(100)) sub_class = str(th.random.randint(100))
if sub_class is None: if sub_class is None:
raise ValueError("Error: the policy {} is not of any known subclasses of BasePolicy!".format(policy)) raise ValueError(f"Error: the policy {policy} is not of any known subclasses of BasePolicy!")
if sub_class not in _policy_registry: if sub_class not in _policy_registry:
_policy_registry[sub_class] = {} _policy_registry[sub_class] = {}
if name in _policy_registry[sub_class]: if name in _policy_registry[sub_class]:
raise ValueError("Error: the name {} is alreay registered for a different policy, will not override." raise ValueError(f"Error: the name {name} is alreay registered for a different policy, will not override.")
.format(name))
_policy_registry[sub_class][name] = policy _policy_registry[sub_class][name] = policy

View file

@ -123,7 +123,7 @@ def json_to_data(json_string, custom_objects=None):
) )
except pickle.UnpicklingError: except pickle.UnpicklingError:
raise RuntimeError( raise RuntimeError(
"Could not deserialize object {}. ".format(data_key) + f"Could not deserialize object {data_key}. " +
"Consider using `custom_objects` argument to replace " + "Consider using `custom_objects` argument to replace " +
"this object." "this object."
) )

View file

@ -1,6 +1,6 @@
from abc import ABCMeta, abstractmethod
import inspect import inspect
import pickle import pickle
from abc import ABC, abstractmethod
import cloudpickle import cloudpickle
@ -27,7 +27,7 @@ class NotSteppingError(Exception):
Exception.__init__(self, msg) Exception.__init__(self, msg)
class VecEnv(object): class VecEnv(ABC):
""" """
An abstract asynchronous, vectorized environment. An abstract asynchronous, vectorized environment.
@ -39,8 +39,6 @@ class VecEnv(object):
'render.modes': ['human', 'rgb_array'] 'render.modes': ['human', 'rgb_array']
} }
__metaclass__ = ABCMeta
def __init__(self, num_envs, observation_space, action_space): def __init__(self, num_envs, observation_space, action_space):
self.num_envs = num_envs self.num_envs = num_envs
self.observation_space = observation_space self.observation_space = observation_space
@ -112,7 +110,7 @@ class VecEnv(object):
raise NotImplementedError() raise NotImplementedError()
@abstractmethod @abstractmethod
def env_method(self, method_name, *method_args, **method_kwargs): def env_method(self, method_name, *method_args, indices=None, **method_kwargs):
""" """
Call instance methods of vectorized environments. Call instance methods of vectorized environments.
@ -148,6 +146,17 @@ class VecEnv(object):
""" """
raise NotImplementedError() raise NotImplementedError()
def seed(self, seed, indices=None):
"""
:param seed: (int or [int])
:param indices: ([int])
"""
indices = self._get_indices(indices)
if not hasattr(seed, 'len'):
seed = [seed] * len(indices)
assert len(seed) == len(indices)
return [self.env_method('seed', seed[i], indices=i) for i in indices]
@property @property
def unwrapped(self): def unwrapped(self):
if isinstance(self, VecEnvWrapper): if isinstance(self, VecEnvWrapper):
@ -163,7 +172,7 @@ class VecEnv(object):
:return: (str or None) name of module whose attribute is being shadowed, if any. :return: (str or None) name of module whose attribute is being shadowed, if any.
""" """
if hasattr(self, name) and already_found: if hasattr(self, name) and already_found:
return "{0}.{1}".format(type(self).__module__, type(self).__name__) return f"{type(self).__module__}.{type(self).__name__}"
else: else:
return None return None
@ -222,8 +231,8 @@ class VecEnvWrapper(VecEnv):
def set_attr(self, attr_name, value, indices=None): def set_attr(self, attr_name, value, indices=None):
return self.venv.set_attr(attr_name, value, indices) return self.venv.set_attr(attr_name, value, indices)
def env_method(self, method_name, *method_args, **method_kwargs): def env_method(self, method_name, *method_args, indices=None, **method_kwargs):
return self.venv.env_method(method_name, *method_args, **method_kwargs) return self.venv.env_method(method_name, *method_args, indices=indices, **method_kwargs)
def __getattr__(self, name): def __getattr__(self, name):
"""Find attribute from wrapped venv(s) if this wrapper does not have it. """Find attribute from wrapped venv(s) if this wrapper does not have it.
@ -232,10 +241,10 @@ class VecEnvWrapper(VecEnv):
""" """
blocked_class = self.getattr_depth_check(name, already_found=False) blocked_class = self.getattr_depth_check(name, already_found=False)
if blocked_class is not None: if blocked_class is not None:
own_class = "{0}.{1}".format(type(self).__module__, type(self).__name__) own_class = f"{type(self).__module__}.{type(self).__name__}"
format_str = ("Error: Recursive attribute lookup for {0} from {1} is " error_str = (f"Error: Recursive attribute lookup for {name} from {own_class} is "
"ambiguous and hides attribute from {2}") "ambiguous and hides attribute from {blocked_class}")
raise AttributeError(format_str.format(name, own_class, blocked_class)) raise AttributeError(error_str)
return self.getattr_recursive(name) return self.getattr_recursive(name)
@ -274,7 +283,7 @@ class VecEnvWrapper(VecEnv):
all_attributes = self._get_all_attributes() all_attributes = self._get_all_attributes()
if name in all_attributes and already_found: if name in all_attributes and already_found:
# this venv's attribute is being hidden because of a higher venv. # this venv's attribute is being hidden because of a higher venv.
shadowed_wrapper_class = "{0}.{1}".format(type(self).__module__, type(self).__name__) shadowed_wrapper_class = f"{type(self).__module__}.{type(self).__name__}"
elif name in all_attributes and not already_found: elif name in all_attributes and not already_found:
# we have found the first reference to the attribute. Now check for duplicates. # we have found the first reference to the attribute. Now check for duplicates.
shadowed_wrapper_class = self.venv.getattr_depth_check(name, True) shadowed_wrapper_class = self.venv.getattr_depth_check(name, True)

View file

@ -3,7 +3,7 @@ from copy import deepcopy
import numpy as np import numpy as np
from torchy_baselines.common.vec_env import VecEnv from torchy_baselines.common.vec_env.base_vec_env import VecEnv
from torchy_baselines.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info from torchy_baselines.common.vec_env.util import copy_obs_dict, dict_to_obs, obs_space_info
@ -54,17 +54,6 @@ class DummyVecEnv(VecEnv):
self._save_obs(env_idx, obs) self._save_obs(env_idx, obs)
return self._obs_from_buf() return self._obs_from_buf()
def seed(self, seed, indices=None):
"""
:param seed: (int or [int])
:param indices: ([int])
"""
indices = self._get_indices(indices)
if not hasattr(seed, 'len'):
seed = [seed] * len(indices)
assert len(seed) == len(indices)
return [self.envs[i].seed(seed[i]) for i in indices]
def close(self): def close(self):
for env in self.envs: for env in self.envs:
env.close() env.close()
@ -99,11 +88,8 @@ class DummyVecEnv(VecEnv):
for env_i in target_envs: for env_i in target_envs:
setattr(env_i, attr_name, value) setattr(env_i, attr_name, value)
def env_method(self, method_name, *method_args, **method_kwargs): def env_method(self, method_name, *method_args, indices=None, **method_kwargs):
"""Call instance methods of vectorized environments.""" """Call instance methods of vectorized environments."""
indices = method_kwargs.get('indices')
if 'indices' in method_kwargs:
del method_kwargs['indices']
target_envs = self._get_target_envs(indices) target_envs = self._get_target_envs(indices)
return [getattr(env_i, method_name)(*method_args, **method_kwargs) for env_i in target_envs] return [getattr(env_i, method_name)(*method_args, **method_kwargs) for env_i in target_envs]

View file

@ -4,7 +4,7 @@ from collections import OrderedDict
import gym import gym
import numpy as np import numpy as np
from torchy_baselines.common.vec_env import VecEnv, CloudpickleWrapper from torchy_baselines.common.vec_env.base_vec_env import VecEnv, CloudpickleWrapper
def _worker(remote, parent_remote, env_fn_wrapper): def _worker(remote, parent_remote, env_fn_wrapper):
@ -111,7 +111,7 @@ class SubprocVecEnv(VecEnv):
for work_remote, remote, env_fn in zip(self.work_remotes, self.remotes, env_fns): for work_remote, remote, env_fn in zip(self.work_remotes, self.remotes, env_fns):
args = (work_remote, remote, CloudpickleWrapper(env_fn)) args = (work_remote, remote, CloudpickleWrapper(env_fn))
# daemon=True: if the main process crashes, we should not cause things to hang # daemon=True: if the main process crashes, we should not cause things to hang
process = ctx.Process(target=_worker, args=args, daemon=True) process = ctx.Process(target=_worker, args=args, daemon=True) # pytype:disable=attribute-error
process.start() process.start()
self.processes.append(process) self.processes.append(process)
work_remote.close() work_remote.close()
@ -153,8 +153,7 @@ class SubprocVecEnv(VecEnv):
for pipe in self.remotes: for pipe in self.remotes:
# gather images from subprocesses # gather images from subprocesses
# `mode` will be taken into account later # `mode` will be taken into account later
kwargs.update({'mode': 'rgb_array'}) pipe.send(('render', (args, {'mode': 'rgb_array', **kwargs})))
pipe.send(('render', (args, kwargs)))
imgs = [pipe.recv() for pipe in self.remotes] imgs = [pipe.recv() for pipe in self.remotes]
# Create a big image by tiling images from subprocesses # Create a big image by tiling images from subprocesses
bigimg = tile_images(imgs) bigimg = tile_images(imgs)
@ -188,22 +187,8 @@ class SubprocVecEnv(VecEnv):
for remote in target_remotes: for remote in target_remotes:
remote.recv() remote.recv()
def seed(self, seed, indices=None): def env_method(self, method_name, *method_args, indices=None, **method_kwargs):
"""
:param seed: (int or [int])
:param indices: ([int])
"""
indices = self._get_indices(indices)
if not hasattr(seed, 'len'):
seed = [seed] * len(indices)
assert len(seed) == len(indices)
return [self.env_method('seed', seed[i], indices=i) for i in indices]
def env_method(self, method_name, *method_args, **method_kwargs):
"""Call instance methods of vectorized environments.""" """Call instance methods of vectorized environments."""
indices = method_kwargs.get('indices')
if 'indices' in method_kwargs:
del method_kwargs['indices']
target_remotes = self._get_target_remotes(indices) target_remotes = self._get_target_remotes(indices)
for remote in target_remotes: for remote in target_remotes:
remote.send(('env_method', (method_name, method_args, method_kwargs))) remote.send(('env_method', (method_name, method_args, method_kwargs)))

View file

@ -15,7 +15,7 @@ def copy_obs_dict(obs):
:param obs: (OrderedDict<ndarray>): a dict of numpy arrays. :param obs: (OrderedDict<ndarray>): a dict of numpy arrays.
:return (OrderedDict<ndarray>) a dict of copied numpy arrays. :return (OrderedDict<ndarray>) a dict of copied numpy arrays.
""" """
assert isinstance(obs, OrderedDict), "unexpected type for observations '{}'".format(type(obs)) assert isinstance(obs, OrderedDict), f"unexpected type for observations '{type(obs)}'"
return OrderedDict([(k, np.copy(v)) for k, v in obs.items()]) return OrderedDict([(k, np.copy(v)) for k, v in obs.items()])
@ -61,7 +61,7 @@ def obs_space_info(obs_space):
elif isinstance(obs_space, gym.spaces.Tuple): elif isinstance(obs_space, gym.spaces.Tuple):
subspaces = {i: space for i, space in enumerate(obs_space.spaces)} subspaces = {i: space for i, space in enumerate(obs_space.spaces)}
else: else:
assert not hasattr(obs_space, 'spaces'), "Unsupported structured space '{}'".format(type(obs_space)) assert not hasattr(obs_space, 'spaces'), f"Unsupported structured space '{type(obs_space)}'"
subspaces = {None: obs_space} subspaces = {None: obs_space}
keys = [] keys = []
shapes = {} shapes = {}

View file

@ -3,18 +3,18 @@ import warnings
import numpy as np import numpy as np
from gym import spaces from gym import spaces
from torchy_baselines.common.vec_env import VecEnvWrapper from torchy_baselines.common.vec_env.base_vec_env import VecEnv, VecEnvWrapper
class VecFrameStack(VecEnvWrapper): class VecFrameStack(VecEnvWrapper):
""" """
Frame stacking wrapper for vectorized environment Frame stacking wrapper for vectorized environment
:param venv: (VecEnv) the vectorized environment to wrap :param venv: the vectorized environment to wrap
:param n_stack: (int) Number of frames to stack :param n_stack: Number of frames to stack
""" """
def __init__(self, venv, n_stack): def __init__(self, venv: VecEnv, n_stack: int):
self.venv = venv self.venv = venv
self.n_stack = n_stack self.n_stack = n_stack
wrapped_obs_space = venv.observation_space wrapped_obs_space = venv.observation_space

View file

@ -2,7 +2,7 @@ import pickle
import numpy as np import numpy as np
from torchy_baselines.common.vec_env import VecEnvWrapper from torchy_baselines.common.vec_env.base_vec_env import VecEnvWrapper
from torchy_baselines.common.running_mean_std import RunningMeanStd from torchy_baselines.common.running_mean_std import RunningMeanStd
@ -182,7 +182,7 @@ class VecNormalize(VecEnvWrapper):
:param path: (str) path to log dir :param path: (str) path to log dir
""" """
for rms, name in zip([self.obs_rms, self.ret_rms], ['obs_rms', 'ret_rms']): for rms, name in zip([self.obs_rms, self.ret_rms], ['obs_rms', 'ret_rms']):
with open("{}/{}.pkl".format(path, name), 'wb') as file_handler: with open(f"{path}/{name}.pkl", 'wb') as file_handler:
pickle.dump(rms, file_handler) pickle.dump(rms, file_handler)
def load_running_average(self, path): def load_running_average(self, path):
@ -190,5 +190,5 @@ class VecNormalize(VecEnvWrapper):
:param path: (str) path to log dir :param path: (str) path to log dir
""" """
for name in ['obs_rms', 'ret_rms']: for name in ['obs_rms', 'ret_rms']:
with open("{}/{}.pkl".format(path, name), 'rb') as file_handler: with open(f"{path}/{name}.pkl", 'rb') as file_handler:
setattr(self, name, pickle.load(file_handler)) setattr(self, name, pickle.load(file_handler))

View file

@ -76,18 +76,20 @@ class PPOPolicy(BasePolicy):
self.sde_feature_extractor = None self.sde_feature_extractor = None
self.sde_net_arch = sde_net_arch self.sde_net_arch = sde_net_arch
self.use_sde = use_sde
# Action distribution # Action distribution
self.action_dist = make_proba_distribution(action_space, use_sde=use_sde, dist_kwargs=dist_kwargs) self.action_dist = make_proba_distribution(action_space, use_sde=use_sde, dist_kwargs=dist_kwargs)
self._build(learning_rate) self._build(learning_rate)
def reset_noise(self, n_envs=1): def reset_noise(self, n_envs: int = 1):
""" """
Sample new weights for the exploration matrix. Sample new weights for the exploration matrix.
:param n_envs: (int) :param n_envs: (int)
""" """
assert isinstance(self.action_dist, StateDependentNoiseDistribution), 'reset_noise() is only available when using SDE'
self.action_dist.sample_weights(self.log_std, batch_size=n_envs) self.action_dist.sample_weights(self.log_std, batch_size=n_envs)
def _build(self, learning_rate): def _build(self, learning_rate):

View file

@ -25,7 +25,7 @@ class PPO(BaseRLModel):
Proximal Policy Optimization algorithm (PPO) (clip version) Proximal Policy Optimization algorithm (PPO) (clip version)
Paper: https://arxiv.org/abs/1707.06347 Paper: https://arxiv.org/abs/1707.06347
Code: This implementation borrows code from OpenAI spinningup (https://github.com/openai/spinningup/) Code: This implementation borrows code from OpenAI Spinning Up (https://github.com/openai/spinningup/)
https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail and https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail and
and Stable Baselines (PPO2 from https://github.com/hill-a/stable-baselines) and Stable Baselines (PPO2 from https://github.com/hill-a/stable-baselines)

View file

View file

@ -94,6 +94,7 @@ class Actor(BaseNetwork):
:return: (th.Tensor) :return: (th.Tensor)
""" """
assert isinstance(self.action_dist, StateDependentNoiseDistribution), 'get_std() is only available when using SDE'
return self.action_dist.get_std(self.log_std) return self.action_dist.get_std(self.log_std)
def reset_noise(self, batch_size=1): def reset_noise(self, batch_size=1):
@ -102,6 +103,7 @@ class Actor(BaseNetwork):
:param batch_size: (int) :param batch_size: (int)
""" """
assert isinstance(self.action_dist, StateDependentNoiseDistribution), 'reset_noise() is only available when using SDE'
self.action_dist.sample_weights(self.log_std, batch_size=batch_size) self.action_dist.sample_weights(self.log_std, batch_size=batch_size)
def _get_latent(self, obs): def _get_latent(self, obs):

View file

@ -39,7 +39,7 @@ class SAC(BaseRLModel):
:param gradient_steps: (int) How many gradient update after each step :param gradient_steps: (int) How many gradient update after each step
:param n_episodes_rollout: (int) Update the model every `n_episodes_rollout` episodes. :param n_episodes_rollout: (int) Update the model every `n_episodes_rollout` episodes.
Note that this cannot be used at the same time as `train_freq` Note that this cannot be used at the same time as `train_freq`
:param target_entropy: (str or float) target entropy when learning ent_coef (ent_coef = 'auto') :param target_entropy: (str or float) target entropy when learning `ent_coef` (`ent_coef = 'auto'`)
:param action_noise: (ActionNoise) the action noise type (None by default), this can help :param action_noise: (ActionNoise) the action noise type (None by default), this can help
for hard exploration problem. Cf common.noise for the different action noise type. for hard exploration problem. Cf common.noise for the different action noise type.
:param gamma: (float) the discount factor :param gamma: (float) the discount factor
@ -161,7 +161,7 @@ class SAC(BaseRLModel):
""" """
return self.unscale_action(self.select_action(observation)) return self.unscale_action(self.select_action(observation))
def train(self, gradient_steps, batch_size=64): def train(self, gradient_steps: int, batch_size: int = 64):
# Update optimizers learning rate # Update optimizers learning rate
optimizers = [self.actor.optimizer, self.critic.optimizer] optimizers = [self.actor.optimizer, self.critic.optimizer]
if self.ent_coef_optimizer is not None: if self.ent_coef_optimizer is not None:
@ -169,6 +169,9 @@ class SAC(BaseRLModel):
self._update_learning_rate(optimizers) self._update_learning_rate(optimizers)
ent_coef_loss, ent_coef = th.zeros(1), th.zeros(1)
actor_loss, critic_loss = th.zeros(1), th.zeros(1)
for gradient_step in range(gradient_steps): for gradient_step in range(gradient_steps):
# Sample replay buffer # Sample replay buffer
replay_data = self.replay_buffer.sample(batch_size, env=self._vec_normalize_env) replay_data = self.replay_buffer.sample(batch_size, env=self._vec_normalize_env)

View file

@ -31,7 +31,7 @@ class TD3(BaseRLModel):
:param gradient_steps: (int) How many gradient update after each step :param gradient_steps: (int) How many gradient update after each step
:param n_episodes_rollout: (int) Update the model every `n_episodes_rollout` episodes. :param n_episodes_rollout: (int) Update the model every `n_episodes_rollout` episodes.
Note that this cannot be used at the same time as `train_freq` Note that this cannot be used at the same time as `train_freq`
:param tau: (float) the soft update coefficient ("polyak update" of the target networks, between 0 and 1) :param tau: (float) the soft update coefficient ("Polyak update" of the target networks, between 0 and 1)
:param action_noise: (ActionNoise) the action noise type. Cf common.noise for the different action noise type. :param action_noise: (ActionNoise) the action noise type. Cf common.noise for the different action noise type.
:param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy :param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy
(smoothing noise) (smoothing noise)