Merge branch 'master' into feat/sde-features

This commit is contained in:
Antonin Raffin 2019-12-05 16:33:41 +01:00
commit 0117cc37f4
13 changed files with 720 additions and 110 deletions

View file

@ -22,6 +22,7 @@ sys.path.insert(0, os.path.abspath('..'))
class Mock(MagicMock): class Mock(MagicMock):
__subclasses__ = [] __subclasses__ = []
@classmethod @classmethod
def __getattr__(cls, name): def __getattr__(cls, name):
return MagicMock() return MagicMock()

View file

@ -5,6 +5,7 @@ import pytest
from torchy_baselines import PPO from torchy_baselines import PPO
@pytest.mark.parametrize('net_arch', [ @pytest.mark.parametrize('net_arch', [
[12, dict(vf=[16], pi=[8])], [12, dict(vf=[16], pi=[8])],
[4], [4],

View file

@ -6,7 +6,6 @@ import numpy as np
from torchy_baselines import A2C, CEMRL, PPO, SAC, TD3 from torchy_baselines import A2C, CEMRL, PPO, SAC, TD3
from torchy_baselines.common.noise import NormalActionNoise from torchy_baselines.common.noise import NormalActionNoise
action_noise = NormalActionNoise(np.zeros(1), 0.1 * np.ones(1)) action_noise = NormalActionNoise(np.zeros(1), 0.1 * np.ones(1))
@ -16,7 +15,7 @@ def test_td3():
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
model.save("test_save") model.save("test_save")
model.load("test_save") model.load("test_save")
os.remove("test_save.pth") os.remove("test_save.zip")
def test_cemrl(): def test_cemrl():
@ -25,7 +24,7 @@ def test_cemrl():
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
model.save("test_save") model.save("test_save")
model.load("test_save") model.load("test_save")
os.remove("test_save.pth") os.remove("test_save.zip")
@pytest.mark.parametrize("model_class", [A2C, PPO]) @pytest.mark.parametrize("model_class", [A2C, PPO])
@ -33,9 +32,10 @@ def test_cemrl():
def test_onpolicy(model_class, env_id): def test_onpolicy(model_class, env_id):
model = model_class('MlpPolicy', env_id, policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True) model = model_class('MlpPolicy', env_id, policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True)
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
# model.save("test_save") model.save("test_save")
# model.load("test_save") model.load("test_save")
# os.remove("test_save.pth") os.remove("test_save.zip")
@pytest.mark.parametrize("ent_coef", ['auto', 0.01]) @pytest.mark.parametrize("ent_coef", ['auto', 0.01])
def test_sac(ent_coef): def test_sac(ent_coef):
@ -43,3 +43,6 @@ def test_sac(ent_coef):
learning_starts=100, verbose=1, create_eval_env=True, ent_coef=ent_coef, learning_starts=100, verbose=1, create_eval_env=True, ent_coef=ent_coef,
action_noise=NormalActionNoise(np.zeros(1), np.zeros(1))) action_noise=NormalActionNoise(np.zeros(1), np.zeros(1)))
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
model.save("test_save")
model.load("test_save")
os.remove("test_save.zip")

156
tests/test_save_load.py Normal file
View file

@ -0,0 +1,156 @@
import os
import pytest
from copy import deepcopy
import numpy as np
import torch as th
from torchy_baselines import A2C, CEMRL, PPO, SAC, TD3
from torchy_baselines.common.vec_env import DummyVecEnv
from torchy_baselines.common.identity_env import IdentityEnvBox, IdentityEnv
MODEL_LIST = [
CEMRL,
PPO,
A2C,
TD3,
SAC,
]
@pytest.mark.parametrize("model_class", MODEL_LIST)
def test_save_load(model_class):
"""
Test if 'save' and 'load' saves and loads model correctly
and if 'load_parameters' and 'get_policy_parameters' work correctly
''warning does not test function of optimizer parameter load
:param model_class: (BaseRLModel) A RL model
"""
env = DummyVecEnv([lambda: IdentityEnvBox(10)])
# create model
model = model_class('MlpPolicy', env, policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True)
model.learn(total_timesteps=500, eval_freq=250)
env.reset()
observations = np.array([env.step(env.action_space.sample())[0] for _ in range(10)])
observations = np.squeeze(observations)
# Get dictionary of current parameters
params = deepcopy(model.get_policy_parameters())
opt_params = deepcopy(model.get_opt_parameters())
# Modify all parameters to be random values
random_params = dict((param_name, th.rand_like(param)) for param_name, param in params.items())
# Update model parameters with the new random values
model.load_parameters(random_params, opt_params)
new_params = model.get_policy_parameters()
# Check that all params are different now
for k in params:
assert not th.allclose(params[k], new_params[k]), "Parameters did not change as expected."
params = new_params
# get selected actions
selected_actions = [model.predict(observation, deterministic=True) for observation in observations]
# Check
model.save("test_save.zip")
del model
model = model_class.load("test_save", env=env)
# check if params are still the same after load
new_params = model.get_policy_parameters()
# Check that all params are the same as before save load procedure now
for k in params:
assert th.allclose(params[k], new_params[k]), "Model parameters not the same after save and load."
# check if optimizer params are still the same after load
new_opt_params = model.get_opt_parameters()
# check if keys are the same
assert opt_params.keys() == new_opt_params.keys()
# check if values are the same: only tested for Adam and RMSProp so far
# comparing states not implemented so far. hashes of state_entries are not the same for equal tensors
# comparing every sub_entry does not work because of bool value of Tensor with more than one value is ambiguous
# so far only comparing param_lists
for optimizer, opt_state in opt_params.items():
for param_group_idx, param_group in enumerate(opt_state['param_groups']):
for param_key, param_value in param_group.items():
if param_key == 'params': # don't know how to handle params correctly, therefore only check if we have the same amount
assert len(param_value) == len(
new_opt_params[optimizer]['param_groups'][param_group_idx][param_key])
else:
assert param_value == new_opt_params[optimizer]['param_groups'][param_group_idx][param_key]
# check if model still selects the same actions
new_selected_actions = [model.predict(observation, deterministic=True) for observation in observations]
# for i in range(len(selected_actionsselected_actions)):
assert np.allclose(selected_actions, new_selected_actions)
# check if learn still works
model.learn(total_timesteps=1000, eval_freq=500)
# clear file from os
os.remove("test_save.zip")
@pytest.mark.parametrize("model_class", MODEL_LIST)
def test_set_env(model_class):
"""
Test if set_env function does work correct
:param model_class: (BaseRLModel) A RL model
"""
env = DummyVecEnv([lambda: IdentityEnvBox(10)])
env2 = DummyVecEnv([lambda: IdentityEnvBox(10)])
env3 = IdentityEnvBox(10)
# create model
model = model_class('MlpPolicy', env, policy_kwargs=dict(net_arch=[16]), create_eval_env=True)
# learn
model.learn(total_timesteps=1000, eval_freq=500)
# change env
model.set_env(env2)
# learn again
model.learn(total_timesteps=1000, eval_freq=500)
# change env test wrapping
model.set_env(env3)
# learn again
model.learn(total_timesteps=1000, eval_freq=500)
@pytest.mark.parametrize("model_class", MODEL_LIST)
def test_exclude_include_saved_params(model_class):
"""
Test if exclude and include parameters of save() work
:param model_class: (BaseRLModel) A RL model
"""
env = DummyVecEnv([lambda: IdentityEnvBox(10)])
# create model, set verbose as 2, which is not standard
model = model_class('MlpPolicy', env, policy_kwargs=dict(net_arch=[16]), verbose=2, create_eval_env=True)
# Check if exclude works
model.save("test_save.zip", exclude=["verbose"])
del model
model = model_class.load("test_save")
# check if verbose was not saved
assert model.verbose != 2
# set verbose as something different then standard settings
model.verbose = 2
# Check if include works
model.save("test_save.zip", exclude=["verbose"], include=["verbose"])
del model
model = model_class.load("test_save")
assert model.verbose == 2
# clear file from os
os.remove("test_save.zip")

View file

@ -59,8 +59,10 @@ class CustomGymEnv(gym.Env):
@pytest.mark.parametrize('vec_env_wrapper', VEC_ENV_WRAPPERS) @pytest.mark.parametrize('vec_env_wrapper', VEC_ENV_WRAPPERS)
def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper): def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper):
"""Test access to methods/attributes of vectorized environments""" """Test access to methods/attributes of vectorized environments"""
def make_env(): def make_env():
return CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2))) return CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2)))
vec_env = vec_env_class([make_env for _ in range(N_ENVS)]) vec_env = vec_env_class([make_env for _ in range(N_ENVS)])
if vec_env_wrapper is not None: if vec_env_wrapper is not None:
@ -92,7 +94,6 @@ def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper):
assert (env_method_subset[1] == np.ones((1, 3))).all() assert (env_method_subset[1] == np.ones((1, 3))).all()
assert len(env_method_subset) == 2 assert len(env_method_subset) == 2
# Test to change value for all the environments # Test to change value for all the environments
setattr_result = vec_env.set_attr('current_step', 42, indices=None) setattr_result = vec_env.set_attr('current_step', 42, indices=None)
getattr_result = vec_env.get_attr('current_step') getattr_result = vec_env.get_attr('current_step')
@ -193,8 +194,10 @@ SPACES = collections.OrderedDict([
('continuous', gym.spaces.Box(low=np.zeros(2), high=np.ones(2))), ('continuous', gym.spaces.Box(low=np.zeros(2), high=np.ones(2))),
]) ])
def check_vecenv_spaces(vec_env_class, space, obs_assert): def check_vecenv_spaces(vec_env_class, space, obs_assert):
"""Helper method to check observation spaces in vectorized environments.""" """Helper method to check observation spaces in vectorized environments."""
def make_env(): def make_env():
return CustomGymEnv(space) return CustomGymEnv(space)
@ -228,6 +231,7 @@ def test_vecenv_single_space(vec_env_class, space):
class _UnorderedDictSpace(gym.spaces.Dict): class _UnorderedDictSpace(gym.spaces.Dict):
"""Like DictSpace, but returns an unordered dict when sampling.""" """Like DictSpace, but returns an unordered dict when sampling."""
def sample(self): def sample(self):
return dict(super().sample()) return dict(super().sample())
@ -301,14 +305,17 @@ class CustomWrapperB(VecNormalize):
def name_test(self): def name_test(self):
return self.__class__ return self.__class__
class CustomWrapperBB(CustomWrapperB): class CustomWrapperBB(CustomWrapperB):
def __init__(self, venv): def __init__(self, venv):
CustomWrapperB.__init__(self, venv) CustomWrapperB.__init__(self, venv)
self.var_bb = 'bb' self.var_bb = 'bb'
def test_vecenv_wrapper_getattr(): def test_vecenv_wrapper_getattr():
def make_env(): def make_env():
return CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2))) return CustomGymEnv(gym.spaces.Box(low=np.zeros(2), high=np.ones(2)))
vec_env = DummyVecEnv([make_env for _ in range(N_ENVS)]) vec_env = DummyVecEnv([make_env for _ in range(N_ENVS)])
wrapped = CustomWrapperA(CustomWrapperBB(vec_env)) wrapped = CustomWrapperA(CustomWrapperBB(vec_env))
assert wrapped.var_a == 'a' assert wrapped.var_a == 'a'

View file

@ -190,16 +190,3 @@ class CEMRL(TD3):
self.es.tell(self.es_params, self.fitnesses) self.es.tell(self.es_params, self.fitnesses)
timesteps_since_eval += actor_steps timesteps_since_eval += actor_steps
return self return self
def save(self, path):
if not path.endswith('.pth'):
path += '.pth'
th.save(self.policy.state_dict(), path)
def load(self, path, env=None, **_kwargs):
if not path.endswith('.pth'):
path += '.pth'
if env is not None:
pass
self.policy.load_state_dict(th.load(path))
self._create_aliases()

View file

@ -1,6 +1,9 @@
import time import time
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from collections import deque from collections import deque
import os
import io
import zipfile
import gym import gym
import torch as th import torch as th
@ -11,6 +14,7 @@ from torchy_baselines.common.utils import set_random_seed, get_schedule_fn, upda
from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv, unwrap_vec_normalize
from torchy_baselines.common.monitor import Monitor from torchy_baselines.common.monitor import Monitor
from torchy_baselines.common import logger from torchy_baselines.common import logger
from torchy_baselines.common.save_util import data_to_json, json_to_data
class BaseRLModel(object): class BaseRLModel(object):
@ -42,9 +46,9 @@ class BaseRLModel(object):
verbose=0, device='auto', support_multi_env=False, verbose=0, device='auto', support_multi_env=False,
create_eval_env=False, monitor_wrapper=True, seed=None, use_sde=False): create_eval_env=False, monitor_wrapper=True, seed=None, use_sde=False):
if isinstance(policy, str) and policy_base is not None: if isinstance(policy, str) and policy_base is not None:
self.policy = get_policy_from_name(policy_base, policy) self.policy_class = get_policy_from_name(policy_base, policy)
else: else:
self.policy = policy self.policy_class = policy
if device == 'auto': if device == 'auto':
device = 'cuda' if th.cuda.is_available() else 'cpu' device = 'cuda' if th.cuda.is_available() else 'cpu'
@ -62,7 +66,6 @@ class BaseRLModel(object):
self.action_space = None self.action_space = None
self.n_envs = None self.n_envs = None
self.num_timesteps = 0 self.num_timesteps = 0
self.params = None
self.eval_env = None self.eval_env = None
self.replay_buffer = None self.replay_buffer = None
self.seed = seed self.seed = seed
@ -190,29 +193,63 @@ class BaseRLModel(object):
""" """
return self.env return self.env
@staticmethod
def check_env(env, observation_space, action_space):
"""
Checks the validity of the environment and returns if it is coherent
Checked parameters:
- observation_space
- action_space
:return: (bool) True if environment seems to be coherent
"""
if observation_space != env.observation_space:
return False
if action_space != env.action_space:
return False
# return true if no check failed
return True
def set_env(self, env): def set_env(self, env):
""" """
Checks the validity of the environment, and if it is coherent, set it as the current environment. Checks the validity of the environment, and if it is coherent, set it as the current environment.
Furthermore wrap any non vectorized env into a vectorized
checked parameters:
- observation_space
- action_space
:param env: (gym.Env) The environment for learning a policy :param env: (gym.Env) The environment for learning a policy
""" """
pass if self.check_env(env, self.observation_space, self.action_space) is False:
raise ValueError("The given environment is not compatible with model: observation and action spaces do not match")
def get_parameter_list(self): # it must be coherent now
""" # if it is not a VecEnv, make it a VecEnv
Get pytorch Variables of model's parameters if not isinstance(env, VecEnv):
if self.verbose >= 1:
This includes all variables necessary for continuing training (saving / loading). print("Wrapping the env in a DummyVecEnv.")
env = DummyVecEnv([lambda: env])
:return: (list) List of pytorch Variables self.n_envs = env.num_envs
""" self.env = env
pass
def get_parameters(self): def get_parameters(self):
""" """
Get current model parameters as dictionary of variable name -> ndarray. Returns policy and optimizer parameters as a tuple
:return: (dict,dict) policy_parameters, opt_parameters
"""
return self.get_policy_parameters(), self.get_opt_parameters()
:return: (OrderedDict) Dictionary of variable name -> ndarray of model's parameters. def get_policy_parameters(self):
"""
Get current model policy parameters as dictionary of variable name -> tensors.
:return: (dict) Dictionary of variable name -> tensor of model's policy parameters.
"""
return self.policy.state_dict()
@abstractmethod
def get_opt_parameters(self):
"""
Get current model optimizer parameters as dictionary of variable names -> tensors
:return: (dict) Dictionary of variable name -> tensor of model's optimizer parameters
""" """
raise NotImplementedError() raise NotImplementedError()
@ -266,50 +303,121 @@ class BaseRLModel(object):
""" """
pass pass
def load_parameters(self, load_path_or_dict, exact_match=True): def load_parameters(self, load_dict, opt_params=None):
""" """
Load model parameters from a file or a dictionary Load model parameters from a dictionary
load_dict should contain all keys from torch.model.state_dict()
If opt_params are given this does also load agent's optimizer-parameters, but can only be handled in child classes.
Dictionary keys should be tensorflow variable names, which can be obtained
with ``get_parameters`` function. If ``exact_match`` is True, dictionary
should contain keys for all model's parameters, otherwise RunTimeError
is raised. If False, only variables included in the dictionary will be updated.
This does not load agent's hyper-parameters. :param load_dict: (dict) dict of parameters from model.state_dict()
:param opt_params: (dict of dicts) dict of optimizer state_dicts should be handled in child_class
.. warning::
This function does not update trainer/optimizer variables (e.g. momentum).
As such training after using this function may lead to less-than-optimal results.
:param load_path_or_dict: (str or file-like or dict) Save parameter location
or dict of parameters as variable.name -> ndarrays to be loaded.
:param exact_match: (bool) If True, expects load dictionary to contain keys for
all variables in the model. If False, loads parameters only for variables
mentioned in the dictionary. Defaults to True.
""" """
raise NotImplementedError() if opt_params is not None:
raise ValueError("Optimizer Parameters where given but no overloaded load function exists for this class")
@abstractmethod self.policy.load_state_dict(load_dict)
def save(self, save_path):
"""
Save the current parameters to file
:param save_path: (str or file-like object) the save location
"""
raise NotImplementedError()
@classmethod @classmethod
@abstractmethod
def load(cls, load_path, env=None, **kwargs): def load(cls, load_path, env=None, **kwargs):
""" """
Load the model from file Load the model from a zip-file
:param load_path: (str or file-like) the saved parameter location :param load_path: (str) the location of the saved data
:param env: (Gym Envrionment) the new environment to run the loaded model on :param env: (Gym Envrionment) the new environment to run the loaded model on
(can be None if you only need prediction from a trained model) (can be None if you only need prediction from a trained model) has priority over any saved environment
:param kwargs: extra arguments to change the model when loading :param kwargs: extra arguments to change the model when loading
""" """
raise NotImplementedError() data, params, opt_params = cls._load_from_file(load_path)
if 'policy_kwargs' in kwargs and kwargs['policy_kwargs'] != data['policy_kwargs']:
raise ValueError("The specified policy kwargs do not equal the stored policy kwargs."
"Stored kwargs: {}, specified kwargs: {}".format(data['policy_kwargs'],
kwargs['policy_kwargs']))
# check if observation space and action space are part of the saved parameters
if ("observation_space" not in data or "action_space" not in data) and "env" not in data:
raise ValueError("The observation_space and action_space was not given, can't verify new environments")
# check if given env is valid
if env is not None and cls.check_env(env, data["observation_space"], data["action_space"]) is False:
raise ValueError("The given environment does not comply to the model")
# if no new env was given use stored env if possible
if env is None and "env" in data:
env = data["env"]
# first create model, but only setup if a env was given
model = cls(policy=data["policy_class"], env=env, _init_setup_model=env is not None)
# load parameters
model.__dict__.update(data)
model.__dict__.update(kwargs)
model.load_parameters(params, opt_params)
return model
@staticmethod
def _load_from_file(load_path, load_data=True):
""" Load model data from a .zip archive
:param load_path: (str) Where to load the model from
:param load_data: (bool) Whether we should load and return data
(class parameters). Mainly used by 'load_parameters' to only load model parameters (weights)
:return: (dict),(dict),(dict) Class parameters, model parameters (state_dict) and dict of optimizer parameters (dict of state_dict)
"""
# Check if file exists if load_path is a string
if isinstance(load_path, str):
if not os.path.exists(load_path):
if os.path.exists(load_path + ".zip"):
load_path += ".zip"
else:
raise ValueError("Error: the file {} could not be found".format(load_path))
# Open the zip archive and load data
try:
with zipfile.ZipFile(load_path, "r") as archive:
namelist = archive.namelist()
# If data or parameters is not in the
# zip archive, assume they were stored
# as None (_save_to_file_zip allows this).
data = None
params = None
opt_params = None
if "data" in namelist and load_data:
# Load class parameters and convert to string
json_data = archive.read("data").decode()
data = json_to_data(json_data)
if "params.pth" in namelist:
# Load parameters with build in torch function
with archive.open("params.pth", mode="r") as param_file:
# File has to be seekable, but param_file is not, so load in BytesIO first
# fixed in python >= 3.7
file_content = io.BytesIO()
file_content.write(param_file.read())
# go to start of file
file_content.seek(0)
params = th.load(file_content)
# check for all other .pth files
other_files = [file_name for file_name in namelist if
os.path.splitext(file_name)[1] == ".pth" and file_name != "params.pth"]
# if there are any other files which end with .pth and aren't "params.pth"
# assume that they each are optimizer parameters
if len(other_files) > 0:
opt_params = dict()
for file_path in other_files:
with archive.open(file_path, mode="r") as opt_param_file:
# File has to be seekable, but opt_param_file is not, so load in BytesIO first
# fixed in python >= 3.7
file_content = io.BytesIO()
file_content.write(opt_param_file.read())
# go to start of file
file_content.seek(0)
# save the parameters in dict with file name but trim file ending
opt_params[os.path.splitext(file_path)[0]] = th.load(file_content)
except zipfile.BadZipFile:
# load_path wasn't a zip file
raise ValueError("Error: the file {} wasn't a zip-file".format(load_path))
return data, params, opt_params
def set_random_seed(self, seed=None): def set_random_seed(self, seed=None):
""" """
@ -481,7 +589,7 @@ class BaseRLModel(object):
# Display training infos # Display training infos
if self.verbose >= 1 and log_interval is not None and ( if self.verbose >= 1 and log_interval is not None and (
episode_num + total_episodes) % log_interval == 0: episode_num + total_episodes) % log_interval == 0:
fps = int(num_timesteps / (time.time() - self.start_time)) fps = int(num_timesteps / (time.time() - self.start_time))
logger.logkv("episodes", episode_num + total_episodes) logger.logkv("episodes", episode_num + total_episodes)
if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0: if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0:
@ -514,3 +622,79 @@ class BaseRLModel(object):
self.rollout_data['returns'][step] = last_return self.rollout_data['returns'][step] = last_return
return mean_reward, total_steps, total_episodes, obs return mean_reward, total_steps, total_episodes, obs
@staticmethod
def _save_to_file_zip(save_path, data=None, params=None, opt_params=None):
"""Save model to a zip archive
:param save_path: (str) Where to store the model
:param data: (dict) Class parameters being stored
:param params: (dict) Model parameters being stored expected to be state_dict
:param opt_params: (dict) Optimizer parameters being stored expected to contain an entry for every
optimizer with its name and the state_dict
"""
# data/params can be None, so do not
# try to serialize them blindly
if data is not None:
serialized_data = data_to_json(data)
# Check postfix if save_path is a string
if isinstance(save_path, str):
_, ext = os.path.splitext(save_path)
if ext == "":
save_path += ".zip"
# Create a zip-archive and write our objects
# there. This works when save_path is either
# str or a file-like
with zipfile.ZipFile(save_path, "w") as archive:
# Do not try to save "None" elements
if data is not None:
archive.writestr("data", serialized_data)
if params is not None:
with archive.open('params.pth', mode="w") as param_file:
th.save(params, param_file)
if opt_params is not None:
for file_name, dict in opt_params.items():
with archive.open(file_name + '.pth', mode="w") as opt_param_file:
th.save(dict, opt_param_file)
@staticmethod
def excluded_save_params():
"""
Returns the names of the parameters that should be excluded by default
when saving the model.
:return: ([str]) List of parameters that should be excluded from save
"""
return ["env", "eval_env", "replay_buffer", "rollout_buffer", "_vec_normalize_env"]
def save(self, path, exclude=None, include=None):
"""
Save all the attributes of the object and the model parameters in a zip-file.
:param path: (str) path to the file where the rl agent should be saved
:param exclude: ([str]) name of parameters that should be excluded in addition to the default one
:param include: ([str]) name of parameters that might be excluded but should be included anyway
"""
# copy parameter list so we don't mutate the original dict
data = self.__dict__.copy()
# use standard list of excluded parameters if none given
if exclude is None:
exclude = self.excluded_save_params()
else:
# append standard exclude params to the given params
exclude.extend([param for param in self.excluded_save_params() if param not in exclude])
# do not exclude params if they are specifically included
if include is not None:
exclude = [param_name for param_name in exclude if param_name not in include]
# remove parameter entries of parameters which are to be excluded
for param_name in exclude:
if param_name in data:
data.pop(param_name, None)
params_to_save = self.get_policy_parameters()
opt_params_to_save = self.get_opt_parameters()
self._save_to_file_zip(path, data=data, params=params_to_save, opt_params=opt_params_to_save)

View file

@ -0,0 +1,105 @@
import numpy as np
from gym import Env
from gym.spaces import Discrete, MultiDiscrete, MultiBinary, Box
class IdentityEnv(Env):
def __init__(self, dim, ep_length=100):
"""
Identity environment for testing purposes
:param dim: (int) the size of the dimensions you want to learn
:param ep_length: (int) the length of each episodes in timesteps
"""
self.action_space = Discrete(dim)
self.observation_space = self.action_space
self.ep_length = ep_length
self.current_step = 0
self.dim = dim
self.reset()
def reset(self):
self.current_step = 0
self._choose_next_state()
return self.state
def step(self, action):
reward = self._get_reward(action)
self._choose_next_state()
self.current_step += 1
done = self.current_step >= self.ep_length
return self.state, reward, done, {}
def _choose_next_state(self):
self.state = self.action_space.sample()
def _get_reward(self, action):
return 1 if np.all(self.state == action) else 0
def render(self, mode='human'):
pass
class IdentityEnvBox(IdentityEnv):
def __init__(self, low=-1, high=1, eps=0.05, ep_length=100):
"""
Identity environment for testing purposes
:param dim: (int) the size of the dimensions you want to learn
:param low: (float) the lower bound of the box dim
:param high: (float) the upper bound of the box dim
:param eps: (float) the epsilon bound for correct value
:param ep_length: (int) the length of each episodes in timesteps
"""
super(IdentityEnvBox, self).__init__(1, ep_length)
self.action_space = Box(low=low, high=high, shape=(1,), dtype=np.float32)
self.observation_space = self.action_space
self.eps = eps
self.reset()
def reset(self):
self.current_step = 0
self._choose_next_state()
return self.state
def step(self, action):
reward = self._get_reward(action)
self._choose_next_state()
self.current_step += 1
done = self.current_step >= self.ep_length
return self.state, reward, done, {}
def _choose_next_state(self):
self.state = self.observation_space.sample()
def _get_reward(self, action):
return 1 if (self.state - self.eps) <= action <= (self.state + self.eps) else 0
class IdentityEnvMultiDiscrete(IdentityEnv):
def __init__(self, dim, ep_length=100):
"""
Identity environment for testing purposes
:param dim: (int) the size of the dimensions you want to learn
:param ep_length: (int) the length of each episodes in timesteps
"""
super(IdentityEnvMultiDiscrete, self).__init__(dim, ep_length)
self.action_space = MultiDiscrete([dim, dim])
self.observation_space = self.action_space
self.reset()
class IdentityEnvMultiBinary(IdentityEnv):
def __init__(self, dim, ep_length=100):
"""
Identity environment for testing purposes
:param dim: (int) the size of the dimensions you want to learn
:param ep_length: (int) the length of each episodes in timesteps
"""
super(IdentityEnvMultiBinary, self).__init__(dim, ep_length)
self.action_space = MultiBinary(dim)
self.observation_space = self.action_space
self.reset()

View file

@ -0,0 +1,134 @@
"""
Save util taken from stable_baselines
used to serialize data (class parameters) of model classes
"""
import json
import base64
import pickle
import cloudpickle
def is_json_serializable(item):
"""
Test if an object is serializable into JSON
:param item: (object) The object to be tested for JSON serialization.
:return: (bool) True if object is JSON serializable, false otherwise.
"""
# Try with try-except struct.
json_serializable = True
try:
_ = json.dumps(item)
except TypeError:
json_serializable = False
return json_serializable
def data_to_json(data):
"""
Turn data (class parameters) into a JSON string for storing
:param data: (Dict) Dictionary of class parameters to be
stored. Items that are not JSON serializable will be
pickled with Cloudpickle and stored as bytearray in
the JSON file
:return: (str) JSON string of the data serialized.
"""
# First, check what elements can not be JSONfied,
# and turn them into byte-strings
serializable_data = {}
for data_key, data_item in data.items():
# See if object is JSON serializable
if is_json_serializable(data_item):
# All good, store as it is
serializable_data[data_key] = data_item
else:
# Not serializable, cloudpickle it into
# bytes and convert to base64 string for storing.
# Also store type of the class for consumption
# from other languages/humans, so we have an
# idea what was being stored.
base64_encoded = base64.b64encode(
cloudpickle.dumps(data_item)
).decode()
# Use ":" to make sure we do
# not override these keys
# when we include variables of the object later
cloudpickle_serialization = {
":type:": str(type(data_item)),
":serialized:": base64_encoded
}
# Add first-level JSON-serializable items of the
# object for further details (but not deeper than this to
# avoid deep nesting).
# First we check that object has attributes (not all do,
# e.g. numpy scalars)
if hasattr(data_item, "__dict__") or isinstance(data_item, dict):
# Take elements from __dict__ for custom classes
item_generator = (
data_item.items if isinstance(data_item, dict) else data_item.__dict__.items
)
for variable_name, variable_item in item_generator():
# Check if serializable. If not, just include the
# string-representation of the object.
if is_json_serializable(variable_item):
cloudpickle_serialization[variable_name] = variable_item
else:
cloudpickle_serialization[variable_name] = str(variable_item)
serializable_data[data_key] = cloudpickle_serialization
json_string = json.dumps(serializable_data, indent=4)
return json_string
def json_to_data(json_string, custom_objects=None):
"""
Turn JSON serialization of class-parameters back into dictionary.
:param json_string: (str) JSON serialization of the class-parameters
that should be loaded.
:param custom_objects: (dict) Dictionary of objects to replace
upon loading. If a variable is present in this dictionary as a
key, it will not be deserialized and the corresponding item
will be used instead. Similar to custom_objects in
`keras.models.load_model`. Useful when you have an object in
file that can not be deserialized.
:return: (dict) Loaded class parameters.
"""
if custom_objects is not None and not isinstance(custom_objects, dict):
raise ValueError("custom_objects argument must be a dict or None")
json_dict = json.loads(json_string)
# This will be filled with deserialized data
return_data = {}
for data_key, data_item in json_dict.items():
if custom_objects is not None and data_key in custom_objects.keys():
# If item is provided in custom_objects, replace
# the one from JSON with the one in custom_objects
return_data[data_key] = custom_objects[data_key]
elif isinstance(data_item, dict) and ":serialized:" in data_item.keys():
# If item is dictionary with ":serialized:"
# key, this means it is serialized with cloudpickle.
serialization = data_item[":serialized:"]
# Try-except deserialization in case we run into
# errors. If so, we can tell bit more information to
# user.
try:
deserialized_object = cloudpickle.loads(
base64.b64decode(serialization.encode())
)
except pickle.UnpicklingError:
raise RuntimeError(
"Could not deserialize object {}. ".format(data_key) +
"Consider using `custom_objects` argument to replace " +
"this object."
)
return_data[data_key] = deserialized_object
else:
# Read as it is
return_data[data_key] = data_item
return return_data

View file

@ -42,6 +42,7 @@ def _worker(remote, parent_remote, env_fn_wrapper):
except EOFError: except EOFError:
break break
def tile_images(img_nhwc): def tile_images(img_nhwc):
""" """
Tile N images into one big PxQ image Tile N images into one big PxQ image
@ -68,7 +69,6 @@ def tile_images(img_nhwc):
return out_image return out_image
class SubprocVecEnv(VecEnv): class SubprocVecEnv(VecEnv):
""" """
Creates a multiprocess vectorized wrapper for multiple environments, distributing each environment to its own Creates a multiprocess vectorized wrapper for multiple environments, distributing each environment to its own

View file

@ -119,9 +119,9 @@ class PPO(BaseRLModel):
self.rollout_buffer = RolloutBuffer(self.n_steps, state_dim, action_dim, self.device, self.rollout_buffer = RolloutBuffer(self.n_steps, state_dim, action_dim, self.device,
gamma=self.gamma, gae_lambda=self.gae_lambda, n_envs=self.n_envs) gamma=self.gamma, gae_lambda=self.gae_lambda, n_envs=self.n_envs)
self.policy = self.policy(self.observation_space, self.action_space, self.policy = self.policy_class(self.observation_space, self.action_space,
self.learning_rate, use_sde=self.use_sde, device=self.device, self.learning_rate, use_sde=self.use_sde, device=self.device,
**self.policy_kwargs) **self.policy_kwargs)
self.policy = self.policy.to(self.device) self.policy = self.policy.to(self.device)
self.clip_range = get_schedule_fn(self.clip_range) self.clip_range = get_schedule_fn(self.clip_range)
@ -308,14 +308,22 @@ class PPO(BaseRLModel):
return self return self
def save(self, path): def get_opt_parameters(self):
if not path.endswith('.pth'): """
path += '.pth' Returns a dict of all the optimizers and their parameters
th.save(self.policy.state_dict(), path)
:return: (dict) of optimizer names and their state_dict
"""
return {"opt": self.policy.optimizer.state_dict()}
def load(self, path, env=None, **_kwargs): def load_parameters(self, load_dict, opt_params):
if not path.endswith('.pth'): """
path += '.pth' Load model parameters and optimizer parameters from a dictionary
if env is not None: load_dict should contain all keys from torch.model.state_dict()
pass This does not load agent's hyper-parameters.
self.policy.load_state_dict(th.load(path))
:param load_dict: (dict) dict of parameters from model.state_dict()
:param opt_params: (dict of dicts) dict of optimizer state_dicts
"""
self.policy.optimizer.load_state_dict(opt_params["opt"])
self.policy.load_state_dict(load_dict)

View file

@ -56,6 +56,7 @@ class SAC(BaseRLModel):
Setting it to auto, the code will be run on the GPU if possible. Setting it to auto, the code will be run on the GPU if possible.
:param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
""" """
def __init__(self, policy, env, learning_rate=3e-4, buffer_size=int(1e6), def __init__(self, policy, env, learning_rate=3e-4, buffer_size=int(1e6),
learning_starts=100, batch_size=256, learning_starts=100, batch_size=256,
tau=0.005, ent_coef='auto', target_update_interval=1, tau=0.005, ent_coef='auto', target_update_interval=1,
@ -128,8 +129,8 @@ class SAC(BaseRLModel):
self.ent_coef = th.tensor(float(self.ent_coef)).to(self.device) self.ent_coef = th.tensor(float(self.ent_coef)).to(self.device)
self.replay_buffer = ReplayBuffer(self.buffer_size, obs_dim, action_dim, self.device) self.replay_buffer = ReplayBuffer(self.buffer_size, obs_dim, action_dim, self.device)
self.policy = self.policy(self.observation_space, self.action_space, learning_rate=self.learning_rate, self.policy = self.policy_class(self.observation_space, self.action_space,
use_sde=self.use_sde, device=self.device, **self.policy_kwargs) self.learning_rate, use_sde=self.use_sde, device=self.device, **self.policy_kwargs)
self.policy = self.policy.to(self.device) self.policy = self.policy.to(self.device)
self._create_aliases() self._create_aliases()
@ -295,15 +296,28 @@ class SAC(BaseRLModel):
return self return self
def save(self, path): def get_opt_parameters(self):
if not path.endswith('.pth'): """
path += '.pth' Returns a dict of all the optimizers and their parameters
th.save(self.policy.state_dict(), path)
def load(self, path, env=None, **_kwargs): :return: (Dict) of optimizer names and their state_dict
if not path.endswith('.pth'): """
path += '.pth' opt_dict = {"actor": self.actor.optimizer.state_dict(), "critic": self.critic.optimizer.state_dict()}
if env is not None: if self.ent_coef_optimizer is not None:
pass opt_dict.update({"ent_coef_optimizer": self.ent_coef_optimizer.state_dict()})
self.policy.load_state_dict(th.load(path)) return opt_dict
self._create_aliases()
def load_parameters(self, load_dict, opt_params):
"""
Load model parameters and optimizer parameters from a dictionary
load_dict should contain all keys from torch.model.state_dict()
This does not load agent's hyper-parameters.
:param load_dict: (dict) dict of parameters from model.state_dict()
:param opt_params: (dict of dicts) dict of optimizer state_dicts should be handled in child_class
"""
self.actor.optimizer.load_state_dict(opt_params["actor"])
self.critic.optimizer.load_state_dict(opt_params["critic"])
if "ent_coef_optimizer" in opt_params:
self.ent_coef_optimizer.load_state_dict(opt_params["ent_coef_optimizer"])
self.policy.load_state_dict(load_dict)

View file

@ -92,8 +92,8 @@ class TD3(BaseRLModel):
obs_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0] obs_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0]
self.set_random_seed(self.seed) self.set_random_seed(self.seed)
self.replay_buffer = ReplayBuffer(self.buffer_size, obs_dim, action_dim, self.device) self.replay_buffer = ReplayBuffer(self.buffer_size, obs_dim, action_dim, self.device)
self.policy = self.policy(self.observation_space, self.action_space, self.learning_rate, self.policy = self.policy_class(self.observation_space, self.action_space,
use_sde=self.use_sde, device=self.device, **self.policy_kwargs) self.learning_rate, use_sde=self.use_sde, device=self.device, **self.policy_kwargs)
self.policy = self.policy.to(self.device) self.policy = self.policy.to(self.device)
self._create_aliases() self._create_aliases()
@ -161,7 +161,9 @@ class TD3(BaseRLModel):
for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()): for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data) target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
def train_actor(self, gradient_steps=1, batch_size=100, tau_actor=0.005, tau_critic=0.005, replay_data=None): def train_actor(self, gradient_steps=1, batch_size=100, tau_actor=0.005,
tau_critic=0.005,
replay_data=None):
# Update optimizer learning rate # Update optimizer learning rate
self._update_learning_rate(self.actor.optimizer) self._update_learning_rate(self.actor.optimizer)
@ -294,15 +296,23 @@ class TD3(BaseRLModel):
return self return self
def save(self, path): def get_opt_parameters(self):
if not path.endswith('.pth'): """
path += '.pth' Returns a dict of all the optimizers and their parameters
th.save(self.policy.state_dict(), path)
def load(self, path, env=None, **_kwargs): :return: (Dict) of optimizer names and their state_dict
if not path.endswith('.pth'): """
path += '.pth' return {"actor": self.actor.optimizer.state_dict(), "critic": self.critic.optimizer.state_dict()}
if env is not None:
pass def load_parameters(self, load_dict, opt_params):
self.policy.load_state_dict(th.load(path)) """
self._create_aliases() Load model parameters and optimizer parameters from a dictionary
load_dict should contain all keys from torch.model.state_dict()
This does not load agent's hyper-parameters.
:param load_dict: (dict) dict of parameters from model.state_dict()
:param opt_params: (dict of dicts) dict of optimizer state_dicts should be handled in child_class
"""
self.actor.optimizer.load_state_dict(opt_params["actor"])
self.critic.optimizer.load_state_dict(opt_params["critic"])
self.policy.load_state_dict(load_dict)