Independent save/load for policies

This commit is contained in:
Antonin RAFFIN 2020-04-20 15:59:44 +02:00
parent 864c976d4d
commit f347474e6a
4 changed files with 63 additions and 12 deletions

View file

@ -3,16 +3,19 @@
Changelog Changelog
========== ==========
Pre-Release 0.5.0a0 (WIP) Pre-Release 0.5.0a1 (WIP)
------------------------------ ------------------------------
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- Previous loading of policy weights is broken and replace by the new saving/loading for policy
New Features: New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
- Added ``optimizer`` and ``optimizer_kwargs`` to ``policy_kwargs`` in order to easily - Added ``optimizer`` and ``optimizer_kwargs`` to ``policy_kwargs`` in order to easily
customizer optimizers customizer optimizers
- Complete independent save/load for policies
Bug Fixes: Bug Fixes:
^^^^^^^^^^ ^^^^^^^^^^

View file

@ -180,9 +180,11 @@ def test_save_load_policy(model_class):
observations = observations.reshape(10, -1) observations = observations.reshape(10, -1)
policy = model.policy policy = model.policy
actor = None policy_class = policy.__class__
actor, actor_class = None, None
if model_class in [SAC, TD3]: if model_class in [SAC, TD3]:
actor = policy.actor actor = policy.actor
actor_class = actor.__class__
# Get dictionary of current parameters # Get dictionary of current parameters
params = deepcopy(policy.state_dict()) params = deepcopy(policy.state_dict())
@ -207,14 +209,17 @@ def test_save_load_policy(model_class):
selected_actions_actor, _ = actor.predict(observations, deterministic=True) selected_actions_actor, _ = actor.predict(observations, deterministic=True)
# Save and load policy # Save and load policy
policy.save("./logs/policy_weights.pkl") policy.save("./logs/policy.pkl")
# Save and load actor # Save and load actor
if actor is not None: if actor is not None:
actor.save("./logs/actor_weights.pkl") actor.save("./logs/actor.pkl")
policy.load("./logs/policy_weights.pkl") del policy, actor
if actor is not None:
actor.load("./logs/actor_weights.pkl")
policy = policy_class.load("./logs/policy.pkl")
if actor_class is not None:
actor = actor_class.load("./logs/actor.pkl")
# check if params are still the same after load # check if params are still the same after load
new_params = policy.state_dict() new_params = policy.state_dict()
@ -227,12 +232,12 @@ def test_save_load_policy(model_class):
new_selected_actions, _ = policy.predict(observations, deterministic=True) new_selected_actions, _ = policy.predict(observations, deterministic=True)
assert np.allclose(selected_actions, new_selected_actions, 1e-4) assert np.allclose(selected_actions, new_selected_actions, 1e-4)
if actor is not None: if actor_class is not None:
new_selected_actions_actor, _ = actor.predict(observations, deterministic=True) new_selected_actions_actor, _ = actor.predict(observations, deterministic=True)
assert np.allclose(selected_actions_actor, new_selected_actions_actor, 1e-4) assert np.allclose(selected_actions_actor, new_selected_actions_actor, 1e-4)
assert np.allclose(selected_actions_actor, new_selected_actions, 1e-4) assert np.allclose(selected_actions_actor, new_selected_actions, 1e-4)
# clear file from os # clear file from os
os.remove("./logs/policy_weights.pkl") os.remove("./logs/policy.pkl")
if actor is not None: if actor_class is not None:
os.remove("./logs/actor_weights.pkl") os.remove("./logs/actor.pkl")

View file

@ -203,7 +203,7 @@ class BasePolicy(nn.Module):
return dict( return dict(
observation_space=self.observation_space, observation_space=self.observation_space,
action_space=self.action_space, action_space=self.action_space,
# Passed to the constructor by child classes # Passed to the constructor by child class
# squash_output=self.squash_output, # squash_output=self.squash_output,
# features_extractor=self.features_extractor # features_extractor=self.features_extractor
normalize_images=self.normalize_images, normalize_images=self.normalize_images,

View file

@ -66,6 +66,15 @@ class Actor(BasePolicy):
self.latent_pi = nn.Sequential(*latent_pi_net) self.latent_pi = nn.Sequential(*latent_pi_net)
self.use_sde = use_sde self.use_sde = use_sde
self.sde_features_extractor = None self.sde_features_extractor = None
self.sde_net_arch = sde_net_arch
self.net_arch = net_arch
self.features_dim = features_dim
self.activation_fn = activation_fn
self.log_std_init = log_std_init
self.sde_net_arch = sde_net_arch
self.use_expln = use_expln
self.full_std = full_std
self.clip_mean = clip_mean
if self.use_sde: if self.use_sde:
latent_sde_dim = net_arch[-1] latent_sde_dim = net_arch[-1]
@ -88,6 +97,23 @@ class Actor(BasePolicy):
self.mu = nn.Linear(net_arch[-1], action_dim) self.mu = nn.Linear(net_arch[-1], action_dim)
self.log_std = nn.Linear(net_arch[-1], action_dim) self.log_std = nn.Linear(net_arch[-1], action_dim)
def _get_data(self) -> Dict[str, Any]:
data = super()._get_data()
data.update(dict(
net_arch=self.net_arch,
features_dim=self.features_dim,
activation_fn=self.activation_fn,
use_sde=self.use_sde,
log_std_init=self.log_std_init,
full_std=self.full_std,
sde_net_arch=self.sde_net_arch,
use_expln=self.use_expln,
features_extractor=self.features_extractor,
clip_mean=self.clip_mean
))
return data
def get_std(self) -> th.Tensor: def get_std(self) -> th.Tensor:
""" """
Retrieve the standard deviation of the action distribution. Retrieve the standard deviation of the action distribution.
@ -286,6 +312,23 @@ class SACPolicy(BasePolicy):
self.critic.optimizer = self.optimizer_class(self.critic.parameters(), lr=lr_schedule(1), self.critic.optimizer = self.optimizer_class(self.critic.parameters(), lr=lr_schedule(1),
**self.optimizer_kwargs) **self.optimizer_kwargs)
def _get_data(self) -> Dict[str, Any]:
data = super()._get_data()
data.update(dict(
net_arch=self.net_args['net_arch'],
activation_fn=self.net_args['activation_fn'],
use_sde=self.actor_kwargs['use_sde'],
log_std_init=self.actor_kwargs['log_std_init'],
sde_net_arch=self.actor_kwargs['sde_net_arch'],
use_expln=self.actor_kwargs['use_expln'],
clip_mean=self.actor_kwargs['clip_mean'],
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
optimizer=self.optimizer_class,
optimizer_kwargs=self.optimizer_kwargs
))
return data
def make_actor(self) -> Actor: def make_actor(self) -> Actor:
return Actor(**self.actor_kwargs).to(self.device) return Actor(**self.actor_kwargs).to(self.device)