Remove sde_net_arch + Simplify policy (#584)

* Remove `sde_net_arch` + Simplify policy

* Add warning at load time
This commit is contained in:
Antonin RAFFIN 2021-09-28 21:32:54 +02:00 committed by GitHub
parent 89af49ca91
commit 201fbffa8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 59 additions and 93 deletions

View file

@ -4,16 +4,19 @@ Changelog
========== ==========
Release 1.2.1a1 (WIP) Release 1.2.1a2 (WIP)
--------------------------- ---------------------------
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- ``sde_net_arch`` argument in policies is deprecated and will be removed in a future version.
- ``_get_latent`` (``ActorCriticPolicy``) was removed
New Features: New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
- Added methods ``get_distribution`` and ``predict_values`` for ``ActorCriticPolicy`` for A2C/PPO/TRPO (@cyprienc) - Added methods ``get_distribution`` and ``predict_values`` for ``ActorCriticPolicy`` for A2C/PPO/TRPO (@cyprienc)
- Added methods ``forward_actor`` and ``forward_critic`` for ``MlpExtractor``
Bug Fixes: Bug Fixes:
^^^^^^^^^^ ^^^^^^^^^^

View file

@ -2,6 +2,7 @@
import collections import collections
import copy import copy
import warnings
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from functools import partial from functools import partial
from typing import Any, Dict, List, Optional, Tuple, Type, Union from typing import Any, Dict, List, Optional, Tuple, Type, Union
@ -171,6 +172,15 @@ class BaseModel(nn.Module, ABC):
""" """
device = get_device(device) device = get_device(device)
saved_variables = th.load(path, map_location=device) saved_variables = th.load(path, map_location=device)
# Allow to load policy saved with older version of SB3
if "sde_net_arch" in saved_variables["data"]:
warnings.warn(
"sde_net_arch is deprecated, please downgrade to SB3 v1.2.0 if you need such parameter.",
DeprecationWarning,
)
del saved_variables["data"]["sde_net_arch"]
# Create policy object # Create policy object
model = cls(**saved_variables["data"]) # pytype: disable=not-instantiable model = cls(**saved_variables["data"]) # pytype: disable=not-instantiable
# Load weights # Load weights
@ -458,11 +468,12 @@ class ActorCriticPolicy(BasePolicy):
"full_std": full_std, "full_std": full_std,
"squash_output": squash_output, "squash_output": squash_output,
"use_expln": use_expln, "use_expln": use_expln,
"learn_features": sde_net_arch is not None, "learn_features": False,
} }
self.sde_features_extractor = None if sde_net_arch is not None:
self.sde_net_arch = sde_net_arch warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
self.use_sde = use_sde self.use_sde = use_sde
self.dist_kwargs = dist_kwargs self.dist_kwargs = dist_kwargs
@ -484,7 +495,6 @@ class ActorCriticPolicy(BasePolicy):
log_std_init=self.log_std_init, log_std_init=self.log_std_init,
squash_output=default_none_kwargs["squash_output"], squash_output=default_none_kwargs["squash_output"],
full_std=default_none_kwargs["full_std"], full_std=default_none_kwargs["full_std"],
sde_net_arch=self.sde_net_arch,
use_expln=default_none_kwargs["use_expln"], use_expln=default_none_kwargs["use_expln"],
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
ortho_init=self.ortho_init, ortho_init=self.ortho_init,
@ -531,26 +541,15 @@ class ActorCriticPolicy(BasePolicy):
latent_dim_pi = self.mlp_extractor.latent_dim_pi latent_dim_pi = self.mlp_extractor.latent_dim_pi
# Separate features extractor for gSDE
if self.sde_net_arch is not None:
self.sde_features_extractor, latent_sde_dim = create_sde_features_extractor(
self.features_dim, self.sde_net_arch, self.activation_fn
)
if isinstance(self.action_dist, DiagGaussianDistribution): if isinstance(self.action_dist, DiagGaussianDistribution):
self.action_net, self.log_std = self.action_dist.proba_distribution_net( self.action_net, self.log_std = self.action_dist.proba_distribution_net(
latent_dim=latent_dim_pi, log_std_init=self.log_std_init latent_dim=latent_dim_pi, log_std_init=self.log_std_init
) )
elif isinstance(self.action_dist, StateDependentNoiseDistribution): elif isinstance(self.action_dist, StateDependentNoiseDistribution):
latent_sde_dim = latent_dim_pi if self.sde_net_arch is None else latent_sde_dim
self.action_net, self.log_std = self.action_dist.proba_distribution_net( self.action_net, self.log_std = self.action_dist.proba_distribution_net(
latent_dim=latent_dim_pi, latent_sde_dim=latent_sde_dim, log_std_init=self.log_std_init latent_dim=latent_dim_pi, latent_sde_dim=latent_dim_pi, log_std_init=self.log_std_init
) )
elif isinstance(self.action_dist, CategoricalDistribution): elif isinstance(self.action_dist, (CategoricalDistribution, MultiCategoricalDistribution, BernoulliDistribution)):
self.action_net = self.action_dist.proba_distribution_net(latent_dim=latent_dim_pi)
elif isinstance(self.action_dist, MultiCategoricalDistribution):
self.action_net = self.action_dist.proba_distribution_net(latent_dim=latent_dim_pi)
elif isinstance(self.action_dist, BernoulliDistribution):
self.action_net = self.action_dist.proba_distribution_net(latent_dim=latent_dim_pi) self.action_net = self.action_dist.proba_distribution_net(latent_dim=latent_dim_pi)
else: else:
raise NotImplementedError(f"Unsupported distribution '{self.action_dist}'.") raise NotImplementedError(f"Unsupported distribution '{self.action_dist}'.")
@ -583,39 +582,21 @@ class ActorCriticPolicy(BasePolicy):
:param deterministic: Whether to sample or use deterministic actions :param deterministic: Whether to sample or use deterministic actions
:return: action, value and log probability of the action :return: action, value and log probability of the action
""" """
latent_pi, latent_vf, latent_sde = self._get_latent(obs) # Preprocess the observation if needed
features = self.extract_features(obs)
latent_pi, latent_vf = self.mlp_extractor(features)
# Evaluate the values for the given observations # Evaluate the values for the given observations
values = self.value_net(latent_vf) values = self.value_net(latent_vf)
distribution = self._get_action_dist_from_latent(latent_pi, latent_sde=latent_sde) distribution = self._get_action_dist_from_latent(latent_pi)
actions = distribution.get_actions(deterministic=deterministic) actions = distribution.get_actions(deterministic=deterministic)
log_prob = distribution.log_prob(actions) log_prob = distribution.log_prob(actions)
return actions, values, log_prob return actions, values, log_prob
def _get_latent(self, obs: th.Tensor) -> Tuple[th.Tensor, th.Tensor, th.Tensor]: def _get_action_dist_from_latent(self, latent_pi: th.Tensor) -> Distribution:
"""
Get the latent code (i.e., activations of the last layer of each network)
for the different networks.
:param obs: Observation
:return: Latent codes
for the actor, the value function and for gSDE function
"""
# Preprocess the observation if needed
features = self.extract_features(obs)
latent_pi, latent_vf = self.mlp_extractor(features)
# Features for sde
latent_sde = latent_pi
if self.sde_features_extractor is not None:
latent_sde = self.sde_features_extractor(features)
return latent_pi, latent_vf, latent_sde
def _get_action_dist_from_latent(self, latent_pi: th.Tensor, latent_sde: Optional[th.Tensor] = None) -> Distribution:
""" """
Retrieve action distribution given the latent codes. Retrieve action distribution given the latent codes.
:param latent_pi: Latent code for the actor :param latent_pi: Latent code for the actor
:param latent_sde: Latent code for the gSDE exploration function
:return: Action distribution :return: Action distribution
""" """
mean_actions = self.action_net(latent_pi) mean_actions = self.action_net(latent_pi)
@ -632,7 +613,7 @@ class ActorCriticPolicy(BasePolicy):
# Here mean_actions are the logits (before rounding to get the binary actions) # Here mean_actions are the logits (before rounding to get the binary actions)
return self.action_dist.proba_distribution(action_logits=mean_actions) return self.action_dist.proba_distribution(action_logits=mean_actions)
elif isinstance(self.action_dist, StateDependentNoiseDistribution): elif isinstance(self.action_dist, StateDependentNoiseDistribution):
return self.action_dist.proba_distribution(mean_actions, self.log_std, latent_sde) return self.action_dist.proba_distribution(mean_actions, self.log_std, latent_pi)
else: else:
raise ValueError("Invalid action distribution") raise ValueError("Invalid action distribution")
@ -644,9 +625,7 @@ class ActorCriticPolicy(BasePolicy):
:param deterministic: Whether to use stochastic or deterministic actions :param deterministic: Whether to use stochastic or deterministic actions
:return: Taken action according to the policy :return: Taken action according to the policy
""" """
latent_pi, _, latent_sde = self._get_latent(observation) return self.get_distribution(observation).get_actions(deterministic=deterministic)
distribution = self._get_action_dist_from_latent(latent_pi, latent_sde)
return distribution.get_actions(deterministic=deterministic)
def evaluate_actions(self, obs: th.Tensor, actions: th.Tensor) -> Tuple[th.Tensor, th.Tensor, th.Tensor]: def evaluate_actions(self, obs: th.Tensor, actions: th.Tensor) -> Tuple[th.Tensor, th.Tensor, th.Tensor]:
""" """
@ -658,8 +637,10 @@ class ActorCriticPolicy(BasePolicy):
:return: estimated value, log likelihood of taking those actions :return: estimated value, log likelihood of taking those actions
and entropy of the action distribution. and entropy of the action distribution.
""" """
latent_pi, latent_vf, latent_sde = self._get_latent(obs) # Preprocess the observation if needed
distribution = self._get_action_dist_from_latent(latent_pi, latent_sde) features = self.extract_features(obs)
latent_pi, latent_vf = self.mlp_extractor(features)
distribution = self._get_action_dist_from_latent(latent_pi)
log_prob = distribution.log_prob(actions) log_prob = distribution.log_prob(actions)
values = self.value_net(latent_vf) values = self.value_net(latent_vf)
return values, log_prob, distribution.entropy() return values, log_prob, distribution.entropy()
@ -671,8 +652,9 @@ class ActorCriticPolicy(BasePolicy):
:param obs: :param obs:
:return: the action distribution. :return: the action distribution.
""" """
latent_pi, _, latent_sde = self._get_latent(obs) features = self.extract_features(obs)
return self._get_action_dist_from_latent(latent_pi, latent_sde) latent_pi = self.mlp_extractor.forward_actor(features)
return self._get_action_dist_from_latent(latent_pi)
def predict_values(self, obs: th.Tensor) -> th.Tensor: def predict_values(self, obs: th.Tensor) -> th.Tensor:
""" """
@ -681,7 +663,8 @@ class ActorCriticPolicy(BasePolicy):
:param obs: :param obs:
:return: the estimated values. :return: the estimated values.
""" """
_, latent_vf, _ = self._get_latent(obs) features = self.extract_features(obs)
latent_vf = self.mlp_extractor.forward_critic(features)
return self.value_net(latent_vf) return self.value_net(latent_vf)
@ -911,27 +894,6 @@ class ContinuousCritic(BaseModel):
return self.q_networks[0](th.cat([features, actions], dim=1)) return self.q_networks[0](th.cat([features, actions], dim=1))
def create_sde_features_extractor(
features_dim: int, sde_net_arch: List[int], activation_fn: Type[nn.Module]
) -> Tuple[nn.Sequential, int]:
"""
Create the neural network that will be used to extract features
for the gSDE exploration function.
:param features_dim:
:param sde_net_arch:
:param activation_fn:
:return:
"""
# Special case: when using states as features (i.e. sde_net_arch is an empty list)
# don't use any activation function
sde_activation = activation_fn if len(sde_net_arch) > 0 else None
latent_sde_net = create_mlp(features_dim, -1, sde_net_arch, activation_fn=sde_activation, squash_output=False)
latent_sde_dim = sde_net_arch[-1] if len(sde_net_arch) > 0 else features_dim
sde_features_extractor = nn.Sequential(*latent_sde_net)
return sde_features_extractor, latent_sde_dim
_policy_registry = dict() # type: Dict[Type[BasePolicy], Dict[str, Type[BasePolicy]]] _policy_registry = dict() # type: Dict[Type[BasePolicy], Dict[str, Type[BasePolicy]]]

View file

@ -227,6 +227,12 @@ class MlpExtractor(nn.Module):
shared_latent = self.shared_net(features) shared_latent = self.shared_net(features)
return self.policy_net(shared_latent), self.value_net(shared_latent) return self.policy_net(shared_latent), self.value_net(shared_latent)
def forward_actor(self, features: th.Tensor) -> th.Tensor:
return self.policy_net(self.shared_net(features))
def forward_critic(self, features: th.Tensor) -> th.Tensor:
return self.value_net(self.shared_net(features))
class CombinedExtractor(BaseFeaturesExtractor): class CombinedExtractor(BaseFeaturesExtractor):
""" """

View file

@ -1,3 +1,4 @@
import warnings
from typing import Any, Dict, List, Optional, Tuple, Type, Union from typing import Any, Dict, List, Optional, Tuple, Type, Union
import gym import gym
@ -5,7 +6,7 @@ import torch as th
from torch import nn from torch import nn
from stable_baselines3.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution from stable_baselines3.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution
from stable_baselines3.common.policies import BasePolicy, ContinuousCritic, create_sde_features_extractor, register_policy from stable_baselines3.common.policies import BasePolicy, ContinuousCritic, register_policy
from stable_baselines3.common.preprocessing import get_action_dim from stable_baselines3.common.preprocessing import get_action_dim
from stable_baselines3.common.torch_layers import ( from stable_baselines3.common.torch_layers import (
BaseFeaturesExtractor, BaseFeaturesExtractor,
@ -75,7 +76,6 @@ class Actor(BasePolicy):
# Save arguments to re-create object at loading # Save arguments to re-create object at loading
self.use_sde = use_sde self.use_sde = use_sde
self.sde_features_extractor = None self.sde_features_extractor = None
self.sde_net_arch = sde_net_arch
self.net_arch = net_arch self.net_arch = net_arch
self.features_dim = features_dim self.features_dim = features_dim
self.activation_fn = activation_fn self.activation_fn = activation_fn
@ -85,24 +85,20 @@ class Actor(BasePolicy):
self.full_std = full_std self.full_std = full_std
self.clip_mean = clip_mean self.clip_mean = clip_mean
if sde_net_arch is not None:
warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
action_dim = get_action_dim(self.action_space) action_dim = get_action_dim(self.action_space)
latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn) latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn)
self.latent_pi = nn.Sequential(*latent_pi_net) self.latent_pi = nn.Sequential(*latent_pi_net)
last_layer_dim = net_arch[-1] if len(net_arch) > 0 else features_dim last_layer_dim = net_arch[-1] if len(net_arch) > 0 else features_dim
if self.use_sde: if self.use_sde:
latent_sde_dim = last_layer_dim
# Separate features extractor for gSDE
if sde_net_arch is not None:
self.sde_features_extractor, latent_sde_dim = create_sde_features_extractor(
features_dim, sde_net_arch, activation_fn
)
self.action_dist = StateDependentNoiseDistribution( self.action_dist = StateDependentNoiseDistribution(
action_dim, full_std=full_std, use_expln=use_expln, learn_features=True, squash_output=True action_dim, full_std=full_std, use_expln=use_expln, learn_features=True, squash_output=True
) )
self.mu, self.log_std = self.action_dist.proba_distribution_net( self.mu, self.log_std = self.action_dist.proba_distribution_net(
latent_dim=last_layer_dim, latent_sde_dim=latent_sde_dim, log_std_init=log_std_init latent_dim=last_layer_dim, latent_sde_dim=last_layer_dim, log_std_init=log_std_init
) )
# Avoid numerical issues by limiting the mean of the Gaussian # Avoid numerical issues by limiting the mean of the Gaussian
# to be in [-clip_mean, clip_mean] # to be in [-clip_mean, clip_mean]
@ -124,7 +120,6 @@ class Actor(BasePolicy):
use_sde=self.use_sde, use_sde=self.use_sde,
log_std_init=self.log_std_init, log_std_init=self.log_std_init,
full_std=self.full_std, full_std=self.full_std,
sde_net_arch=self.sde_net_arch,
use_expln=self.use_expln, use_expln=self.use_expln,
features_extractor=self.features_extractor, features_extractor=self.features_extractor,
clip_mean=self.clip_mean, clip_mean=self.clip_mean,
@ -169,10 +164,7 @@ class Actor(BasePolicy):
mean_actions = self.mu(latent_pi) mean_actions = self.mu(latent_pi)
if self.use_sde: if self.use_sde:
latent_sde = latent_pi return mean_actions, self.log_std, dict(latent_sde=latent_pi)
if self.sde_features_extractor is not None:
latent_sde = self.sde_features_extractor(features)
return mean_actions, self.log_std, dict(latent_sde=latent_sde)
# Unstructured exploration (Original implementation) # Unstructured exploration (Original implementation)
log_std = self.log_std(latent_pi) log_std = self.log_std(latent_pi)
# Original Implementation to cap the standard deviation # Original Implementation to cap the standard deviation
@ -273,10 +265,13 @@ class SACPolicy(BasePolicy):
"normalize_images": normalize_images, "normalize_images": normalize_images,
} }
self.actor_kwargs = self.net_args.copy() self.actor_kwargs = self.net_args.copy()
if sde_net_arch is not None:
warnings.warn("sde_net_arch is deprecated and will be removed in SB3 v2.4.0.", DeprecationWarning)
sde_kwargs = { sde_kwargs = {
"use_sde": use_sde, "use_sde": use_sde,
"log_std_init": log_std_init, "log_std_init": log_std_init,
"sde_net_arch": sde_net_arch,
"use_expln": use_expln, "use_expln": use_expln,
"clip_mean": clip_mean, "clip_mean": clip_mean,
} }
@ -329,7 +324,6 @@ class SACPolicy(BasePolicy):
activation_fn=self.net_args["activation_fn"], activation_fn=self.net_args["activation_fn"],
use_sde=self.actor_kwargs["use_sde"], use_sde=self.actor_kwargs["use_sde"],
log_std_init=self.actor_kwargs["log_std_init"], log_std_init=self.actor_kwargs["log_std_init"],
sde_net_arch=self.actor_kwargs["sde_net_arch"],
use_expln=self.actor_kwargs["use_expln"], use_expln=self.actor_kwargs["use_expln"],
clip_mean=self.actor_kwargs["clip_mean"], clip_mean=self.actor_kwargs["clip_mean"],
n_critics=self.critic_kwargs["n_critics"], n_critics=self.critic_kwargs["n_critics"],

View file

@ -1 +1 @@
1.2.1a1 1.2.1a2

View file

@ -60,9 +60,9 @@ def test_sde_check():
@pytest.mark.parametrize("model_class", [SAC, A2C, PPO]) @pytest.mark.parametrize("model_class", [SAC, A2C, PPO])
@pytest.mark.parametrize("sde_net_arch", [None, [32, 16], []])
@pytest.mark.parametrize("use_expln", [False, True]) @pytest.mark.parametrize("use_expln", [False, True])
def test_state_dependent_offpolicy_noise(model_class, sde_net_arch, use_expln): def test_state_dependent_noise(model_class, use_expln):
kwargs = {"learning_starts": 0} if model_class == SAC else {"n_steps": 64}
model = model_class( model = model_class(
"MlpPolicy", "MlpPolicy",
"Pendulum-v0", "Pendulum-v0",
@ -70,9 +70,10 @@ def test_state_dependent_offpolicy_noise(model_class, sde_net_arch, use_expln):
seed=None, seed=None,
create_eval_env=True, create_eval_env=True,
verbose=1, verbose=1,
policy_kwargs=dict(log_std_init=-2, sde_net_arch=sde_net_arch, use_expln=use_expln, net_arch=[64]), policy_kwargs=dict(log_std_init=-2, use_expln=use_expln, net_arch=[64]),
**kwargs,
) )
model.learn(total_timesteps=int(300), eval_freq=250) model.learn(total_timesteps=255, eval_freq=250)
model.policy.reset_noise() model.policy.reset_noise()
if model_class == SAC: if model_class == SAC:
model.policy.actor.get_std() model.policy.actor.get_std()