SDE on latent_pi

This commit is contained in:
Antonin Raffin 2019-10-31 11:44:27 +01:00
parent 862ae666b5
commit 925afe784c
3 changed files with 73 additions and 41 deletions

View file

@ -112,7 +112,6 @@ class A2C(PPO):
# Optimization step # Optimization step
self.policy.optimizer.zero_grad() self.policy.optimizer.zero_grad()
loss.backward() loss.backward()
# Clip grad norm # Clip grad norm
th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
self.policy.optimizer.step() self.policy.optimizer.step()

View file

@ -2,6 +2,7 @@ import numpy as np
import torch as th import torch as th
import torch.nn as nn import torch.nn as nn
from torch.distributions import Normal, Categorical from torch.distributions import Normal, Categorical
import torch.nn.functional as F
from gym import spaces from gym import spaces
class Distribution(object): class Distribution(object):
@ -168,20 +169,20 @@ class CategoricalDistribution(Distribution):
class StateDependentNoiseDistribution(Distribution): class StateDependentNoiseDistribution(Distribution):
def __init__(self, features_dim, action_dim, use_expln=False, def __init__(self, action_dim, use_expln=False,
squash_output=True, epsilon=1e-6): squash_output=False, epsilon=1e-6):
super(StateDependentNoiseDistribution, self).__init__() super(StateDependentNoiseDistribution, self).__init__()
self.distribution = None self.distribution = None
self.action_dim = action_dim self.action_dim = action_dim
self.features_dim = features_dim
self.mean_actions = None self.mean_actions = None
self.log_std = None self.log_std = None
self.weights_dist = None self.weights_dist = None
self.noise_weights = None self.exploration_mat = None
self.gaussian_action = None
self.use_expln = use_expln self.use_expln = use_expln
self.squash_output = squash_output if squash_output:
self.epsilon = epsilon self.bijector = TanhBijector(epsilon)
else:
self.bijector = None
def get_std(self, log_std): def get_std(self, log_std):
if self.use_expln: if self.use_expln:
@ -196,71 +197,103 @@ class StateDependentNoiseDistribution(Distribution):
def sample_weights(self, log_std): def sample_weights(self, log_std):
self.weights_dist = Normal(th.zeros_like(log_std), self.get_std(log_std)) self.weights_dist = Normal(th.zeros_like(log_std), self.get_std(log_std))
self.noise_weights = self.weights_dist.rsample() self.exploration_mat = self.weights_dist.rsample()
def proba_distribution_net(self, latent_dim, log_std_init=-1): def proba_distribution_net(self, latent_dim, log_std_init=-1):
mean_actions = nn.Linear(latent_dim, self.action_dim) mean_actions = nn.Linear(latent_dim, self.action_dim)
# TODO: log_std_init depending on the number of layers? # TODO: log_std_init depending on the number of layers?
log_std = nn.Parameter(th.ones(self.features_dim, self.action_dim) * log_std_init) log_std = nn.Parameter(th.ones(latent_dim, self.action_dim) * log_std_init)
self.sample_weights(log_std) self.sample_weights(log_std)
return mean_actions, log_std return mean_actions, log_std
def proba_distribution(self, mean_actions, log_std, observations, deterministic=False): def proba_distribution(self, mean_actions, log_std, latent_pi, deterministic=False):
variance = th.mm(observations ** 2, self.get_std(log_std) ** 2) # TODO: try without detach
variance = th.mm(latent_pi.detach() ** 2, self.get_std(log_std) ** 2)
self.distribution = Normal(mean_actions, th.sqrt(variance)) self.distribution = Normal(mean_actions, th.sqrt(variance))
if deterministic: if deterministic:
action = self.mode() action = self.mode()
else: else:
action = self.sample(observations) action = self.sample(latent_pi)
return action, self return action, self
def mode(self): def mode(self):
self.gaussian_action = self.distribution.mean action = self.distribution.mean
if self.squash_output: if self.bijector is not None:
return th.tanh(self.gaussian_action) return self.bijector.forward(action)
return self.gaussian_action return action
def sample(self, observations): def sample(self, latent_pi):
noise = th.mm(observations, self.noise_weights) noise = th.mm(latent_pi.detach(), self.exploration_mat)
self.gaussian_action = self.distribution.mean + noise action = self.distribution.mean + noise
if self.squash_output: if self.bijector is not None:
return th.tanh(self.gaussian_action) return self.bijector.forward(action)
return self.gaussian_action return action
def entropy(self): def entropy(self):
# TODO: account for the squashing? # TODO: account for the squashing?
return self.distribution.entropy() return self.distribution.entropy()
def log_prob_from_params(self, mean_actions, log_std, observations): def log_prob_from_params(self, mean_actions, log_std, latent_pi):
action, _ = self.proba_distribution(mean_actions, log_std, observations) action, _ = self.proba_distribution(mean_actions, log_std, latent_pi)
log_prob = self.log_prob(action) log_prob = self.log_prob(action)
return action, log_prob return action, log_prob
def log_prob(self, action): def log_prob(self, action):
if self.squash_output: if self.bijector is not None:
gaussian_action = self.gaussian_action gaussian_action = self.bijector.inverse(action)
else: else:
gaussian_action = action gaussian_action = action
# log likelihood for a gaussian # log likelihood for a gaussian
log_prob = self.distribution.log_prob(gaussian_action) log_prob = self.distribution.log_prob(gaussian_action)
# log_prob = self.distribution.log_prob(action)
if len(log_prob.shape) > 1: if len(log_prob.shape) > 1:
log_prob = log_prob.sum(axis=1) log_prob = log_prob.sum(axis=1)
else: else:
log_prob = log_prob.sum() log_prob = log_prob.sum()
if self.squash_output:
if self.bijector is not None:
# Squash correction (from original SAC implementation) # Squash correction (from original SAC implementation)
log_prob -= th.sum(th.log(1 - action ** 2 + self.epsilon), dim=1) log_prob -= th.sum(self.bijector.log_prob_correction(gaussian_action), dim=1)
return log_prob return log_prob
def make_proba_distribution(action_space, features_dim=None, use_sde=False): class TanhBijector(object):
def __init__(self, epsilon=1e-6):
super(TanhBijector, self).__init__()
self.epsilon = epsilon
def forward(self, x):
return th.tanh(x)
def inverse(self, action):
"""
Inverse tanh.
From https://github.com/tensorflow/agents:
0.99999997 is the maximum value such that atanh(x) is valid for both
float32 and float64
:param action: (th.Tensor)
:return: (th.Tensor)
"""
# Inverse tanh
# Naive implementation (not stable): 0.5 * torch.log((1 + x ) / (1 - x))
# We use numpy to avoid numerical instability
# Note: Using numpy, we do not keep the gradient
clipped_action = np.clip(action.cpu().numpy(), -0.99999997, 0.99999997)
return th.from_numpy(np.arctanh(clipped_action)).to(action.device)
def log_prob_correction(self, x):
# Squash correction (from original SAC implementation)
return th.log(1 - th.tanh(x) ** 2 + self.epsilon)
def make_proba_distribution(action_space, use_sde=False):
""" """
Return an instance of Distribution for the correct type of action space Return an instance of Distribution for the correct type of action space
:param action_space: (Gym Space) the input action space :param action_space: (Gym Space) the input action space
:param feature_dim: (int) Dimension of the feature vector
:param use_sde: (bool) Force the use of StateDependentNoiseDistribution :param use_sde: (bool) Force the use of StateDependentNoiseDistribution
instead of DiagGaussianDistribution instead of DiagGaussianDistribution
:return: (Distribution) the approriate Distribution object :return: (Distribution) the approriate Distribution object
@ -268,7 +301,7 @@ def make_proba_distribution(action_space, features_dim=None, use_sde=False):
if isinstance(action_space, spaces.Box): if isinstance(action_space, spaces.Box):
assert len(action_space.shape) == 1, "Error: the action space must be a vector" assert len(action_space.shape) == 1, "Error: the action space must be a vector"
if use_sde: if use_sde:
return StateDependentNoiseDistribution(features_dim, action_space.shape[0]) return StateDependentNoiseDistribution(action_space.shape[0])
return DiagGaussianDistribution(action_space.shape[0]) return DiagGaussianDistribution(action_space.shape[0])
elif isinstance(action_space, spaces.Discrete): elif isinstance(action_space, spaces.Discrete):
return CategoricalDistribution(action_space.n) return CategoricalDistribution(action_space.n)

View file

@ -125,7 +125,7 @@ class PPOPolicy(BasePolicy):
self.features_dim = self.obs_dim self.features_dim = self.obs_dim
self.log_std_init = log_std_init self.log_std_init = log_std_init
# Action distribution # Action distribution
self.action_dist = make_proba_distribution(action_space, self.features_dim, use_sde=use_sde) self.action_dist = make_proba_distribution(action_space, use_sde=use_sde)
self._build(learning_rate) self._build(learning_rate)
@ -161,15 +161,15 @@ class PPOPolicy(BasePolicy):
obs = th.FloatTensor(obs).to(self.device) obs = th.FloatTensor(obs).to(self.device)
latent_pi, latent_vf = self._get_latent(obs) latent_pi, latent_vf = self._get_latent(obs)
value = self.value_net(latent_vf) value = self.value_net(latent_vf)
action, action_distribution = self._get_action_dist_from_latent(latent_pi, obs, deterministic=deterministic) action, action_distribution = self._get_action_dist_from_latent(latent_pi, deterministic=deterministic)
log_prob = action_distribution.log_prob(action) log_prob = action_distribution.log_prob(action)
return action, value, log_prob return action, value, log_prob
def _get_latent(self, obs): def _get_latent(self, obs):
return self.mlp_extractor(self.features_extractor(obs)) return self.mlp_extractor(self.features_extractor(obs))
def _get_action_dist_from_latent(self, latent, obs, deterministic=False): def _get_action_dist_from_latent(self, latent_pi, deterministic=False):
mean_actions = self.action_net(latent) mean_actions = self.action_net(latent_pi)
if isinstance(self.action_dist, DiagGaussianDistribution): if isinstance(self.action_dist, DiagGaussianDistribution):
return self.action_dist.proba_distribution(mean_actions, self.log_std, deterministic=deterministic) return self.action_dist.proba_distribution(mean_actions, self.log_std, deterministic=deterministic)
@ -178,16 +178,16 @@ class PPOPolicy(BasePolicy):
return self.action_dist.proba_distribution(mean_actions, deterministic=deterministic) return self.action_dist.proba_distribution(mean_actions, deterministic=deterministic)
elif isinstance(self.action_dist, StateDependentNoiseDistribution): elif isinstance(self.action_dist, StateDependentNoiseDistribution):
return self.action_dist.proba_distribution(mean_actions, self.log_std, obs, deterministic=deterministic) return self.action_dist.proba_distribution(mean_actions, self.log_std, latent_pi, deterministic=deterministic)
def actor_forward(self, obs, deterministic=False): def actor_forward(self, obs, deterministic=False):
latent_pi, _ = self._get_latent(obs) latent_pi, _ = self._get_latent(obs)
action, _ = self._get_action_dist_from_latent(latent_pi, obs, deterministic=deterministic) action, _ = self._get_action_dist_from_latent(latent_pi, deterministic=deterministic)
return action.detach().cpu().numpy() return action.detach().cpu().numpy()
def get_policy_stats(self, obs, action): def get_policy_stats(self, obs, action, deterministic=False):
latent_pi, latent_vf = self._get_latent(obs) latent_pi, latent_vf = self._get_latent(obs)
_, action_distribution = self._get_action_dist_from_latent(latent_pi, obs) _, action_distribution = self._get_action_dist_from_latent(latent_pi, deterministic=deterministic)
log_prob = action_distribution.log_prob(action) log_prob = action_distribution.log_prob(action)
value = self.value_net(latent_vf) value = self.value_net(latent_vf)
return value, log_prob, action_distribution.entropy() return value, log_prob, action_distribution.entropy()