mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-19 10:00:55 +00:00
Reformat and type the distributions
This commit is contained in:
parent
f1a4fa2d3f
commit
aa8b4eb22a
4 changed files with 75 additions and 54 deletions
|
|
@ -225,13 +225,14 @@ class BaseRLModel(ABC):
|
||||||
return self._vec_normalize_env
|
return self._vec_normalize_env
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check_env(env, observation_space: gym.spaces.Space, action_space: gym.spaces.Space) -> bool:
|
def check_env(env: GymEnv, observation_space: gym.spaces.Space, action_space: gym.spaces.Space) -> bool:
|
||||||
"""
|
"""
|
||||||
Checks the validity of the environment and returns if it is consistent.
|
Checks the validity of the environment and returns if it is consistent.
|
||||||
Checked parameters:
|
Checked parameters:
|
||||||
- observation_space
|
- observation_space
|
||||||
- action_space
|
- action_space
|
||||||
|
|
||||||
|
:param env: (GymEnv)
|
||||||
:param observation_space: (gym.spaces.Space)
|
:param observation_space: (gym.spaces.Space)
|
||||||
:param action_space: (gym.spaces.Space)
|
:param action_space: (gym.spaces.Space)
|
||||||
:return: (bool) True if environment seems to be coherent
|
:return: (bool) True if environment seems to be coherent
|
||||||
|
|
@ -404,7 +405,6 @@ class BaseRLModel(ABC):
|
||||||
# return clipped_actions, state
|
# return clipped_actions, state
|
||||||
return clipped_actions
|
return clipped_actions
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def load(cls, load_path: str, env: Optional[GymEnv] = None, **kwargs):
|
def load(cls, load_path: str, env: Optional[GymEnv] = None, **kwargs):
|
||||||
"""
|
"""
|
||||||
|
|
@ -774,6 +774,7 @@ class OffPolicyRLModel(BaseRLModel):
|
||||||
:param use_sde_at_warmup: (bool) Whether to use SDE instead of uniform sampling
|
:param use_sde_at_warmup: (bool) Whether to use SDE instead of uniform sampling
|
||||||
during the warm up phase (before learning starts)
|
during the warm up phase (before learning starts)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self,
|
def __init__(self,
|
||||||
policy: Type[BasePolicy],
|
policy: Type[BasePolicy],
|
||||||
env: Union[GymEnv, str],
|
env: Union[GymEnv, str],
|
||||||
|
|
@ -790,8 +791,8 @@ class OffPolicyRLModel(BaseRLModel):
|
||||||
use_sde_at_warmup: bool = False):
|
use_sde_at_warmup: bool = False):
|
||||||
|
|
||||||
super(OffPolicyRLModel, self).__init__(policy, env, policy_base, policy_kwargs, verbose,
|
super(OffPolicyRLModel, self).__init__(policy, env, policy_base, policy_kwargs, verbose,
|
||||||
device, support_multi_env, create_eval_env, monitor_wrapper,
|
device, support_multi_env, create_eval_env, monitor_wrapper,
|
||||||
seed, use_sde, sde_sample_freq)
|
seed, use_sde, sde_sample_freq)
|
||||||
# For SDE only
|
# For SDE only
|
||||||
self.rollout_data = None
|
self.rollout_data = None
|
||||||
self.on_policy_exploration = False
|
self.on_policy_exploration = False
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from typing import Optional
|
from typing import Optional, Tuple, Dict, Any
|
||||||
|
|
||||||
|
import gym
|
||||||
import torch as th
|
import torch as th
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from torch.distributions import Normal, Categorical
|
from torch.distributions import Normal, Categorical
|
||||||
|
|
@ -45,14 +46,15 @@ class DiagGaussianDistribution(Distribution):
|
||||||
:param action_dim: (int) Number of continuous actions
|
:param action_dim: (int) Number of continuous actions
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, action_dim):
|
def __init__(self, action_dim: int):
|
||||||
super(DiagGaussianDistribution, self).__init__()
|
super(DiagGaussianDistribution, self).__init__()
|
||||||
self.distribution = None
|
self.distribution = None
|
||||||
self.action_dim = action_dim
|
self.action_dim = action_dim
|
||||||
self.mean_actions = None
|
self.mean_actions = None
|
||||||
self.log_std = None
|
self.log_std = None
|
||||||
|
|
||||||
def proba_distribution_net(self, latent_dim, log_std_init=0.0):
|
def proba_distribution_net(self, latent_dim: int,
|
||||||
|
log_std_init: float = 0.0) -> Tuple[nn.Module, nn.Parameter]:
|
||||||
"""
|
"""
|
||||||
Create the layers and parameter that represent the distribution:
|
Create the layers and parameter that represent the distribution:
|
||||||
one output will be the mean of the gaussian, the other parameter will be the
|
one output will be the mean of the gaussian, the other parameter will be the
|
||||||
|
|
@ -64,10 +66,12 @@ class DiagGaussianDistribution(Distribution):
|
||||||
"""
|
"""
|
||||||
mean_actions = nn.Linear(latent_dim, self.action_dim)
|
mean_actions = nn.Linear(latent_dim, self.action_dim)
|
||||||
# TODO: allow action dependent std
|
# TODO: allow action dependent std
|
||||||
log_std = nn.Parameter(th.ones(self.action_dim) * log_std_init)
|
log_std = nn.Parameter(th.ones(self.action_dim) * log_std_init, requires_grad=True)
|
||||||
return mean_actions, log_std
|
return mean_actions, log_std
|
||||||
|
|
||||||
def proba_distribution(self, mean_actions, log_std, deterministic=False):
|
def proba_distribution(self, mean_actions: th.Tensor,
|
||||||
|
log_std: th.Tensor,
|
||||||
|
deterministic: bool = False) -> Tuple[th.Tensor, 'DiagGaussianDistribution']:
|
||||||
"""
|
"""
|
||||||
Create and sample for the distribution given its parameters (mean, std)
|
Create and sample for the distribution given its parameters (mean, std)
|
||||||
|
|
||||||
|
|
@ -84,29 +88,29 @@ class DiagGaussianDistribution(Distribution):
|
||||||
action = self.sample()
|
action = self.sample()
|
||||||
return action, self
|
return action, self
|
||||||
|
|
||||||
def mode(self):
|
def mode(self) -> th.Tensor:
|
||||||
return self.distribution.mean
|
return self.distribution.mean
|
||||||
|
|
||||||
def sample(self):
|
def sample(self) -> th.Tensor:
|
||||||
return self.distribution.rsample()
|
return self.distribution.rsample()
|
||||||
|
|
||||||
def entropy(self):
|
def entropy(self) -> th.Tensor:
|
||||||
return self.distribution.entropy()
|
return self.distribution.entropy()
|
||||||
|
|
||||||
def log_prob_from_params(self, mean_actions, log_std):
|
def log_prob_from_params(self, mean_actions: th.Tensor, log_std: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
|
||||||
"""
|
"""
|
||||||
Compute the log probabilty of taking an action
|
Compute the log probabilty of taking an action
|
||||||
given the distribution parameters.
|
given the distribution parameters.
|
||||||
|
|
||||||
:param mean_actions: (th.Tensor)
|
:param mean_actions: (th.Tensor)
|
||||||
:param log_std: (th.Tensor)
|
:param log_std: (th.Tensor)
|
||||||
:return: (th.Tensor, th.Tensor)
|
:return: (Tuple[th.Tensor, th.Tensor])
|
||||||
"""
|
"""
|
||||||
action, _ = self.proba_distribution(mean_actions, log_std)
|
action, _ = self.proba_distribution(mean_actions, log_std)
|
||||||
log_prob = self.log_prob(action)
|
log_prob = self.log_prob(action)
|
||||||
return action, log_prob
|
return action, log_prob
|
||||||
|
|
||||||
def log_prob(self, action):
|
def log_prob(self, action: th.Tensor) -> th.Tensor:
|
||||||
"""
|
"""
|
||||||
Get the log probabilty of an action given a distribution.
|
Get the log probabilty of an action given a distribution.
|
||||||
Note that you must call `proba_distribution()` method
|
Note that you must call `proba_distribution()` method
|
||||||
|
|
@ -132,7 +136,7 @@ class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
|
||||||
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, action_dim, epsilon=1e-6):
|
def __init__(self, action_dim: int, epsilon: float = 1e-6):
|
||||||
super(SquashedDiagGaussianDistribution, self).__init__(action_dim)
|
super(SquashedDiagGaussianDistribution, self).__init__(action_dim)
|
||||||
# Avoid NaN (prevents division by zero or log of zero)
|
# Avoid NaN (prevents division by zero or log of zero)
|
||||||
self.epsilon = epsilon
|
self.epsilon = epsilon
|
||||||
|
|
@ -143,26 +147,26 @@ class SquashedDiagGaussianDistribution(DiagGaussianDistribution):
|
||||||
deterministic)
|
deterministic)
|
||||||
return action, self
|
return action, self
|
||||||
|
|
||||||
def mode(self):
|
def mode(self) -> th.Tensor:
|
||||||
self.gaussian_action = self.distribution.mean
|
self.gaussian_action = self.distribution.mean
|
||||||
# Squash the output
|
# Squash the output
|
||||||
return th.tanh(self.gaussian_action)
|
return th.tanh(self.gaussian_action)
|
||||||
|
|
||||||
def entropy(self):
|
def entropy(self) -> Optional[th.Tensor]:
|
||||||
# No analytical form,
|
# No analytical form,
|
||||||
# entropy needs to be estimated using -log_prob.mean()
|
# entropy needs to be estimated using -log_prob.mean()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def sample(self):
|
def sample(self) -> th.Tensor:
|
||||||
self.gaussian_action = self.distribution.rsample()
|
self.gaussian_action = self.distribution.rsample()
|
||||||
return th.tanh(self.gaussian_action)
|
return th.tanh(self.gaussian_action)
|
||||||
|
|
||||||
def log_prob_from_params(self, mean_actions, log_std):
|
def log_prob_from_params(self, mean_actions, log_std) -> Tuple[th.Tensor, th.Tensor]:
|
||||||
action, _ = self.proba_distribution(mean_actions, log_std)
|
action, _ = self.proba_distribution(mean_actions, log_std)
|
||||||
log_prob = self.log_prob(action, self.gaussian_action)
|
log_prob = self.log_prob(action, self.gaussian_action)
|
||||||
return action, log_prob
|
return action, log_prob
|
||||||
|
|
||||||
def log_prob(self, action, gaussian_action=None):
|
def log_prob(self, action: th.Tensor, gaussian_action: Optional[th.Tensor] = None) -> th.Tensor:
|
||||||
# Inverse tanh
|
# Inverse tanh
|
||||||
# Naive implementation (not stable): 0.5 * torch.log((1 + x) / (1 - x))
|
# Naive implementation (not stable): 0.5 * torch.log((1 + x) / (1 - x))
|
||||||
# We use numpy to avoid numerical instability
|
# We use numpy to avoid numerical instability
|
||||||
|
|
@ -185,12 +189,12 @@ class CategoricalDistribution(Distribution):
|
||||||
:param action_dim: (int) Number of discrete actions
|
:param action_dim: (int) Number of discrete actions
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, action_dim):
|
def __init__(self, action_dim: int):
|
||||||
super(CategoricalDistribution, self).__init__()
|
super(CategoricalDistribution, self).__init__()
|
||||||
self.distribution = None
|
self.distribution = None
|
||||||
self.action_dim = action_dim
|
self.action_dim = action_dim
|
||||||
|
|
||||||
def proba_distribution_net(self, latent_dim):
|
def proba_distribution_net(self, latent_dim: int) -> nn.Module:
|
||||||
"""
|
"""
|
||||||
Create the layer that represents the distribution:
|
Create the layer that represents the distribution:
|
||||||
it will be the logits of the Categorical distribution.
|
it will be the logits of the Categorical distribution.
|
||||||
|
|
@ -202,7 +206,8 @@ class CategoricalDistribution(Distribution):
|
||||||
action_logits = nn.Linear(latent_dim, self.action_dim)
|
action_logits = nn.Linear(latent_dim, self.action_dim)
|
||||||
return action_logits
|
return action_logits
|
||||||
|
|
||||||
def proba_distribution(self, action_logits, deterministic=False):
|
def proba_distribution(self, action_logits: th.Tensor,
|
||||||
|
deterministic: bool = False) -> Tuple[th.Tensor, 'CategoricalDistribution']:
|
||||||
self.distribution = Categorical(logits=action_logits)
|
self.distribution = Categorical(logits=action_logits)
|
||||||
if deterministic:
|
if deterministic:
|
||||||
action = self.mode()
|
action = self.mode()
|
||||||
|
|
@ -210,21 +215,21 @@ class CategoricalDistribution(Distribution):
|
||||||
action = self.sample()
|
action = self.sample()
|
||||||
return action, self
|
return action, self
|
||||||
|
|
||||||
def mode(self):
|
def mode(self) -> th.Tensor:
|
||||||
return th.argmax(self.distribution.probs, dim=1)
|
return th.argmax(self.distribution.probs, dim=1)
|
||||||
|
|
||||||
def sample(self):
|
def sample(self) -> th.Tensor:
|
||||||
return self.distribution.sample()
|
return self.distribution.sample()
|
||||||
|
|
||||||
def entropy(self):
|
def entropy(self) -> th.Tensor:
|
||||||
return self.distribution.entropy()
|
return self.distribution.entropy()
|
||||||
|
|
||||||
def log_prob_from_params(self, action_logits):
|
def log_prob_from_params(self, action_logits: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
|
||||||
action, _ = self.proba_distribution(action_logits)
|
action, _ = self.proba_distribution(action_logits)
|
||||||
log_prob = self.log_prob(action)
|
log_prob = self.log_prob(action)
|
||||||
return action, log_prob
|
return action, log_prob
|
||||||
|
|
||||||
def log_prob(self, action):
|
def log_prob(self, action: th.Tensor) -> th.Tensor:
|
||||||
log_prob = self.distribution.log_prob(action)
|
log_prob = self.distribution.log_prob(action)
|
||||||
return log_prob
|
return log_prob
|
||||||
|
|
||||||
|
|
@ -249,8 +254,12 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, action_dim, full_std=True, use_expln=False,
|
def __init__(self, action_dim: int,
|
||||||
squash_output=False, learn_features=False, epsilon=1e-6):
|
full_std: bool = True,
|
||||||
|
use_expln: bool = False,
|
||||||
|
squash_output: bool = False,
|
||||||
|
learn_features: bool = False,
|
||||||
|
epsilon: float = 1e-6):
|
||||||
super(StateDependentNoiseDistribution, self).__init__()
|
super(StateDependentNoiseDistribution, self).__init__()
|
||||||
self.distribution = None
|
self.distribution = None
|
||||||
self.action_dim = action_dim
|
self.action_dim = action_dim
|
||||||
|
|
@ -269,7 +278,7 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
else:
|
else:
|
||||||
self.bijector = None
|
self.bijector = None
|
||||||
|
|
||||||
def get_std(self, log_std):
|
def get_std(self, log_std: th.Tensor) -> th.Tensor:
|
||||||
"""
|
"""
|
||||||
Get the standard deviation from the learned parameter
|
Get the standard deviation from the learned parameter
|
||||||
(log of it by default). This ensures that the std is positive.
|
(log of it by default). This ensures that the std is positive.
|
||||||
|
|
@ -294,7 +303,7 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
# Reduce the number of parameters:
|
# Reduce the number of parameters:
|
||||||
return th.ones(self.latent_sde_dim, self.action_dim).to(log_std.device) * std
|
return th.ones(self.latent_sde_dim, self.action_dim).to(log_std.device) * std
|
||||||
|
|
||||||
def sample_weights(self, log_std, batch_size=1):
|
def sample_weights(self, log_std: th.Tensor, batch_size: int = 1) -> None:
|
||||||
"""
|
"""
|
||||||
Sample weights for the noise exploration matrix,
|
Sample weights for the noise exploration matrix,
|
||||||
using a centered Gaussian distribution.
|
using a centered Gaussian distribution.
|
||||||
|
|
@ -307,7 +316,8 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
self.exploration_mat = self.weights_dist.rsample()
|
self.exploration_mat = self.weights_dist.rsample()
|
||||||
self.exploration_matrices = self.weights_dist.rsample((batch_size,))
|
self.exploration_matrices = self.weights_dist.rsample((batch_size,))
|
||||||
|
|
||||||
def proba_distribution_net(self, latent_dim, log_std_init=-2.0, latent_sde_dim=None):
|
def proba_distribution_net(self, latent_dim: int, log_std_init: float = -2.0,
|
||||||
|
latent_sde_dim: Optional[th.Tensor] = None) -> Tuple[nn.Module, nn.Parameter]:
|
||||||
"""
|
"""
|
||||||
Create the layers and parameter that represent the distribution:
|
Create the layers and parameter that represent the distribution:
|
||||||
one output will be the deterministic action, the other parameter will be the
|
one output will be the deterministic action, the other parameter will be the
|
||||||
|
|
@ -327,12 +337,15 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
# Reduce the number of parameters if needed
|
# Reduce the number of parameters if needed
|
||||||
log_std = th.ones(self.latent_sde_dim, self.action_dim) if self.full_std else th.ones(self.latent_sde_dim, 1)
|
log_std = th.ones(self.latent_sde_dim, self.action_dim) if self.full_std else th.ones(self.latent_sde_dim, 1)
|
||||||
# Transform it to a parameter so it can be optimized
|
# Transform it to a parameter so it can be optimized
|
||||||
log_std = nn.Parameter(log_std * log_std_init)
|
log_std = nn.Parameter(log_std * log_std_init, requires_grad=True)
|
||||||
# Sample an exploration matrix
|
# Sample an exploration matrix
|
||||||
self.sample_weights(log_std)
|
self.sample_weights(log_std)
|
||||||
return mean_actions_net, log_std
|
return mean_actions_net, log_std
|
||||||
|
|
||||||
def proba_distribution(self, mean_actions, log_std, latent_sde, deterministic=False):
|
def proba_distribution(self, mean_actions: th.Tensor,
|
||||||
|
log_std: th.Tensor,
|
||||||
|
latent_sde: th.Tensor,
|
||||||
|
deterministic: bool = False) -> Tuple[th.Tensor, 'StateDependentNoiseDistribution']:
|
||||||
"""
|
"""
|
||||||
Create and sample for the distribution given its parameters (mean, std)
|
Create and sample for the distribution given its parameters (mean, std)
|
||||||
|
|
||||||
|
|
@ -340,7 +353,7 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
:param log_std: (th.Tensor)
|
:param log_std: (th.Tensor)
|
||||||
:param latent_sde: (th.Tensor)
|
:param latent_sde: (th.Tensor)
|
||||||
:param deterministic: (bool)
|
:param deterministic: (bool)
|
||||||
:return: (th.Tensor)
|
:return: (Tuple[th.Tensor, Distribution])
|
||||||
"""
|
"""
|
||||||
# Stop gradient if we don't want to influence the features
|
# Stop gradient if we don't want to influence the features
|
||||||
latent_sde = latent_sde if self.learn_features else latent_sde.detach()
|
latent_sde = latent_sde if self.learn_features else latent_sde.detach()
|
||||||
|
|
@ -353,13 +366,13 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
action = self.sample(latent_sde)
|
action = self.sample(latent_sde)
|
||||||
return action, self
|
return action, self
|
||||||
|
|
||||||
def mode(self):
|
def mode(self) -> th.Tensor:
|
||||||
action = self.distribution.mean
|
action = self.distribution.mean
|
||||||
if self.bijector is not None:
|
if self.bijector is not None:
|
||||||
return self.bijector.forward(action)
|
return self.bijector.forward(action)
|
||||||
return action
|
return action
|
||||||
|
|
||||||
def get_noise(self, latent_sde):
|
def get_noise(self, latent_sde: th.Tensor) -> th.Tensor:
|
||||||
latent_sde = latent_sde if self.learn_features else latent_sde.detach()
|
latent_sde = latent_sde if self.learn_features else latent_sde.detach()
|
||||||
# Default case: only one exploration matrix
|
# Default case: only one exploration matrix
|
||||||
if len(latent_sde) == 1 or len(latent_sde) != len(self.exploration_matrices):
|
if len(latent_sde) == 1 or len(latent_sde) != len(self.exploration_matrices):
|
||||||
|
|
@ -371,26 +384,28 @@ class StateDependentNoiseDistribution(Distribution):
|
||||||
noise = th.bmm(latent_sde, self.exploration_matrices)
|
noise = th.bmm(latent_sde, self.exploration_matrices)
|
||||||
return noise.squeeze(1)
|
return noise.squeeze(1)
|
||||||
|
|
||||||
def sample(self, latent_sde):
|
def sample(self, latent_sde: th.Tensor) -> th.Tensor:
|
||||||
noise = self.get_noise(latent_sde)
|
noise = self.get_noise(latent_sde)
|
||||||
action = self.distribution.mean + noise
|
action = self.distribution.mean + noise
|
||||||
if self.bijector is not None:
|
if self.bijector is not None:
|
||||||
return self.bijector.forward(action)
|
return self.bijector.forward(action)
|
||||||
return action
|
return action
|
||||||
|
|
||||||
def entropy(self):
|
def entropy(self) -> Optional[th.Tensor]:
|
||||||
# No analytical form,
|
# No analytical form,
|
||||||
# entropy needs to be estimated using -log_prob.mean()
|
# entropy needs to be estimated using -log_prob.mean()
|
||||||
if self.bijector is not None:
|
if self.bijector is not None:
|
||||||
return None
|
return None
|
||||||
return self.distribution.entropy()
|
return self.distribution.entropy()
|
||||||
|
|
||||||
def log_prob_from_params(self, mean_actions, log_std, latent_sde):
|
def log_prob_from_params(self, mean_actions: th.Tensor,
|
||||||
|
log_std: th.Tensor,
|
||||||
|
latent_sde: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
|
||||||
action, _ = self.proba_distribution(mean_actions, log_std, latent_sde)
|
action, _ = self.proba_distribution(mean_actions, log_std, latent_sde)
|
||||||
log_prob = self.log_prob(action)
|
log_prob = self.log_prob(action)
|
||||||
return action, log_prob
|
return action, log_prob
|
||||||
|
|
||||||
def log_prob(self, action):
|
def log_prob(self, action: th.Tensor) -> th.Tensor:
|
||||||
if self.bijector is not None:
|
if self.bijector is not None:
|
||||||
gaussian_action = self.bijector.inverse(action)
|
gaussian_action = self.bijector.inverse(action)
|
||||||
else:
|
else:
|
||||||
|
|
@ -418,16 +433,16 @@ class TanhBijector(object):
|
||||||
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
:param epsilon: (float) small value to avoid NaN due to numerical imprecision.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, epsilon=1e-6):
|
def __init__(self, epsilon: float = 1e-6):
|
||||||
super(TanhBijector, self).__init__()
|
super(TanhBijector, self).__init__()
|
||||||
self.epsilon = epsilon
|
self.epsilon = epsilon
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def forward(x):
|
def forward(x: th.Tensor) -> th.Tensor:
|
||||||
return th.tanh(x)
|
return th.tanh(x)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def atanh(x):
|
def atanh(x: th.Tensor) -> th.Tensor:
|
||||||
"""
|
"""
|
||||||
Inverse of Tanh
|
Inverse of Tanh
|
||||||
|
|
||||||
|
|
@ -437,7 +452,7 @@ class TanhBijector(object):
|
||||||
return 0.5 * (x.log1p() - (-x).log1p())
|
return 0.5 * (x.log1p() - (-x).log1p())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def inverse(y):
|
def inverse(y: th.Tensor) -> th.Tensor:
|
||||||
"""
|
"""
|
||||||
Inverse tanh.
|
Inverse tanh.
|
||||||
|
|
||||||
|
|
@ -448,19 +463,21 @@ class TanhBijector(object):
|
||||||
# Clip the action to avoid NaN
|
# Clip the action to avoid NaN
|
||||||
return TanhBijector.atanh(y.clamp(min=-1. + eps, max=1. - eps))
|
return TanhBijector.atanh(y.clamp(min=-1. + eps, max=1. - eps))
|
||||||
|
|
||||||
def log_prob_correction(self, x):
|
def log_prob_correction(self, x: th.Tensor) -> th.Tensor:
|
||||||
# Squash correction (from original SAC implementation)
|
# Squash correction (from original SAC implementation)
|
||||||
return th.log(1.0 - th.tanh(x) ** 2 + self.epsilon)
|
return th.log(1.0 - th.tanh(x) ** 2 + self.epsilon)
|
||||||
|
|
||||||
|
|
||||||
def make_proba_distribution(action_space, use_sde=False, dist_kwargs=None):
|
def make_proba_distribution(action_space: gym.spaces.Space,
|
||||||
|
use_sde: bool = False,
|
||||||
|
dist_kwargs: Optional[Dict[str, Any]] = None) -> Distribution:
|
||||||
"""
|
"""
|
||||||
Return an instance of Distribution for the correct type of action space
|
Return an instance of Distribution for the correct type of action space
|
||||||
|
|
||||||
:param action_space: (Gym Space) the input action space
|
:param action_space: (gym.spaces.Space) the input action space
|
||||||
:param use_sde: (bool) Force the use of StateDependentNoiseDistribution
|
:param use_sde: (bool) Force the use of StateDependentNoiseDistribution
|
||||||
instead of DiagGaussianDistribution
|
instead of DiagGaussianDistribution
|
||||||
:param dist_kwargs: (dict) Keyword arguments to pass to the probabilty distribution
|
:param dist_kwargs: (Optional[Dict[str, Any]]) Keyword arguments to pass to the probabilty distribution
|
||||||
:return: (Distribution) the approriate Distribution object
|
:return: (Distribution) the approriate Distribution object
|
||||||
"""
|
"""
|
||||||
if dist_kwargs is None:
|
if dist_kwargs is None:
|
||||||
|
|
@ -478,5 +495,6 @@ def make_proba_distribution(action_space, use_sde=False, dist_kwargs=None):
|
||||||
# elif isinstance(action_space, spaces.MultiBinary):
|
# elif isinstance(action_space, spaces.MultiBinary):
|
||||||
# return BernoulliDistribution(action_space.n, **dist_kwargs)
|
# return BernoulliDistribution(action_space.n, **dist_kwargs)
|
||||||
else:
|
else:
|
||||||
raise NotImplementedError(f"Error: probability distribution, not implemented for action space of type {type(action_space)}."
|
raise NotImplementedError("Error: probability distribution, not implemented for action space"
|
||||||
|
f"of type {type(action_space)}."
|
||||||
" Must be of type Gym Spaces: Box, Discrete, MultiDiscrete or MultiBinary.")
|
" Must be of type Gym Spaces: Box, Discrete, MultiDiscrete or MultiBinary.")
|
||||||
|
|
|
||||||
|
|
@ -2,12 +2,12 @@ from typing import Tuple, Callable, List, Optional
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import matplotlib
|
# import matplotlib
|
||||||
|
# matplotlib.use('TkAgg') # Can change to 'Agg' for non-interactive mode
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
from torchy_baselines.common.monitor import load_results
|
from torchy_baselines.common.monitor import load_results
|
||||||
|
|
||||||
# matplotlib.use('TkAgg') # Can change to 'Agg' for non-interactive mode
|
|
||||||
plt.rcParams['svg.fonttype'] = 'none'
|
plt.rcParams['svg.fonttype'] = 'none'
|
||||||
|
|
||||||
X_TIMESTEPS = 'timesteps'
|
X_TIMESTEPS = 'timesteps'
|
||||||
|
|
|
||||||
|
|
@ -161,6 +161,8 @@ class PPOPolicy(BasePolicy):
|
||||||
elif isinstance(self.action_dist, StateDependentNoiseDistribution):
|
elif isinstance(self.action_dist, StateDependentNoiseDistribution):
|
||||||
return self.action_dist.proba_distribution(mean_actions, self.log_std, latent_sde,
|
return self.action_dist.proba_distribution(mean_actions, self.log_std, latent_sde,
|
||||||
deterministic=deterministic)
|
deterministic=deterministic)
|
||||||
|
else:
|
||||||
|
raise ValueError('Invalid action distribution')
|
||||||
|
|
||||||
def predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor:
|
def predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor:
|
||||||
latent_pi, _, latent_sde = self._get_latent(observation)
|
latent_pi, _, latent_sde = self._get_latent(observation)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue