mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-18 09:54:10 +00:00
Start cleanup + update docstrings
This commit is contained in:
parent
95c741c707
commit
5d353d598c
2 changed files with 43 additions and 11 deletions
|
|
@ -1,17 +1,25 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import gym
|
||||||
import torch as th
|
import torch as th
|
||||||
from torch.distributions import Normal
|
from torch.distributions import Normal
|
||||||
|
|
||||||
from torchy_baselines import A2C
|
from torchy_baselines import A2C
|
||||||
|
from torchy_baselines.common.vec_env import DummyVecEnv, VecNormalize
|
||||||
|
from torchy_baselines.common.monitor import Monitor
|
||||||
|
|
||||||
|
|
||||||
def test_state_dependent_exploration():
|
def test_state_dependent_exploration():
|
||||||
|
"""
|
||||||
|
Check that the gradient correspond to the expected one
|
||||||
|
"""
|
||||||
n_states = 2
|
n_states = 2
|
||||||
state_dim = 3
|
state_dim = 3
|
||||||
# TODO: fix for action_dim > 1
|
# TODO: fix for action_dim > 1
|
||||||
action_dim = 1
|
action_dim = 1
|
||||||
sigma = th.ones(state_dim, action_dim, requires_grad=True)
|
sigma = th.ones(state_dim, 1, requires_grad=True)
|
||||||
|
# Reduce the number of parameters
|
||||||
|
# sigma_ = th.ones(state_dim, action_dim) * sigma_
|
||||||
|
|
||||||
# weights_dist = Normal(th.zeros_like(log_sigma), th.exp(log_sigma))
|
# weights_dist = Normal(th.zeros_like(log_sigma), th.exp(log_sigma))
|
||||||
th.manual_seed(2)
|
th.manual_seed(2)
|
||||||
|
|
@ -42,16 +50,11 @@ def test_state_dependent_exploration():
|
||||||
|
|
||||||
@pytest.mark.parametrize("model_class", [A2C])
|
@pytest.mark.parametrize("model_class", [A2C])
|
||||||
def test_state_dependent_noise(model_class):
|
def test_state_dependent_noise(model_class):
|
||||||
import gym
|
|
||||||
from torchy_baselines.common.vec_env import DummyVecEnv, VecNormalize
|
|
||||||
from torchy_baselines.common.monitor import Monitor
|
|
||||||
|
|
||||||
# env_id = 'Pendulum-v0'
|
|
||||||
env_id = 'MountainCarContinuous-v0'
|
env_id = 'MountainCarContinuous-v0'
|
||||||
# env_id = 'LunarLanderContinuous-v2'
|
|
||||||
env = VecNormalize(DummyVecEnv([lambda: Monitor(gym.make(env_id))]), norm_reward=True)
|
env = VecNormalize(DummyVecEnv([lambda: Monitor(gym.make(env_id))]), norm_reward=True)
|
||||||
eval_env = VecNormalize(DummyVecEnv([lambda: Monitor(gym.make(env_id))]), training=False, norm_reward=False)
|
eval_env = VecNormalize(DummyVecEnv([lambda: Monitor(gym.make(env_id))]), training=False, norm_reward=False)
|
||||||
model = model_class('MlpPolicy', env, n_steps=200, max_grad_norm=1, use_rms_prop=False,
|
|
||||||
use_sde=True, ent_coef=0.00, verbose=1, create_eval_env=True, learning_rate=3e-4,
|
model = model_class('MlpPolicy', env, n_steps=200, use_sde=True, ent_coef=0.00, verbose=1, learning_rate=3e-4,
|
||||||
policy_kwargs=dict(log_std_init=0.0, ortho_init=False, net_arch=[256, dict(pi=[256], vf=[256])]), seed=None)
|
policy_kwargs=dict(log_std_init=0.0, ortho_init=False), seed=None)
|
||||||
model.learn(total_timesteps=int(20000), log_interval=5, eval_freq=10000, eval_env=eval_env)
|
model.learn(total_timesteps=int(1000), log_interval=5, eval_freq=500, eval_env=eval_env)
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,11 @@ class Distribution(object):
|
||||||
|
|
||||||
|
|
||||||
class DiagGaussianDistribution(Distribution):
|
class DiagGaussianDistribution(Distribution):
|
||||||
|
"""
|
||||||
|
Gaussian distribution with diagonal covariance matrix.
|
||||||
|
|
||||||
|
:param action_dim: (int) Number of actions
|
||||||
|
"""
|
||||||
def __init__(self, action_dim):
|
def __init__(self, action_dim):
|
||||||
super(DiagGaussianDistribution, self).__init__()
|
super(DiagGaussianDistribution, self).__init__()
|
||||||
self.distribution = None
|
self.distribution = None
|
||||||
|
|
@ -53,12 +58,28 @@ class DiagGaussianDistribution(Distribution):
|
||||||
self.log_std = None
|
self.log_std = None
|
||||||
|
|
||||||
def proba_distribution_net(self, latent_dim, log_std_init=0.0):
|
def proba_distribution_net(self, latent_dim, log_std_init=0.0):
|
||||||
|
"""
|
||||||
|
Create the layers and parameter that represent the distribution:
|
||||||
|
one output will be the mean of the gaussian, the other parameter will be the
|
||||||
|
standard deviation (log std in fact to allow negative values)
|
||||||
|
|
||||||
|
:param latent_dim: (int) Dimension og the last layer of the policy (before the action layer)
|
||||||
|
:param log_std_init: (float) Initial value for the log standard deviation
|
||||||
|
"""
|
||||||
mean_actions = nn.Linear(latent_dim, self.action_dim)
|
mean_actions = nn.Linear(latent_dim, self.action_dim)
|
||||||
# TODO: allow action dependent std
|
# TODO: allow action dependent std
|
||||||
log_std = nn.Parameter(th.ones(self.action_dim) * log_std_init)
|
log_std = nn.Parameter(th.ones(self.action_dim) * log_std_init)
|
||||||
return mean_actions, log_std
|
return mean_actions, log_std
|
||||||
|
|
||||||
def proba_distribution(self, mean_actions, log_std, deterministic=False):
|
def proba_distribution(self, mean_actions, log_std, deterministic=False):
|
||||||
|
"""
|
||||||
|
Create and sample for the distribution given its parameters (mean, std)
|
||||||
|
|
||||||
|
:param mean_actions: (th.Tensor)
|
||||||
|
:param log_std: (th.Tensor)
|
||||||
|
:param deterministic: (bool)
|
||||||
|
:return: (th.Tensor)
|
||||||
|
"""
|
||||||
action_std = th.ones_like(mean_actions) * log_std.exp()
|
action_std = th.ones_like(mean_actions) * log_std.exp()
|
||||||
self.distribution = Normal(mean_actions, action_std)
|
self.distribution = Normal(mean_actions, action_std)
|
||||||
if deterministic:
|
if deterministic:
|
||||||
|
|
@ -77,6 +98,14 @@ class DiagGaussianDistribution(Distribution):
|
||||||
return self.distribution.entropy()
|
return self.distribution.entropy()
|
||||||
|
|
||||||
def log_prob_from_params(self, mean_actions, log_std):
|
def log_prob_from_params(self, mean_actions, log_std):
|
||||||
|
"""
|
||||||
|
Compute the log probabilty of taking an action
|
||||||
|
given the distribution parameters.
|
||||||
|
|
||||||
|
:param mean_actions: (th.Tensor)
|
||||||
|
:param log_std: (th.Tensor)
|
||||||
|
:return: (th.Tensor, th.Tensor)
|
||||||
|
"""
|
||||||
action, _ = self.proba_distribution(mean_actions, log_std)
|
action, _ = self.proba_distribution(mean_actions, log_std)
|
||||||
log_prob = self.log_prob(action)
|
log_prob = self.log_prob(action)
|
||||||
return action, log_prob
|
return action, log_prob
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue