mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-18 09:54:10 +00:00
Add noise clipping
This commit is contained in:
parent
f4546837c3
commit
715865a0fe
2 changed files with 6 additions and 4 deletions
|
|
@ -351,6 +351,7 @@ class BaseRLModel(object):
|
||||||
# Add noise to the action (improve exploration)
|
# Add noise to the action (improve exploration)
|
||||||
if action_noise is not None:
|
if action_noise is not None:
|
||||||
# NOTE: in the original implementation of TD3, the noise was applied to the unscaled action
|
# NOTE: in the original implementation of TD3, the noise was applied to the unscaled action
|
||||||
|
# Update(October 2019): Not anymore
|
||||||
action = np.clip(action + action_noise(), -1, 1)
|
action = np.clip(action + action_noise(), -1, 1)
|
||||||
|
|
||||||
# Rescale and perform action
|
# Rescale and perform action
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from torchy_baselines.common.policies import BasePolicy, register_policy, create
|
||||||
|
|
||||||
class Actor(BaseNetwork):
|
class Actor(BaseNetwork):
|
||||||
def __init__(self, obs_dim, action_dim, net_arch, activation_fn=nn.ReLU,
|
def __init__(self, obs_dim, action_dim, net_arch, activation_fn=nn.ReLU,
|
||||||
use_sde=False, log_std_init=0.0, clip_noise=0.1):
|
use_sde=False, log_std_init=-2, clip_noise=0.1):
|
||||||
super(Actor, self).__init__()
|
super(Actor, self).__init__()
|
||||||
|
|
||||||
self.latent_pi, self.log_std = None, None
|
self.latent_pi, self.log_std = None, None
|
||||||
|
|
@ -36,8 +36,8 @@ class Actor(BaseNetwork):
|
||||||
if deterministic:
|
if deterministic:
|
||||||
return self.actor_net(latent_pi)
|
return self.actor_net(latent_pi)
|
||||||
noise = th.mm(latent_pi.detach(), self.exploration_mat)
|
noise = th.mm(latent_pi.detach(), self.exploration_mat)
|
||||||
# noise = th.clamp(noise, -self.clip_noise, self.clip_noise)
|
noise = th.clamp(noise, -self.clip_noise, self.clip_noise)
|
||||||
# TODO: fix clipping
|
# TODO: fix clipping with squashing ?
|
||||||
return th.clamp(self.actor_net(latent_pi) + noise, -1, 1)
|
return th.clamp(self.actor_net(latent_pi) + noise, -1, 1)
|
||||||
else:
|
else:
|
||||||
return self.actor_net(obs)
|
return self.actor_net(obs)
|
||||||
|
|
@ -67,7 +67,7 @@ class Critic(BaseNetwork):
|
||||||
class TD3Policy(BasePolicy):
|
class TD3Policy(BasePolicy):
|
||||||
def __init__(self, observation_space, action_space,
|
def __init__(self, observation_space, action_space,
|
||||||
learning_rate, net_arch=None, device='cpu',
|
learning_rate, net_arch=None, device='cpu',
|
||||||
activation_fn=nn.ReLU, use_sde=False, log_std_init=0.0):
|
activation_fn=nn.ReLU, use_sde=False, log_std_init=-2, clip_noise=0.1):
|
||||||
super(TD3Policy, self).__init__(observation_space, action_space, device)
|
super(TD3Policy, self).__init__(observation_space, action_space, device)
|
||||||
|
|
||||||
if net_arch is None:
|
if net_arch is None:
|
||||||
|
|
@ -86,6 +86,7 @@ class TD3Policy(BasePolicy):
|
||||||
self.actor_kwargs = self.net_args.copy()
|
self.actor_kwargs = self.net_args.copy()
|
||||||
self.actor_kwargs['use_sde'] = use_sde
|
self.actor_kwargs['use_sde'] = use_sde
|
||||||
self.actor_kwargs['log_std_init'] = log_std_init
|
self.actor_kwargs['log_std_init'] = log_std_init
|
||||||
|
self.actor_kwargs['clip_noise'] = clip_noise
|
||||||
|
|
||||||
self.actor, self.actor_target = None, None
|
self.actor, self.actor_target = None, None
|
||||||
self.critic, self.critic_target = None, None
|
self.critic, self.critic_target = None, None
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue