Remove SDE support for TD3

This commit is contained in:
Antonin RAFFIN 2020-05-08 15:00:34 +02:00
parent 9a9870fa9f
commit c20af230f7
6 changed files with 40 additions and 372 deletions

View file

@ -3,12 +3,13 @@
Changelog Changelog
========== ==========
Pre-Release 0.6.0a6 (WIP) Pre-Release 0.6.0a7 (WIP)
------------------------------ ------------------------------
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- Remove State-Dependent Exploration (SDE) support for ``TD3``
New Features: New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^

View file

@ -686,6 +686,7 @@ class OffPolicyRLModel(BaseRLModel):
Default: -1 (only sample at the beginning of the rollout) Default: -1 (only sample at the beginning of the rollout)
:param use_sde_at_warmup: (bool) Whether to use SDE instead of uniform sampling :param use_sde_at_warmup: (bool) Whether to use SDE instead of uniform sampling
during the warm up phase (before learning starts) during the warm up phase (before learning starts)
:param sde_support: (bool) Whether the model support SDE or not
""" """
def __init__(self, def __init__(self,
@ -705,7 +706,8 @@ class OffPolicyRLModel(BaseRLModel):
seed: Optional[int] = None, seed: Optional[int] = None,
use_sde: bool = False, use_sde: bool = False,
sde_sample_freq: int = -1, sde_sample_freq: int = -1,
use_sde_at_warmup: bool = False): use_sde_at_warmup: bool = False,
sde_support: bool = True):
super(OffPolicyRLModel, self).__init__(policy, env, policy_base, learning_rate, super(OffPolicyRLModel, self).__init__(policy, env, policy_base, learning_rate,
policy_kwargs, verbose, policy_kwargs, verbose,
@ -717,11 +719,10 @@ class OffPolicyRLModel(BaseRLModel):
self.actor = None self.actor = None
self.replay_buffer = None # type: Optional[ReplayBuffer] self.replay_buffer = None # type: Optional[ReplayBuffer]
# Update policy keyword arguments # Update policy keyword arguments
self.policy_kwargs['use_sde'] = self.use_sde if sde_support:
self.policy_kwargs['use_sde'] = self.use_sde
self.policy_kwargs['device'] = self.device self.policy_kwargs['device'] = self.device
# For SDE only # For SDE only
self.rollout_data = None
self.on_policy_exploration = False
self.use_sde_at_warmup = use_sde_at_warmup self.use_sde_at_warmup = use_sde_at_warmup
def _setup_model(self): def _setup_model(self):
@ -786,12 +787,8 @@ class OffPolicyRLModel(BaseRLModel):
assert isinstance(env, VecEnv), "You must pass a VecEnv" assert isinstance(env, VecEnv), "You must pass a VecEnv"
assert env.num_envs == 1, "OffPolicyRLModel only support single environment" assert env.num_envs == 1, "OffPolicyRLModel only support single environment"
self.rollout_data = None
if self.use_sde: if self.use_sde:
self.actor.reset_noise() self.actor.reset_noise()
# Reset rollout data
if self.on_policy_exploration:
self.rollout_data = {key: [] for key in ['observations', 'actions', 'rewards', 'dones', 'values']}
callback.on_rollout_start() callback.on_rollout_start()
continue_training = True continue_training = True
@ -816,23 +813,25 @@ class OffPolicyRLModel(BaseRLModel):
unscaled_action, _ = self.predict(self._last_obs, deterministic=False) unscaled_action, _ = self.predict(self._last_obs, deterministic=False)
# Rescale the action from [low, high] to [-1, 1] # Rescale the action from [low, high] to [-1, 1]
scaled_action = self.policy.scale_action(unscaled_action) if isinstance(self.action_space, gym.spaces.Box):
scaled_action = self.policy.scale_action(unscaled_action)
if self.use_sde: # Add noise to the action (improve exploration)
# When using SDE, the action can be out of bounds if action_noise is not None:
# TODO: fix with squashing and account for that in the proba distribution # NOTE: in the original implementation of TD3, the noise was applied to the unscaled action
clipped_action = np.clip(scaled_action, -1, 1) # Update(October 2019): Not anymore
clipped_action = np.clip(scaled_action + action_noise(), -1, 1)
# We store the scaled action in the buffer
buffer_action = clipped_action
action = self.policy.unscale_action(clipped_action)
else: else:
clipped_action = scaled_action # Discrete case, no need to normalize or clip
buffer_action = unscaled_action
# Add noise to the action (improve exploration) action = buffer_action
if action_noise is not None:
# NOTE: in the original implementation of TD3, the noise was applied to the unscaled action
# Update(October 2019): Not anymore
clipped_action = np.clip(clipped_action + action_noise(), -1, 1)
# Rescale and perform action # Rescale and perform action
new_obs, reward, done, infos = env.step(self.policy.unscale_action(clipped_action)) new_obs, reward, done, infos = env.step(action)
# Only stop training if return value is False, not when it is None. # Only stop training if return value is False, not when it is None.
if callback.on_step() is False: if callback.on_step() is False:
@ -853,16 +852,7 @@ class OffPolicyRLModel(BaseRLModel):
# Avoid changing the original ones # Avoid changing the original ones
self._last_original_obs, new_obs_, reward_ = self._last_obs, new_obs, reward self._last_original_obs, new_obs_, reward_ = self._last_obs, new_obs, reward
replay_buffer.add(self._last_original_obs, new_obs_, clipped_action, reward_, done) replay_buffer.add(self._last_original_obs, new_obs_, buffer_action, reward_, done)
if self.rollout_data is not None:
# Assume only one env
self.rollout_data['observations'].append(self._last_obs[0].copy())
self.rollout_data['actions'].append(scaled_action[0].copy())
self.rollout_data['rewards'].append(reward[0].copy())
self.rollout_data['dones'].append(done[0].copy())
obs_tensor = th.FloatTensor(self._last_obs).to(self.device)
self.rollout_data['values'].append(self.vf_net(obs_tensor)[0].cpu().detach().numpy())
self._last_obs = new_obs self._last_obs = new_obs
# Save the unnormalized observation # Save the unnormalized observation
@ -880,6 +870,7 @@ class OffPolicyRLModel(BaseRLModel):
self._episode_num += 1 self._episode_num += 1
episode_rewards.append(episode_reward) episode_rewards.append(episode_reward)
total_timesteps.append(episode_timesteps) total_timesteps.append(episode_timesteps)
if action_noise is not None: if action_noise is not None:
action_noise.reset() action_noise.reset()
@ -890,7 +881,6 @@ class OffPolicyRLModel(BaseRLModel):
if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0: if len(self.ep_info_buffer) > 0 and len(self.ep_info_buffer[0]) > 0:
logger.logkv('ep_rew_mean', self.safe_mean([ep_info['r'] for ep_info in self.ep_info_buffer])) logger.logkv('ep_rew_mean', self.safe_mean([ep_info['r'] for ep_info in self.ep_info_buffer]))
logger.logkv('ep_len_mean', self.safe_mean([ep_info['l'] for ep_info in self.ep_info_buffer])) logger.logkv('ep_len_mean', self.safe_mean([ep_info['l'] for ep_info in self.ep_info_buffer]))
# logger.logkv("n_updates", n_updates)
logger.logkv("fps", fps) logger.logkv("fps", fps)
logger.logkv('time_elapsed', int(time.time() - self.start_time)) logger.logkv('time_elapsed', int(time.time() - self.start_time))
logger.logkv("total timesteps", self.num_timesteps) logger.logkv("total timesteps", self.num_timesteps)
@ -903,27 +893,6 @@ class OffPolicyRLModel(BaseRLModel):
mean_reward = np.mean(episode_rewards) if total_episodes > 0 else 0.0 mean_reward = np.mean(episode_rewards) if total_episodes > 0 else 0.0
# Post processing
if self.rollout_data is not None:
for key in ['observations', 'actions', 'rewards', 'dones', 'values']:
self.rollout_data[key] = th.FloatTensor(np.array(self.rollout_data[key])).to(self.device)
self.rollout_data['returns'] = self.rollout_data['rewards'].clone()
self.rollout_data['advantage'] = self.rollout_data['rewards'].clone()
# Compute return and advantage
last_return = 0.0
for step in reversed(range(len(self.rollout_data['rewards']))):
if step == len(self.rollout_data['rewards']) - 1:
next_non_terminal = 1.0 - done[0]
next_value = self.vf_net(th.FloatTensor(self._last_obs).to(self.device))[0].detach()
last_return = self.rollout_data['rewards'][step] + next_non_terminal * next_value
else:
next_non_terminal = 1.0 - self.rollout_data['dones'][step + 1]
last_return = self.rollout_data['rewards'][step] + self.gamma * last_return * next_non_terminal
self.rollout_data['returns'][step] = last_return
self.rollout_data['advantage'] = self.rollout_data['returns'] - self.rollout_data['values']
callback.on_rollout_end() callback.on_rollout_end()
return RolloutReturn(mean_reward, total_steps, total_episodes, continue_training) return RolloutReturn(mean_reward, total_steps, total_episodes, continue_training)

View file

@ -6,9 +6,7 @@ import torch.nn as nn
from stable_baselines3.common.preprocessing import get_action_dim from stable_baselines3.common.preprocessing import get_action_dim
from stable_baselines3.common.policies import (BasePolicy, register_policy, create_mlp, from stable_baselines3.common.policies import (BasePolicy, register_policy, create_mlp,
create_sde_features_extractor, NatureCNN, NatureCNN, BaseFeaturesExtractor, FlattenExtractor)
BaseFeaturesExtractor, FlattenExtractor)
from stable_baselines3.common.distributions import StateDependentNoiseDistribution
class Actor(BasePolicy): class Actor(BasePolicy):
@ -22,18 +20,6 @@ class Actor(BasePolicy):
(a CNN when using images, a nn.Flatten() layer otherwise) (a CNN when using images, a nn.Flatten() layer otherwise)
:param features_dim: (int) Number of features :param features_dim: (int) Number of features
:param activation_fn: (Type[nn.Module]) Activation function :param activation_fn: (Type[nn.Module]) Activation function
:param use_sde: (bool) Whether to use State Dependent Exploration or not
:param log_std_init: (float) Initial value for the log standard deviation
:param clip_noise: (float) Clip the magnitude of the noise
:param lr_sde: (float) Learning rate for the standard deviation of the noise
:param full_std: (bool) Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using SDE.
:param sde_net_arch: ([int]) Network architecture for extracting features
when using SDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: (bool) Use ``expln()`` function instead of ``exp()`` when using SDE to ensure
a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param normalize_images: (bool) Whether to normalize images or not, :param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default) dividing by 255.0 (True by default)
:param device: (Union[th.device, str]) Device on which the code should run. :param device: (Union[th.device, str]) Device on which the code should run.
@ -45,65 +31,25 @@ class Actor(BasePolicy):
features_extractor: nn.Module, features_extractor: nn.Module,
features_dim: int, features_dim: int,
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False,
log_std_init: float = -3,
clip_noise: Optional[float] = None,
lr_sde: float = 3e-4,
full_std: bool = False,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False,
normalize_images: bool = True, normalize_images: bool = True,
device: Union[th.device, str] = 'auto'): device: Union[th.device, str] = 'auto'):
super(Actor, self).__init__(observation_space, action_space, super(Actor, self).__init__(observation_space, action_space,
features_extractor=features_extractor, features_extractor=features_extractor,
normalize_images=normalize_images, normalize_images=normalize_images,
device=device, device=device,
squash_output=not use_sde) squash_output=True)
self.latent_pi, self.log_std = None, None
self.weights_dist, self.exploration_mat = None, None
self.use_sde, self.sde_optimizer = use_sde, None
self.full_std = full_std
self.sde_features_extractor = None
self.features_extractor = features_extractor self.features_extractor = features_extractor
self.normalize_images = normalize_images self.normalize_images = normalize_images
self.net_arch = net_arch self.net_arch = net_arch
self.features_dim = features_dim self.features_dim = features_dim
self.activation_fn = activation_fn self.activation_fn = activation_fn
self.clip_noise = clip_noise
self.lr_sde = lr_sde
self.log_std_init = log_std_init
self.sde_net_arch = sde_net_arch
self.use_expln = use_expln
self.full_std = full_std
action_dim = get_action_dim(self.action_space) action_dim = get_action_dim(self.action_space)
actor_net = create_mlp(features_dim, action_dim, net_arch, activation_fn, squash_output=True)
if use_sde: # Deterministic action
latent_pi_net = create_mlp(features_dim, -1, net_arch, activation_fn, squash_output=False) self.mu = nn.Sequential(*actor_net)
self.latent_pi = nn.Sequential(*latent_pi_net)
latent_sde_dim = net_arch[-1]
learn_features = sde_net_arch is not None
# Separate feature extractor for SDE
if sde_net_arch is not None:
self.sde_features_extractor, latent_sde_dim = create_sde_features_extractor(features_dim, sde_net_arch,
activation_fn)
# Create state dependent noise matrix (SDE)
self.action_dist = StateDependentNoiseDistribution(action_dim, full_std=full_std, use_expln=use_expln,
squash_output=False, learn_features=learn_features)
action_net, self.log_std = self.action_dist.proba_distribution_net(latent_dim=net_arch[-1],
latent_sde_dim=latent_sde_dim,
log_std_init=log_std_init)
# Squash output
self.mu = nn.Sequential(action_net, nn.Tanh())
self.sde_optimizer = th.optim.Adam([self.log_std], lr=lr_sde)
self.reset_noise()
else:
actor_net = create_mlp(features_dim, action_dim, net_arch, activation_fn, squash_output=True)
self.mu = nn.Sequential(*actor_net)
def _get_data(self) -> Dict[str, Any]: def _get_data(self) -> Dict[str, Any]:
data = super()._get_data() data = super()._get_data()
@ -112,73 +58,14 @@ class Actor(BasePolicy):
net_arch=self.net_arch, net_arch=self.net_arch,
features_dim=self.features_dim, features_dim=self.features_dim,
activation_fn=self.activation_fn, activation_fn=self.activation_fn,
use_sde=self.use_sde,
log_std_init=self.log_std_init,
clip_noise=self.clip_noise,
lr_sde=self.lr_sde,
full_std=self.full_std,
sde_net_arch=self.sde_net_arch,
use_expln=self.use_expln,
features_extractor=self.features_extractor features_extractor=self.features_extractor
)) ))
return data return data
def get_std(self) -> th.Tensor:
"""
Retrieve the standard deviation of the action distribution.
Only useful when using SDE.
It corresponds to ``th.exp(log_std)`` in the normal case,
but is slightly different when using ``expln`` function
(cf StateDependentNoiseDistribution doc).
:return: (th.Tensor)
"""
return self.action_dist.get_std(self.log_std)
def _get_latent(self, obs: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
features = self.extract_features(obs)
latent_pi = self.latent_pi(features)
latent_sde = self.sde_features_extractor(features) if self.sde_features_extractor is not None else latent_pi
return latent_pi, latent_sde
def evaluate_actions(self, obs: th.Tensor, actions: th.Tensor) -> Tuple[th.Tensor, th.Tensor]:
"""
Evaluate actions according to the current policy,
given the observations. Only useful when using SDE.
:param obs: (th.Tensor)
:param actions: (th.Tensor)
:return: (th.Tensor, th.Tensor) log likelihood of taking those actions
and entropy of the action distribution.
"""
latent_pi, latent_sde = self._get_latent(obs)
mean_actions = self.mu(latent_pi)
distribution = self.action_dist.proba_distribution(mean_actions, self.log_std, latent_sde)
log_prob = distribution.log_prob(actions)
return log_prob, distribution.entropy()
def reset_noise(self) -> None:
"""
Sample new weights for the exploration matrix, when using SDE.
"""
self.action_dist.sample_weights(self.log_std)
def forward(self, obs: th.Tensor, deterministic: bool = True) -> th.Tensor: def forward(self, obs: th.Tensor, deterministic: bool = True) -> th.Tensor:
if self.use_sde: # assert deterministic, 'The TD3 actor only outputs deterministic actions'
latent_pi, latent_sde = self._get_latent(obs) features = self.extract_features(obs)
if deterministic: return self.mu(features)
return self.mu(latent_pi)
noise = self.action_dist.get_noise(latent_sde)
if self.clip_noise is not None:
noise = th.clamp(noise, -self.clip_noise, self.clip_noise)
# TODO: Replace with squashing -> need to account for that in the sde update
# -> set squash_output=True in the action_dist?
# NOTE: the clipping is done in the rollout for now
return self.mu(latent_pi) + noise
else:
features = self.extract_features(obs)
return self.mu(features)
def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor: def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor:
return self.forward(observation, deterministic=deterministic) return self.forward(observation, deterministic=deterministic)
@ -234,43 +121,6 @@ class Critic(BasePolicy):
return self.q1_net(th.cat([features, actions], dim=1)) return self.q1_net(th.cat([features, actions], dim=1))
class ValueFunction(BasePolicy):
"""
Value function for TD3 when doing on-policy exploration with SDE.
:param observation_space: (gym.spaces.Space) Obervation space
:param action_space: (gym.spaces.Space) Action space
:param features_extractor: (nn.Module) Network to extract features
(a CNN when using images, a nn.Flatten() layer otherwise)
:param features_dim: (int) Number of features
:param net_arch: (Optional[List[int]]) Network architecture
:param activation_fn: (Type[nn.Module]) Activation function
:param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default)
"""
def __init__(self, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space,
features_extractor: nn.Module,
features_dim: int,
net_arch: Optional[List[int]] = None,
activation_fn: Type[nn.Module] = nn.Tanh,
normalize_images: bool = True):
super(ValueFunction, self).__init__(observation_space, action_space,
features_extractor=features_extractor,
normalize_images=normalize_images)
if net_arch is None:
net_arch = [64, 64]
vf_net = create_mlp(features_dim, 1, net_arch, activation_fn)
self.vf_net = nn.Sequential(*vf_net)
def forward(self, obs: th.Tensor) -> th.Tensor:
with th.no_grad():
features = self.extract_features(obs)
return self.vf_net(features)
class TD3Policy(BasePolicy): class TD3Policy(BasePolicy):
""" """
Policy class (with both actor and critic) for TD3. Policy class (with both actor and critic) for TD3.
@ -281,14 +131,6 @@ class TD3Policy(BasePolicy):
:param net_arch: (Optional[List[int]]) The specification of the policy and value networks. :param net_arch: (Optional[List[int]]) The specification of the policy and value networks.
:param device: (Union[th.device, str]) Device on which the code should run. :param device: (Union[th.device, str]) Device on which the code should run.
:param activation_fn: (Type[nn.Module]) Activation function :param activation_fn: (Type[nn.Module]) Activation function
:param use_sde: (bool) Whether to use State Dependent Exploration or not
:param log_std_init: (float) Initial value for the log standard deviation
:param sde_net_arch: ([int]) Network architecture for extracting features
when using SDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: (bool) Use ``expln()`` function instead of ``exp()`` when using SDE to ensure
a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use. :param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param features_extractor_kwargs: (Optional[Dict[str, Any]]) Keyword arguments :param features_extractor_kwargs: (Optional[Dict[str, Any]]) Keyword arguments
to pass to the feature extractor. to pass to the feature extractor.
@ -305,12 +147,6 @@ class TD3Policy(BasePolicy):
net_arch: Optional[List[int]] = None, net_arch: Optional[List[int]] = None,
device: Union[th.device, str] = 'auto', device: Union[th.device, str] = 'auto',
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False,
log_std_init: float = -3,
clip_noise: Optional[float] = None,
lr_sde: float = 3e-4,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor, features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor,
features_extractor_kwargs: Optional[Dict[str, Any]] = None, features_extractor_kwargs: Optional[Dict[str, Any]] = None,
normalize_images: bool = True, normalize_images: bool = True,
@ -347,23 +183,9 @@ class TD3Policy(BasePolicy):
'normalize_images': normalize_images, 'normalize_images': normalize_images,
'device': device 'device': device
} }
self.actor_kwargs = self.net_args.copy()
sde_kwargs = {
'use_sde': use_sde,
'log_std_init': log_std_init,
'clip_noise': clip_noise,
'lr_sde': lr_sde,
'sde_net_arch': sde_net_arch,
'use_expln': use_expln
}
self.actor_kwargs.update(sde_kwargs)
self.actor, self.actor_target = None, None self.actor, self.actor_target = None, None
self.critic, self.critic_target = None, None self.critic, self.critic_target = None, None
# For SDE only
self.use_sde = use_sde
self.vf_net = None
self.log_std_init = log_std_init
self._build(lr_schedule) self._build(lr_schedule)
def _build(self, lr_schedule: Callable) -> None: def _build(self, lr_schedule: Callable) -> None:
@ -377,11 +199,6 @@ class TD3Policy(BasePolicy):
self.critic_target.load_state_dict(self.critic.state_dict()) self.critic_target.load_state_dict(self.critic.state_dict())
self.critic.optimizer = self.optimizer_class(self.critic.parameters(), lr=lr_schedule(1), self.critic.optimizer = self.optimizer_class(self.critic.parameters(), lr=lr_schedule(1),
**self.optimizer_kwargs) **self.optimizer_kwargs)
if self.use_sde:
self.vf_net = ValueFunction(self.observation_space, self.action_space,
features_extractor=self.features_extractor,
features_dim=self.features_dim)
self.actor.sde_optimizer.add_param_group({'params': self.vf_net.parameters()}) # pytype: disable=attribute-error
def _get_data(self) -> Dict[str, Any]: def _get_data(self) -> Dict[str, Any]:
data = super()._get_data() data = super()._get_data()
@ -389,12 +206,6 @@ class TD3Policy(BasePolicy):
data.update(dict( data.update(dict(
net_arch=self.net_args['net_arch'], net_arch=self.net_args['net_arch'],
activation_fn=self.net_args['activation_fn'], activation_fn=self.net_args['activation_fn'],
use_sde=self.actor_kwargs['use_sde'],
log_std_init=self.actor_kwargs['log_std_init'],
clip_noise=self.actor_kwargs['clip_noise'],
lr_sde=self.actor_kwargs['lr_sde'],
sde_net_arch=self.actor_kwargs['sde_net_arch'],
use_expln=self.actor_kwargs['use_expln'],
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
optimizer_class=self.optimizer_class, optimizer_class=self.optimizer_class,
optimizer_kwargs=self.optimizer_kwargs, optimizer_kwargs=self.optimizer_kwargs,
@ -403,11 +214,8 @@ class TD3Policy(BasePolicy):
)) ))
return data return data
def reset_noise(self) -> None:
return self.actor.reset_noise()
def make_actor(self) -> Actor: def make_actor(self) -> Actor:
return Actor(**self.actor_kwargs).to(self.device) return Actor(**self.net_args).to(self.device)
def make_critic(self) -> Critic: def make_critic(self) -> Critic:
return Critic(**self.net_args).to(self.device) return Critic(**self.net_args).to(self.device)
@ -432,14 +240,6 @@ class CnnPolicy(TD3Policy):
:param net_arch: (Optional[List[int]]) The specification of the policy and value networks. :param net_arch: (Optional[List[int]]) The specification of the policy and value networks.
:param device: (Union[th.device, str]) Device on which the code should run. :param device: (Union[th.device, str]) Device on which the code should run.
:param activation_fn: (Type[nn.Module]) Activation function :param activation_fn: (Type[nn.Module]) Activation function
:param use_sde: (bool) Whether to use State Dependent Exploration or not
:param log_std_init: (float) Initial value for the log standard deviation
:param sde_net_arch: ([int]) Network architecture for extracting features
when using SDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: (bool) Use ``expln()`` function instead of ``exp()`` when using SDE to ensure
a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use. :param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param features_extractor_kwargs: (Optional[Dict[str, Any]]) Keyword arguments :param features_extractor_kwargs: (Optional[Dict[str, Any]]) Keyword arguments
to pass to the feature extractor. to pass to the feature extractor.
@ -456,12 +256,6 @@ class CnnPolicy(TD3Policy):
net_arch: Optional[List[int]] = None, net_arch: Optional[List[int]] = None,
device: Union[th.device, str] = 'auto', device: Union[th.device, str] = 'auto',
activation_fn: Type[nn.Module] = nn.ReLU, activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False,
log_std_init: float = -3,
clip_noise: Optional[float] = None,
lr_sde: float = 3e-4,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN, features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN,
features_extractor_kwargs: Optional[Dict[str, Any]] = None, features_extractor_kwargs: Optional[Dict[str, Any]] = None,
normalize_images: bool = True, normalize_images: bool = True,
@ -473,12 +267,6 @@ class CnnPolicy(TD3Policy):
net_arch, net_arch,
device, device,
activation_fn, activation_fn,
use_sde,
log_std_init,
clip_noise,
lr_sde,
sde_net_arch,
use_expln,
features_extractor_class, features_extractor_class,
features_extractor_kwargs, features_extractor_kwargs,
normalize_images, normalize_images,

View file

@ -39,15 +39,6 @@ class TD3(OffPolicyRLModel):
:param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy :param target_policy_noise: (float) Standard deviation of Gaussian noise added to target policy
(smoothing noise) (smoothing noise)
:param target_noise_clip: (float) Limit for absolute value of target policy smoothing noise. :param target_noise_clip: (float) Limit for absolute value of target policy smoothing noise.
:param use_sde: (bool) Whether to use State Dependent Exploration (SDE)
instead of action noise exploration (default: False)
:param sde_sample_freq: (int) Sample a new noise matrix every n steps when using SDE
Default: -1 (only sample at the beginning of the rollout)
:param sde_max_grad_norm: (float)
:param sde_ent_coef: (float)
:param sde_log_std_scheduler: (callable)
:param use_sde_at_warmup: (bool) Whether to use SDE instead of uniform sampling
during the warm up phase (before learning starts)
:param create_eval_env: (bool) Whether to create a second environment that will be :param create_eval_env: (bool) Whether to create a second environment that will be
used for evaluating the agent periodically. (Only available when passing string for the environment) used for evaluating the agent periodically. (Only available when passing string for the environment)
:param policy_kwargs: (dict) additional arguments to be passed to the policy on creation :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
@ -73,12 +64,6 @@ class TD3(OffPolicyRLModel):
policy_delay: int = 2, policy_delay: int = 2,
target_policy_noise: float = 0.2, target_policy_noise: float = 0.2,
target_noise_clip: float = 0.5, target_noise_clip: float = 0.5,
use_sde: bool = False,
sde_sample_freq: int = -1,
sde_max_grad_norm: float = 1,
sde_ent_coef: float = 0.0,
sde_log_std_scheduler: Optional[Callable] = None,
use_sde_at_warmup: bool = False,
tensorboard_log: Optional[str] = None, tensorboard_log: Optional[str] = None,
create_eval_env: bool = False, create_eval_env: bool = False,
policy_kwargs: Dict[str, Any] = None, policy_kwargs: Dict[str, Any] = None,
@ -91,8 +76,7 @@ class TD3(OffPolicyRLModel):
buffer_size, learning_starts, batch_size, buffer_size, learning_starts, batch_size,
policy_kwargs, verbose, device, policy_kwargs, verbose, device,
create_eval_env=create_eval_env, seed=seed, create_eval_env=create_eval_env, seed=seed,
use_sde=use_sde, sde_sample_freq=sde_sample_freq, sde_support=False)
use_sde_at_warmup=use_sde_at_warmup)
self.train_freq = train_freq self.train_freq = train_freq
self.gradient_steps = gradient_steps self.gradient_steps = gradient_steps
@ -104,13 +88,6 @@ class TD3(OffPolicyRLModel):
self.target_noise_clip = target_noise_clip self.target_noise_clip = target_noise_clip
self.target_policy_noise = target_policy_noise self.target_policy_noise = target_policy_noise
# State Dependent Exploration
self.sde_max_grad_norm = sde_max_grad_norm
self.sde_ent_coef = sde_ent_coef
self.sde_log_std_scheduler = sde_log_std_scheduler
self.on_policy_exploration = True
self.sde_vf = None
if _init_setup_model: if _init_setup_model:
self._setup_model() self._setup_model()
@ -123,7 +100,6 @@ class TD3(OffPolicyRLModel):
self.actor_target = self.policy.actor_target self.actor_target = self.policy.actor_target
self.critic = self.policy.critic self.critic = self.policy.critic
self.critic_target = self.policy.critic_target self.critic_target = self.policy.critic_target
self.vf_net = self.policy.vf_net
def train(self, gradient_steps: int, batch_size: int = 100, policy_delay: int = 2) -> None: def train(self, gradient_steps: int, batch_size: int = 100, policy_delay: int = 2) -> None:
@ -178,52 +154,6 @@ class TD3(OffPolicyRLModel):
self._n_updates += gradient_steps self._n_updates += gradient_steps
logger.logkv("n_updates", self._n_updates) logger.logkv("n_updates", self._n_updates)
def train_sde(self) -> None:
# Update optimizer learning rate
# self._update_learning_rate(self.policy.optimizer)
# Unpack
obs, action, advantage, returns = [self.rollout_data[key] for key in
['observations', 'actions', 'advantage', 'returns']]
log_prob, entropy = self.actor.evaluate_actions(obs, action)
values = self.vf_net(obs).flatten()
# Normalize advantage
# if self.normalize_advantage:
# advantage = (advantage - advantage.mean()) / (advantage.std() + 1e-8)
# Value loss using the TD(gae_lambda) target
value_loss = F.mse_loss(returns, values)
# A2C loss
policy_loss = -(advantage * log_prob).mean()
# Entropy loss favor exploration
if entropy is None:
# Approximate entropy when no analytical form
entropy_loss = -log_prob.mean()
else:
entropy_loss = -th.mean(entropy)
vf_coef = 0.5
loss = policy_loss + self.sde_ent_coef * entropy_loss + vf_coef * value_loss
# Optimization step
self.actor.sde_optimizer.zero_grad()
loss.backward()
assert not th.isnan(log_prob).any(), log_prob
assert not th.isnan(entropy).any()
assert not th.isnan(self.actor.log_std.grad).any()
assert not th.isnan(self.actor.log_std).any()
# Clip grad norm
th.nn.utils.clip_grad_norm_([self.actor.log_std], self.sde_max_grad_norm)
self.actor.sde_optimizer.step()
del self.rollout_data
def learn(self, def learn(self,
total_timesteps: int, total_timesteps: int,
callback: MaybeCallback = None, callback: MaybeCallback = None,
@ -255,16 +185,6 @@ class TD3(OffPolicyRLModel):
self._update_current_progress(self.num_timesteps, total_timesteps) self._update_current_progress(self.num_timesteps, total_timesteps)
if self.num_timesteps > 0 and self.num_timesteps > self.learning_starts: if self.num_timesteps > 0 and self.num_timesteps > self.learning_starts:
if self.use_sde:
if self.sde_log_std_scheduler is not None:
# Call the scheduler
value = self.sde_log_std_scheduler(self._current_progress)
self.actor.log_std.data = th.ones_like(self.actor.log_std) * value
else:
# On-policy gradient
self.train_sde()
gradient_steps = self.gradient_steps if self.gradient_steps > 0 else rollout.episode_timesteps gradient_steps = self.gradient_steps if self.gradient_steps > 0 else rollout.episode_timesteps
self.train(gradient_steps, batch_size=self.batch_size, policy_delay=self.policy_delay) self.train(gradient_steps, batch_size=self.batch_size, policy_delay=self.policy_delay)
@ -280,7 +200,7 @@ class TD3(OffPolicyRLModel):
:return: (List[str]) List of parameters that should be excluded from save :return: (List[str]) List of parameters that should be excluded from save
""" """
# Exclude aliases # Exclude aliases
return super(TD3, self).excluded_save_params() + ["actor", "critic", "vf_net", "actor_target", "critic_target"] return super(TD3, self).excluded_save_params() + ["actor", "critic", "actor_target", "critic_target"]
def get_torch_variables(self) -> Tuple[List[str], List[str]]: def get_torch_variables(self) -> Tuple[List[str], List[str]]:
""" """

View file

@ -1 +1 @@
0.6.0a6 0.6.0a7

View file

@ -2,7 +2,7 @@ import pytest
import torch as th import torch as th
from torch.distributions import Normal from torch.distributions import Normal
from stable_baselines3 import A2C, TD3, SAC, PPO from stable_baselines3 import A2C, SAC, PPO
def test_state_dependent_exploration_grad(): def test_state_dependent_exploration_grad():
@ -54,20 +54,10 @@ def test_state_dependent_exploration_grad():
assert sigma_hat.grad.allclose(grad) assert sigma_hat.grad.allclose(grad)
@pytest.mark.parametrize("model_class", [TD3, SAC, A2C, PPO]) @pytest.mark.parametrize("model_class", [SAC, A2C, PPO])
@pytest.mark.parametrize("sde_net_arch", [None, [32, 16], []]) @pytest.mark.parametrize("sde_net_arch", [None, [32, 16], []])
@pytest.mark.parametrize("use_expln", [False, True]) @pytest.mark.parametrize("use_expln", [False, True])
def test_state_dependent_offpolicy_noise(model_class, sde_net_arch, use_expln): def test_state_dependent_offpolicy_noise(model_class, sde_net_arch, use_expln):
model = model_class('MlpPolicy', 'Pendulum-v0', use_sde=True, seed=None, create_eval_env=True, model = model_class('MlpPolicy', 'Pendulum-v0', use_sde=True, seed=None, create_eval_env=True,
verbose=1, policy_kwargs=dict(log_std_init=-2, sde_net_arch=sde_net_arch, use_expln=use_expln)) verbose=1, policy_kwargs=dict(log_std_init=-2, sde_net_arch=sde_net_arch, use_expln=use_expln))
model.learn(total_timesteps=int(500), eval_freq=250) model.learn(total_timesteps=int(500), eval_freq=250)
def test_scheduler():
def scheduler(progress):
return -2.0 * progress + 1
model = TD3('MlpPolicy', 'Pendulum-v0', use_sde=True, seed=None, create_eval_env=True,
verbose=1, sde_log_std_scheduler=scheduler)
model.learn(total_timesteps=int(1000), eval_freq=500)
assert th.isclose(model.actor.log_std, th.ones_like(model.actor.log_std)).all()