Start CNN support (failing for SAC)

This commit is contained in:
Antonin RAFFIN 2020-04-21 16:22:46 +02:00
parent 6d2fc328c9
commit 93c2a01f91
9 changed files with 273 additions and 24 deletions

View file

@ -3,7 +3,7 @@
Changelog Changelog
========== ==========
Pre-Release 0.5.0a1 (WIP) Pre-Release 0.5.0a2 (WIP)
------------------------------ ------------------------------
Breaking Changes: Breaking Changes:
@ -15,6 +15,7 @@ New Features:
- Added ``optimizer`` and ``optimizer_kwargs`` to ``policy_kwargs`` in order to easily - Added ``optimizer`` and ``optimizer_kwargs`` to ``policy_kwargs`` in order to easily
customizer optimizers customizer optimizers
- Complete independent save/load for policies - Complete independent save/load for policies
- Add ``CnnPolicies`` to support images as input (caveat: only support Atari resolution for now)
Bug Fixes: Bug Fixes:

16
tests/test_cnn.py Normal file
View file

@ -0,0 +1,16 @@
import pytest
from torchy_baselines import A2C, PPO, SAC, TD3
from torchy_baselines.common.identity_env import FakeImageEnv
@pytest.mark.parametrize('model_class', [A2C, PPO, SAC])
def test_cnn(model_class):
# Fake grayscale with frameskip
env = FakeImageEnv(screen_height=84, screen_width=84, n_channels=1,
discrete = model_class not in {SAC, TD3})
if model_class in {A2C, PPO}:
kwargs = dict(n_steps=100)
else:
kwargs = dict(buffer_size=500)
_ = model_class('CnnPolicy', env, **kwargs).learn(500)

View file

@ -7,6 +7,7 @@ from torchy_baselines import A2C, PPO, SAC, TD3
@pytest.mark.parametrize('net_arch', [ @pytest.mark.parametrize('net_arch', [
[12, dict(vf=[16], pi=[8])], [12, dict(vf=[16], pi=[8])],
[4], [4],
[],
[4, 4], [4, 4],
[12, dict(vf=[8, 4], pi=[8])], [12, dict(vf=[8, 4], pi=[8])],
[12, dict(vf=[8], pi=[8, 4])], [12, dict(vf=[8], pi=[8, 4])],

View file

@ -1,5 +1,6 @@
import numpy as np from typing import List
import numpy as np
from gym import Env from gym import Env
from gym.spaces import Discrete, MultiDiscrete, MultiBinary, Box from gym.spaces import Discrete, MultiDiscrete, MultiBinary, Box
@ -102,3 +103,37 @@ class IdentityEnvMultiBinary(IdentityEnv):
self.action_space = MultiBinary(dim) self.action_space = MultiBinary(dim)
self.observation_space = self.action_space self.observation_space = self.action_space
self.reset() self.reset()
class FakeImageEnv(Env):
def __init__(self, action_dim: int = 6,
screen_height: int = 210,
screen_width: int = 160,
n_channels: int = 3,
discrete: bool = True):
"""
Fake atari environment for testing purposes.
"""
self.observation_space = Box(low=0, high=255, shape=(screen_height, screen_width, n_channels), dtype=np.uint8)
if discrete:
self.action_space = Discrete(action_dim)
else:
self.action_space = Box(low=-1, high=1, shape=(5,), dtype=np.float32)
self.ep_length = 10
def reset(self):
self.current_step = 0
return self.observation_space.sample()
def step(self, action: int):
reward = 0.0
self.current_step += 1
done = self.current_step >= self.ep_length
return self.observation_space.sample(), reward, done, {}
def render(self, mode='human'):
pass
def get_action_meanings(self) -> List[str]:
return ['NOOP']

View file

@ -7,7 +7,7 @@ import torch as th
import torch.nn as nn import torch.nn as nn
import numpy as np import numpy as np
from torchy_baselines.common.preprocessing import preprocess_obs from torchy_baselines.common.preprocessing import preprocess_obs, get_obs_dim
from torchy_baselines.common.utils import get_device, get_schedule_fn from torchy_baselines.common.utils import get_device, get_schedule_fn
@ -289,7 +289,8 @@ def create_mlp(input_dim: int,
modules.append(activation_fn()) modules.append(activation_fn())
if output_dim > 0: if output_dim > 0:
modules.append(nn.Linear(net_arch[-1], output_dim)) last_layer_dim = net_arch[-1] if len(net_arch) > 0 else input_dim
modules.append(nn.Linear(last_layer_dim, output_dim))
if squash_output: if squash_output:
modules.append(nn.Tanh()) modules.append(nn.Tanh())
return modules return modules
@ -453,3 +454,44 @@ class MlpExtractor(nn.Module):
""" """
shared_latent = self.shared_net(features) shared_latent = self.shared_net(features)
return self.policy_net(shared_latent), self.value_net(shared_latent) return self.policy_net(shared_latent), self.value_net(shared_latent)
class BaseFeaturesExtractor(nn.Module):
def __init__(self, observation_space: gym.Space, features_dim: int = 0):
super(BaseFeaturesExtractor, self).__init__()
assert features_dim > 0
self._observation_space = observation_space
self._features_dim = features_dim
@property
def features_dim(self) -> int:
return self._features_dim
def forward(self, observations: th.Tensor) -> th.Tensor:
raise NotImplementedError()
class FlattenExtractor(BaseFeaturesExtractor):
def __init__(self, observation_space: gym.Space):
super(FlattenExtractor, self).__init__(observation_space, get_obs_dim(observation_space))
self.flatten = nn.Flatten()
def forward(self, observations: th.Tensor) -> th.Tensor:
return self.flatten(observations)
class NatureCNN(BaseFeaturesExtractor):
def __init__(self, observation_space: gym.Space,
features_dim: int = 512):
super(NatureCNN, self).__init__(observation_space, features_dim)
# TODO: custom init?
# we assume WxHxC images
# TODO: compute shape before flatten
n_input_channels = observation_space.shape[-1]
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, 32, 8, stride=4), nn.ReLU(),
nn.Conv2d(32, 64, 4, stride=2), nn.ReLU(),
nn.Conv2d(64, 32, 3, stride=1), nn.ReLU(), nn.Flatten(),
nn.Linear(32 * 7 * 7, features_dim), nn.ReLU())
def forward(self, observations: th.Tensor) -> th.Tensor:
return self.cnn(observations)

View file

@ -47,8 +47,12 @@ def preprocess_obs(obs: th.Tensor, observation_space: spaces.Space,
:return: (th.Tensor) :return: (th.Tensor)
""" """
if isinstance(observation_space, spaces.Box): if isinstance(observation_space, spaces.Box):
if is_image_space(observation_space) and normalize_images: if is_image_space(observation_space):
return obs.float() / 255.0 # Re-order from BxWxHxC to BxCxWxH
obs = obs.permute(0, 3, 1, 2) # .contiguous()?
if normalize_images:
return obs.float() / 255.0
return obs.float()
return obs.float() return obs.float()
elif isinstance(observation_space, spaces.Discrete): elif isinstance(observation_space, spaces.Discrete):
# One hot encoding and convert to float to avoid errors # One hot encoding and convert to float to avoid errors

View file

@ -6,9 +6,9 @@ import torch as th
import torch.nn as nn import torch.nn as nn
import numpy as np import numpy as np
from torchy_baselines.common.preprocessing import get_obs_dim
from torchy_baselines.common.policies import (BasePolicy, register_policy, MlpExtractor, from torchy_baselines.common.policies import (BasePolicy, register_policy, MlpExtractor,
create_sde_features_extractor) create_sde_features_extractor, NatureCNN,
BaseFeaturesExtractor, FlattenExtractor)
from torchy_baselines.common.distributions import (make_proba_distribution, Distribution, from torchy_baselines.common.distributions import (make_proba_distribution, Distribution,
DiagGaussianDistribution, CategoricalDistribution, DiagGaussianDistribution, CategoricalDistribution,
StateDependentNoiseDistribution) StateDependentNoiseDistribution)
@ -37,6 +37,7 @@ class PPOPolicy(BasePolicy):
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param squash_output: (bool) Whether to squash the output using a tanh function, :param squash_output: (bool) Whether to squash the output using a tanh function,
this allows to ensure boundaries when using SDE. this allows to ensure boundaries when using SDE.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param normalize_images: (bool) Whether to normalize images or not, :param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default) dividing by 255.0 (True by default)
:param optimizer: (Type[th.optim.Optimizer]) The optimizer to use, :param optimizer: (Type[th.optim.Optimizer]) The optimizer to use,
@ -58,6 +59,7 @@ class PPOPolicy(BasePolicy):
sde_net_arch: Optional[List[int]] = None, sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
squash_output: bool = False, squash_output: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor,
normalize_images: bool = True, normalize_images: bool = True,
optimizer: Type[th.optim.Optimizer] = th.optim.Adam, optimizer: Type[th.optim.Optimizer] = th.optim.Adam,
optimizer_kwargs: Optional[Dict[str, Any]] = None): optimizer_kwargs: Optional[Dict[str, Any]] = None):
@ -65,7 +67,10 @@ class PPOPolicy(BasePolicy):
# Default network architecture, from stable-baselines # Default network architecture, from stable-baselines
if net_arch is None: if net_arch is None:
net_arch = [dict(pi=[64, 64], vf=[64, 64])] if features_extractor_class == FlattenExtractor:
net_arch = [dict(pi=[64, 64], vf=[64, 64])]
else:
net_arch = []
self.net_arch = net_arch self.net_arch = net_arch
self.activation_fn = activation_fn self.activation_fn = activation_fn
@ -79,9 +84,11 @@ class PPOPolicy(BasePolicy):
self.optimizer_class = optimizer self.optimizer_class = optimizer
self.optimizer_kwargs = optimizer_kwargs self.optimizer_kwargs = optimizer_kwargs
self.ortho_init = ortho_init self.ortho_init = ortho_init
# In the future, feature_extractor will be replaced with a CNN
self.features_extractor = nn.Flatten() self.features_extractor_class = features_extractor_class
self.features_dim = get_obs_dim(self.observation_space) self.features_extractor = features_extractor_class(self.observation_space)
self.features_dim = self.features_extractor.features_dim
self.normalize_images = normalize_images self.normalize_images = normalize_images
self.log_std_init = log_std_init self.log_std_init = log_std_init
dist_kwargs = None dist_kwargs = None
@ -119,7 +126,8 @@ class PPOPolicy(BasePolicy):
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
optimizer=self.optimizer_class, optimizer=self.optimizer_class,
optimizer_kwargs=self.optimizer_kwargs, optimizer_kwargs=self.optimizer_kwargs,
ortho_init=self.ortho_init ortho_init=self.ortho_init,
features_extractor_class=self.features_extractor_class
)) ))
return data return data
@ -266,4 +274,75 @@ class PPOPolicy(BasePolicy):
MlpPolicy = PPOPolicy MlpPolicy = PPOPolicy
class CnnPolicy(PPOPolicy):
"""
CnnPolicy class (with both actor and critic) for A2C and derivates (PPO).
:param observation_space: (gym.spaces.Space) Observation space
:param action_space: (gym.spaces.Space) Action space
:param lr_schedule: (Callable) Learning rate schedule (could be constant)
:param net_arch: ([int or dict]) The specification of the policy and value networks.
:param device: (str or th.device) Device on which the code should run.
:param activation_fn: (Type[nn.Module]) Activation function
:param ortho_init: (bool) Whether to use or not orthogonal initialization
:param use_sde: (bool) Whether to use State Dependent Exploration or not
:param log_std_init: (float) Initial value for the log standard deviation
:param full_std: (bool) Whether to use (n_features x n_actions) parameters
for the std instead of only (n_features,) when using SDE
:param sde_net_arch: ([int]) Network architecture for extracting features
when using SDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: (bool) Use ``expln()`` function instead of ``exp()`` to ensure
a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param squash_output: (bool) Whether to squash the output using a tanh function,
this allows to ensure boundaries when using SDE.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default)
:param optimizer: (Type[th.optim.Optimizer]) The optimizer to use,
``th.optim.Adam`` by default
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer
"""
def __init__(self,
observation_space: gym.spaces.Space,
action_space: gym.spaces.Space,
lr_schedule: Callable,
net_arch: Optional[List[Union[int, Dict[str, List[int]]]]] = None,
device: Union[th.device, str] = 'auto',
activation_fn: Type[nn.Module] = nn.Tanh,
ortho_init: bool = True,
use_sde: bool = False,
log_std_init: float = 0.0,
full_std: bool = True,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False,
squash_output: bool = False,
features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN,
normalize_images: bool = True,
optimizer: Type[th.optim.Optimizer] = th.optim.Adam,
optimizer_kwargs: Optional[Dict[str, Any]] = None):
super(CnnPolicy, self).__init__(observation_space,
action_space,
lr_schedule,
net_arch,
device,
activation_fn,
ortho_init,
use_sde,
log_std_init,
full_std,
sde_net_arch,
use_expln,
squash_output,
features_extractor_class,
normalize_images,
optimizer,
optimizer_kwargs)
register_policy("MlpPolicy", MlpPolicy) register_policy("MlpPolicy", MlpPolicy)
register_policy("CnnPolicy", CnnPolicy)

View file

@ -4,9 +4,10 @@ import gym
import torch as th import torch as th
import torch.nn as nn import torch.nn as nn
from torchy_baselines.common.preprocessing import get_action_dim, get_obs_dim from torchy_baselines.common.preprocessing import get_action_dim
from torchy_baselines.common.policies import (BasePolicy, register_policy, create_mlp, from torchy_baselines.common.policies import (BasePolicy, register_policy, create_mlp,
create_sde_features_extractor) create_sde_features_extractor, NatureCNN,
BaseFeaturesExtractor, FlattenExtractor)
from torchy_baselines.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution from torchy_baselines.common.distributions import SquashedDiagGaussianDistribution, StateDependentNoiseDistribution
# CAP the standard deviation of the actor # CAP the standard deviation of the actor
@ -75,9 +76,11 @@ class Actor(BasePolicy):
self.use_expln = use_expln self.use_expln = use_expln
self.full_std = full_std self.full_std = full_std
self.clip_mean = clip_mean self.clip_mean = clip_mean
last_layer_dim = net_arch[-1] if len(net_arch) > 0 else features_dim
if self.use_sde: if self.use_sde:
latent_sde_dim = net_arch[-1] latent_sde_dim = last_layer_dim
# Separate feature extractor for SDE # Separate feature extractor for SDE
if sde_net_arch is not None: if sde_net_arch is not None:
self.sde_features_extractor, latent_sde_dim = create_sde_features_extractor(features_dim, sde_net_arch, self.sde_features_extractor, latent_sde_dim = create_sde_features_extractor(features_dim, sde_net_arch,
@ -94,8 +97,8 @@ class Actor(BasePolicy):
self.mu = nn.Sequential(self.mu, nn.Hardtanh(min_val=-clip_mean, max_val=clip_mean)) self.mu = nn.Sequential(self.mu, nn.Hardtanh(min_val=-clip_mean, max_val=clip_mean))
else: else:
self.action_dist = SquashedDiagGaussianDistribution(action_dim) self.action_dist = SquashedDiagGaussianDistribution(action_dim)
self.mu = nn.Linear(net_arch[-1], action_dim) self.mu = nn.Linear(last_layer_dim, action_dim)
self.log_std = nn.Linear(net_arch[-1], action_dim) self.log_std = nn.Linear(last_layer_dim, action_dim)
def _get_data(self) -> Dict[str, Any]: def _get_data(self) -> Dict[str, Any]:
data = super()._get_data() data = super()._get_data()
@ -239,6 +242,7 @@ class SACPolicy(BasePolicy):
a positive standard deviation (cf paper). It allows to keep variance a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough. above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param clip_mean: (float) Clip the mean output when using SDE to avoid numerical instability. :param clip_mean: (float) Clip the mean output when using SDE to avoid numerical instability.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param normalize_images: (bool) Whether to normalize images or not, :param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default) dividing by 255.0 (True by default)
:param optimizer: (Type[th.optim.Optimizer]) The optimizer to use, :param optimizer: (Type[th.optim.Optimizer]) The optimizer to use,
@ -257,13 +261,17 @@ class SACPolicy(BasePolicy):
sde_net_arch: Optional[List[int]] = None, sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False, use_expln: bool = False,
clip_mean: float = 2.0, clip_mean: float = 2.0,
features_extractor_class: Type[BaseFeaturesExtractor] = FlattenExtractor,
normalize_images: bool = True, normalize_images: bool = True,
optimizer: Type[th.optim.Optimizer] = th.optim.Adam, optimizer: Type[th.optim.Optimizer] = th.optim.Adam,
optimizer_kwargs: Optional[Dict[str, Any]] = None): optimizer_kwargs: Optional[Dict[str, Any]] = None):
super(SACPolicy, self).__init__(observation_space, action_space, device, squash_output=True) super(SACPolicy, self).__init__(observation_space, action_space, device, squash_output=True)
if net_arch is None: if net_arch is None:
net_arch = [256, 256] if features_extractor_class == FlattenExtractor:
net_arch = [256, 256]
else:
net_arch = []
if optimizer_kwargs is None: if optimizer_kwargs is None:
optimizer_kwargs = {} optimizer_kwargs = {}
@ -271,9 +279,9 @@ class SACPolicy(BasePolicy):
self.optimizer_class = optimizer self.optimizer_class = optimizer
self.optimizer_kwargs = optimizer_kwargs self.optimizer_kwargs = optimizer_kwargs
# In the future, features_extractor will be replaced with a CNN self.features_extractor_class = features_extractor_class
self.features_extractor = nn.Flatten() self.features_extractor = features_extractor_class(self.observation_space)
self.features_dim = get_obs_dim(self.observation_space) self.features_dim = self.features_extractor.features_dim
self.net_arch = net_arch self.net_arch = net_arch
self.activation_fn = activation_fn self.activation_fn = activation_fn
@ -325,7 +333,8 @@ class SACPolicy(BasePolicy):
clip_mean=self.actor_kwargs['clip_mean'], clip_mean=self.actor_kwargs['clip_mean'],
lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone lr_schedule=self._dummy_schedule, # dummy lr schedule, not needed for loading policy alone
optimizer=self.optimizer_class, optimizer=self.optimizer_class,
optimizer_kwargs=self.optimizer_kwargs optimizer_kwargs=self.optimizer_kwargs,
features_extractor_class=self.features_extractor_class
)) ))
return data return data
@ -344,4 +353,66 @@ class SACPolicy(BasePolicy):
MlpPolicy = SACPolicy MlpPolicy = SACPolicy
class CnnPolicy(SACPolicy):
"""
Policy class (with both actor and critic) for SAC.
:param observation_space: (gym.spaces.Space) Observation space
:param action_space: (gym.spaces.Space) Action space
:param lr_schedule: (callable) Learning rate schedule (could be constant)
:param net_arch: (Optional[List[int]]) The specification of the policy and value networks.
:param device: (str or th.device) Device on which the code should run.
:param activation_fn: (Type[nn.Module]) Activation function
:param use_sde: (bool) Whether to use State Dependent Exploration or not
:param log_std_init: (float) Initial value for the log standard deviation
:param sde_net_arch: ([int]) Network architecture for extracting features
when using SDE. If None, the latent features from the policy will be used.
Pass an empty list to use the states as features.
:param use_expln: (bool) Use ``expln()`` function instead of ``exp()`` when using SDE to ensure
a positive standard deviation (cf paper). It allows to keep variance
above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.
:param clip_mean: (float) Clip the mean output when using SDE to avoid numerical instability.
:param features_extractor_class: (Type[BaseFeaturesExtractor]) Features extractor to use.
:param normalize_images: (bool) Whether to normalize images or not,
dividing by 255.0 (True by default)
:param optimizer: (Type[th.optim.Optimizer]) The optimizer to use,
``th.optim.Adam`` by default
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer
"""
def __init__(self, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space,
lr_schedule: Callable,
net_arch: Optional[List[int]] = None,
device: Union[th.device, str] = 'auto',
activation_fn: Type[nn.Module] = nn.ReLU,
use_sde: bool = False,
log_std_init: float = -3,
sde_net_arch: Optional[List[int]] = None,
use_expln: bool = False,
clip_mean: float = 2.0,
features_extractor_class: Type[BaseFeaturesExtractor] = NatureCNN,
normalize_images: bool = True,
optimizer: Type[th.optim.Optimizer] = th.optim.Adam,
optimizer_kwargs: Optional[Dict[str, Any]] = None):
super(CnnPolicy, self).__init__(observation_space,
action_space,
lr_schedule,
net_arch,
device,
activation_fn,
use_sde,
log_std_init,
sde_net_arch,
use_expln,
clip_mean,
features_extractor_class,
normalize_images,
optimizer,
optimizer_kwargs)
register_policy("MlpPolicy", MlpPolicy) register_policy("MlpPolicy", MlpPolicy)
register_policy("CnnPolicy", CnnPolicy)

View file

@ -1 +1 @@
0.5.0a1 0.5.0a2