Cleanup + reformat code

This commit is contained in:
Antonin RAFFIN 2020-05-08 15:10:46 +02:00
parent c20af230f7
commit 413a2386d9
8 changed files with 52 additions and 44 deletions

View file

@ -240,8 +240,7 @@ class BaseRLModel(ABC):
if (observation_space != env.observation_space if (observation_space != env.observation_space
# Special cases for images that need to be transposed # Special cases for images that need to be transposed
and not (is_image_space(env.observation_space) and not (is_image_space(env.observation_space)
and observation_space == VecTransposeImage.transpose_space(env.observation_space) and observation_space == VecTransposeImage.transpose_space(env.observation_space))):
)):
raise ValueError(f'Observation spaces do not match: {observation_space} != {env.observation_space}') raise ValueError(f'Observation spaces do not match: {observation_space} != {env.observation_space}')
if action_space != env.action_space: if action_space != env.action_space:
raise ValueError(f'Action spaces do not match: {action_space} != {env.action_space}') raise ValueError(f'Action spaces do not match: {action_space} != {env.action_space}')
@ -820,11 +819,11 @@ class OffPolicyRLModel(BaseRLModel):
if action_noise is not None: if action_noise is not None:
# NOTE: in the original implementation of TD3, the noise was applied to the unscaled action # NOTE: in the original implementation of TD3, the noise was applied to the unscaled action
# Update(October 2019): Not anymore # Update(October 2019): Not anymore
clipped_action = np.clip(scaled_action + action_noise(), -1, 1) scaled_action = np.clip(scaled_action + action_noise(), -1, 1)
# We store the scaled action in the buffer # We store the scaled action in the buffer
buffer_action = clipped_action buffer_action = scaled_action
action = self.policy.unscale_action(clipped_action) action = self.policy.unscale_action(scaled_action)
else: else:
# Discrete case, no need to normalize or clip # Discrete case, no need to normalize or clip
buffer_action = unscaled_action buffer_action = unscaled_action

View file

@ -20,6 +20,7 @@ class KVWriter(object):
""" """
Key Value writer Key Value writer
""" """
def writekvs(self, kvs: Dict) -> None: def writekvs(self, kvs: Dict) -> None:
""" """
write a dictionary to file write a dictionary to file
@ -39,6 +40,7 @@ class SeqWriter(object):
""" """
sequence writer sequence writer
""" """
def writeseq(self, seq: List): def writeseq(self, seq: List):
""" """
write an array to file write an array to file
@ -49,7 +51,7 @@ class SeqWriter(object):
class HumanOutputFormat(KVWriter, SeqWriter): class HumanOutputFormat(KVWriter, SeqWriter):
def __init__(self, filename_or_file: Union [str, TextIO]): def __init__(self, filename_or_file: Union[str, TextIO]):
""" """
log to a file, in a human readable format log to a file, in a human readable format

View file

@ -166,7 +166,7 @@ class BasePolicy(nn.Module):
module.bias.data.fill_(0.0) module.bias.data.fill_(0.0)
@staticmethod @staticmethod
def _dummy_schedule(progress: float) -> float: def _dummy_schedule(_progress: float) -> float:
""" (float) Useful for pickling policy.""" """ (float) Useful for pickling policy."""
return 0.0 return 0.0

View file

@ -11,8 +11,9 @@ from stable_baselines3.common.vec_env.util import copy_obs_dict, dict_to_obs, ob
class DummyVecEnv(VecEnv): class DummyVecEnv(VecEnv):
""" """
Creates a simple vectorized wrapper for multiple environments, calling each environment in sequence on the current Creates a simple vectorized wrapper for multiple environments, calling each environment in sequence on the current
Python process. This is useful for computationally simple environment such as ````cartpole-v1````, as the overhead of Python process. This is useful for computationally simple environment such as ``cartpole-v1``,
multiprocess or multithread outweighs the environment computation time. This can also be used for RL methods that as the overhead of multiprocess or multithread outweighs the environment computation time.
This can also be used for RL methods that
require a vectorized environment, but that you want a single environments to train with. require a vectorized environment, but that you want a single environments to train with.
:param env_fns: ([Gym Environment]) the list of environments to vectorize :param env_fns: ([Gym Environment]) the list of environments to vectorize

View file

@ -47,6 +47,7 @@ class PPOPolicy(BasePolicy):
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments, :param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer excluding the learning rate, to pass to the optimizer
""" """
def __init__(self, def __init__(self,
observation_space: gym.spaces.Space, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
@ -145,7 +146,8 @@ class PPOPolicy(BasePolicy):
:param n_envs: (int) :param n_envs: (int)
""" """
assert isinstance(self.action_dist, StateDependentNoiseDistribution), 'reset_noise() is only available when using SDE' assert isinstance(self.action_dist,
StateDependentNoiseDistribution), 'reset_noise() is only available when using SDE'
self.action_dist.sample_weights(self.log_std, batch_size=n_envs) self.action_dist.sample_weights(self.log_std, batch_size=n_envs)
def _build(self, lr_schedule: Callable) -> None: def _build(self, lr_schedule: Callable) -> None:
@ -319,6 +321,7 @@ class CnnPolicy(PPOPolicy):
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments, :param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer excluding the learning rate, to pass to the optimizer
""" """
def __init__(self, def __init__(self,
observation_space: gym.spaces.Space, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,

View file

@ -24,6 +24,7 @@ class Actor(BasePolicy):
dividing by 255.0 (True by default) dividing by 255.0 (True by default)
:param device: (Union[th.device, str]) Device on which the code should run. :param device: (Union[th.device, str]) Device on which the code should run.
""" """
def __init__(self, def __init__(self,
observation_space: gym.spaces.Space, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
@ -39,7 +40,6 @@ class Actor(BasePolicy):
device=device, device=device,
squash_output=True) squash_output=True)
self.features_extractor = features_extractor self.features_extractor = features_extractor
self.normalize_images = normalize_images self.normalize_images = normalize_images
self.net_arch = net_arch self.net_arch = net_arch
@ -87,6 +87,7 @@ class Critic(BasePolicy):
dividing by 255.0 (True by default) dividing by 255.0 (True by default)
:param device: (Union[th.device, str]) Device on which the code should run. :param device: (Union[th.device, str]) Device on which the code should run.
""" """
def __init__(self, observation_space: gym.spaces.Space, def __init__(self, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
net_arch: List[int], net_arch: List[int],
@ -141,6 +142,7 @@ class TD3Policy(BasePolicy):
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments, :param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer excluding the learning rate, to pass to the optimizer
""" """
def __init__(self, observation_space: gym.spaces.Space, def __init__(self, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
lr_schedule: Callable, lr_schedule: Callable,
@ -250,6 +252,7 @@ class CnnPolicy(TD3Policy):
:param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments, :param optimizer_kwargs: (Optional[Dict[str, Any]]) Additional keyword arguments,
excluding the learning rate, to pass to the optimizer excluding the learning rate, to pass to the optimizer
""" """
def __init__(self, observation_space: gym.spaces.Space, def __init__(self, observation_space: gym.spaces.Space,
action_space: gym.spaces.Space, action_space: gym.spaces.Space,
lr_schedule: Callable, lr_schedule: Callable,