Merge pull request #49 from Antonin-Raffin/refactor/buffers

Refactor buffers
This commit is contained in:
Raffin, Antonin 2020-02-03 16:03:24 +01:00 committed by GitHub Enterprise
commit 9d52a7d7d6
10 changed files with 328 additions and 108 deletions

View file

@ -42,7 +42,7 @@ make type
``` ```
pip install -e .[docs] pip install -e .[docs]
make docs make doc
``` ```
Spell check for the documentation: Spell check for the documentation:
@ -58,7 +58,7 @@ To cite this repository in publications:
``` ```
@misc{torchy-baselines, @misc{torchy-baselines,
author = {Raffin, Antonin and Dormann, Noah and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi}, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah},
title = {Torchy Baselines}, title = {Torchy Baselines},
year = {2019}, year = {2019},
publisher = {GitHub}, publisher = {GitHub},

View file

@ -49,7 +49,7 @@ To cite this project in publications:
.. code-block:: bibtex .. code-block:: bibtex
@misc{torchy-baselines, @misc{torchy-baselines,
author = {Raffin, Antonin and Dormann, Noah and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi}, author = {Raffin, Antonin and Hill, Ashley and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Dormann, Noah},
title = {Torchy Baselines}, title = {Torchy Baselines},
year = {2019}, year = {2019},
publisher = {GitHub}, publisher = {GitHub},

View file

@ -3,13 +3,14 @@
Changelog Changelog
========== ==========
Pre-Release 0.2.0a0 (WIP) Pre-Release 0.2.0a1 (WIP)
------------------------------ ------------------------------
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- Python 2 support was dropped, Torchy Baselines now requires Python 3.6 or above - Python 2 support was dropped, Torchy Baselines now requires Python 3.6 or above
- Return type of `evaluation.evaluate_policy()` has been changed - Return type of `evaluation.evaluate_policy()` has been changed
- Refactored the replay buffer to avoid transformation between PyTorch and NumPy
New Features: New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^

View file

@ -10,7 +10,9 @@ setup(name='torchy_baselines',
'gym[classic_control]>=0.10.9', 'gym[classic_control]>=0.10.9',
'numpy', 'numpy',
'torch>=1.2.0', 'torch>=1.2.0',
'cloudpickle' 'cloudpickle',
# For reading logs
'pandas'
], ],
extras_require={ extras_require={
'tests': [ 'tests': [
@ -32,8 +34,6 @@ setup(name='torchy_baselines',
'extra': [ 'extra': [
# For render # For render
'opencv-python', 'opencv-python',
# For reading logs
'pandas'
] ]
}, },
description='Pytorch version of Stable Baselines, implementations of reinforcement learning algorithms.', description='Pytorch version of Stable Baselines, implementations of reinforcement learning algorithms.',
@ -45,7 +45,7 @@ setup(name='torchy_baselines',
license="MIT", license="MIT",
long_description="", long_description="",
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
version="0.2.0a0", version="0.2.0a1",
) )
# python setup.py sdist # python setup.py sdist

85
tests/test_monitor.py Normal file
View file

@ -0,0 +1,85 @@
import uuid
import json
import os
import pandas
import gym
from torchy_baselines.common.monitor import Monitor, get_monitor_files, load_results
def test_monitor():
"""
test the monitor wrapper
"""
env = gym.make("CartPole-v1")
env.seed(0)
monitor_file = "/tmp/stable_baselines-test-{}.monitor.csv".format(uuid.uuid4())
monitor_env = Monitor(env, monitor_file)
monitor_env.reset()
for _ in range(1000):
_, _, done, _ = monitor_env.step(0)
if done:
monitor_env.reset()
file_handler = open(monitor_file, 'rt')
first_line = file_handler.readline()
assert first_line.startswith('#')
metadata = json.loads(first_line[1:])
assert metadata['env_id'] == "CartPole-v1"
assert set(metadata.keys()) == {'env_id', 't_start'}, "Incorrect keys in monitor metadata"
last_logline = pandas.read_csv(file_handler, index_col=None)
assert set(last_logline.keys()) == {'l', 't', 'r'}, "Incorrect keys in monitor logline"
file_handler.close()
os.remove(monitor_file)
def test_monitor_load_results(tmp_path):
"""
test load_results on log files produced by the monitor wrapper
"""
tmp_path = str(tmp_path)
env1 = gym.make("CartPole-v1")
env1.seed(0)
monitor_file1 = os.path.join(tmp_path, "stable_baselines-test-{}.monitor.csv".format(uuid.uuid4()))
monitor_env1 = Monitor(env1, monitor_file1)
monitor_files = get_monitor_files(tmp_path)
assert len(monitor_files) == 1
assert monitor_file1 in monitor_files
monitor_env1.reset()
episode_count1 = 0
for _ in range(1000):
_, _, done, _ = monitor_env1.step(monitor_env1.action_space.sample())
if done:
episode_count1 += 1
monitor_env1.reset()
results_size1 = len(load_results(os.path.join(tmp_path)).index)
assert results_size1 == episode_count1
env2 = gym.make("CartPole-v1")
env2.seed(0)
monitor_file2 = os.path.join(tmp_path, "stable_baselines-test-{}.monitor.csv".format(uuid.uuid4()))
monitor_env2 = Monitor(env2, monitor_file2)
monitor_files = get_monitor_files(tmp_path)
assert len(monitor_files) == 2
assert monitor_file1 in monitor_files
assert monitor_file2 in monitor_files
monitor_env2.reset()
episode_count2 = 0
for _ in range(1000):
_, _, done, _ = monitor_env2.step(monitor_env2.action_space.sample())
if done:
episode_count2 += 1
monitor_env2.reset()
results_size2 = len(load_results(os.path.join(tmp_path)).index)
assert results_size2 == (results_size1 + episode_count2)
os.remove(monitor_file1)
os.remove(monitor_file2)

View file

@ -4,4 +4,4 @@ from torchy_baselines.ppo import PPO
from torchy_baselines.sac import SAC from torchy_baselines.sac import SAC
from torchy_baselines.td3 import TD3 from torchy_baselines.td3 import TD3
__version__ = "0.2.0a0" __version__ = "0.2.0a1"

View file

@ -119,8 +119,8 @@ class A2C(PPO):
th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
self.policy.optimizer.step() self.policy.optimizer.step()
explained_var = explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), explained_var = explained_variance(self.rollout_buffer.returns.flatten(),
self.rollout_buffer.values.flatten().cpu().numpy()) self.rollout_buffer.values.flatten())
logger.logkv("explained_variance", explained_var) logger.logkv("explained_variance", explained_var)
logger.logkv("entropy", entropy.mean().item()) logger.logkv("entropy", entropy.mean().item())

View file

@ -1,6 +1,10 @@
from typing import Union, Optional, Tuple, Generator
import numpy as np import numpy as np
import torch as th import torch as th
from torchy_baselines.common.vec_env import VecNormalize
class BaseBuffer(object): class BaseBuffer(object):
""" """
@ -9,10 +13,16 @@ class BaseBuffer(object):
:param buffer_size: (int) Max number of element in the buffer :param buffer_size: (int) Max number of element in the buffer
:param obs_dim: (int) Dimension of the observation :param obs_dim: (int) Dimension of the observation
:param action_dim: (int) Dimension of the action space :param action_dim: (int) Dimension of the action space
:param device: (th.device) :param device: (Union[th.device, str]) PyTorch device
to which the values will be converted
:param n_envs: (int) Number of parallel environments :param n_envs: (int) Number of parallel environments
""" """
def __init__(self, buffer_size, obs_dim, action_dim, device='cpu', n_envs=1): def __init__(self,
buffer_size: int,
obs_dim: int,
action_dim: int,
device: Union[th.device, str] = 'cpu',
n_envs: int = 1):
super(BaseBuffer, self).__init__() super(BaseBuffer, self).__init__()
self.buffer_size = buffer_size self.buffer_size = buffer_size
self.obs_dim = obs_dim self.obs_dim = obs_dim
@ -23,21 +33,21 @@ class BaseBuffer(object):
self.n_envs = n_envs self.n_envs = n_envs
@staticmethod @staticmethod
def swap_and_flatten(tensor): def swap_and_flatten(arr: np.ndarray) -> np.ndarray:
""" """
Swap and then flatten axes 0 (buffer_size) and 1 (n_envs) Swap and then flatten axes 0 (buffer_size) and 1 (n_envs)
to convert shape from [n_steps, n_envs, ...] (when ... is the shape of the features) to convert shape from [n_steps, n_envs, ...] (when ... is the shape of the features)
to [n_steps * n_envs, ...] (which maintain the order) to [n_steps * n_envs, ...] (which maintain the order)
:param tensor: (th.Tensor) :param arr: (np.ndarray)
:return: (th.Tensor) :return: (np.ndarray)
""" """
shape = tensor.shape shape = arr.shape
if len(shape) < 3: if len(shape) < 3:
shape = shape + (1,) shape = shape + (1,)
return tensor.transpose(0, 1).reshape(shape[0] * shape[1], *shape[2:]) return arr.swapaxes(0, 1).reshape(shape[0] * shape[1], *shape[2:])
def size(self): def size(self) -> int:
""" """
:return: (int) The current size of the buffer :return: (int) The current size of the buffer
""" """
@ -45,55 +55,75 @@ class BaseBuffer(object):
return self.buffer_size return self.buffer_size
return self.pos return self.pos
def add(self, *args, **kwargs): def add(self, *args, **kwargs) -> None:
""" """
Add elements to the buffer. Add elements to the buffer.
""" """
raise NotImplementedError() raise NotImplementedError()
def reset(self): def reset(self) -> None:
""" """
Reset the buffer. Reset the buffer.
""" """
self.pos = 0 self.pos = 0
self.full = False self.full = False
def sample(self, batch_size, env=None): def sample(self,
batch_size: int,
env: Optional[VecNormalize] = None
) -> Tuple[th.Tensor, ...]:
""" """
:param batch_size: (int) Number of element to sample :param batch_size: (int) Number of element to sample
:param env: (VecNormalize) [Optional] associated gym VecEnv :param env: (Optional[VecNormalize]) associated gym VecEnv
to normalize the observations/rewards when sampling to normalize the observations/rewards when sampling
""" """
upper_bound = self.buffer_size if self.full else self.pos upper_bound = self.buffer_size if self.full else self.pos
batch_inds = th.LongTensor( batch_inds = np.random.randint(0, upper_bound, size=batch_size)
np.random.randint(0, upper_bound, size=batch_size))
return self._get_samples(batch_inds, env=env) return self._get_samples(batch_inds, env=env)
def _get_samples(self, batch_inds, env=None): def _get_samples(self,
batch_inds: np.ndarray,
env: Optional[VecNormalize] = None
) -> Tuple[th.Tensor, ...]:
""" """
:param batch_inds: (th.Tensor) :param batch_inds: (th.Tensor)
:param env: (gym.Env) :param env: (Optional[VecNormalize])
:return: ([th.Tensor]) :return: ([th.Tensor])
""" """
raise NotImplementedError() raise NotImplementedError()
def to_torch(self, array: np.ndarray, copy: bool = True) -> th.Tensor:
"""
Convert a numpy array to a PyTorch tensor.
Note: it copies the data by default
:param array: (np.ndarray)
:param copy: (bool) Whether to copy or not the data
(may be useful to avoid changing things be reference)
:return: (th.Tensor)
"""
if copy:
return th.tensor(array).to(self.device)
return th.as_tensor(array).to(self.device)
@staticmethod @staticmethod
def _normalize_obs(obs, env=None): def _normalize_obs(obs: np.ndarray,
env: Optional[VecNormalize] = None) -> np.ndarray:
if env is not None: if env is not None:
# TODO: get rid of pytorch - numpy conversion return env.normalize_obs(obs).astype(np.float32)
return th.FloatTensor(env.normalize_obs(obs.numpy()))
return obs return obs
def _normalize_reward(self, reward, env=None): def _normalize_reward(self,
reward: np.ndarray,
env: Optional[VecNormalize] = None) -> np.ndarray:
if env is not None: if env is not None:
return th.FloatTensor(env.normalize_reward(reward.numpy())) return env.normalize_reward(reward).astype(np.float32)
return reward return reward
class ReplayBuffer(BaseBuffer): class ReplayBuffer(BaseBuffer):
""" """
Replay buffer used in off-policy algorithms like SAC/TD3. Replay buffer used in off-policy algorithms like SAC/TD3.
Adapted from from https://github.com/apourchot/CEM-RL
:param buffer_size: (int) Max number of element in the buffer :param buffer_size: (int) Max number of element in the buffer
:param obs_dim: (int) Dimension of the observation :param obs_dim: (int) Dimension of the observation
@ -101,35 +131,51 @@ class ReplayBuffer(BaseBuffer):
:param device: (th.device) :param device: (th.device)
:param n_envs: (int) Number of parallel environments :param n_envs: (int) Number of parallel environments
""" """
def __init__(self, buffer_size, obs_dim, action_dim, device='cpu', n_envs=1): def __init__(self,
buffer_size: int,
obs_dim: int,
action_dim: int,
device: Union[th.device, str] = 'cpu',
n_envs: int = 1):
super(ReplayBuffer, self).__init__(buffer_size, obs_dim, action_dim, device, n_envs=n_envs) super(ReplayBuffer, self).__init__(buffer_size, obs_dim, action_dim, device, n_envs=n_envs)
assert n_envs == 1 assert n_envs == 1, "Replay buffer only support single environment for now"
self.observations = th.zeros(self.buffer_size, self.n_envs, self.obs_dim)
self.actions = th.zeros(self.buffer_size, self.n_envs, self.action_dim)
self.next_observations = th.zeros(self.buffer_size, self.n_envs, self.obs_dim)
self.rewards = th.zeros(self.buffer_size, self.n_envs)
self.dones = th.zeros(self.buffer_size, self.n_envs)
def add(self, obs, next_obs, action, reward, done): self.observations = np.zeros((self.buffer_size, self.n_envs, self.obs_dim), dtype=np.float32)
self.actions = np.zeros((self.buffer_size, self.n_envs, self.action_dim), dtype=np.float32)
self.next_observations = np.zeros((self.buffer_size, self.n_envs, self.obs_dim), dtype=np.float32)
self.rewards = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.dones = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
def add(self,
obs: np.ndarray,
next_obs: np.ndarray,
action: np.ndarray,
reward: np.ndarray,
done: np.ndarray) -> None:
# Copy to avoid modification by reference # Copy to avoid modification by reference
self.observations[self.pos] = th.FloatTensor(np.array(obs).copy()) self.observations[self.pos] = np.array(obs).copy()
self.next_observations[self.pos] = th.FloatTensor(np.array(next_obs).copy()) self.next_observations[self.pos] = np.array(next_obs).copy()
self.actions[self.pos] = th.FloatTensor(np.array(action).copy()) self.actions[self.pos] = np.array(action).copy()
self.rewards[self.pos] = th.FloatTensor(np.array(reward).copy()) self.rewards[self.pos] = np.array(reward).copy()
self.dones[self.pos] = th.FloatTensor(np.array(done).copy()) self.dones[self.pos] = np.array(done).copy()
self.pos += 1 self.pos += 1
if self.pos == self.buffer_size: if self.pos == self.buffer_size:
self.full = True self.full = True
self.pos = 0 self.pos = 0
def _get_samples(self, batch_inds, env=None): def _get_samples(self,
return (self._normalize_obs(self.observations[batch_inds, 0, :], env).to(self.device), batch_inds: np.ndarray,
self.actions[batch_inds, 0, :].to(self.device), env: Optional[VecNormalize] = None
self._normalize_obs(self.next_observations[batch_inds, 0, :], env).to(self.device), ) -> Tuple[th.Tensor, ...]:
self.dones[batch_inds].to(self.device), data = (self._normalize_obs(self.observations[batch_inds, 0, :], env),
self._normalize_reward(self.rewards[batch_inds], env).to(self.device)) self.actions[batch_inds, 0, :],
self._normalize_obs(self.next_observations[batch_inds, 0, :], env),
self.dones[batch_inds],
self._normalize_reward(self.rewards[batch_inds], env))
return tuple(map(self.to_torch, data))
class RolloutBuffer(BaseBuffer): class RolloutBuffer(BaseBuffer):
@ -145,10 +191,16 @@ class RolloutBuffer(BaseBuffer):
:param gamma: (float) Discount factor :param gamma: (float) Discount factor
:param n_envs: (int) Number of parallel environments :param n_envs: (int) Number of parallel environments
""" """
def __init__(self, buffer_size, obs_dim, action_dim, device='cpu', def __init__(self,
gae_lambda=1, gamma=0.99, n_envs=1): buffer_size: int,
obs_dim: int,
action_dim: int,
device: Union[th.device, str] = 'cpu',
gae_lambda: float = 1,
gamma: float = 0.99,
n_envs: int = 1):
super(RolloutBuffer, self).__init__(buffer_size, obs_dim, action_dim, device, n_envs=n_envs) super(RolloutBuffer, self).__init__(buffer_size, obs_dim, action_dim, device, n_envs=n_envs)
# TODO: try the buffer on the gpu?
self.gae_lambda = gae_lambda self.gae_lambda = gae_lambda
self.gamma = gamma self.gamma = gamma
self.observations, self.actions, self.rewards, self.advantages = None, None, None, None self.observations, self.actions, self.rewards, self.advantages = None, None, None, None
@ -156,35 +208,41 @@ class RolloutBuffer(BaseBuffer):
self.generator_ready = False self.generator_ready = False
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self.observations = th.zeros(self.buffer_size, self.n_envs, self.obs_dim) self.observations = np.zeros((self.buffer_size, self.n_envs, self.obs_dim), dtype=np.float32)
self.actions = th.zeros(self.buffer_size, self.n_envs, self.action_dim) self.actions = np.zeros((self.buffer_size, self.n_envs, self.action_dim), dtype=np.float32)
self.rewards = th.zeros(self.buffer_size, self.n_envs) self.rewards = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.returns = th.zeros(self.buffer_size, self.n_envs) self.returns = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.dones = th.zeros(self.buffer_size, self.n_envs) self.dones = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.values = th.zeros(self.buffer_size, self.n_envs) self.values = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.log_probs = th.zeros(self.buffer_size, self.n_envs) self.log_probs = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.advantages = th.zeros(self.buffer_size, self.n_envs) self.advantages = np.zeros((self.buffer_size, self.n_envs), dtype=np.float32)
self.generator_ready = False self.generator_ready = False
super(RolloutBuffer, self).reset() super(RolloutBuffer, self).reset()
def compute_returns_and_advantage(self, last_value, dones=False, use_gae=True): def compute_returns_and_advantage(self,
last_value: th.Tensor,
dones: np.ndarray,
use_gae: bool = True) -> None:
""" """
Post-processing step: compute the returns (sum of discounted rewards) Post-processing step: compute the returns (sum of discounted rewards)
and advantage (A(s) = R - V(S)). and advantage (A(s) = R - V(S)).
Adapted from Stable-Baselines PPO2. Adapted from Stable-Baselines PPO2.
:param last_value: (th.Tensor) :param last_value: (th.Tensor)
:param dones: ([bool]) :param dones: (np.ndarray)
:param use_gae: (bool) Whether to use Generalized Advantage Estimation :param use_gae: (bool) Whether to use Generalized Advantage Estimation
or normal advantage for advantage computation. or normal advantage for advantage computation.
""" """
# convert to numpy
last_value = last_value.clone().cpu().numpy().flatten()
if use_gae: if use_gae:
last_gae_lam = 0 last_gae_lam = 0
for step in reversed(range(self.buffer_size)): for step in reversed(range(self.buffer_size)):
if step == self.buffer_size - 1: if step == self.buffer_size - 1:
next_non_terminal = th.FloatTensor(1.0 - dones) next_non_terminal = 1.0 - dones
next_value = last_value.clone().cpu().flatten() next_value = last_value
else: else:
next_non_terminal = 1.0 - self.dones[step + 1] next_non_terminal = 1.0 - self.dones[step + 1]
next_value = self.values[step + 1] next_value = self.values[step + 1]
@ -199,8 +257,8 @@ class RolloutBuffer(BaseBuffer):
last_return = 0.0 last_return = 0.0
for step in reversed(range(self.buffer_size)): for step in reversed(range(self.buffer_size)):
if step == self.buffer_size - 1: if step == self.buffer_size - 1:
next_non_terminal = th.FloatTensor(1.0 - dones) next_non_terminal = 1.0 - dones
next_value = last_value.clone().cpu().flatten() next_value = last_value
last_return = self.rewards[step] + next_non_terminal * next_value last_return = self.rewards[step] + next_non_terminal * next_value
else: else:
next_non_terminal = 1.0 - self.dones[step + 1] next_non_terminal = 1.0 - self.dones[step + 1]
@ -208,7 +266,13 @@ class RolloutBuffer(BaseBuffer):
self.returns[step] = last_return self.returns[step] = last_return
self.advantages = self.returns - self.values self.advantages = self.returns - self.values
def add(self, obs, action, reward, done, value, log_prob): def add(self,
obs: np.ndarray,
action: np.ndarray,
reward: np.ndarray,
done: np.ndarray,
value: th.Tensor,
log_prob: th.Tensor) -> None:
""" """
:param obs: (np.ndarray) Observation :param obs: (np.ndarray) Observation
:param action: (np.ndarray) Action :param action: (np.ndarray) Action
@ -223,19 +287,19 @@ class RolloutBuffer(BaseBuffer):
# Reshape 0-d tensor to avoid error # Reshape 0-d tensor to avoid error
log_prob = log_prob.reshape(-1, 1) log_prob = log_prob.reshape(-1, 1)
self.observations[self.pos] = th.FloatTensor(np.array(obs).copy()) self.observations[self.pos] = np.array(obs).copy()
self.actions[self.pos] = th.FloatTensor(np.array(action).copy()) self.actions[self.pos] = np.array(action).copy()
self.rewards[self.pos] = th.FloatTensor(np.array(reward).copy()) self.rewards[self.pos] = np.array(reward).copy()
self.dones[self.pos] = th.FloatTensor(np.array(done).copy()) self.dones[self.pos] = np.array(done).copy()
self.values[self.pos] = th.FloatTensor(value.clone().cpu().flatten()) self.values[self.pos] = value.clone().cpu().numpy().flatten()
self.log_probs[self.pos] = th.FloatTensor(log_prob.cpu().clone()) self.log_probs[self.pos] = log_prob.clone().cpu().numpy()
self.pos += 1 self.pos += 1
if self.pos == self.buffer_size: if self.pos == self.buffer_size:
self.full = True self.full = True
def get(self, batch_size=None): def get(self, batch_size: Optional[int] = None) -> Generator[Tuple[th.Tensor, ...], None, None]:
assert self.full assert self.full, ''
indices = th.randperm(self.buffer_size * self.n_envs) indices = np.random.permutation(self.buffer_size * self.n_envs)
# Prepare the data # Prepare the data
if not self.generator_ready: if not self.generator_ready:
for tensor in ['observations', 'actions', 'values', for tensor in ['observations', 'actions', 'values',
@ -252,10 +316,12 @@ class RolloutBuffer(BaseBuffer):
yield self._get_samples(indices[start_idx:start_idx + batch_size]) yield self._get_samples(indices[start_idx:start_idx + batch_size])
start_idx += batch_size start_idx += batch_size
def _get_samples(self, batch_inds, env=None): def _get_samples(self, batch_inds: np.ndarray,
return (self.observations[batch_inds].to(self.device), env: Optional[VecNormalize] = None) -> Tuple[th.Tensor, ...]:
self.actions[batch_inds].to(self.device), data = (self.observations[batch_inds],
self.values[batch_inds].flatten().to(self.device), self.actions[batch_inds],
self.log_probs[batch_inds].flatten().to(self.device), self.values[batch_inds].flatten(),
self.advantages[batch_inds].flatten().to(self.device), self.log_probs[batch_inds].flatten(),
self.returns[batch_inds].flatten().to(self.device)) self.advantages[batch_inds].flatten(),
self.returns[batch_inds].flatten())
return tuple(map(self.to_torch, data))

View file

@ -1,29 +1,37 @@
""" __all__ = ['Monitor', 'get_monitor_files', 'load_results']
Taken from stable-baselines
"""
import csv import csv
import json import json
import os import os
import time import time
from glob import glob
from typing import Tuple, Dict, Any, List, Optional
from gym.core import Wrapper import gym
import pandas
import numpy as np
class Monitor(Wrapper): class Monitor(gym.Wrapper):
EXT = "monitor.csv" EXT = "monitor.csv"
file_handler = None file_handler = None
def __init__(self, env, filename=None, allow_early_resets=True, reset_keywords=(), info_keywords=()): def __init__(self,
env: gym.Env,
filename: Optional[str],
allow_early_resets: bool = True,
reset_keywords=(),
info_keywords=()):
""" """
A monitor wrapper for Gym environments, it is used to know the episode reward, length, time and other data. A monitor wrapper for Gym environments, it is used to know the episode reward, length, time and other data.
:param env: (Gym environment) The environment :param env: (gym.Env) The environment
:param filename: (str) the location to save a log file, can be None for no log :param filename: (Optional[str]) the location to save a log file, can be None for no log
:param allow_early_resets: (bool) allows the reset of the environment before it is done :param allow_early_resets: (bool) allows the reset of the environment before it is done
:param reset_keywords: (tuple) extra keywords for the reset call, if extra parameters are needed at reset :param reset_keywords: (tuple) extra keywords for the reset call, if extra parameters are needed at reset
:param info_keywords: (tuple) extra information to log, from the information return of environment.step :param info_keywords: (tuple) extra information to log, from the information return of environment.step
""" """
Wrapper.__init__(self, env=env) super(Monitor, self).__init__(env=env)
self.t_start = time.time() self.t_start = time.time()
if filename is None: if filename is None:
self.file_handler = None self.file_handler = None
@ -52,12 +60,12 @@ class Monitor(Wrapper):
self.total_steps = 0 self.total_steps = 0
self.current_reset_info = {} # extra info about the current episode, that was passed in during reset() self.current_reset_info = {} # extra info about the current episode, that was passed in during reset()
def reset(self, **kwargs): def reset(self, **kwargs) -> np.ndarray:
""" """
Calls the Gym environment reset. Can only be called if the environment is over, or if allow_early_resets is True Calls the Gym environment reset. Can only be called if the environment is over, or if allow_early_resets is True
:param kwargs: Extra keywords saved for the next episode. only if defined by reset_keywords :param kwargs: Extra keywords saved for the next episode. only if defined by reset_keywords
:return: ([int] or [float]) the first observation of the environment :return: (np.ndarray) the first observation of the environment
""" """
if not self.allow_early_resets and not self.needs_reset: if not self.allow_early_resets and not self.needs_reset:
raise RuntimeError("Tried to reset an environment before done. If you want to allow early resets, " raise RuntimeError("Tried to reset an environment before done. If you want to allow early resets, "
@ -67,16 +75,16 @@ class Monitor(Wrapper):
for key in self.reset_keywords: for key in self.reset_keywords:
value = kwargs.get(key) value = kwargs.get(key)
if value is None: if value is None:
raise ValueError('Expected you to pass kwarg %s into reset' % key) raise ValueError('Expected you to pass kwarg {} into reset'.format(key))
self.current_reset_info[key] = value self.current_reset_info[key] = value
return self.env.reset(**kwargs) return self.env.reset(**kwargs)
def step(self, action): def step(self, action: np.ndarray) -> Tuple[np.ndarray, float, bool, Dict[Any, Any]]:
""" """
Step the environment with the given action Step the environment with the given action
:param action: ([int] or [float]) the action :param action: (np.ndarray) the action
:return: ([int] or [float], [float], [bool], dict) observation, reward, done, information :return: (Tuple[np.ndarray, float, bool, Dict[Any, Any]]) observation, reward, done, information
""" """
if self.needs_reset: if self.needs_reset:
raise RuntimeError("Tried to step environment that needs reset") raise RuntimeError("Tried to step environment that needs reset")
@ -104,10 +112,11 @@ class Monitor(Wrapper):
""" """
Closes the environment Closes the environment
""" """
super(Monitor, self).close()
if self.file_handler is not None: if self.file_handler is not None:
self.file_handler.close() self.file_handler.close()
def get_total_steps(self): def get_total_steps(self) -> int:
""" """
Returns the total number of timesteps Returns the total number of timesteps
@ -115,7 +124,7 @@ class Monitor(Wrapper):
""" """
return self.total_steps return self.total_steps
def get_episode_rewards(self): def get_episode_rewards(self) -> List[float]:
""" """
Returns the rewards of all the episodes Returns the rewards of all the episodes
@ -123,7 +132,7 @@ class Monitor(Wrapper):
""" """
return self.episode_rewards return self.episode_rewards
def get_episode_lengths(self): def get_episode_lengths(self) -> List[int]:
""" """
Returns the number of timesteps of all the episodes Returns the number of timesteps of all the episodes
@ -131,10 +140,69 @@ class Monitor(Wrapper):
""" """
return self.episode_lengths return self.episode_lengths
def get_episode_times(self): def get_episode_times(self) -> List[float]:
""" """
Returns the runtime in seconds of all the episodes Returns the runtime in seconds of all the episodes
:return: ([float]) :return: ([float])
""" """
return self.episode_times return self.episode_times
class LoadMonitorResultsError(Exception):
"""
Raised when loading the monitor log fails.
"""
pass
def get_monitor_files(path: str) -> List[str]:
"""
get all the monitor files in the given path
:param path: (str) the logging folder
:return: ([str]) the log files
"""
return glob(os.path.join(path, "*" + Monitor.EXT))
def load_results(path: str) -> pandas.DataFrame:
"""
Load all Monitor logs from a given directory path matching ``*monitor.csv`` and ``*monitor.json``
:param path: (str) the directory path containing the log file(s)
:return: (pandas.DataFrame) the logged data
"""
# get both csv and (old) json files
monitor_files = (glob(os.path.join(path, "*monitor.json")) + get_monitor_files(path))
if not monitor_files:
raise LoadMonitorResultsError("no monitor files of the form *%s found in %s" % (Monitor.EXT, path))
data_frames = []
headers = []
for file_name in monitor_files:
with open(file_name, 'rt') as file_handler:
if file_name.endswith('csv'):
first_line = file_handler.readline()
assert first_line[0] == '#'
header = json.loads(first_line[1:])
data_frame = pandas.read_csv(file_handler, index_col=None)
headers.append(header)
elif file_name.endswith('json'): # Deprecated json format
episodes = []
lines = file_handler.readlines()
header = json.loads(lines[0])
headers.append(header)
for line in lines[1:]:
episode = json.loads(line)
episodes.append(episode)
data_frame = pandas.DataFrame(episodes)
else:
assert 0, 'unreachable'
data_frame['t'] += header['t_start']
data_frames.append(data_frame)
data_frame = pandas.concat(data_frames)
data_frame.sort_values('t', inplace=True)
data_frame.reset_index(inplace=True)
data_frame['t'] -= min(header['t_start'] for header in headers)
# data_frame.headers = headers # HACK to preserve backwards compatibility
return data_frame

View file

@ -275,8 +275,8 @@ class PPO(BaseRLModel):
np.mean(approx_kl_divs))) np.mean(approx_kl_divs)))
break break
explained_var = explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), explained_var = explained_variance(self.rollout_buffer.returns.flatten(),
self.rollout_buffer.values.flatten().cpu().numpy()) self.rollout_buffer.values.flatten())
logger.logkv("explained_variance", explained_var) logger.logkv("explained_variance", explained_var)
# TODO: gather stats for the entropy and other losses? # TODO: gather stats for the entropy and other losses?