Added support for vector envs in evaluation (#447)

* added vector env support to evaluate_policy

* fixed linting and documentation

* updated changelog

* fixed code style issue

* added tests for vec env

* fixed formatting

* renamed observations

* added comments for vector evaluation

* fixed issues

* Cleanup + bump version

* Add comment

* Fix wrong count of episodes

Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
Co-authored-by: Antonin Raffin <antonin.raffin@dlr.de>
This commit is contained in:
Benjamin Black 2021-05-28 04:40:29 -06:00 committed by GitHub
parent 88e1be9ff5
commit a038044d11
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 115 additions and 60 deletions

View file

@ -184,7 +184,7 @@ EvalCallback
^^^^^^^^^^^^ ^^^^^^^^^^^^
Evaluate periodically the performance of an agent, using a separate test environment. Evaluate periodically the performance of an agent, using a separate test environment.
It will save the best model if ``best_model_save_path`` folder is specified and save the evaluations results in a numpy archive (`evaluations.npz`) if ``log_path`` folder is specified. It will save the best model if ``best_model_save_path`` folder is specified and save the evaluations results in a numpy archive (``evaluations.npz``) if ``log_path`` folder is specified.
.. note:: .. note::
@ -222,7 +222,7 @@ CallbackList
^^^^^^^^^^^^ ^^^^^^^^^^^^
Class for chaining callbacks, they will be called sequentially. Class for chaining callbacks, they will be called sequentially.
Alternatively, you can pass directly a list of callbacks to the `learn()` method, it will be converted automatically to a ``CallbackList``. Alternatively, you can pass directly a list of callbacks to the ``learn()`` method, it will be converted automatically to a ``CallbackList``.
.. code-block:: python .. code-block:: python

View file

@ -4,7 +4,7 @@ Changelog
========== ==========
Release 1.1.0a9 (WIP) Release 1.1.0a10 (WIP)
--------------------------- ---------------------------
**Dict observation support, timeout handling and refactored HER** **Dict observation support, timeout handling and refactored HER**
@ -45,6 +45,7 @@ New Features:
- Added support for image observation when using ``HER`` - Added support for image observation when using ``HER``
- Added ``replay_buffer_class`` and ``replay_buffer_kwargs`` arguments to off-policy algorithms - Added ``replay_buffer_class`` and ``replay_buffer_kwargs`` arguments to off-policy algorithms
- Added ``kl_divergence`` helper for ``Distribution`` classes (@09tangriro) - Added ``kl_divergence`` helper for ``Distribution`` classes (@09tangriro)
- Added support for vector environments with ``num_envs > 1`` (@benblack769)
- Added ``wrapper_kwargs`` argument to ``make_vec_env`` (@amy12xx) - Added ``wrapper_kwargs`` argument to ``make_vec_env`` (@amy12xx)
Bug Fixes: Bug Fixes:
@ -696,4 +697,4 @@ And all the contributors:
@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37 @andyshih12 @RaphaelWag @xicocaio @tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37 @andyshih12 @RaphaelWag @xicocaio
@diditforlulz273 @liorcohen5 @ManifoldFR @mloo3 @SwamyDev @wmmc88 @megan-klaiber @thisray @diditforlulz273 @liorcohen5 @ManifoldFR @mloo3 @SwamyDev @wmmc88 @megan-klaiber @thisray
@tfederico @hn2 @LucasAlegre @AptX395 @zampanteymedio @JadenTravnik @decodyng @ardabbour @lorenz-h @mschweizer @lorepieri8 @vwxyzjn @tfederico @hn2 @LucasAlegre @AptX395 @zampanteymedio @JadenTravnik @decodyng @ardabbour @lorenz-h @mschweizer @lorepieri8 @vwxyzjn
@ShangqunYu @PierreExeter @JacopoPan @ltbd78 @tom-doerr @Atlis @liusida @09tangriro @amy12xx @juancroldan @ShangqunYu @PierreExeter @JacopoPan @ltbd78 @tom-doerr @Atlis @liusida @09tangriro @amy12xx @juancroldan @benblack769

View file

@ -318,9 +318,6 @@ class EvalCallback(EventCallback):
if not isinstance(eval_env, VecEnv): if not isinstance(eval_env, VecEnv):
eval_env = DummyVecEnv([lambda: eval_env]) eval_env = DummyVecEnv([lambda: eval_env])
if isinstance(eval_env, VecEnv):
assert eval_env.num_envs == 1, "You must pass only one environment for evaluation"
self.eval_env = eval_env self.eval_env = eval_env
self.best_model_save_path = best_model_save_path self.best_model_save_path = best_model_save_path
# Logs will be written in ``evaluations.npz`` # Logs will be written in ``evaluations.npz``
@ -355,9 +352,6 @@ class EvalCallback(EventCallback):
:param globals_: :param globals_:
""" """
info = locals_["info"] info = locals_["info"]
# VecEnv: unpack
if not isinstance(info, dict):
info = info[0]
if locals_["done"]: if locals_["done"]:
maybe_is_success = info.get("is_success") maybe_is_success = info.get("is_success")

View file

@ -5,7 +5,7 @@ import gym
import numpy as np import numpy as np
from stable_baselines3.common import base_class from stable_baselines3.common import base_class
from stable_baselines3.common.vec_env import VecEnv, VecMonitor, is_vecenv_wrapped from stable_baselines3.common.vec_env import DummyVecEnv, VecEnv, VecMonitor, is_vecenv_wrapped
def evaluate_policy( def evaluate_policy(
@ -21,7 +21,10 @@ def evaluate_policy(
) -> Union[Tuple[float, float], Tuple[List[float], List[int]]]: ) -> Union[Tuple[float, float], Tuple[List[float], List[int]]]:
""" """
Runs policy for ``n_eval_episodes`` episodes and returns average reward. Runs policy for ``n_eval_episodes`` episodes and returns average reward.
This is made to work only with one env. If a vector env is passed in, this divides the episodes to evaluate onto the
different elements of the vector env. This static division of work is done to
remove bias. See https://github.com/DLR-RM/stable-baselines3/issues/402 for more
details and discussion.
.. note:: .. note::
If environment has not been wrapped with ``Monitor`` wrapper, reward and If environment has not been wrapped with ``Monitor`` wrapper, reward and
@ -32,8 +35,7 @@ def evaluate_policy(
wrapper before anything else. wrapper before anything else.
:param model: The RL agent you want to evaluate. :param model: The RL agent you want to evaluate.
:param env: The gym environment. In the case of a ``VecEnv`` :param env: The gym environment or ``VecEnv`` environment.
this must contain only one environment.
:param n_eval_episodes: Number of episode to evaluate the agent :param n_eval_episodes: Number of episode to evaluate the agent
:param deterministic: Whether to use deterministic or stochastic actions :param deterministic: Whether to use deterministic or stochastic actions
:param render: Whether to render the environment or not :param render: Whether to render the environment or not
@ -52,14 +54,12 @@ def evaluate_policy(
""" """
is_monitor_wrapped = False is_monitor_wrapped = False
# Avoid circular import # Avoid circular import
from stable_baselines3.common.env_util import is_wrapped
from stable_baselines3.common.monitor import Monitor from stable_baselines3.common.monitor import Monitor
if isinstance(env, VecEnv): if not isinstance(env, VecEnv):
assert env.num_envs == 1, "You must pass only one environment when using this function" env = DummyVecEnv([lambda: env])
is_monitor_wrapped = is_vecenv_wrapped(env, VecMonitor) or env.env_is_wrapped(Monitor)[0]
else: is_monitor_wrapped = is_vecenv_wrapped(env, VecMonitor) or env.env_is_wrapped(Monitor)[0]
is_monitor_wrapped = is_wrapped(env, Monitor)
if not is_monitor_wrapped and warn: if not is_monitor_wrapped and warn:
warnings.warn( warnings.warn(
@ -69,41 +69,58 @@ def evaluate_policy(
UserWarning, UserWarning,
) )
episode_rewards, episode_lengths = [], [] n_envs = env.num_envs
not_reseted = True episode_rewards = []
while len(episode_rewards) < n_eval_episodes: episode_lengths = []
# Number of loops here might differ from true episodes
# played, if underlying wrappers modify episode lengths.
# Avoid double reset, as VecEnv are reset automatically.
if not isinstance(env, VecEnv) or not_reseted:
obs = env.reset()
not_reseted = False
done, state = False, None
episode_reward = 0.0
episode_length = 0
while not done:
action, state = model.predict(obs, state=state, deterministic=deterministic)
obs, reward, done, info = env.step(action)
episode_reward += reward
if callback is not None:
callback(locals(), globals())
episode_length += 1
if render:
env.render()
if is_monitor_wrapped: episode_counts = np.zeros(n_envs, dtype="int")
# Do not trust "done" with episode endings. # Divides episodes among different sub environments in the vector as evenly as possible
# Remove vecenv stacking (if any) episode_count_targets = np.array([(n_eval_episodes + i) // n_envs for i in range(n_envs)], dtype="int")
if isinstance(env, VecEnv):
info = info[0] current_rewards = np.zeros(n_envs)
if "episode" in info.keys(): current_lengths = np.zeros(n_envs, dtype="int")
# Monitor wrapper includes "episode" key in info if environment observations = env.reset()
# has been wrapped with it. Use those rewards instead. states = None
episode_rewards.append(info["episode"]["r"]) while (episode_counts < episode_count_targets).any():
episode_lengths.append(info["episode"]["l"]) actions, states = model.predict(observations, state=states, deterministic=deterministic)
else: observations, rewards, dones, infos = env.step(actions)
episode_rewards.append(episode_reward) current_rewards += rewards
episode_lengths.append(episode_length) current_lengths += 1
for i in range(n_envs):
if episode_counts[i] < episode_count_targets[i]:
# unpack values so that the callback can access the local variables
reward = rewards[i]
done = dones[i]
info = infos[i]
if callback is not None:
callback(locals(), globals())
if dones[i]:
if is_monitor_wrapped:
# Atari wrapper can send a "done" signal when
# the agent loses a life, but it does not correspond
# to the true end of episode
if "episode" in info.keys():
# Do not trust "done" with episode endings.
# Monitor wrapper includes "episode" key in info if environment
# has been wrapped with it. Use those rewards instead.
episode_rewards.append(info["episode"]["r"])
episode_lengths.append(info["episode"]["l"])
# Only increment at the real end of an episode
episode_counts[i] += 1
else:
episode_rewards.append(current_rewards[i])
episode_lengths.append(current_lengths[i])
episode_counts[i] += 1
current_rewards[i] = 0
current_lengths[i] = 0
if states is not None:
states[i] *= 0
if render:
env.render()
mean_reward = np.mean(episode_rewards) mean_reward = np.mean(episode_rewards)
std_reward = np.std(episode_rewards) std_reward = np.std(episode_rewards)

View file

@ -1 +1 @@
1.1.0a9 1.1.0a10

View file

@ -15,7 +15,7 @@ from stable_baselines3.common.callbacks import (
StopTrainingOnRewardThreshold, StopTrainingOnRewardThreshold,
) )
from stable_baselines3.common.env_util import make_vec_env from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3.common.envs import BitFlippingEnv from stable_baselines3.common.envs import BitFlippingEnv, IdentityEnv
from stable_baselines3.common.vec_env import DummyVecEnv from stable_baselines3.common.vec_env import DummyVecEnv
@ -102,10 +102,27 @@ def select_env(model_class) -> str:
return "Pendulum-v0" return "Pendulum-v0"
def test_eval_callback_vec_env():
# tests that eval callback does not crash when given a vector
n_eval_envs = 3
train_env = IdentityEnv()
eval_env = DummyVecEnv([lambda: IdentityEnv()] * n_eval_envs)
model = A2C("MlpPolicy", train_env, seed=0)
eval_callback = EvalCallback(
eval_env,
eval_freq=100,
warn=False,
)
model.learn(300, callback=eval_callback)
assert eval_callback.last_mean_reward == 100.0
def test_eval_success_logging(tmp_path): def test_eval_success_logging(tmp_path):
n_bits = 2 n_bits = 2
n_envs = 2
env = BitFlippingEnv(n_bits=n_bits) env = BitFlippingEnv(n_bits=n_bits)
eval_env = DummyVecEnv([lambda: BitFlippingEnv(n_bits=n_bits)]) eval_env = DummyVecEnv([lambda: BitFlippingEnv(n_bits=n_bits)] * n_envs)
eval_callback = EvalCallback( eval_callback = EvalCallback(
eval_env, eval_env,
eval_freq=250, eval_freq=250,

View file

@ -197,11 +197,37 @@ class AlwaysDoneWrapper(gym.Wrapper):
return self.last_obs return self.last_obs
@pytest.mark.parametrize("n_envs", [1, 2, 5, 7])
def test_evaluate_vector_env(n_envs):
# Tests that the number of episodes evaluated is correct
n_eval_episodes = 6
env = make_vec_env("CartPole-v1", n_envs)
model = A2C("MlpPolicy", "CartPole-v1", seed=0)
class CountCallback:
def __init__(self):
self.count = 0
def __call__(self, locals_, globals_):
if locals_["done"]:
self.count += 1
count_callback = CountCallback()
evaluate_policy(model, env, n_eval_episodes, callback=count_callback)
assert count_callback.count == n_eval_episodes
@pytest.mark.parametrize("vec_env_class", [None, DummyVecEnv, SubprocVecEnv]) @pytest.mark.parametrize("vec_env_class", [None, DummyVecEnv, SubprocVecEnv])
def test_evaluate_policy_monitors(vec_env_class): def test_evaluate_policy_monitors(vec_env_class):
# Make numpy warnings throw exception
np.seterr(all="raise")
# Test that results are correct with monitor environments. # Test that results are correct with monitor environments.
# Also test VecEnvs # Also test VecEnvs
n_eval_episodes = 2 n_eval_episodes = 3
n_envs = 2
env_id = "CartPole-v0" env_id = "CartPole-v0"
model = A2C("MlpPolicy", env_id, seed=0) model = A2C("MlpPolicy", env_id, seed=0)
@ -217,9 +243,9 @@ def test_evaluate_policy_monitors(vec_env_class):
env = wrapper_class(env) env = wrapper_class(env)
else: else:
if with_monitor: if with_monitor:
env = vec_env_class([lambda: wrapper_class(Monitor(gym.make(env_id)))]) env = vec_env_class([lambda: wrapper_class(Monitor(gym.make(env_id)))] * n_envs)
else: else:
env = vec_env_class([lambda: wrapper_class(gym.make(env_id))]) env = vec_env_class([lambda: wrapper_class(gym.make(env_id))] * n_envs)
return env return env
# Test that evaluation with VecEnvs works as expected # Test that evaluation with VecEnvs works as expected