Add support for setting options at reset with VecEnv (#1606)

* Update signatures, and test with options

* Update changelog and black formatting

* Finish implementation (fixes, doc, tests)

* Use deepcopy to avoid side effects (modif by reference)

* Fix for mypy

---------

Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
This commit is contained in:
Hosseinkhan Rémy 2023-10-23 13:38:48 +02:00 committed by GitHub
parent 2ddf015cd9
commit aab545901f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 70 additions and 9 deletions

View file

@ -89,8 +89,8 @@ SB3 VecEnv API is actually close to Gym 0.21 API but differs to Gym 0.26+ API:
if no mode is passed or ``mode="rgb_array"`` is passed when calling ``vec_env.render`` then we use the default mode, otherwise, we use the OpenCV display. if no mode is passed or ``mode="rgb_array"`` is passed when calling ``vec_env.render`` then we use the default mode, otherwise, we use the OpenCV display.
Note that if ``render_mode != "rgb_array"``, you can only call ``vec_env.render()`` (without argument or with ``mode=env.render_mode``). Note that if ``render_mode != "rgb_array"``, you can only call ``vec_env.render()`` (without argument or with ``mode=env.render_mode``).
- the ``reset()`` method doesn't take any parameter. If you want to seed the pseudo-random generator, - the ``reset()`` method doesn't take any parameter. If you want to seed the pseudo-random generator or pass options,
you should call ``vec_env.seed(seed=seed)`` and ``obs = vec_env.reset()`` afterward. you should call ``vec_env.seed(seed=seed)``/``vec_env.set_options(options)`` and ``obs = vec_env.reset()`` afterward (seed and options are discared after each call to ``reset()``).
- methods and attributes of the underlying Gym envs can be accessed, called and set using ``vec_env.get_attr("attribute_name")``, - methods and attributes of the underlying Gym envs can be accessed, called and set using ``vec_env.get_attr("attribute_name")``,
``vec_env.env_method("method_name", args1, args2, kwargs1=kwargs1)`` and ``vec_env.set_attr("attribute_name", new_value)``. ``vec_env.env_method("method_name", args1, args2, kwargs1=kwargs1)`` and ``vec_env.set_attr("attribute_name", new_value)``.

View file

@ -15,6 +15,7 @@ New Features:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
- Improved error message of the ``env_checker`` for env wrongly detected as GoalEnv (``compute_reward()`` is defined) - Improved error message of the ``env_checker`` for env wrongly detected as GoalEnv (``compute_reward()`` is defined)
- Improved error message when mixing Gym API with VecEnv API (see GH#1694) - Improved error message when mixing Gym API with VecEnv API (see GH#1694)
- Add support for setting ``options`` at reset with VecEnv via the ``set_options()`` method. Same as seeds logic, options are reset at the end of an episode (@ReHoss)
Bug Fixes: Bug Fixes:
^^^^^^^^^^ ^^^^^^^^^^
@ -1465,6 +1466,6 @@ And all the contributors:
@Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede
@carlosluis @arjun-kg @tlpss @JonathanKuelz @Gabo-Tor @iwishiwasaneagle @carlosluis @arjun-kg @tlpss @JonathanKuelz @Gabo-Tor @iwishiwasaneagle
@Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875 @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875
@anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong @anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong @ReHoss
@DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError @harveybellini @DavyMorgan @FieteO @jonasreiher @npit @WeberSamuel @troiganto @DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError @harveybellini @DavyMorgan @FieteO @jonasreiher @npit @WeberSamuel @troiganto
@lutogniew @lbergmann1 @lukashass @BertrandDecoster @pseudo-rnd-thoughts @stefanbschneider @kyle-he @PatrickHelm @corentinlger @lutogniew @lbergmann1 @lukashass @BertrandDecoster @pseudo-rnd-thoughts @stefanbschneider @kyle-he @PatrickHelm @corentinlger

View file

@ -1,6 +1,7 @@
import inspect import inspect
import warnings import warnings
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from copy import deepcopy
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Type, Union from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Type, Union
import cloudpickle import cloudpickle
@ -67,6 +68,8 @@ class VecEnv(ABC):
self.reset_infos: List[Dict[str, Any]] = [{} for _ in range(num_envs)] self.reset_infos: List[Dict[str, Any]] = [{} for _ in range(num_envs)]
# seeds to be used in the next call to env.reset() # seeds to be used in the next call to env.reset()
self._seeds: List[Optional[int]] = [None for _ in range(num_envs)] self._seeds: List[Optional[int]] = [None for _ in range(num_envs)]
# options to be used in the next call to env.reset()
self._options: List[Dict[str, Any]] = [{} for _ in range(num_envs)]
try: try:
render_modes = self.get_attr("render_mode") render_modes = self.get_attr("render_mode")
@ -95,6 +98,12 @@ class VecEnv(ABC):
""" """
self._seeds = [None for _ in range(self.num_envs)] self._seeds = [None for _ in range(self.num_envs)]
def _reset_options(self) -> None:
"""
Reset the options that are going to be used at the next reset.
"""
self._options = [{} for _ in range(self.num_envs)]
@abstractmethod @abstractmethod
def reset(self) -> VecEnvObs: def reset(self) -> VecEnvObs:
""" """
@ -283,6 +292,22 @@ class VecEnv(ABC):
self._seeds = [seed + idx for idx in range(self.num_envs)] self._seeds = [seed + idx for idx in range(self.num_envs)]
return self._seeds return self._seeds
def set_options(self, options: Optional[Union[List[Dict], Dict]] = None) -> None:
"""
Set environment options for all environments.
If a dict is passed instead of a list, the same options will be used for all environments.
WARNING: Those options will only be passed to the environment at the next reset.
:param options: A dictionary of environment options to pass to each environment at the next reset.
"""
if options is None:
options = {}
# Use deepcopy to avoid side effects
if isinstance(options, dict):
self._options = deepcopy([options] * self.num_envs)
else:
self._options = deepcopy(options)
@property @property
def unwrapped(self) -> "VecEnv": def unwrapped(self) -> "VecEnv":
if isinstance(self, VecEnvWrapper): if isinstance(self, VecEnvWrapper):
@ -354,6 +379,9 @@ class VecEnvWrapper(VecEnv):
def seed(self, seed: Optional[int] = None) -> Sequence[Union[None, int]]: def seed(self, seed: Optional[int] = None) -> Sequence[Union[None, int]]:
return self.venv.seed(seed) return self.venv.seed(seed)
def set_options(self, options: Optional[Union[List[Dict], Dict]] = None) -> None:
return self.venv.set_options(options)
def close(self) -> None: def close(self) -> None:
return self.venv.close() return self.venv.close()

View file

@ -73,10 +73,12 @@ class DummyVecEnv(VecEnv):
def reset(self) -> VecEnvObs: def reset(self) -> VecEnvObs:
for env_idx in range(self.num_envs): for env_idx in range(self.num_envs):
obs, self.reset_infos[env_idx] = self.envs[env_idx].reset(seed=self._seeds[env_idx]) maybe_options = {"options": self._options[env_idx]} if self._options[env_idx] else {}
obs, self.reset_infos[env_idx] = self.envs[env_idx].reset(seed=self._seeds[env_idx], **maybe_options)
self._save_obs(env_idx, obs) self._save_obs(env_idx, obs)
# Seeds are only used once # Seeds and options are only used once
self._reset_seeds() self._reset_seeds()
self._reset_options()
return self._obs_from_buf() return self._obs_from_buf()
def close(self) -> None: def close(self) -> None:

View file

@ -42,7 +42,8 @@ def _worker(
observation, reset_info = env.reset() observation, reset_info = env.reset()
remote.send((observation, reward, done, info, reset_info)) remote.send((observation, reward, done, info, reset_info))
elif cmd == "reset": elif cmd == "reset":
observation, reset_info = env.reset(seed=data) maybe_options = {"options": data[1]} if data[1] else {}
observation, reset_info = env.reset(seed=data[0], **maybe_options)
remote.send((observation, reset_info)) remote.send((observation, reset_info))
elif cmd == "render": elif cmd == "render":
remote.send(env.render()) remote.send(env.render())
@ -132,11 +133,12 @@ class SubprocVecEnv(VecEnv):
def reset(self) -> VecEnvObs: def reset(self) -> VecEnvObs:
for env_idx, remote in enumerate(self.remotes): for env_idx, remote in enumerate(self.remotes):
remote.send(("reset", self._seeds[env_idx])) remote.send(("reset", (self._seeds[env_idx], self._options[env_idx])))
results = [remote.recv() for remote in self.remotes] results = [remote.recv() for remote in self.remotes]
obs, self.reset_infos = zip(*results) obs, self.reset_infos = zip(*results)
# Seeds are only used once # Seeds and options are only used once
self._reset_seeds() self._reset_seeds()
self._reset_options()
return _flatten_obs(obs, self.observation_space) return _flatten_obs(obs, self.observation_space)
def close(self) -> None: def close(self) -> None:

View file

@ -30,11 +30,13 @@ class CustomGymEnv(gym.Env):
self.current_step = 0 self.current_step = 0
self.ep_length = 4 self.ep_length = 4
self.render_mode = render_mode self.render_mode = render_mode
self.current_options: Optional[Dict] = None
def reset(self, *, seed: Optional[int] = None, options: Optional[Dict] = None): def reset(self, *, seed: Optional[int] = None, options: Optional[Dict] = None):
if seed is not None: if seed is not None:
self.seed(seed) self.seed(seed)
self.current_step = 0 self.current_step = 0
self.current_options = options
self._choose_next_state() self._choose_next_state()
return self.state, {} return self.state, {}
@ -160,6 +162,25 @@ def test_vecenv_custom_calls(vec_env_class, vec_env_wrapper):
assert getattr_result == [12] + [0 for _ in range(N_ENVS - 2)] + [12] assert getattr_result == [12] + [0 for _ in range(N_ENVS - 2)] + [12]
assert vec_env.get_attr("current_step", indices=[-1]) == [12] assert vec_env.get_attr("current_step", indices=[-1]) == [12]
# Checks that options are correctly passed
assert vec_env.get_attr("current_options")[0] is None
# Same options for all envs
options = {"hello": 1}
vec_env.set_options(options)
assert vec_env.get_attr("current_options")[0] is None
# Only effective at reset
vec_env.reset()
assert vec_env.get_attr("current_options") == [options] * N_ENVS
vec_env.reset()
# Options are reset
assert vec_env.get_attr("current_options")[0] is None
# Use a list of options, different for the first env
options = [{"hello": 1}] * N_ENVS
options[0] = {"other_option": 2}
vec_env.set_options(options)
vec_env.reset()
assert vec_env.get_attr("current_options") == options
vec_env.close() vec_env.close()
@ -487,7 +508,14 @@ def test_vec_deterministic(vec_env_class):
vec_env.seed(3) vec_env.seed(3)
new_obs = vec_env.reset() new_obs = vec_env.reset()
assert np.allclose(new_obs, obs) assert np.allclose(new_obs, obs)
vec_env.close() # Test with VecNormalize (VecEnvWrapper should call self.venv.seed())
vec_normalize = VecNormalize(vec_env)
vec_normalize.seed(3)
obs = vec_env.reset()
vec_normalize.seed(3)
new_obs = vec_env.reset()
assert np.allclose(new_obs, obs)
vec_normalize.close()
# Similar test but with make_vec_env # Similar test but with make_vec_env
vec_env_1 = make_vec_env("Pendulum-v1", n_envs=N_ENVS, vec_env_cls=vec_env_class, seed=0) vec_env_1 = make_vec_env("Pendulum-v1", n_envs=N_ENVS, vec_env_cls=vec_env_class, seed=0)
vec_env_2 = make_vec_env("Pendulum-v1", n_envs=N_ENVS, vec_env_cls=vec_env_class, seed=0) vec_env_2 = make_vec_env("Pendulum-v1", n_envs=N_ENVS, vec_env_cls=vec_env_class, seed=0)