Fix wrong truncation in HER replay buffer (#1543)

* fix episode start idx that leads to wrong episode length

* add episode length test

* Update changelog

* Reformat files

* Use replay_buffer.dones to test HER truncation warning

* truncate_last_trajectory: sample truncated episode and handle infinite horizon tasks

* make test_truncate_last_trajectory independent of learning

* Add timeout comment HER truncate_last_trajectory

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>

* Update version.txt

* Update version

---------

Co-authored-by: Quentin Gallouédec <45557362+qgallouedec@users.noreply.github.com>
This commit is contained in:
Lara Bergmann 2023-06-07 15:57:12 +02:00 committed by GitHub
parent 4fcda6b2d4
commit 32778ddc94
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 142 additions and 18 deletions

View file

@ -3,7 +3,7 @@
Changelog Changelog
========== ==========
Release 2.0.0a11 (WIP) Release 2.0.0a12 (WIP)
-------------------------- --------------------------
**Gymnasium support** **Gymnasium support**
@ -41,6 +41,7 @@ Bug Fixes:
- Fixed loading DQN changes ``target_update_interval`` (@tobirohrer) - Fixed loading DQN changes ``target_update_interval`` (@tobirohrer)
- Fixed env checker to properly reset the env before calling ``step()`` when checking - Fixed env checker to properly reset the env before calling ``step()`` when checking
for ``Inf`` and ``NaN`` (@lutogniew) for ``Inf`` and ``NaN`` (@lutogniew)
- Fixed HER ``truncate_last_trajectory()`` (@lbergmann1)
Deprecations: Deprecations:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^
@ -1350,4 +1351,4 @@ And all the contributors:
@Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875 @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875
@anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong @anand-bala @hughperkins @sidney-tio @AlexPasqua @dominicgkerr @Akhilez @Rocamonde @tobirohrer @ZikangXiong
@DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError @harveybellini @DavyMorgan @FieteO @jonasreiher @npit @WeberSamuel @troiganto @DavyMorgan @luizapozzobon @Bonifatius94 @theSquaredError @harveybellini @DavyMorgan @FieteO @jonasreiher @npit @WeberSamuel @troiganto
@lutogniew @lutogniew @lbergmann1

View file

@ -162,16 +162,24 @@ class HerReplayBuffer(DictReplayBuffer):
# When episode ends, compute and store the episode length # When episode ends, compute and store the episode length
for env_idx in range(self.n_envs): for env_idx in range(self.n_envs):
if done[env_idx]: if done[env_idx]:
episode_start = self._current_ep_start[env_idx] self._compute_episode_length(env_idx)
episode_end = self.pos
if episode_end < episode_start: def _compute_episode_length(self, env_idx: int) -> None:
# Occurs when the buffer becomes full, the storage resumes at the """
# beginning of the buffer. This can happen in the middle of an episode. Compute and store the episode length for environment with index env_idx
episode_end += self.buffer_size
episode_indices = np.arange(episode_start, episode_end) % self.buffer_size :param env_idx: index of the environment for which the episode length should be computed
self.ep_length[episode_indices, env_idx] = episode_end - episode_start """
# Update the current episode start episode_start = self._current_ep_start[env_idx]
self._current_ep_start[env_idx] = self.pos episode_end = self.pos
if episode_end < episode_start:
# Occurs when the buffer becomes full, the storage resumes at the
# beginning of the buffer. This can happen in the middle of an episode.
episode_end += self.buffer_size
episode_indices = np.arange(episode_start, episode_end) % self.buffer_size
self.ep_length[episode_indices, env_idx] = episode_end - episode_start
# Update the current episode start
self._current_ep_start[env_idx] = self.pos
def sample(self, batch_size: int, env: Optional[VecNormalize] = None) -> DictReplayBufferSamples: def sample(self, batch_size: int, env: Optional[VecNormalize] = None) -> DictReplayBufferSamples:
""" """
@ -375,12 +383,19 @@ class HerReplayBuffer(DictReplayBuffer):
If not called, we assume that we continue the same trajectory (same episode). If not called, we assume that we continue the same trajectory (same episode).
""" """
# If we are at the start of an episode, no need to truncate # If we are at the start of an episode, no need to truncate
if (self.ep_start[self.pos] != self.pos).any(): if (self._current_ep_start != self.pos).any():
warnings.warn( warnings.warn(
"The last trajectory in the replay buffer will be truncated.\n" "The last trajectory in the replay buffer will be truncated.\n"
"If you are in the same episode as when the replay buffer was saved,\n" "If you are in the same episode as when the replay buffer was saved,\n"
"you should use `truncate_last_trajectory=False` to avoid that issue." "you should use `truncate_last_trajectory=False` to avoid that issue."
) )
self.ep_start[-1] = self.pos # only consider epsiodes that are not finished
# set done = True for current episodes for env_idx in np.where(self._current_ep_start != self.pos)[0]:
self.dones[self.pos - 1] = True # set done = True for last episodes
self.dones[self.pos - 1, env_idx] = True
# make sure that last episodes can be sampled and
# update next episode start (self._current_ep_start)
self._compute_episode_length(env_idx)
# handle infinite horizon tasks
if self.handle_timeout_termination:
self.timeouts[self.pos - 1, env_idx] = True # not an actual timeout, but it allows bootstrapping

View file

@ -1 +1 @@
2.0.0a11 2.0.0a12

View file

@ -267,7 +267,7 @@ def test_save_load_replay_buffer(n_envs, tmp_path, recwarn, truncate_last_trajec
model.load_replay_buffer(path, truncate_last_traj=truncate_last_trajectory) model.load_replay_buffer(path, truncate_last_traj=truncate_last_trajectory)
if truncate_last_trajectory: if truncate_last_trajectory and (old_replay_buffer.dones[old_replay_buffer.pos - 1] == 0).any():
assert len(recwarn) == 1 assert len(recwarn) == 1
warning = recwarn.pop(UserWarning) warning = recwarn.pop(UserWarning)
assert "The last trajectory in the replay buffer will be truncated" in str(warning.message) assert "The last trajectory in the replay buffer will be truncated" in str(warning.message)
@ -323,6 +323,114 @@ def test_full_replay_buffer():
model.learn(total_timesteps=100) model.learn(total_timesteps=100)
@pytest.mark.parametrize("n_envs", [1, 2])
@pytest.mark.parametrize("n_steps", [4, 5])
@pytest.mark.parametrize("handle_timeout_termination", [False, True])
def test_truncate_last_trajectory(n_envs, recwarn, n_steps, handle_timeout_termination):
"""
Test if 'truncate_last_trajectory' works correctly
"""
# remove gym warnings
warnings.filterwarnings(action="ignore", category=DeprecationWarning)
warnings.filterwarnings(action="ignore", category=UserWarning, module="gym")
n_bits = 4
def env_fn():
return BitFlippingEnv(n_bits=n_bits, continuous=True)
venv = make_vec_env(env_fn, n_envs)
replay_buffer = HerReplayBuffer(
buffer_size=int(1e4),
observation_space=venv.observation_space,
action_space=venv.action_space,
env=venv,
n_envs=n_envs,
n_sampled_goal=2,
goal_selection_strategy="future",
)
observations = venv.reset()
for _ in range(n_steps):
actions = np.random.rand(n_envs, n_bits)
next_observations, rewards, dones, infos = venv.step(actions)
replay_buffer.add(observations, next_observations, actions, rewards, dones, infos)
observations = next_observations
old_replay_buffer = deepcopy(replay_buffer)
pos = replay_buffer.pos
if handle_timeout_termination:
env_idx_not_finished = np.where(replay_buffer._current_ep_start != pos)[0]
# Check that there is no warning
assert len(recwarn) == 0
replay_buffer.truncate_last_trajectory()
if (old_replay_buffer.dones[pos - 1] == 0).any():
# at least one episode in the replay buffer did not finish
assert len(recwarn) == 1
warning = recwarn.pop(UserWarning)
assert "The last trajectory in the replay buffer will be truncated" in str(warning.message)
else:
# all episodes in the replay buffer are finished
assert len(recwarn) == 0
# next episode starts at current pos
assert (replay_buffer._current_ep_start == pos).all()
# done = True for last episodes
assert (replay_buffer.dones[pos - 1] == 1).all()
# for all episodes that are not finished before truncate_last_trajectory: timeouts should be 1
if handle_timeout_termination:
assert (replay_buffer.timeouts[pos - 1, env_idx_not_finished] == 1).all()
# episode length sould be != 0 -> episode can be sampled
assert (replay_buffer.ep_length[pos - 1] != 0).all()
# replay buffer should not have changed after truncate_last_trajectory (except dones[pos-1])
for key in ["observation", "desired_goal", "achieved_goal"]:
assert np.allclose(old_replay_buffer.observations[key], replay_buffer.observations[key])
assert np.allclose(old_replay_buffer.next_observations[key], replay_buffer.next_observations[key])
assert np.allclose(old_replay_buffer.actions, replay_buffer.actions)
assert np.allclose(old_replay_buffer.rewards, replay_buffer.rewards)
# we might change the last done of the last trajectory so we don't compare it
assert np.allclose(old_replay_buffer.dones[: pos - 1], replay_buffer.dones[: pos - 1])
assert np.allclose(old_replay_buffer.dones[pos:], replay_buffer.dones[pos:])
for _ in range(10):
actions = np.random.rand(n_envs, n_bits)
next_observations, rewards, dones, infos = venv.step(actions)
replay_buffer.add(observations, next_observations, actions, rewards, dones, infos)
observations = next_observations
# old oberservations must remain unchanged
for key in ["observation", "desired_goal", "achieved_goal"]:
assert np.allclose(old_replay_buffer.observations[key][:pos], replay_buffer.observations[key][:pos])
assert np.allclose(old_replay_buffer.next_observations[key][:pos], replay_buffer.next_observations[key][:pos])
assert np.allclose(old_replay_buffer.actions[:pos], replay_buffer.actions[:pos])
assert np.allclose(old_replay_buffer.rewards[:pos], replay_buffer.rewards[:pos])
assert np.allclose(old_replay_buffer.dones[: pos - 1], replay_buffer.dones[: pos - 1])
# new oberservations must differ from old observations
end_pos = replay_buffer.pos
for key in ["observation", "desired_goal", "achieved_goal"]:
assert not np.allclose(old_replay_buffer.observations[key][pos:end_pos], replay_buffer.observations[key][pos:end_pos])
assert not np.allclose(
old_replay_buffer.next_observations[key][pos:end_pos], replay_buffer.next_observations[key][pos:end_pos]
)
assert not np.allclose(old_replay_buffer.actions[pos:end_pos], replay_buffer.actions[pos:end_pos])
assert not np.allclose(old_replay_buffer.rewards[pos:end_pos], replay_buffer.rewards[pos:end_pos])
assert not np.allclose(old_replay_buffer.dones[pos - 1 : end_pos], replay_buffer.dones[pos - 1 : end_pos])
# all entries with index >= replay_buffer.pos must remain unchanged
for key in ["observation", "desired_goal", "achieved_goal"]:
assert np.allclose(old_replay_buffer.observations[key][end_pos:], replay_buffer.observations[key][end_pos:])
assert np.allclose(old_replay_buffer.next_observations[key][end_pos:], replay_buffer.next_observations[key][end_pos:])
assert np.allclose(old_replay_buffer.actions[end_pos:], replay_buffer.actions[end_pos:])
assert np.allclose(old_replay_buffer.rewards[end_pos:], replay_buffer.rewards[end_pos:])
assert np.allclose(old_replay_buffer.dones[end_pos:], replay_buffer.dones[end_pos:])
@pytest.mark.parametrize("n_bits", [10]) @pytest.mark.parametrize("n_bits", [10])
def test_performance_her(n_bits): def test_performance_her(n_bits):
""" """