Hotfix PPO + gSDE (#53)

* Fix variable being passed with gradients

* Update changelog

* Bump version

* Fixes #54
This commit is contained in:
Antonin RAFFIN 2020-06-10 18:58:35 +02:00 committed by GitHub
parent b833207142
commit 494ebfd20a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 18 additions and 13 deletions

View file

@ -4,9 +4,11 @@ Changelog
========== ==========
Pre-Release 0.7.0a1 (WIP) Pre-Release 0.7.0 (2020-06-10)
------------------------------ ------------------------------
**Hotfix for PPO/A2C + gSDE, internal refactoring and bug fixes**
Breaking Changes: Breaking Changes:
^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^
- ``render()`` method of ``VecEnvs`` now only accept one argument: ``mode`` - ``render()`` method of ``VecEnvs`` now only accept one argument: ``mode``
@ -38,6 +40,7 @@ Bug Fixes:
- Fixed ``seed()`` method for ``SubprocVecEnv`` - Fixed ``seed()`` method for ``SubprocVecEnv``
- Fixed loading on GPU for testing when using gSDE and ``deterministic=False`` - Fixed loading on GPU for testing when using gSDE and ``deterministic=False``
- Fixed ``register_policy`` to allow re-registering same policy for same sub-class (i.e. assign same value to same key). - Fixed ``register_policy`` to allow re-registering same policy for same sub-class (i.e. assign same value to same key).
- Fixed a bug where the gradient was passed when using ``gSDE`` with ``PPO``/``A2C``, this does not affect ``SAC``
Deprecations: Deprecations:
^^^^^^^^^^^^^ ^^^^^^^^^^^^^

View file

@ -525,7 +525,7 @@ class StateDependentNoiseDistribution(Distribution):
""" """
# Stop gradient if we don't want to influence the features # Stop gradient if we don't want to influence the features
self._latent_sde = latent_sde if self.learn_features else latent_sde.detach() self._latent_sde = latent_sde if self.learn_features else latent_sde.detach()
variance = th.mm(latent_sde ** 2, self.get_std(log_std) ** 2) variance = th.mm(self._latent_sde ** 2, self.get_std(log_std) ** 2)
self.distribution = Normal(mean_actions, th.sqrt(variance + self.epsilon)) self.distribution = Normal(mean_actions, th.sqrt(variance + self.epsilon))
return self return self

View file

@ -1 +1 @@
0.7.0a1 0.7.0

View file

@ -1,5 +1,3 @@
import os
import gym import gym
import pytest import pytest
import numpy as np import numpy as np
@ -138,7 +136,7 @@ def test_sync_vec_normalize():
assert unwrap_vec_normalize(env) is None assert unwrap_vec_normalize(env) is None
env = VecNormalize(env, norm_obs=True, norm_reward=True, clip_obs=10., clip_reward=10.) env = VecNormalize(env, norm_obs=True, norm_reward=True, clip_obs=100., clip_reward=100.)
assert isinstance(unwrap_vec_normalize(env), VecNormalize) assert isinstance(unwrap_vec_normalize(env), VecNormalize)
@ -147,9 +145,13 @@ def test_sync_vec_normalize():
assert isinstance(unwrap_vec_normalize(env), VecNormalize) assert isinstance(unwrap_vec_normalize(env), VecNormalize)
eval_env = DummyVecEnv([make_env]) eval_env = DummyVecEnv([make_env])
eval_env = VecNormalize(eval_env, training=False, norm_obs=True, norm_reward=True, clip_obs=10., clip_reward=10.) eval_env = VecNormalize(eval_env, training=False, norm_obs=True, norm_reward=True,
clip_obs=100., clip_reward=100.)
eval_env = VecFrameStack(eval_env, 1) eval_env = VecFrameStack(eval_env, 1)
env.seed(0)
env.action_space.seed(0)
env.reset() env.reset()
# Initialize running mean # Initialize running mean
latest_reward = None latest_reward = None