Update PyBullet example (#2049)

This commit is contained in:
Antonin RAFFIN 2024-11-29 14:58:09 +01:00 committed by GitHub
parent 98366922b4
commit 897d01d225
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 34 additions and 36 deletions

View file

@ -397,14 +397,14 @@ PyBullet: Normalizing input features
------------------------------------ ------------------------------------
Normalizing input features may be essential to successful training of an RL agent Normalizing input features may be essential to successful training of an RL agent
(by default, images are scaled but not other types of input), (by default, images are scaled, but other types of input are not),
for instance when training on `PyBullet <https://github.com/bulletphysics/bullet3/>`__ environments. For that, a wrapper exists and for instance when training on `PyBullet <https://github.com/bulletphysics/bullet3/>`__ environments.
will compute a running average and standard deviation of input features (it can do the same for rewards). For this, there is a wrapper ``VecNormalize`` that will compute a running average and standard deviation of the input features (it can do the same for rewards).
.. note:: .. note::
you need to install pybullet with ``pip install pybullet`` you need to install pybullet envs with ``pip install pybullet_envs_gymnasium``
.. image:: ../_static/img/colab-badge.svg .. image:: ../_static/img/colab-badge.svg
@ -413,44 +413,41 @@ will compute a running average and standard deviation of input features (it can
.. code-block:: python .. code-block:: python
import os from pathlib import Path
import gymnasium as gym
import pybullet_envs
from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize import pybullet_envs_gymnasium
from stable_baselines3 import PPO
# Note: pybullet is not compatible yet with Gymnasium from stable_baselines3.common.vec_env import VecNormalize
# you might need to use `import rl_zoo3.gym_patches` from stable_baselines3.common.env_util import make_vec_env
# and use gym (not Gymnasium) to instantiate the env from stable_baselines3 import PPO
# Alternatively, you can use the MuJoCo equivalent "HalfCheetah-v4"
vec_env = DummyVecEnv([lambda: gym.make("HalfCheetahBulletEnv-v0")])
# Automatically normalize the input features and reward
vec_env = VecNormalize(vec_env, norm_obs=True, norm_reward=True,
clip_obs=10.)
model = PPO("MlpPolicy", vec_env) # Alternatively, you can use the MuJoCo equivalent "HalfCheetah-v4"
model.learn(total_timesteps=2000) vec_env = make_vec_env("HalfCheetahBulletEnv-v0", n_envs=1)
# Automatically normalize the input features and reward
vec_env = VecNormalize(vec_env, norm_obs=True, norm_reward=True, clip_obs=10.0)
# Don't forget to save the VecNormalize statistics when saving the agent model = PPO("MlpPolicy", vec_env)
log_dir = "/tmp/" model.learn(total_timesteps=2000)
model.save(log_dir + "ppo_halfcheetah")
stats_path = os.path.join(log_dir, "vec_normalize.pkl")
vec_env.save(stats_path)
# To demonstrate loading # Don't forget to save the VecNormalize statistics when saving the agent
del model, vec_env log_dir = Path("/tmp/")
model.save(log_dir / "ppo_halfcheetah")
stats_path = log_dir / "vec_normalize.pkl"
vec_env.save(stats_path)
# Load the saved statistics # To demonstrate loading
vec_env = DummyVecEnv([lambda: gym.make("HalfCheetahBulletEnv-v0")]) del model, vec_env
vec_env = VecNormalize.load(stats_path, vec_env)
# do not update them at test time
vec_env.training = False
# reward normalization is not needed at test time
vec_env.norm_reward = False
# Load the agent # Load the saved statistics
model = PPO.load(log_dir + "ppo_halfcheetah", env=vec_env) vec_env = make_vec_env("HalfCheetahBulletEnv-v0", n_envs=1)
vec_env = VecNormalize.load(stats_path, vec_env)
# do not update them at test time
vec_env.training = False
# reward normalization is not needed at test time
vec_env.norm_reward = False
# Load the agent
model = PPO.load(log_dir / "ppo_halfcheetah", env=vec_env)
Hindsight Experience Replay (HER) Hindsight Experience Replay (HER)

View file

@ -36,7 +36,8 @@ Others:
Documentation: Documentation:
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^
Added Decisions and Dragons to resources. (@jmacglashan) - Added Decisions and Dragons to resources. (@jmacglashan)
- Updated PyBullet example, now compatible with Gymnasium
Release 2.4.0 (2024-11-18) Release 2.4.0 (2024-11-18)
-------------------------- --------------------------