Update PyBullet example (#2049)

This commit is contained in:
Antonin RAFFIN 2024-11-29 14:58:09 +01:00 committed by GitHub
parent 98366922b4
commit 897d01d225
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 34 additions and 36 deletions

View file

@ -397,14 +397,14 @@ PyBullet: Normalizing input features
------------------------------------ ------------------------------------
Normalizing input features may be essential to successful training of an RL agent Normalizing input features may be essential to successful training of an RL agent
(by default, images are scaled but not other types of input), (by default, images are scaled, but other types of input are not),
for instance when training on `PyBullet <https://github.com/bulletphysics/bullet3/>`__ environments. For that, a wrapper exists and for instance when training on `PyBullet <https://github.com/bulletphysics/bullet3/>`__ environments.
will compute a running average and standard deviation of input features (it can do the same for rewards). For this, there is a wrapper ``VecNormalize`` that will compute a running average and standard deviation of the input features (it can do the same for rewards).
.. note:: .. note::
you need to install pybullet with ``pip install pybullet`` you need to install pybullet envs with ``pip install pybullet_envs_gymnasium``
.. image:: ../_static/img/colab-badge.svg .. image:: ../_static/img/colab-badge.svg
@ -413,36 +413,33 @@ will compute a running average and standard deviation of input features (it can
.. code-block:: python .. code-block:: python
import os from pathlib import Path
import gymnasium as gym
import pybullet_envs
from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize import pybullet_envs_gymnasium
from stable_baselines3.common.vec_env import VecNormalize
from stable_baselines3.common.env_util import make_vec_env
from stable_baselines3 import PPO from stable_baselines3 import PPO
# Note: pybullet is not compatible yet with Gymnasium
# you might need to use `import rl_zoo3.gym_patches`
# and use gym (not Gymnasium) to instantiate the env
# Alternatively, you can use the MuJoCo equivalent "HalfCheetah-v4" # Alternatively, you can use the MuJoCo equivalent "HalfCheetah-v4"
vec_env = DummyVecEnv([lambda: gym.make("HalfCheetahBulletEnv-v0")]) vec_env = make_vec_env("HalfCheetahBulletEnv-v0", n_envs=1)
# Automatically normalize the input features and reward # Automatically normalize the input features and reward
vec_env = VecNormalize(vec_env, norm_obs=True, norm_reward=True, vec_env = VecNormalize(vec_env, norm_obs=True, norm_reward=True, clip_obs=10.0)
clip_obs=10.)
model = PPO("MlpPolicy", vec_env) model = PPO("MlpPolicy", vec_env)
model.learn(total_timesteps=2000) model.learn(total_timesteps=2000)
# Don't forget to save the VecNormalize statistics when saving the agent # Don't forget to save the VecNormalize statistics when saving the agent
log_dir = "/tmp/" log_dir = Path("/tmp/")
model.save(log_dir + "ppo_halfcheetah") model.save(log_dir / "ppo_halfcheetah")
stats_path = os.path.join(log_dir, "vec_normalize.pkl") stats_path = log_dir / "vec_normalize.pkl"
vec_env.save(stats_path) vec_env.save(stats_path)
# To demonstrate loading # To demonstrate loading
del model, vec_env del model, vec_env
# Load the saved statistics # Load the saved statistics
vec_env = DummyVecEnv([lambda: gym.make("HalfCheetahBulletEnv-v0")]) vec_env = make_vec_env("HalfCheetahBulletEnv-v0", n_envs=1)
vec_env = VecNormalize.load(stats_path, vec_env) vec_env = VecNormalize.load(stats_path, vec_env)
# do not update them at test time # do not update them at test time
vec_env.training = False vec_env.training = False
@ -450,7 +447,7 @@ will compute a running average and standard deviation of input features (it can
vec_env.norm_reward = False vec_env.norm_reward = False
# Load the agent # Load the agent
model = PPO.load(log_dir + "ppo_halfcheetah", env=vec_env) model = PPO.load(log_dir / "ppo_halfcheetah", env=vec_env)
Hindsight Experience Replay (HER) Hindsight Experience Replay (HER)

View file

@ -36,7 +36,8 @@ Others:
Documentation: Documentation:
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^
Added Decisions and Dragons to resources. (@jmacglashan) - Added Decisions and Dragons to resources. (@jmacglashan)
- Updated PyBullet example, now compatible with Gymnasium
Release 2.4.0 (2024-11-18) Release 2.4.0 (2024-11-18)
-------------------------- --------------------------