mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-19 10:00:55 +00:00
Update SB3 ONNX export documentation (#1816)
This commit is contained in:
parent
a9273f968e
commit
620e58e61f
3 changed files with 57 additions and 38 deletions
|
|
@ -31,53 +31,52 @@ to do inference in another framework.
|
||||||
Export to ONNX
|
Export to ONNX
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
As of June 2021, ONNX format `doesn't support <https://github.com/onnx/onnx/issues/3033>`_ exporting models that use the ``broadcast_tensors`` functionality of pytorch. So in order to export the trained stable-baseline3 models in the ONNX format, we need to first remove the layers that use broadcasting. This can be done by creating a class that removes the unsupported layers.
|
|
||||||
|
|
||||||
The following examples are for ``MlpPolicy`` only, and are general examples. Note that you have to preprocess the observation the same way stable-baselines3 agent does (see ``common.preprocessing.preprocess_obs``).
|
If you are using PyTorch 2.0+ and ONNX Opset 14+, you can easily export SB3 policies using the following code:
|
||||||
|
|
||||||
For PPO, assuming a shared feature extractor.
|
|
||||||
|
|
||||||
.. warning::
|
.. warning::
|
||||||
|
|
||||||
The following example is for continuous actions only.
|
The following returns normalized actions and doesn't include the `post-processing <https://github.com/DLR-RM/stable-baselines3/blob/a9273f968eaf8c6e04302a07d803eebfca6e7e86/stable_baselines3/common/policies.py#L370-L377>`_ step that is done with continuous actions
|
||||||
When using discrete or binary actions, you must do some `post-processing <https://github.com/DLR-RM/stable-baselines3/blob/f3a35aa786ee41ffff599b99fa1607c067e89074/stable_baselines3/common/policies.py#L621-L637>`_
|
(clip or unscale the action to the correct space).
|
||||||
to obtain the action (e.g., convert action logits to action).
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
import torch as th
|
import torch as th
|
||||||
|
from typing import Tuple
|
||||||
|
|
||||||
from stable_baselines3 import PPO
|
from stable_baselines3 import PPO
|
||||||
|
from stable_baselines3.common.policies import BasePolicy
|
||||||
|
|
||||||
|
|
||||||
class OnnxablePolicy(th.nn.Module):
|
class OnnxableSB3Policy(th.nn.Module):
|
||||||
def __init__(self, extractor, action_net, value_net):
|
def __init__(self, policy: BasePolicy):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.extractor = extractor
|
self.policy = policy
|
||||||
self.action_net = action_net
|
|
||||||
self.value_net = value_net
|
|
||||||
|
|
||||||
def forward(self, observation):
|
def forward(self, observation: th.Tensor) -> Tuple[th.Tensor, th.Tensor, th.Tensor]:
|
||||||
# NOTE: You may have to process (normalize) observation in the correct
|
# NOTE: Preprocessing is included, but postprocessing
|
||||||
# way before using this. See `common.preprocessing.preprocess_obs`
|
# (clipping/inscaling actions) is not,
|
||||||
action_hidden, value_hidden = self.extractor(observation)
|
# If needed, you also need to transpose the images so that they are channel first
|
||||||
return self.action_net(action_hidden), self.value_net(value_hidden)
|
# use deterministic=False if you want to export the stochastic policy
|
||||||
|
# policy() returns `actions, values, log_prob` for PPO
|
||||||
|
return self.policy(observation, deterministic=True)
|
||||||
|
|
||||||
|
|
||||||
# Example: model = PPO("MlpPolicy", "Pendulum-v1")
|
# Example: model = PPO("MlpPolicy", "Pendulum-v1")
|
||||||
|
PPO("MlpPolicy", "Pendulum-v1").save("PathToTrainedModel")
|
||||||
model = PPO.load("PathToTrainedModel.zip", device="cpu")
|
model = PPO.load("PathToTrainedModel.zip", device="cpu")
|
||||||
onnxable_model = OnnxablePolicy(
|
|
||||||
model.policy.mlp_extractor, model.policy.action_net, model.policy.value_net
|
onnx_policy = OnnxableSB3Policy(model.policy)
|
||||||
)
|
|
||||||
|
|
||||||
observation_size = model.observation_space.shape
|
observation_size = model.observation_space.shape
|
||||||
dummy_input = th.randn(1, *observation_size)
|
dummy_input = th.randn(1, *observation_size)
|
||||||
th.onnx.export(
|
th.onnx.export(
|
||||||
onnxable_model,
|
onnx_policy,
|
||||||
dummy_input,
|
dummy_input,
|
||||||
"my_ppo_model.onnx",
|
"my_ppo_model.onnx",
|
||||||
opset_version=9,
|
opset_version=17,
|
||||||
input_names=["input"],
|
input_names=["input"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -93,7 +92,13 @@ For PPO, assuming a shared feature extractor.
|
||||||
|
|
||||||
observation = np.zeros((1, *observation_size)).astype(np.float32)
|
observation = np.zeros((1, *observation_size)).astype(np.float32)
|
||||||
ort_sess = ort.InferenceSession(onnx_path)
|
ort_sess = ort.InferenceSession(onnx_path)
|
||||||
action, value = ort_sess.run(None, {"input": observation})
|
actions, values, log_prob = ort_sess.run(None, {"input": observation})
|
||||||
|
|
||||||
|
print(actions, values, log_prob)
|
||||||
|
|
||||||
|
# Check that the predictions are the same
|
||||||
|
with th.no_grad():
|
||||||
|
print(model.policy(th.as_tensor(observation), deterministic=True))
|
||||||
|
|
||||||
|
|
||||||
For SAC the procedure is similar. The example shown only exports the actor network as the actor is sufficient to roll out the trained policies.
|
For SAC the procedure is similar. The example shown only exports the actor network as the actor is sufficient to roll out the trained policies.
|
||||||
|
|
@ -108,23 +113,16 @@ For SAC the procedure is similar. The example shown only exports the actor netwo
|
||||||
class OnnxablePolicy(th.nn.Module):
|
class OnnxablePolicy(th.nn.Module):
|
||||||
def __init__(self, actor: th.nn.Module):
|
def __init__(self, actor: th.nn.Module):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
# Removing the flatten layer because it can't be onnxed
|
self.actor = actor
|
||||||
self.actor = th.nn.Sequential(
|
|
||||||
actor.latent_pi,
|
|
||||||
actor.mu,
|
|
||||||
# For gSDE
|
|
||||||
# th.nn.Hardtanh(min_val=-actor.clip_mean, max_val=actor.clip_mean),
|
|
||||||
# Squash the output
|
|
||||||
th.nn.Tanh(),
|
|
||||||
)
|
|
||||||
|
|
||||||
def forward(self, observation: th.Tensor) -> th.Tensor:
|
def forward(self, observation: th.Tensor) -> th.Tensor:
|
||||||
# NOTE: You may have to process (normalize) observation in the correct
|
# NOTE: You may have to postprocess (unnormalize) actions
|
||||||
# way before using this. See `common.preprocessing.preprocess_obs`
|
# to the correct bounds (see commented code below)
|
||||||
return self.actor(observation)
|
return self.actor(observation, deterministic=True)
|
||||||
|
|
||||||
|
|
||||||
# Example: model = SAC("MlpPolicy", "Pendulum-v1")
|
# Example: model = SAC("MlpPolicy", "Pendulum-v1")
|
||||||
|
SAC("MlpPolicy", "Pendulum-v1").save("PathToTrainedModel.zip")
|
||||||
model = SAC.load("PathToTrainedModel.zip", device="cpu")
|
model = SAC.load("PathToTrainedModel.zip", device="cpu")
|
||||||
onnxable_model = OnnxablePolicy(model.policy.actor)
|
onnxable_model = OnnxablePolicy(model.policy.actor)
|
||||||
|
|
||||||
|
|
@ -134,7 +132,7 @@ For SAC the procedure is similar. The example shown only exports the actor netwo
|
||||||
onnxable_model,
|
onnxable_model,
|
||||||
dummy_input,
|
dummy_input,
|
||||||
"my_sac_actor.onnx",
|
"my_sac_actor.onnx",
|
||||||
opset_version=9,
|
opset_version=17,
|
||||||
input_names=["input"],
|
input_names=["input"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -147,10 +145,23 @@ For SAC the procedure is similar. The example shown only exports the actor netwo
|
||||||
|
|
||||||
observation = np.zeros((1, *observation_size)).astype(np.float32)
|
observation = np.zeros((1, *observation_size)).astype(np.float32)
|
||||||
ort_sess = ort.InferenceSession(onnx_path)
|
ort_sess = ort.InferenceSession(onnx_path)
|
||||||
action = ort_sess.run(None, {"input": observation})
|
scaled_action = ort_sess.run(None, {"input": observation})[0]
|
||||||
|
|
||||||
|
print(scaled_action)
|
||||||
|
|
||||||
|
# Post-process: rescale to correct space
|
||||||
|
# Rescale the action from [-1, 1] to [low, high]
|
||||||
|
# low, high = model.action_space.low, model.action_space.high
|
||||||
|
# post_processed_action = low + (0.5 * (scaled_action + 1.0) * (high - low))
|
||||||
|
|
||||||
|
# Check that the predictions are the same
|
||||||
|
with th.no_grad():
|
||||||
|
print(model.actor(th.as_tensor(observation), deterministic=True))
|
||||||
|
|
||||||
|
|
||||||
|
For more discussion around the topic, please refer to `GH#383 <https://github.com/DLR-RM/stable-baselines3/issues/383>`_ and `GH#1349 <https://github.com/DLR-RM/stable-baselines3/issues/1349>`_.
|
||||||
|
|
||||||
|
|
||||||
For more discussion around the topic refer to this `issue. <https://github.com/DLR-RM/stable-baselines3/issues/383>`_
|
|
||||||
|
|
||||||
Trace/Export to C++
|
Trace/Export to C++
|
||||||
-------------------
|
-------------------
|
||||||
|
|
|
||||||
|
|
@ -252,6 +252,12 @@ A better solution would be to use a squashing function (cf ``SAC``) or a Beta di
|
||||||
Tips and Tricks when implementing an RL algorithm
|
Tips and Tricks when implementing an RL algorithm
|
||||||
=================================================
|
=================================================
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
We have a `video on YouTube about reliable RL <https://www.youtube.com/watch?v=7-PUg9EAa3Y>`_ that covers
|
||||||
|
this section in more details. You can also find the `slides online <https://araffin.github.io/slides/tips-reliable-rl/>`_.
|
||||||
|
|
||||||
|
|
||||||
When you try to reproduce a RL paper by implementing the algorithm, the `nuts and bolts of RL research <http://joschu.net/docs/nuts-and-bolts.pdf>`_
|
When you try to reproduce a RL paper by implementing the algorithm, the `nuts and bolts of RL research <http://joschu.net/docs/nuts-and-bolts.pdf>`_
|
||||||
by John Schulman are quite useful (`video <https://www.youtube.com/watch?v=8EcdaCk9KaQ>`_).
|
by John Schulman are quite useful (`video <https://www.youtube.com/watch?v=8EcdaCk9KaQ>`_).
|
||||||
|
|
||||||
|
|
@ -282,4 +288,4 @@ in RL with discrete actions:
|
||||||
3. Pong (one of the easiest Atari game)
|
3. Pong (one of the easiest Atari game)
|
||||||
4. other Atari games (e.g. Breakout)
|
4. other Atari games (e.g. Breakout)
|
||||||
|
|
||||||
.. _SBX: https://github.com/araffin/sbx
|
.. _SBX: https://github.com/araffin/sbx
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,8 @@ Documentation:
|
||||||
^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^
|
||||||
- Added a paragraph on modifying vectorized environment parameters via setters (@fracapuano)
|
- Added a paragraph on modifying vectorized environment parameters via setters (@fracapuano)
|
||||||
- Updated callback code example
|
- Updated callback code example
|
||||||
|
- Updated export to ONNX documentation, it is now much simpler to export SB3 models with newer ONNX Opset!
|
||||||
|
- Added video link to "Practical Tips for Reliable Reinforcement Learning" video
|
||||||
|
|
||||||
Release 2.2.1 (2023-11-17)
|
Release 2.2.1 (2023-11-17)
|
||||||
--------------------------
|
--------------------------
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue