mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-18 09:54:10 +00:00
Small docstring improvements related to the notion of Rollout (#206)
* Small docstring improvements related to the notion of Rollout * documented changes in changelog.rst, added myself to contributers * Minor edits Co-authored-by: Stefan Heid <stefan.heid@upb.de> Co-authored-by: Antonin RAFFIN <antonin.raffin@ensta.org>
This commit is contained in:
parent
581ee4ad7a
commit
9d463bc476
7 changed files with 19 additions and 8 deletions
|
|
@ -26,6 +26,7 @@ Others:
|
||||||
Documentation:
|
Documentation:
|
||||||
^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^
|
||||||
- Updated algorithm table
|
- Updated algorithm table
|
||||||
|
- Minor docstring improvements regarding rollout (@stheid)
|
||||||
|
|
||||||
|
|
||||||
Pre-Release 0.10.0 (2020-10-28)
|
Pre-Release 0.10.0 (2020-10-28)
|
||||||
|
|
@ -497,7 +498,7 @@ And all the contributors:
|
||||||
@EliasHasle @mrakgr @Bleyddyn @antoine-galataud @junhyeokahn @AdamGleave @keshaviyengar @tperol
|
@EliasHasle @mrakgr @Bleyddyn @antoine-galataud @junhyeokahn @AdamGleave @keshaviyengar @tperol
|
||||||
@XMaster96 @kantneel @Pastafarianist @GerardMaggiolino @PatrickWalter214 @yutingsz @sc420 @Aaahh @billtubbs
|
@XMaster96 @kantneel @Pastafarianist @GerardMaggiolino @PatrickWalter214 @yutingsz @sc420 @Aaahh @billtubbs
|
||||||
@Miffyli @dwiel @miguelrass @qxcv @jaberkow @eavelardev @ruifeng96150 @pedrohbtp @srivatsankrishnan @evilsocket
|
@Miffyli @dwiel @miguelrass @qxcv @jaberkow @eavelardev @ruifeng96150 @pedrohbtp @srivatsankrishnan @evilsocket
|
||||||
@MarvineGothic @jdossgollin @SyllogismRXS @rusu24edward @jbulow @Antymon @seheevic @justinkterry @edbeeching
|
@MarvineGothic @jdossgollin @stheid @SyllogismRXS @rusu24edward @jbulow @Antymon @seheevic @justinkterry @edbeeching
|
||||||
@flodorner @KuKuXia @NeoExtended @PartiallyTyped @mmcenta @richardwu @kinalmehta @rolandgvc @tkelestemur @mloo3
|
@flodorner @KuKuXia @NeoExtended @PartiallyTyped @mmcenta @richardwu @kinalmehta @rolandgvc @tkelestemur @mloo3
|
||||||
@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37 @andyshih12 @RaphaelWag @xicocaio
|
@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37 @andyshih12 @RaphaelWag @xicocaio
|
||||||
@diditforlulz273 @liorcohen5 @ManifoldFR @mloo3 @SwamyDev @wmmc88 @megan-klaiber
|
@diditforlulz273 @liorcohen5 @ManifoldFR @mloo3 @SwamyDev @wmmc88 @megan-klaiber
|
||||||
|
|
|
||||||
|
|
@ -259,6 +259,15 @@ class ReplayBuffer(BaseBuffer):
|
||||||
class RolloutBuffer(BaseBuffer):
|
class RolloutBuffer(BaseBuffer):
|
||||||
"""
|
"""
|
||||||
Rollout buffer used in on-policy algorithms like A2C/PPO.
|
Rollout buffer used in on-policy algorithms like A2C/PPO.
|
||||||
|
It corresponds to ``buffer_size`` transitions collected
|
||||||
|
using the current policy.
|
||||||
|
This experience will be discarded after the policy update.
|
||||||
|
In order to use PPO objective, we also store the current value of each state
|
||||||
|
and the log probability of each taken action.
|
||||||
|
|
||||||
|
The term rollout here refers to the model-free notion and should not
|
||||||
|
be used with the concept of rollout used in model-based RL or planning.
|
||||||
|
Hence, it is only involved in policy and value function training but not action selection.
|
||||||
|
|
||||||
:param buffer_size: Max number of element in the buffer
|
:param buffer_size: Max number of element in the buffer
|
||||||
:param observation_space: Observation space
|
:param observation_space: Observation space
|
||||||
|
|
|
||||||
|
|
@ -363,7 +363,7 @@ class OffPolicyAlgorithm(BaseAlgorithm):
|
||||||
log_interval: Optional[int] = None,
|
log_interval: Optional[int] = None,
|
||||||
) -> RolloutReturn:
|
) -> RolloutReturn:
|
||||||
"""
|
"""
|
||||||
Collect experiences and store them into a ReplayBuffer.
|
Collect experiences and store them into a ``ReplayBuffer``.
|
||||||
|
|
||||||
:param env: The training environment
|
:param env: The training environment
|
||||||
:param callback: Callback that will be called at each step
|
:param callback: Callback that will be called at each step
|
||||||
|
|
|
||||||
|
|
@ -124,7 +124,9 @@ class OnPolicyAlgorithm(BaseAlgorithm):
|
||||||
self, env: VecEnv, callback: BaseCallback, rollout_buffer: RolloutBuffer, n_rollout_steps: int
|
self, env: VecEnv, callback: BaseCallback, rollout_buffer: RolloutBuffer, n_rollout_steps: int
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
Collect rollouts using the current policy and fill a `RolloutBuffer`.
|
Collect experiences using the current policy and fill a ``RolloutBuffer``.
|
||||||
|
The term rollout here refers to the model-free notion and should not
|
||||||
|
be used with the concept of rollout used in model-based RL or planning.
|
||||||
|
|
||||||
:param env: The training environment
|
:param env: The training environment
|
||||||
:param callback: Callback that will be called at each step
|
:param callback: Callback that will be called at each step
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@ class DQN(OffPolicyAlgorithm):
|
||||||
def _on_step(self) -> None:
|
def _on_step(self) -> None:
|
||||||
"""
|
"""
|
||||||
Update the exploration rate and target network if needed.
|
Update the exploration rate and target network if needed.
|
||||||
This method is called in ``collect_rollout()`` after each step in the environment.
|
This method is called in ``collect_rollouts()`` after each step in the environment.
|
||||||
"""
|
"""
|
||||||
if self.num_timesteps % self.target_update_interval == 0:
|
if self.num_timesteps % self.target_update_interval == 0:
|
||||||
polyak_update(self.q_net.parameters(), self.q_net_target.parameters(), self.tau)
|
polyak_update(self.q_net.parameters(), self.q_net_target.parameters(), self.tau)
|
||||||
|
|
|
||||||
|
|
@ -132,8 +132,7 @@ class PPO(OnPolicyAlgorithm):
|
||||||
|
|
||||||
def train(self) -> None:
|
def train(self) -> None:
|
||||||
"""
|
"""
|
||||||
Update policy using the currently gathered
|
Update policy using the currently gathered rollout buffer.
|
||||||
rollout buffer.
|
|
||||||
"""
|
"""
|
||||||
# Update optimizer learning rate
|
# Update optimizer learning rate
|
||||||
self._update_learning_rate(self.policy.optimizer)
|
self._update_learning_rate(self.policy.optimizer)
|
||||||
|
|
@ -147,7 +146,7 @@ class PPO(OnPolicyAlgorithm):
|
||||||
pg_losses, value_losses = [], []
|
pg_losses, value_losses = [], []
|
||||||
clip_fractions = []
|
clip_fractions = []
|
||||||
|
|
||||||
# train for gradient_steps epochs
|
# train for n_epochs epochs
|
||||||
for epoch in range(self.n_epochs):
|
for epoch in range(self.n_epochs):
|
||||||
approx_kl_divs = []
|
approx_kl_divs = []
|
||||||
# Do a complete pass on the rollout buffer
|
# Do a complete pass on the rollout buffer
|
||||||
|
|
|
||||||
|
|
@ -154,7 +154,7 @@ def test_features_extractor_target_net(model_class, share_features_extractor):
|
||||||
model.lr_schedule = lambda _: 0.0
|
model.lr_schedule = lambda _: 0.0
|
||||||
# Re-activate polyak update
|
# Re-activate polyak update
|
||||||
model.tau = 0.01
|
model.tau = 0.01
|
||||||
# Special case for DQN: target net is updated in the `collect_rollout()`
|
# Special case for DQN: target net is updated in the `collect_rollouts()`
|
||||||
# not the `train()` method
|
# not the `train()` method
|
||||||
if model_class == DQN:
|
if model_class == DQN:
|
||||||
model.target_update_interval = 1
|
model.target_update_interval = 1
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue