mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-18 09:54:10 +00:00
Update closer to original implementation for CEMRL
This commit is contained in:
parent
f04754afec
commit
2a660e9a41
3 changed files with 20 additions and 9 deletions
|
|
@ -7,7 +7,7 @@ import numpy as np
|
||||||
class CEM(object):
|
class CEM(object):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Cross-entropy methods.
|
Cross-entropy method with diagonal covariance (separable CEM)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, num_params,
|
def __init__(self, num_params,
|
||||||
|
|
|
||||||
|
|
@ -80,12 +80,15 @@ class CEMRL(TD3):
|
||||||
|
|
||||||
# In the paper: 2 * actor_steps // self.n_grad
|
# In the paper: 2 * actor_steps // self.n_grad
|
||||||
# In the original implementation: actor_steps // self.n_grad
|
# In the original implementation: actor_steps // self.n_grad
|
||||||
# Difference with current implementation:
|
# Difference with TD3 implementation:
|
||||||
# the target critic is updated in the train_critic()
|
# the target critic is updated in the train_critic()
|
||||||
# instead of the train_actor()
|
# instead of the train_actor() and no policy delay
|
||||||
# Issue with this update style: the bigger the population, the slower the code
|
# Issue with this update style: the bigger the population, the slower the code
|
||||||
if self.update_style == 'original':
|
if self.update_style == 'original':
|
||||||
self.train_critic(actor_steps // self.n_grad)
|
self.train_critic(actor_steps // self.n_grad, tau=0.005)
|
||||||
|
self.train_actor(actor_steps, tau_critic=0.0)
|
||||||
|
elif self.update_style == 'original_td3':
|
||||||
|
self.train_critic(actor_steps // self.n_grad, tau=0.0)
|
||||||
self.train_actor(actor_steps)
|
self.train_actor(actor_steps)
|
||||||
else:
|
else:
|
||||||
# Closer to td3: with policy delay
|
# Closer to td3: with policy delay
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,7 @@ class TD3(BaseRLModel):
|
||||||
return self.max_action * self.select_action(observation)
|
return self.max_action * self.select_action(observation)
|
||||||
|
|
||||||
def train_critic(self, n_iterations=1, batch_size=100, discount=0.99,
|
def train_critic(self, n_iterations=1, batch_size=100, discount=0.99,
|
||||||
policy_noise=0.2, noise_clip=0.5, replay_data=None):
|
policy_noise=0.2, noise_clip=0.5, replay_data=None, tau=0.0):
|
||||||
|
|
||||||
for it in range(n_iterations):
|
for it in range(n_iterations):
|
||||||
# Sample replay buffer
|
# Sample replay buffer
|
||||||
|
|
@ -106,7 +106,14 @@ class TD3(BaseRLModel):
|
||||||
critic_loss.backward()
|
critic_loss.backward()
|
||||||
self.critic.optimizer.step()
|
self.critic.optimizer.step()
|
||||||
|
|
||||||
def train_actor(self, n_iterations=1, batch_size=100, tau=0.005, replay_data=None):
|
# Update the frozen target models
|
||||||
|
# Note: by default, for TD3, this update is done in train_actor
|
||||||
|
# however, for CEMRL it is done here
|
||||||
|
if tau > 0:
|
||||||
|
for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
|
||||||
|
target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
|
||||||
|
|
||||||
|
def train_actor(self, n_iterations=1, batch_size=100, tau_actor=0.005, tau_critic=0.005, replay_data=None):
|
||||||
|
|
||||||
for it in range(n_iterations):
|
for it in range(n_iterations):
|
||||||
# Sample replay buffer
|
# Sample replay buffer
|
||||||
|
|
@ -124,11 +131,12 @@ class TD3(BaseRLModel):
|
||||||
self.actor.optimizer.step()
|
self.actor.optimizer.step()
|
||||||
|
|
||||||
# Update the frozen target models
|
# Update the frozen target models
|
||||||
for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
|
if tau_critic > 0:
|
||||||
target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
|
for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):
|
||||||
|
target_param.data.copy_(tau_critic * param.data + (1 - tau_critic) * target_param.data)
|
||||||
|
|
||||||
for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
|
for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
|
||||||
target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
|
target_param.data.copy_(tau_actor * param.data + (1 - tau_actor) * target_param.data)
|
||||||
|
|
||||||
def train(self, n_iterations, batch_size=100, discount=0.99,
|
def train(self, n_iterations, batch_size=100, discount=0.99,
|
||||||
tau=0.005, policy_noise=0.2, noise_clip=0.5, policy_freq=2):
|
tau=0.005, policy_noise=0.2, noise_clip=0.5, policy_freq=2):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue