mirror of
https://github.com/saymrwulf/stable-baselines3.git
synced 2026-11-18 09:54:10 +00:00
Add a parameter to disable ortho init
This commit is contained in:
parent
b4dc9d4e4d
commit
440166fe26
2 changed files with 26 additions and 13 deletions
14
README.md
14
README.md
|
|
@ -6,15 +6,25 @@
|
||||||
|
|
||||||
PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines), a set of improved implementations of reinforcement learning algorithms.
|
PyTorch version of [Stable Baselines](https://github.com/hill-a/stable-baselines), a set of improved implementations of reinforcement learning algorithms.
|
||||||
|
|
||||||
|
## Implemented Algorithms
|
||||||
|
|
||||||
|
- CEM-RL (with TD3)
|
||||||
|
- PPO
|
||||||
|
- SAC
|
||||||
|
- TD3
|
||||||
|
|
||||||
|
|
||||||
|
## Roadmap
|
||||||
|
|
||||||
TODO:
|
TODO:
|
||||||
- save/load
|
- save/load
|
||||||
- predict
|
- predict
|
||||||
- better rescale (min + action * range)
|
- better rescale (min + action * range)
|
||||||
- documentation
|
|
||||||
- flexible mlp
|
- flexible mlp
|
||||||
- logger
|
- logger
|
||||||
- better monitor wrapper?
|
- better monitor wrapper?
|
||||||
- automatic choice for action distribution
|
- automatic choice for action distribution
|
||||||
|
- A2C
|
||||||
|
|
||||||
Later:
|
Later:
|
||||||
- get_parameters / set_parameters
|
- get_parameters / set_parameters
|
||||||
|
|
@ -22,6 +32,6 @@ Later:
|
||||||
- tensorboard support
|
- tensorboard support
|
||||||
- DQN
|
- DQN
|
||||||
- TRPO
|
- TRPO
|
||||||
- A2C
|
|
||||||
- ACER
|
- ACER
|
||||||
|
- DDPG
|
||||||
- HER -> use stable-baselines because does not depends on tf?
|
- HER -> use stable-baselines because does not depends on tf?
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ from torchy_baselines.common.distributions import DiagGaussianDistribution, Squa
|
||||||
class PPOPolicy(BasePolicy):
|
class PPOPolicy(BasePolicy):
|
||||||
def __init__(self, observation_space, action_space,
|
def __init__(self, observation_space, action_space,
|
||||||
learning_rate=1e-3, net_arch=None, device='cpu',
|
learning_rate=1e-3, net_arch=None, device='cpu',
|
||||||
activation_fn=nn.Tanh, adam_epsilon=1e-5):
|
activation_fn=nn.Tanh, adam_epsilon=1e-5, ortho_init=True):
|
||||||
super(PPOPolicy, self).__init__(observation_space, action_space, device)
|
super(PPOPolicy, self).__init__(observation_space, action_space, device)
|
||||||
self.obs_dim = self.observation_space.shape[0]
|
self.obs_dim = self.observation_space.shape[0]
|
||||||
self.action_dim = self.action_space.shape[0]
|
self.action_dim = self.action_space.shape[0]
|
||||||
|
|
@ -20,6 +20,7 @@ class PPOPolicy(BasePolicy):
|
||||||
self.net_arch = net_arch
|
self.net_arch = net_arch
|
||||||
self.activation_fn = activation_fn
|
self.activation_fn = activation_fn
|
||||||
self.adam_epsilon = adam_epsilon
|
self.adam_epsilon = adam_epsilon
|
||||||
|
self.ortho_init = ortho_init
|
||||||
self.net_args = {
|
self.net_args = {
|
||||||
'input_dim': self.obs_dim,
|
'input_dim': self.obs_dim,
|
||||||
'output_dim': -1,
|
'output_dim': -1,
|
||||||
|
|
@ -48,16 +49,18 @@ class PPOPolicy(BasePolicy):
|
||||||
self.action_net, self.log_std = self.action_dist.proba_distribution_net(latent_dim=self.net_arch[-1])
|
self.action_net, self.log_std = self.action_dist.proba_distribution_net(latent_dim=self.net_arch[-1])
|
||||||
self.value_net = nn.Linear(self.net_arch[-1], 1)
|
self.value_net = nn.Linear(self.net_arch[-1], 1)
|
||||||
# Init weights: use orthogonal initialization
|
# Init weights: use orthogonal initialization
|
||||||
for module in [self.pi_net, self.vf_net, self.action_net, self.value_net]:
|
# with small initial weight for the output
|
||||||
# Values from stable-baselines check why
|
if self.ortho_init:
|
||||||
gain = {
|
for module in [self.pi_net, self.vf_net, self.action_net, self.value_net]:
|
||||||
self.pi_net: np.sqrt(2),
|
# Values from stable-baselines check why
|
||||||
self.vf_net: np.sqrt(2),
|
gain = {
|
||||||
self.shared_net: np.sqrt(2),
|
self.pi_net: np.sqrt(2),
|
||||||
self.action_net: 0.01,
|
self.vf_net: np.sqrt(2),
|
||||||
self.value_net: 1
|
self.shared_net: np.sqrt(2),
|
||||||
}[module]
|
self.action_net: 0.01,
|
||||||
module.apply(partial(self.init_weights, gain=gain))
|
self.value_net: 1
|
||||||
|
}[module]
|
||||||
|
module.apply(partial(self.init_weights, gain=gain))
|
||||||
# TODO: support linear decay of the learning rate
|
# TODO: support linear decay of the learning rate
|
||||||
self.optimizer = th.optim.Adam(self.parameters(), lr=learning_rate, eps=self.adam_epsilon)
|
self.optimizer = th.optim.Adam(self.parameters(), lr=learning_rate, eps=self.adam_epsilon)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue