This commit is contained in:
Antonin RAFFIN 2019-09-21 17:17:09 +02:00
parent 3ececcd3a9
commit 2469ff3859
11 changed files with 22 additions and 23 deletions

View file

@ -1,9 +1,8 @@
import os import os
import gym
from torchy_baselines import TD3, CEMRL, PPO from torchy_baselines import TD3, CEMRL, PPO
def test_td3(): def test_td3():
model = TD3('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[64, 64]), model = TD3('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[64, 64]),
start_timesteps=100, verbose=1, create_eval_env=True) start_timesteps=100, verbose=1, create_eval_env=True)
@ -12,6 +11,7 @@ def test_td3():
model.load("test_save") model.load("test_save")
os.remove("test_save.pth") os.remove("test_save.pth")
def test_cemrl(): def test_cemrl():
model = CEMRL('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), pop_size=2, n_grad=1, model = CEMRL('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), pop_size=2, n_grad=1,
start_timesteps=100, verbose=1, create_eval_env=True) start_timesteps=100, verbose=1, create_eval_env=True)
@ -20,6 +20,7 @@ def test_cemrl():
model.load("test_save") model.load("test_save")
os.remove("test_save.pth") os.remove("test_save.pth")
def test_ppo(): def test_ppo():
model = PPO('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True) model = PPO('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True)
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)

View file

@ -1,12 +1,10 @@
import time import time
import torch as th import torch as th
import torch.nn.functional as F
import numpy as np
from torchy_baselines.td3.td3 import TD3
from torchy_baselines.common.evaluation import evaluate_policy
from torchy_baselines.cem_rl.cem import CEM from torchy_baselines.cem_rl.cem import CEM
from torchy_baselines.common.evaluation import evaluate_policy
from torchy_baselines.td3.td3 import TD3
class CEMRL(TD3): class CEMRL(TD3):
@ -16,6 +14,7 @@ class CEMRL(TD3):
Paper: https://arxiv.org/abs/1810.01222 Paper: https://arxiv.org/abs/1810.01222
Code: https://github.com/apourchot/CEM-RL Code: https://github.com/apourchot/CEM-RL
""" """
def __init__(self, policy, env, policy_kwargs=None, verbose=0, def __init__(self, policy, env, policy_kwargs=None, verbose=0,
sigma_init=1e-3, pop_size=10, damp=1e-3, damp_limit=1e-5, sigma_init=1e-3, pop_size=10, damp=1e-3, damp_limit=1e-5,
elitism=False, n_grad=5, policy_freq=2, batch_size=100, elitism=False, n_grad=5, policy_freq=2, batch_size=100,
@ -100,7 +99,7 @@ class CEMRL(TD3):
else: else:
# scales with a bigger population # scales with a bigger population
# but less training steps per agent # but less training steps per agent
n_training_steps == 2 * (actor_steps // self.n_grad) n_training_steps = 2 * (actor_steps // self.n_grad)
for it in range(n_training_steps): for it in range(n_training_steps):
# Sample replay buffer # Sample replay buffer
replay_data = self.replay_buffer.sample(self.batch_size) replay_data = self.replay_buffer.sample(self.batch_size)
@ -148,7 +147,6 @@ class CEMRL(TD3):
print("Total T: {} Episode Num: {} Episode T: {} Reward: {}".format( print("Total T: {} Episode Num: {} Episode T: {} Reward: {}".format(
self.num_timesteps, episode_num, episode_timesteps, episode_reward)) self.num_timesteps, episode_num, episode_timesteps, episode_reward))
self.es.tell(self.es_params, self.fitnesses) self.es.tell(self.es_params, self.fitnesses)
timesteps_since_eval += actor_steps timesteps_since_eval += actor_steps
return self return self

View file

@ -1,6 +1,5 @@
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
import gym import gym
import torch as th import torch as th
import numpy as np import numpy as np
@ -216,7 +215,6 @@ class BaseRLModel(object):
if self.eval_env is not None: if self.eval_env is not None:
self.eval_env.seed(seed) self.eval_env.seed(seed)
def collect_rollouts(self, env, n_episodes=1, action_noise_std=0.0, def collect_rollouts(self, env, n_episodes=1, action_noise_std=0.0,
deterministic=False, callback=None, remove_timelimits=True, deterministic=False, callback=None, remove_timelimits=True,
start_timesteps=0, num_timesteps=0, replay_buffer=None): start_timesteps=0, num_timesteps=0, replay_buffer=None):

View file

@ -52,10 +52,12 @@ class BaseBuffer(object):
def _get_samples(self, batch_inds): def _get_samples(self, batch_inds):
raise NotImplementedError() raise NotImplementedError()
class ReplayBuffer(BaseBuffer): class ReplayBuffer(BaseBuffer):
""" """
Taken from https://github.com/apourchot/CEM-RL Taken from https://github.com/apourchot/CEM-RL
""" """
def __init__(self, buffer_size, state_dim, action_dim, device='cpu', n_envs=1): def __init__(self, buffer_size, state_dim, action_dim, device='cpu', n_envs=1):
super(ReplayBuffer, self).__init__(buffer_size, state_dim, action_dim, device, n_envs=n_envs) super(ReplayBuffer, self).__init__(buffer_size, state_dim, action_dim, device, n_envs=n_envs)

View file

@ -1,6 +1,7 @@
import torch as th import torch as th
from torch.distributions import Normal from torch.distributions import Normal
class Distribution(object): class Distribution(object):
def __init__(self): def __init__(self):
super(Distribution, self).__init__() super(Distribution, self).__init__()

View file

@ -1,8 +1,7 @@
import random import random
import scipy.signal
import torch as th
import numpy as np import numpy as np
import torch as th
def set_random_seed(seed, using_cuda=False): def set_random_seed(seed, using_cuda=False):

View file

@ -1,6 +1,6 @@
# flake8: noqa F401 # flake8: noqa F401
from torchy_baselines.common.vec_env.base_vec_env import AlreadySteppingError, NotSteppingError, VecEnv, VecEnvWrapper, \ from torchy_baselines.common.vec_env.base_vec_env import AlreadySteppingError, NotSteppingError,\
CloudpickleWrapper VecEnv, VecEnvWrapper, CloudpickleWrapper
from torchy_baselines.common.vec_env.dummy_vec_env import DummyVecEnv from torchy_baselines.common.vec_env.dummy_vec_env import DummyVecEnv
from torchy_baselines.common.vec_env.subproc_vec_env import SubprocVecEnv from torchy_baselines.common.vec_env.subproc_vec_env import SubprocVecEnv
from torchy_baselines.common.vec_env.vec_frame_stack import VecFrameStack from torchy_baselines.common.vec_env.vec_frame_stack import VecFrameStack

View file

@ -44,7 +44,6 @@ class PPOPolicy(BasePolicy):
self.log_std = nn.Parameter(th.zeros(self.action_dim)) self.log_std = nn.Parameter(th.zeros(self.action_dim))
# Init weights: use orthogonal initialization # Init weights: use orthogonal initialization
for module in [self.shared_net, self.actor_net, self.value_net]: for module in [self.shared_net, self.actor_net, self.value_net]:
gain = 0.01 if module == self.actor_net else 1.0
# Values from stable-baselines check why # Values from stable-baselines check why
gain = { gain = {
self.shared_net: np.sqrt(2), self.shared_net: np.sqrt(2),
@ -98,6 +97,7 @@ class PPOPolicy(BasePolicy):
def value_forward(self): def value_forward(self):
pass pass
MlpPolicy = PPOPolicy MlpPolicy = PPOPolicy
register_policy("MlpPolicy", MlpPolicy) register_policy("MlpPolicy", MlpPolicy)

View file

@ -79,6 +79,7 @@ class TD3Policy(BasePolicy):
def make_critic(self): def make_critic(self):
return Critic(**self.net_args).to(self.device) return Critic(**self.net_args).to(self.device)
MlpPolicy = TD3Policy MlpPolicy = TD3Policy
register_policy("MlpPolicy", MlpPolicy) register_policy("MlpPolicy", MlpPolicy)

View file

@ -135,8 +135,7 @@ class TD3(BaseRLModel):
for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()): for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):
target_param.data.copy_(tau_actor * param.data + (1 - tau_actor) * target_param.data) target_param.data.copy_(tau_actor * param.data + (1 - tau_actor) * target_param.data)
def train(self, n_iterations, batch_size=100, discount=0.99, def train(self, n_iterations, batch_size=100, policy_freq=2):
tau=0.005, policy_noise=0.2, noise_clip=0.5, policy_freq=2):
for it in range(n_iterations): for it in range(n_iterations):