Full compat for VecEnv + bug fixes for cuda

This commit is contained in:
Antonin Raffin 2019-09-20 16:43:19 +02:00
parent 255ff10bff
commit 0e727a5f72
8 changed files with 59 additions and 57 deletions

View file

@ -4,8 +4,9 @@ import gym
from torchy_baselines import TD3, CEMRL, PPO from torchy_baselines import TD3, CEMRL, PPO
def test_pendulum(): def test_td3():
model = TD3('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[64, 64]), start_timesteps=100, verbose=1) model = TD3('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[64, 64]),
start_timesteps=100, verbose=1, create_eval_env=True)
model.learn(total_timesteps=500, eval_freq=100) model.learn(total_timesteps=500, eval_freq=100)
model.save("test_save") model.save("test_save")
model.load("test_save") model.load("test_save")
@ -13,14 +14,14 @@ def test_pendulum():
def test_cemrl(): def test_cemrl():
model = CEMRL('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), pop_size=2, n_grad=1, model = CEMRL('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), pop_size=2, n_grad=1,
start_timesteps=100, verbose=1) start_timesteps=100, verbose=1, create_eval_env=True)
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
model.save("test_save") model.save("test_save")
model.load("test_save") model.load("test_save")
os.remove("test_save.pth") os.remove("test_save.pth")
def test_ppo(): def test_ppo():
model = PPO('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), verbose=1) model = PPO('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), verbose=1, create_eval_env=True)
model.learn(total_timesteps=1000, eval_freq=500) model.learn(total_timesteps=1000, eval_freq=500)
# model.save("test_save") # model.save("test_save")
# model.load("test_save") # model.load("test_save")

View file

@ -21,12 +21,14 @@ class CEMRL(TD3):
elitism=False, n_grad=5, policy_freq=2, batch_size=100, elitism=False, n_grad=5, policy_freq=2, batch_size=100,
buffer_size=int(1e6), learning_rate=1e-3, seed=0, device='auto', buffer_size=int(1e6), learning_rate=1e-3, seed=0, device='auto',
action_noise_std=0.0, start_timesteps=100, update_style='original', action_noise_std=0.0, start_timesteps=100, update_style='original',
create_eval_env=False,
_init_setup_model=True): _init_setup_model=True):
super(CEMRL, self).__init__(policy, env, policy_kwargs, verbose, super(CEMRL, self).__init__(policy, env, policy_kwargs, verbose,
buffer_size, learning_rate, seed, device, buffer_size, learning_rate, seed, device,
action_noise_std, start_timesteps, action_noise_std, start_timesteps,
policy_freq=policy_freq, batch_size=batch_size, policy_freq=policy_freq, batch_size=batch_size,
create_eval_env=create_eval_env,
_init_setup_model=False) _init_setup_model=False)
self.es = None self.es = None
@ -58,6 +60,7 @@ class CEMRL(TD3):
episode_num = 0 episode_num = 0
evaluations = [] evaluations = []
start_time = time.time() start_time = time.time()
eval_env = self._get_eval_env(eval_env)
while self.num_timesteps < total_timesteps: while self.num_timesteps < total_timesteps:

View file

@ -26,7 +26,7 @@ class BaseRLModel(object):
__metaclass__ = ABCMeta __metaclass__ = ABCMeta
def __init__(self, policy, env, policy_base, policy_kwargs=None, def __init__(self, policy, env, policy_base, policy_kwargs=None,
verbose=0, device='auto', support_multi_env=False): verbose=0, device='auto', support_multi_env=False, create_eval_env=False):
if isinstance(policy, str) and policy_base is not None: if isinstance(policy, str) and policy_base is not None:
self.policy = get_policy_from_name(policy_base, policy) self.policy = get_policy_from_name(policy_base, policy)
else: else:
@ -47,10 +47,13 @@ class BaseRLModel(object):
self.n_envs = None self.n_envs = None
self.num_timesteps = 0 self.num_timesteps = 0
self.params = None self.params = None
self.eval_env = None
self.replay_buffer = None self.replay_buffer = None
if env is not None: if env is not None:
if isinstance(env, str): if isinstance(env, str):
if create_eval_env:
self.eval_env = DummyVecEnv([lambda: gym.make(env)])
if self.verbose >= 1: if self.verbose >= 1:
print("Creating environment from the given name, wrapped in a DummyVecEnv.") print("Creating environment from the given name, wrapped in a DummyVecEnv.")
env = DummyVecEnv([lambda: gym.make(env)]) env = DummyVecEnv([lambda: gym.make(env)])
@ -68,14 +71,15 @@ class BaseRLModel(object):
raise ValueError("Error: the model does not support multiple envs requires a single vectorized" raise ValueError("Error: the model does not support multiple envs requires a single vectorized"
" environment.") " environment.")
# if env is not None: def _get_eval_env(self, eval_env):
# if env is not None: if eval_env is None:
# if isinstance(env, str): eval_env = self.eval_env
# env = gym.make(env)
# self.env = env if eval_env is not None:
# self.n_envs = 1 if not isinstance(eval_env, VecEnv):
# self.observation_space = env.observation_space eval_env = DummyVecEnv([lambda: eval_env])
# self.action_space = env.action_space assert eval_env.num_envs == 1
return eval_env
def get_env(self): def get_env(self):
""" """
@ -216,6 +220,9 @@ class BaseRLModel(object):
episode_rewards = [] episode_rewards = []
total_timesteps = [] total_timesteps = []
assert isinstance(env, VecEnv)
assert env.num_envs == 1
for _ in range(n_episodes): for _ in range(n_episodes):
done = False done = False
# Reset environment # Reset environment
@ -224,7 +231,7 @@ class BaseRLModel(object):
while not done: while not done:
# Select action randomly or according to policy # Select action randomly or according to policy
if num_timesteps < start_timesteps: if num_timesteps < start_timesteps:
action = env.action_space.sample() action = [env.action_space.sample()]
else: else:
action = self.predict(obs, deterministic=deterministic) / self.max_action action = self.predict(obs, deterministic=deterministic) / self.max_action
@ -236,11 +243,12 @@ class BaseRLModel(object):
# Rescale and perform action # Rescale and perform action
new_obs, reward, done, _ = env.step(self.max_action * action) new_obs, reward, done, _ = env.step(self.max_action * action)
if hasattr(self.env, '_max_episode_steps') and remove_timelimits: # TODO: fix for VecEnv
done_bool = 0 if episode_timesteps + 1 == env._max_episode_steps else float(done) # if hasattr(self.env, '_max_episode_steps') and remove_timelimits:
else: # done_bool = 0 if episode_timesteps + 1 == env._max_episode_steps else float(done)
done_bool = float(done) # else:
# done_bool = float(done)
done_bool = [float(done[0])]
episode_reward += reward episode_reward += reward
# Store data in replay buffer # Store data in replay buffer

View file

@ -67,11 +67,12 @@ class ReplayBuffer(BaseBuffer):
self.dones = th.zeros(self.buffer_size, self.n_envs) self.dones = th.zeros(self.buffer_size, self.n_envs)
def add(self, state, next_state, action, reward, done): def add(self, state, next_state, action, reward, done):
self.states[self.pos] = th.FloatTensor(state) # Copy to avoid modification by reference
self.next_states[self.pos] = th.FloatTensor(next_state) self.states[self.pos] = th.FloatTensor(np.array(state))
self.actions[self.pos] = th.FloatTensor(action) self.next_states[self.pos] = th.FloatTensor(np.array(next_state))
self.rewards[self.pos] = th.FloatTensor(reward) self.actions[self.pos] = th.FloatTensor(np.array(action))
self.dones[self.pos] = th.FloatTensor(done) self.rewards[self.pos] = th.FloatTensor(np.array(reward))
self.dones[self.pos] = th.FloatTensor(np.array(done))
self.pos += 1 self.pos += 1
if self.pos == self.buffer_size: if self.pos == self.buffer_size:
@ -90,7 +91,7 @@ class RolloutBuffer(BaseBuffer):
def __init__(self, buffer_size, state_dim, action_dim, device='cpu', def __init__(self, buffer_size, state_dim, action_dim, device='cpu',
lambda_=1, gamma=0.99, n_envs=1): lambda_=1, gamma=0.99, n_envs=1):
super(RolloutBuffer, self).__init__(buffer_size, state_dim, action_dim, device, n_envs=n_envs) super(RolloutBuffer, self).__init__(buffer_size, state_dim, action_dim, device, n_envs=n_envs)
# TODO: try the buffer on the gpu?
self.lambda_ = lambda_ self.lambda_ = lambda_
self.gamma = gamma self.gamma = gamma
self.states, self.actions, self.rewards, self.advantages = None, None, None, None self.states, self.actions, self.rewards, self.advantages = None, None, None, None
@ -118,7 +119,7 @@ class RolloutBuffer(BaseBuffer):
for step in reversed(range(self.buffer_size)): for step in reversed(range(self.buffer_size)):
if step == self.buffer_size - 1: if step == self.buffer_size - 1:
next_non_terminal = th.FloatTensor(1.0 - dones) next_non_terminal = th.FloatTensor(1.0 - dones)
next_value = last_value.flatten() next_value = last_value.clone().cpu().flatten()
else: else:
next_non_terminal = 1.0 - self.dones[step + 1] next_non_terminal = 1.0 - self.dones[step + 1]
next_value = self.values[step + 1] next_value = self.values[step + 1]
@ -128,12 +129,12 @@ class RolloutBuffer(BaseBuffer):
self.returns = self.advantages + self.values self.returns = self.advantages + self.values
def add(self, state, action, reward, done, value, log_prob): def add(self, state, action, reward, done, value, log_prob):
self.values[self.pos] = th.FloatTensor(value.flatten()) self.values[self.pos] = th.FloatTensor(value.clone().cpu().flatten())
self.log_probs[self.pos] = th.FloatTensor(log_prob) self.log_probs[self.pos] = th.FloatTensor(log_prob.cpu().clone())
self.states[self.pos] = th.FloatTensor(state) self.states[self.pos] = th.FloatTensor(np.array(state))
self.actions[self.pos] = th.FloatTensor(action) self.actions[self.pos] = th.FloatTensor(np.array(action))
self.rewards[self.pos] = th.FloatTensor(reward) self.rewards[self.pos] = th.FloatTensor(np.array(reward))
self.dones[self.pos] = th.FloatTensor(done) self.dones[self.pos] = th.FloatTensor(np.array(done))
self.pos += 1 self.pos += 1
if self.pos == self.buffer_size: if self.pos == self.buffer_size:
self.full = True self.full = True

View file

@ -54,7 +54,7 @@ class PPOPolicy(BasePolicy):
def _get_action_dist_from_latent(self, latent, deterministic=False): def _get_action_dist_from_latent(self, latent, deterministic=False):
mean_actions = self.actor_net(latent) mean_actions = self.actor_net(latent)
action_std = th.ones(mean_actions.size()) * self.log_std.exp() action_std = th.ones_like(mean_actions) * self.log_std.exp()
action_distribution = Normal(mean_actions, action_std) action_distribution = Normal(mean_actions, action_std)
# Sample from the gaussian # Sample from the gaussian
if deterministic: if deterministic:
@ -73,13 +73,11 @@ class PPOPolicy(BasePolicy):
return log_prob return log_prob
def actor_forward(self, state, deterministic=False): def actor_forward(self, state, deterministic=False):
state = th.FloatTensor(state).to(self.device)
latent = self.shared_net(state) latent = self.shared_net(state)
action, _ = self._get_action_dist_from_latent(latent, deterministic=deterministic) action, _ = self._get_action_dist_from_latent(latent, deterministic=deterministic)
return action.detach().cpu().numpy() return action.detach().cpu().numpy()
def get_policy_stats(self, state, action): def get_policy_stats(self, state, action):
state = th.FloatTensor(state).to(self.device)
latent = self.shared_net(state) latent = self.shared_net(state)
_, action_distribution = self._get_action_dist_from_latent(latent) _, action_distribution = self._get_action_dist_from_latent(latent)
log_prob = self._get_log_prob(action_distribution, action) log_prob = self._get_log_prob(action_distribution, action)

View file

@ -10,7 +10,6 @@ from torchy_baselines.common.evaluation import evaluate_policy
from torchy_baselines.ppo.policies import PPOPolicy from torchy_baselines.ppo.policies import PPOPolicy
from torchy_baselines.common.buffers import RolloutBuffer from torchy_baselines.common.buffers import RolloutBuffer
from torchy_baselines.common.utils import explained_variance from torchy_baselines.common.utils import explained_variance
from torchy_baselines.common.vec_env import VecEnv, DummyVecEnv
class PPO(BaseRLModel): class PPO(BaseRLModel):
@ -27,11 +26,11 @@ class PPO(BaseRLModel):
n_optim=5, batch_size=64, n_steps=256, n_optim=5, batch_size=64, n_steps=256,
gamma=0.99, lambda_=0.95, clip_range=0.2, gamma=0.99, lambda_=0.95, clip_range=0.2,
ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5, ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5,
target_kl=None, clip_range_vf=None, target_kl=None, clip_range_vf=None, create_eval_env=False,
_init_setup_model=True): _init_setup_model=True):
super(PPO, self).__init__(policy, env, PPOPolicy, policy_kwargs, super(PPO, self).__init__(policy, env, PPOPolicy, policy_kwargs,
verbose, device, support_multi_env=True) verbose, device, create_eval_env=create_eval_env, support_multi_env=True)
self.learning_rate = learning_rate self.learning_rate = learning_rate
self._seed = seed self._seed = seed
@ -53,12 +52,15 @@ class PPO(BaseRLModel):
def _setup_model(self): def _setup_model(self):
state_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0] state_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0]
self.seed(self._seed) # TODO: different seed for each env when n_envs > 1
if self.n_envs == 1:
self.seed(self._seed)
self.rollout_buffer = RolloutBuffer(self.n_steps, state_dim, action_dim, self.device, self.rollout_buffer = RolloutBuffer(self.n_steps, state_dim, action_dim, self.device,
gamma=self.gamma, lambda_=self.lambda_, n_envs=self.n_envs) gamma=self.gamma, lambda_=self.lambda_, n_envs=self.n_envs)
self.policy = self.policy(self.observation_space, self.action_space, self.policy = self.policy(self.observation_space, self.action_space,
self.learning_rate, device=self.device, **self.policy_kwargs) self.learning_rate, device=self.device, **self.policy_kwargs)
self.policy = self.policy.to(self.device)
def select_action(self, observation): def select_action(self, observation):
# Normally not needed # Normally not needed
@ -99,7 +101,6 @@ class PPO(BaseRLModel):
n_steps += 1 n_steps += 1
rollout_buffer.add(obs, actions, rewards, dones, values, log_probs) rollout_buffer.add(obs, actions, rewards, dones, values, log_probs)
obs = new_obs obs = new_obs
rollout_buffer.compute_returns_and_advantage(values, dones=dones) rollout_buffer.compute_returns_and_advantage(values, dones=dones)
@ -123,7 +124,6 @@ class PPO(BaseRLModel):
# ratio between old and new policy, should be one at the first iteration # ratio between old and new policy, should be one at the first iteration
ratio = th.exp(log_prob - old_log_prob) ratio = th.exp(log_prob - old_log_prob)
# clipped surrogate loss # clipped surrogate loss
policy_loss_1 = advantage * ratio policy_loss_1 = advantage * ratio
policy_loss_2 = advantage * th.clamp(ratio, 1 - self.clip_range, 1 + self.clip_range) policy_loss_2 = advantage * th.clamp(ratio, 1 - self.clip_range, 1 + self.clip_range)
@ -136,7 +136,6 @@ class PPO(BaseRLModel):
# Clip the different between old and new value # Clip the different between old and new value
# NOTE: this depends on the reward scaling # NOTE: this depends on the reward scaling
values_pred = old_values + th.clamp(values - old_values, -self.clip_range_vf, self.clip_range_vf) values_pred = old_values + th.clamp(values - old_values, -self.clip_range_vf, self.clip_range_vf)
# Value loss using the TD(lambda_) target # Value loss using the TD(lambda_) target
value_loss = F.mse_loss(return_batch, values_pred) value_loss = F.mse_loss(return_batch, values_pred)
@ -152,7 +151,6 @@ class PPO(BaseRLModel):
# Clip grad norm # Clip grad norm
th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm)
self.policy.optimizer.step() self.policy.optimizer.step()
approx_kl_divs.append(th.mean(old_log_prob - log_prob).detach().cpu().numpy()) approx_kl_divs.append(th.mean(old_log_prob - log_prob).detach().cpu().numpy())
if self.target_kl is not None and np.mean(approx_kl_divs) > 1.5 * self.target_kl: if self.target_kl is not None and np.mean(approx_kl_divs) > 1.5 * self.target_kl:
@ -170,11 +168,7 @@ class PPO(BaseRLModel):
evaluations = [] evaluations = []
start_time = time.time() start_time = time.time()
obs = self.env.reset() obs = self.env.reset()
eval_env = self._get_eval_env(eval_env)
if eval_env is not None and not isinstance(eval_env, VecEnv):
eval_env = DummyVecEnv([lambda: eval_env])
assert eval_env.num_envs == 1
while self.num_timesteps < total_timesteps: while self.num_timesteps < total_timesteps:

View file

@ -27,12 +27,6 @@ class Critic(BaseNetwork):
if net_arch is None: if net_arch is None:
net_arch = [400, 300] net_arch = [400, 300]
# TODO: solve pytorch parameter registration
# for _ in range(n_critics):
# q_net = create_mlp(state_dim + action_dim, 1, net_arch, activation_fn)
# self.q_net = nn.Sequential(*q_net)
# self.q_networks.append(self.q_net)
q1_net = create_mlp(state_dim + action_dim, 1, net_arch, activation_fn) q1_net = create_mlp(state_dim + action_dim, 1, net_arch, activation_fn)
self.q1_net = nn.Sequential(*q1_net) self.q1_net = nn.Sequential(*q1_net)

View file

@ -20,10 +20,11 @@ class TD3(BaseRLModel):
def __init__(self, policy, env, policy_kwargs=None, verbose=0, def __init__(self, policy, env, policy_kwargs=None, verbose=0,
buffer_size=int(1e6), learning_rate=1e-3, seed=0, device='auto', buffer_size=int(1e6), learning_rate=1e-3, seed=0, device='auto',
action_noise_std=0.1, start_timesteps=100, policy_freq=2, action_noise_std=0.1, start_timesteps=100, policy_freq=2,
batch_size=100, batch_size=100, create_eval_env=False,
_init_setup_model=True): _init_setup_model=True):
super(TD3, self).__init__(policy, env, TD3Policy, policy_kwargs, verbose, device) super(TD3, self).__init__(policy, env, TD3Policy, policy_kwargs, verbose, device,
create_eval_env=create_eval_env)
self.max_action = np.abs(self.action_space.high) self.max_action = np.abs(self.action_space.high)
self.action_noise_std = action_noise_std self.action_noise_std = action_noise_std
@ -43,6 +44,7 @@ class TD3(BaseRLModel):
self.replay_buffer = ReplayBuffer(self.buffer_size, state_dim, action_dim, self.device) self.replay_buffer = ReplayBuffer(self.buffer_size, state_dim, action_dim, self.device)
self.policy = self.policy(self.observation_space, self.action_space, self.policy = self.policy(self.observation_space, self.action_space,
self.learning_rate, device=self.device, **self.policy_kwargs) self.learning_rate, device=self.device, **self.policy_kwargs)
self.policy = self.policy.to(self.device)
self._create_aliases() self._create_aliases()
def _create_aliases(self): def _create_aliases(self):
@ -56,7 +58,7 @@ class TD3(BaseRLModel):
observation = np.array(observation) observation = np.array(observation)
with th.no_grad(): with th.no_grad():
observation = th.FloatTensor(observation.reshape(1, -1)).to(self.device) observation = th.FloatTensor(observation.reshape(1, -1)).to(self.device)
return self.actor(observation).cpu().data.numpy().flatten() return self.actor(observation).cpu().data.numpy()
def predict(self, observation, state=None, mask=None, deterministic=True): def predict(self, observation, state=None, mask=None, deterministic=True):
""" """
@ -153,6 +155,7 @@ class TD3(BaseRLModel):
episode_num = 0 episode_num = 0
evaluations = [] evaluations = []
start_time = time.time() start_time = time.time()
eval_env = self._get_eval_env(eval_env)
while self.num_timesteps < total_timesteps: while self.num_timesteps < total_timesteps: