From a9b8276efb702add65e356a610812d04970afb61 Mon Sep 17 00:00:00 2001 From: Antonin Raffin Date: Fri, 20 Sep 2019 18:06:08 +0200 Subject: [PATCH] Attempt to fix loss of perf because of VecEnvs --- tests/test_run.py | 4 ++-- torchy_baselines/common/buffers.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index d2f2165..0583d12 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -7,7 +7,7 @@ from torchy_baselines import TD3, CEMRL, PPO def test_td3(): model = TD3('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[64, 64]), start_timesteps=100, verbose=1, create_eval_env=True) - model.learn(total_timesteps=500, eval_freq=100) + model.learn(total_timesteps=20000, eval_freq=1000) model.save("test_save") model.load("test_save") os.remove("test_save.pth") @@ -15,7 +15,7 @@ def test_td3(): def test_cemrl(): model = CEMRL('MlpPolicy', 'Pendulum-v0', policy_kwargs=dict(net_arch=[16]), pop_size=2, n_grad=1, start_timesteps=100, verbose=1, create_eval_env=True) - model.learn(total_timesteps=1000, eval_freq=500) + model.learn(total_timesteps=20000, eval_freq=1000) model.save("test_save") model.load("test_save") os.remove("test_save.pth") diff --git a/torchy_baselines/common/buffers.py b/torchy_baselines/common/buffers.py index 304384b..9dfdb97 100644 --- a/torchy_baselines/common/buffers.py +++ b/torchy_baselines/common/buffers.py @@ -68,11 +68,11 @@ class ReplayBuffer(BaseBuffer): def add(self, state, next_state, action, reward, done): # Copy to avoid modification by reference - self.states[self.pos] = th.FloatTensor(np.array(state)) - self.next_states[self.pos] = th.FloatTensor(np.array(next_state)) - self.actions[self.pos] = th.FloatTensor(np.array(action)) - self.rewards[self.pos] = th.FloatTensor(np.array(reward)) - self.dones[self.pos] = th.FloatTensor(np.array(done)) + self.states[self.pos] = th.FloatTensor(np.array(state).copy()) + self.next_states[self.pos] = th.FloatTensor(np.array(next_state).copy()) + self.actions[self.pos] = th.FloatTensor(np.array(action).copy()) + self.rewards[self.pos] = th.FloatTensor(np.array(reward).copy()) + self.dones[self.pos] = th.FloatTensor(np.array(done).copy()) self.pos += 1 if self.pos == self.buffer_size: @@ -131,10 +131,10 @@ class RolloutBuffer(BaseBuffer): def add(self, state, action, reward, done, value, log_prob): self.values[self.pos] = th.FloatTensor(value.clone().cpu().flatten()) self.log_probs[self.pos] = th.FloatTensor(log_prob.cpu().clone()) - self.states[self.pos] = th.FloatTensor(np.array(state)) - self.actions[self.pos] = th.FloatTensor(np.array(action)) - self.rewards[self.pos] = th.FloatTensor(np.array(reward)) - self.dones[self.pos] = th.FloatTensor(np.array(done)) + self.states[self.pos] = th.FloatTensor(np.array(state).copy()) + self.actions[self.pos] = th.FloatTensor(np.array(action).copy()) + self.rewards[self.pos] = th.FloatTensor(np.array(reward).copy()) + self.dones[self.pos] = th.FloatTensor(np.array(done).copy()) self.pos += 1 if self.pos == self.buffer_size: self.full = True