From 6cce61d18339ba8b75dce51e0d05cc42735b751f Mon Sep 17 00:00:00 2001
From: Antonin Raffin <antonin.raffin@dlr.de>
Date: Fri, 6 Sep 2019 14:11:27 +0200
Subject: [PATCH] Fix random exploration

---
 torchy_baselines/cem_rl/cem_rl.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/torchy_baselines/cem_rl/cem_rl.py b/torchy_baselines/cem_rl/cem_rl.py
index f4a7944..c7d8e5c 100644
--- a/torchy_baselines/cem_rl/cem_rl.py
+++ b/torchy_baselines/cem_rl/cem_rl.py
@@ -194,6 +194,8 @@ class CEMRL(TD3):
 
                     obs = new_obs
                     episode_timesteps += 1
+                    # Note: if put on the outer, it will explore start_timesteps for each actor
+                    self.num_timesteps += 1
 
                 if self.verbose > 1:
                     print("Total T: {} Episode Num: {} Episode T: {} Reward: {}".format(
@@ -204,7 +206,7 @@ class CEMRL(TD3):
 
             self.es.tell(self.es_params, self.fitnesses)
 
-            self.num_timesteps += actor_steps
+            # self.num_timesteps += actor_steps
             timesteps_since_eval += actor_steps
         return self