Add append mode to Monitor (#1037)

* Added option to override or use existing CSVs

* Updated changelog for Monitor override

* Changed default value to override

* Simplify code and add test

* Update version

* Fix for pytype

Co-authored-by: Antonin Raffin <antonin.raffin@ensta.org>

docs/misc/changelog.rst

@@ -3,7 +3,7 @@
 Changelog
 ==========
 
-Release 1.6.1a3 (WIP)
+Release 1.6.1a4 (WIP)
 ---------------------------
 
 Breaking Changes:
@@ -14,6 +14,7 @@ New Features:
 ^^^^^^^^^^^^^
 - Support logging hyperparameters to tensorboard (@timothe-chaumont)
 - Added checkpoints for replay buffer and ``VecNormalize`` statistics (@anand-bala)
+- Added option for ``Monitor`` to append to existing file instead of overriding (@sidney-tio)
 
 SB3-Contrib
 ^^^^^^^^^^^
@@ -1030,4 +1031,4 @@ And all the contributors:
 @simoninithomas @armandpl @manuel-delverme @Gautam-J @gianlucadecola @buoyancy99 @caburu @xy9485
 @Gregwar @ycheng517 @quantitative-technologies @bcollazo @git-thor @TibiGG @cool-RR @MWeltevrede
 @Melanol @qgallouedec @francescoluciano @jlp-ue @burakdmb @timothe-chaumont @honglu2875
-@anand-bala @hughperkins
+@anand-bala @hughperkins @sidney-tio

stable_baselines3/common/monitor.py

@@ -24,6 +24,8 @@ class Monitor(gym.Wrapper):
     :param reset_keywords: extra keywords for the reset call,
         if extra parameters are needed at reset
     :param info_keywords: extra information to log, from the information return of env.step()
+    :param override_existing: appends to file if ``filename`` exists, otherwise
+        override existing files (default)
     """
 
     EXT = "monitor.csv"
@@ -35,6 +37,7 @@ class Monitor(gym.Wrapper):
         allow_early_resets: bool = True,
         reset_keywords: Tuple[str, ...] = (),
         info_keywords: Tuple[str, ...] = (),
+        override_existing: bool = True,
     ):
         super().__init__(env=env)
         self.t_start = time.time()
@@ -43,6 +46,7 @@ class Monitor(gym.Wrapper):
                 filename,
                 header={"t_start": self.t_start, "env_id": env.spec and env.spec.id},
                 extra_keys=reset_keywords + info_keywords,
+                override_existing=override_existing,
             )
         else:
             self.results_writer = None
@@ -163,6 +167,8 @@ class ResultsWriter:
     :param header: the header dictionary object of the saved csv
     :param reset_keywords: the extra information to log, typically is composed of
         ``reset_keywords`` and ``info_keywords``
+    :param override_existing: appends to file if ``filename`` exists, otherwise
+        override existing files (default)
     """
 
     def __init__(
@@ -170,6 +176,7 @@ class ResultsWriter:
         filename: str = "",
         header: Optional[Dict[str, Union[float, str]]] = None,
         extra_keys: Tuple[str, ...] = (),
+        override_existing: bool = True,
     ):
         if header is None:
             header = {}
@@ -178,11 +185,15 @@ class ResultsWriter:
                 filename = os.path.join(filename, Monitor.EXT)
             else:
                 filename = filename + "." + Monitor.EXT
+        # Append mode when not overridding existing file
+        mode = "w" if override_existing else "a"
         # Prevent newline issue on Windows, see GH issue #692
-        self.file_handler = open(filename, "wt", newline="\n")
-        self.file_handler.write("#%s\n" % json.dumps(header))
+        self.file_handler = open(filename, f"{mode}t", newline="\n")
         self.logger = csv.DictWriter(self.file_handler, fieldnames=("r", "l", "t") + extra_keys)
-        self.logger.writeheader()
+        if override_existing:
+            self.file_handler.write(f"#{json.dumps(header)}\n")
+            self.logger.writeheader()
+
         self.file_handler.flush()
 
     def write_row(self, epinfo: Dict[str, Union[float, int]]) -> None:

stable_baselines3/her/her_replay_buffer.py

@@ -226,7 +226,7 @@ class HerReplayBuffer(DictReplayBuffer):
             maybe_vec_env=None,
             online_sampling=False,
             n_sampled_goal=n_sampled_goal,
-        )
+        )  # pytype: disable=bad-return-type
 
     def sample_goals(
         self,

stable_baselines3/version.txt

@@ -1 +1 @@
-1.6.1a3
+1.6.1a4

tests/test_monitor.py

@@ -83,13 +83,16 @@ def test_monitor_load_results(tmp_path):
     assert monitor_file1 in monitor_files
     assert monitor_file2 in monitor_files
 
-    monitor_env2.reset()
     episode_count2 = 0
-    for _ in range(1000):
-        _, _, done, _ = monitor_env2.step(monitor_env2.action_space.sample())
-        if done:
-            episode_count2 += 1
-            monitor_env2.reset()
+    for _ in range(2):
+        # Test appending to existing file
+        monitor_env2 = Monitor(env2, monitor_file2, override_existing=False)
+        monitor_env2.reset()
+        for _ in range(1000):
+            _, _, done, _ = monitor_env2.step(monitor_env2.action_space.sample())
+            if done:
+                episode_count2 += 1
+                monitor_env2.reset()
 
     results_size2 = len(load_results(os.path.join(tmp_path)).index)