From 0d41bc13560ac8ed010d8bcc6407d3318a494646 Mon Sep 17 00:00:00 2001 From: Antonin Raffin Date: Tue, 29 Oct 2019 15:15:11 +0100 Subject: [PATCH] Add more logging --- torchy_baselines/a2c/a2c.py | 18 ++++++++++++------ torchy_baselines/ppo/ppo.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/torchy_baselines/a2c/a2c.py b/torchy_baselines/a2c/a2c.py index 3aa1589..6b51227 100644 --- a/torchy_baselines/a2c/a2c.py +++ b/torchy_baselines/a2c/a2c.py @@ -76,9 +76,6 @@ class A2C(PPO): eps=self.rms_prop_eps, weight_decay=0) def train(self, gradient_steps, batch_size=None): - if self.use_sde: - logger.logkv("noise net std", th.exp(self.policy.log_std).mean().item()) - # Update optimizer learning rate self._update_learning_rate(self.policy.optimizer) # A2C with gradient_steps > 1 does not make sense @@ -118,10 +115,19 @@ class A2C(PPO): # Clip grad norm th.nn.utils.clip_grad_norm_(self.policy.parameters(), self.max_grad_norm) self.policy.optimizer.step() - # approx_kl_divs.append(th.mean(old_log_prob - log_prob).detach().cpu().numpy()) - # print(explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), - # self.rollout_buffer.values.flatten().cpu().numpy())) + explained_var = explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), + self.rollout_buffer.values.flatten().cpu().numpy()) + + logger.logkv("explained_variance", explained_var) + logger.logkv("entropy", entropy.mean().item()) + logger.logkv("policy_loss", policy_loss.item()) + logger.logkv("value_loss", value_loss.item()) + + if self.use_sde: + logger.logkv("noise net std", th.exp(self.policy.log_std).mean().item()) + # print(th.exp(self.policy.log_std).detach()) + def learn(self, total_timesteps, callback=None, log_interval=100, eval_env=None, eval_freq=-1, n_eval_episodes=5, tb_log_name="A2C", reset_num_timesteps=True): diff --git a/torchy_baselines/ppo/ppo.py b/torchy_baselines/ppo/ppo.py index 7fa318d..8171eb5 100644 --- a/torchy_baselines/ppo/ppo.py +++ b/torchy_baselines/ppo/ppo.py @@ -245,8 +245,14 @@ class PPO(BaseRLModel): print("Early stopping at step {} due to reaching max kl: {:.2f}".format(it, np.mean(approx_kl_divs))) break - # print(explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), - # self.rollout_buffer.values.flatten().cpu().numpy())) + explained_var = explained_variance(self.rollout_buffer.returns.flatten().cpu().numpy(), + self.rollout_buffer.values.flatten().cpu().numpy()) + + logger.logkv("explained_variance", explained_var) + # TODO: gather stats for the entropy and other losses? + logger.logkv("entropy", entropy.mean().item()) + logger.logkv("policy_loss", policy_loss.item()) + logger.logkv("value_loss", value_loss.item()) def learn(self, total_timesteps, callback=None, log_interval=1, eval_env=None, eval_freq=-1, n_eval_episodes=5, tb_log_name="PPO", reset_num_timesteps=True):