Add learning rate schedule

torchy_baselines/a2c/a2c.py

@@ -69,11 +69,13 @@ class A2C(PPO):
         super(A2C, self)._setup_model()
         if self.use_rms_prop:
             self.policy.optimizer = th.optim.RMSprop(self.policy.parameters(),
-                                                     lr=self.learning_rate, alpha=0.99,
+                                                     lr=self.learning_rate(1), alpha=0.99,
                                                      eps=self.rms_prop_eps, weight_decay=0)
 
     def train(self, gradient_steps, batch_size=None):
 
+        # Update optimizer learning rate
+        self._update_learning_rate(self.policy.optimizer)
         # A2C with gradient_steps > 1 does not make sense
         assert gradient_steps == 1
         # We do not use minibatches for A2C

torchy_baselines/cem_rl/cem_rl.py

@@ -78,7 +78,7 @@ class CEMRL(TD3):
                     # set params
                     self.actor.load_from_vector(self.es_params[i])
                     self.actor_target.load_from_vector(self.es_params[i])
-                    self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=self.learning_rate)
+                    self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=self.learning_rate(self._current_progress))
 
                     # In the paper: 2 * actor_steps // self.n_grad
                     # In the original implementation: actor_steps // self.n_grad
@@ -153,6 +153,7 @@ class CEMRL(TD3):
                     print("Total T: {} Episode Num: {} Episode T: {} Reward: {}".format(
                         self.num_timesteps, episode_num, episode_timesteps, episode_reward))
 
+            self._update_current_progress(self.num_timesteps, total_timesteps)
             self.es.tell(self.es_params, self.fitnesses)
             timesteps_since_eval += actor_steps
         return self

torchy_baselines/common/base_class.py

@@ -7,7 +7,7 @@ import torch as th
 import numpy as np
 
 from torchy_baselines.common.policies import get_policy_from_name
-from torchy_baselines.common.utils import set_random_seed
+from torchy_baselines.common.utils import set_random_seed, get_schedule_fn, update_learning_rate
 from torchy_baselines.common.vec_env import DummyVecEnv, VecEnv
 from torchy_baselines.common.monitor import Monitor
 from torchy_baselines.common import logger
@@ -57,6 +57,9 @@ class BaseRLModel(object):
         self.replay_buffer = None
         self.seed = seed
         self.action_noise = None
+        # Track the training progress (from 1 to 0)
+        # this is used to update the learning rate
+        self._current_progress = 1
 
         if env is not None:
             if isinstance(env, str):
@@ -112,6 +115,27 @@ class BaseRLModel(object):
         low, high = self.action_space.low, self.action_space.high
         return low + (0.5 * (scaled_action + 1.0) * (high -  low))
 
+    def _setup_learning_rate(self):
+        """Transform to callable if needed."""
+        self.learning_rate = get_schedule_fn(self.learning_rate)
+
+    def _update_current_progress(self, num_timesteps, total_timesteps):
+        """
+        Compute current progress (from 1 to 0)
+
+        :param num_timesteps: (int)
+        :param total_timesteps: (int)
+        """
+        self._current_progress = 1.0 - float(num_timesteps) / float(total_timesteps)
+
+    def _update_learning_rate(self, optimizers):
+        logger.logkv("learning_rate", self.learning_rate(self._current_progress))
+
+        if not isinstance(optimizers, list):
+            optimizers = [optimizers]
+        for optimizer in optimizers:
+            update_learning_rate(optimizer, self.learning_rate(self._current_progress))
+
     @staticmethod
     def safe_mean(arr):
         """

torchy_baselines/common/utils.py

@@ -38,3 +38,48 @@ def explained_variance(y_pred, y_true):
     assert y_true.ndim == 1 and y_pred.ndim == 1
     var_y = np.var(y_true)
     return np.nan if var_y == 0 else 1 - np.var(y_true - y_pred) / var_y
+
+
+def update_learning_rate(optimizer, learning_rate):
+    """
+    Update the learning rate for a given optimizer.
+    Useful when doing linear schedule.
+
+    :param optimizer: (th.optim.Optimizer)
+    :param learning_rate: (float)
+    """
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = learning_rate
+
+
+def get_schedule_fn(value_schedule):
+    """
+    Transform (if needed) learning rate and clip range (for PPO)
+    to callable.
+
+    :param value_schedule: (callable or float)
+    :return: (function)
+    """
+    # If the passed schedule is a float
+    # create a constant function
+    if isinstance(value_schedule, (float, int)):
+        # Cast to float to avoid errors
+        value_schedule = constant_fn(float(value_schedule))
+    else:
+        assert callable(value_schedule)
+    return value_schedule
+
+
+def constant_fn(val):
+    """
+    Create a function that returns a constant
+    It is useful for learning rate schedule (to avoid code duplication)
+
+    :param val: (float)
+    :return: (function)
+    """
+
+    def func(_):
+        return val
+
+    return func

torchy_baselines/ppo/policies.py

@@ -100,7 +100,7 @@ class MlpExtractor(nn.Module):
 
 class PPOPolicy(BasePolicy):
     def __init__(self, observation_space, action_space,
-                 learning_rate=1e-3, net_arch=None, device='cpu',
+                 learning_rate, net_arch=None, device='cpu',
                  activation_fn=nn.Tanh, adam_epsilon=1e-5, ortho_init=True):
         super(PPOPolicy, self).__init__(observation_space, action_space, device)
         self.obs_dim = self.observation_space.shape[0]
@@ -149,7 +149,7 @@ class PPOPolicy(BasePolicy):
                 }[module]
                 module.apply(partial(self.init_weights, gain=gain))
         # TODO: support linear decay of the learning rate
-        self.optimizer = th.optim.Adam(self.parameters(), lr=learning_rate, eps=self.adam_epsilon)
+        self.optimizer = th.optim.Adam(self.parameters(), lr=learning_rate(1), eps=self.adam_epsilon)
 
     def forward(self, obs, deterministic=False):
         if not isinstance(obs, th.Tensor):

torchy_baselines/ppo/ppo.py

@@ -16,7 +16,7 @@ import numpy as np
 from torchy_baselines.common.base_class import BaseRLModel
 from torchy_baselines.common.evaluation import evaluate_policy
 from torchy_baselines.common.buffers import RolloutBuffer
-from torchy_baselines.common.utils import explained_variance
+from torchy_baselines.common.utils import explained_variance, get_schedule_fn
 from torchy_baselines.common.vec_env import VecNormalize
 from torchy_baselines.common import logger
 from torchy_baselines.ppo.policies import PPOPolicy
@@ -36,14 +36,16 @@ class PPO(BaseRLModel):
     :param policy: (PPOPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, ...)
     :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
     :param learning_rate: (float or callable) The learning rate, it can be a function
+        of the current progress (from 1 to 0)
     :param n_steps: (int) The number of steps to run for each environment per update
         (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
     :param batch_size: (int) Minibatch size
     :param n_epochs: (int) Number of epoch when optimizing the surrogate loss
     :param gamma: (float) Discount factor
     :param gae_lambda: (float) Factor for trade-off of bias vs variance for Generalized Advantage Estimator
-    :param clip_range: (float or callable) Clipping parameter, it can be a function
+    :param clip_range: (float or callable) Clipping parameter, it can be a function of the current progress (from 1 to 0).
-    :param clip_range_vf: (float or callable) Clipping parameter for the value function, it can be a function.
+    :param clip_range_vf: (float or callable) Clipping parameter for the value function,
+        it can be a function of the current progress (from 1 to 0).
         This is a parameter specific to the OpenAI implementation. If None is passed (default),
         no clipping will be done on the value function.
         IMPORTANT: this clipping depends on the reward scaling.
@@ -84,12 +86,12 @@ class PPO(BaseRLModel):
         self.gamma = gamma
         self.gae_lambda = gae_lambda
         self.clip_range = clip_range
+        self.clip_range_vf = clip_range_vf
         self.ent_coef = ent_coef
         self.vf_coef = vf_coef
         self.max_grad_norm = max_grad_norm
         self.rollout_buffer = None
         self.target_kl = target_kl
-        self.clip_range_vf = clip_range_vf
         self.tensorboard_log = tensorboard_log
         self.tb_writer = None
 
@@ -97,6 +99,7 @@ class PPO(BaseRLModel):
             self._setup_model()
 
     def _setup_model(self):
+        self._setup_learning_rate()
         # TODO: preprocessing: one hot vector for obs discrete
         state_dim = self.observation_space.shape[0]
         if isinstance(self.action_space, spaces.Box):
@@ -116,6 +119,10 @@ class PPO(BaseRLModel):
                                   self.learning_rate, device=self.device, **self.policy_kwargs)
         self.policy = self.policy.to(self.device)
 
+        self.clip_range = get_schedule_fn(self.clip_range)
+        if self.clip_range_vf is not None:
+            self.clip_range_vf = get_schedule_fn(self.clip_range_vf)
+
     def select_action(self, observation):
         # Normally not needed
         observation = np.array(observation)
@@ -169,6 +176,15 @@ class PPO(BaseRLModel):
         return obs
 
     def train(self, gradient_steps, batch_size=64):
+        # Update optimizer learning rate
+        self._update_learning_rate(self.policy.optimizer)
+        # Compute current clip range
+        clip_range = self.clip_range(self._current_progress)
+        logger.logkv("clip_range", clip_range)
+        if self.clip_range_vf is not None:
+            clip_range_vf = self.clip_range_vf(self._current_progress)
+            logger.logkv("clip_range_vf", clip_range_vf)
+
 
         for gradient_step in range(gradient_steps):
             approx_kl_divs = []
@@ -190,7 +206,7 @@ class PPO(BaseRLModel):
                 ratio = th.exp(log_prob - old_log_prob)
                 # clipped surrogate loss
                 policy_loss_1 = advantage * ratio
-                policy_loss_2 = advantage * th.clamp(ratio, 1 - self.clip_range, 1 + self.clip_range)
+                policy_loss_2 = advantage * th.clamp(ratio, 1 - clip_range, 1 + clip_range)
                 policy_loss = -th.min(policy_loss_1, policy_loss_2).mean()
 
                 if self.clip_range_vf is None:
@@ -199,7 +215,7 @@ class PPO(BaseRLModel):
                 else:
                     # Clip the different between old and new value
                     # NOTE: this depends on the reward scaling
-                    values_pred = old_values + th.clamp(values - old_values, -self.clip_range_vf, self.clip_range_vf)
+                    values_pred = old_values + th.clamp(values - old_values, -clip_range_vf, clip_range_vf)
                 # Value loss using the TD(gae_lambda) target
                 value_loss = F.mse_loss(return_batch, values_pred)
 
@@ -244,6 +260,7 @@ class PPO(BaseRLModel):
             iteration += 1
             self.num_timesteps += self.n_steps * self.n_envs
             timesteps_since_eval += self.n_steps * self.n_envs
+            self._update_current_progress(self.num_timesteps, total_timesteps)
 
             # Display training infos
             if self.verbose >= 1 and log_interval is not None and iteration % log_interval == 0:

torchy_baselines/sac/policies.py

@@ -63,7 +63,7 @@ class Critic(BaseNetwork):
 
 class SACPolicy(BasePolicy):
     def __init__(self, observation_space, action_space,
-                 learning_rate=3e-4, net_arch=None, device='cpu',
+                 learning_rate, net_arch=None, device='cpu',
                  activation_fn=nn.ReLU):
         super(SACPolicy, self).__init__(observation_space, action_space, device)
 
@@ -87,12 +87,12 @@ class SACPolicy(BasePolicy):
 
     def _build(self, learning_rate):
         self.actor = self.make_actor()
-        self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=learning_rate)
+        self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=learning_rate(1))
 
         self.critic = self.make_critic()
         self.critic_target = self.make_critic()
         self.critic_target.load_state_dict(self.critic.state_dict())
-        self.critic.optimizer = th.optim.Adam(self.critic.parameters(), lr=learning_rate)
+        self.critic.optimizer = th.optim.Adam(self.critic.parameters(), lr=learning_rate(1))
 
     def make_actor(self):
         return Actor(**self.net_args).to(self.device)

torchy_baselines/sac/sac.py

@@ -88,6 +88,7 @@ class SAC(BaseRLModel):
             self._setup_model()
 
     def _setup_model(self):
+        self._setup_learning_rate()
         obs_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0]
         if self.seed is not None:
             self.set_random_seed(self.seed)
@@ -114,7 +115,7 @@ class SAC(BaseRLModel):
             # Note: we optimize the log of the entropy coeff which is slightly different from the paper
             # as discussed in https://github.com/rail-berkeley/softlearning/issues/37
             self.log_ent_coef = th.log(th.ones(1, device=self.device) * init_value).requires_grad_(True)
-            self.ent_coef_optimizer = th.optim.Adam([self.log_ent_coef], lr=self.learning_rate)
+            self.ent_coef_optimizer = th.optim.Adam([self.log_ent_coef], lr=self.learning_rate(1))
         else:
             # Force conversion to float
             # this will throw an error if a malformed string (different from 'auto')
@@ -152,6 +153,13 @@ class SAC(BaseRLModel):
         return self.unscale_action(self.select_action(observation))
 
     def train(self, gradient_steps, batch_size=64):
+        # Update optimizers learning rate
+        optimizers = [self.actor.optimizer, self.critic.optimizer]
+        if self.ent_coef_optimizer is not None:
+            optimizers += [self.ent_coef_optimizer]
+
+        self._update_learning_rate(optimizers)
+
         for gradient_step in range(gradient_steps):
             # Sample replay buffer
             replay_data = self.replay_buffer.sample(batch_size)
@@ -245,6 +253,7 @@ class SAC(BaseRLModel):
             self.num_timesteps += episode_timesteps
             episode_num += n_episodes
             timesteps_since_eval += episode_timesteps
+            self._update_current_progress(self.num_timesteps, total_timesteps)
 
             if self.num_timesteps > 0 and self.num_timesteps > self.learning_starts:
                 if self.verbose > 1:

torchy_baselines/td3/policies.py

@@ -39,7 +39,7 @@ class Critic(BaseNetwork):
 
 class TD3Policy(BasePolicy):
     def __init__(self, observation_space, action_space,
-                 learning_rate=1e-3, net_arch=None, device='cpu',
+                 learning_rate, net_arch=None, device='cpu',
                  activation_fn=nn.ReLU):
         super(TD3Policy, self).__init__(observation_space, action_space, device)
 
@@ -64,12 +64,12 @@ class TD3Policy(BasePolicy):
         self.actor = self.make_actor()
         self.actor_target = self.make_actor()
         self.actor_target.load_state_dict(self.actor.state_dict())
-        self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=learning_rate)
+        self.actor.optimizer = th.optim.Adam(self.actor.parameters(), lr=learning_rate(1))
 
         self.critic = self.make_critic()
         self.critic_target = self.make_critic()
         self.critic_target.load_state_dict(self.critic.state_dict())
-        self.critic.optimizer = th.optim.Adam(self.critic.parameters(), lr=learning_rate)
+        self.critic.optimizer = th.optim.Adam(self.critic.parameters(), lr=learning_rate(1))
 
     def make_actor(self):
         return Actor(**self.net_args).to(self.device)

torchy_baselines/td3/td3.py

@@ -75,6 +75,7 @@ class TD3(BaseRLModel):
             self._setup_model()
 
     def _setup_model(self):
+        self._setup_learning_rate()
         obs_dim, action_dim = self.observation_space.shape[0], self.action_space.shape[0]
         self.set_random_seed(self.seed)
         self.replay_buffer = ReplayBuffer(self.buffer_size, obs_dim, action_dim, self.device)
@@ -109,6 +110,8 @@ class TD3(BaseRLModel):
         return self.unscale_action(self.select_action(observation))
 
     def train_critic(self, gradient_steps=1, batch_size=100, replay_data=None, tau=0.0):
+        # Update optimizer learning rate
+        self._update_learning_rate(self.critic.optimizer)
 
         for gradient_step in range(gradient_steps):
             # Sample replay buffer
@@ -146,6 +149,8 @@ class TD3(BaseRLModel):
                     target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
 
     def train_actor(self, gradient_steps=1, batch_size=100, tau_actor=0.005, tau_critic=0.005, replay_data=None):
+        # Update optimizer learning rate
+        self._update_learning_rate(self.actor.optimizer)
 
         for gradient_step in range(gradient_steps):
             # Sample replay buffer
@@ -208,6 +213,7 @@ class TD3(BaseRLModel):
             episode_num += n_episodes
             self.num_timesteps += episode_timesteps
             timesteps_since_eval += episode_timesteps
+            self._update_current_progress(self.num_timesteps, total_timesteps)
 
             if self.num_timesteps > 0 and self.num_timesteps > self.learning_starts:
                 if self.verbose > 1: