diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index b9b501b..6f4dd74 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -30,6 +30,8 @@ Bug Fixes:
 ^^^^^^^^^^
 - Fixed a bug in the ``close()`` method of ``SubprocVecEnv``, causing wrappers further down in the wrapper stack to not be closed. (@NeoExtended)
 - Fix target for updating q values in SAC: the entropy term was not conditioned by terminals states
+- Use ``cloudpickle.load`` instead of ``pickle.load`` in ``CloudpickleWrapper``. (@shwang)
+- Fixed a bug with orthogonal initialization when `bias=False` in custom policy (@rk37)
 
 Deprecations:
 ^^^^^^^^^^^^^
@@ -353,4 +355,4 @@ And all the contributors:
 @Miffyli @dwiel @miguelrass @qxcv @jaberkow @eavelardev @ruifeng96150 @pedrohbtp @srivatsankrishnan @evilsocket
 @MarvineGothic @jdossgollin @SyllogismRXS @rusu24edward @jbulow @Antymon @seheevic @justinkterry @edbeeching
 @flodorner @KuKuXia @NeoExtended @PartiallyTyped @mmcenta @richardwu @kinalmehta @rolandgvc @tkelestemur @mloo3
-@tirafesi @blurLake @koulakis @joeljosephjin
+@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37
diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py
index acadd88..01b788f 100644
--- a/stable_baselines3/common/policies.py
+++ b/stable_baselines3/common/policies.py
@@ -188,7 +188,8 @@ class BasePolicy(BaseModel):
         """
         if isinstance(module, (nn.Linear, nn.Conv2d)):
             nn.init.orthogonal_(module.weight, gain=gain)
-            module.bias.data.fill_(0.0)
+            if module.bias is not None:
+                module.bias.data.fill_(0.0)
 
     @abstractmethod
     def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor:
diff --git a/stable_baselines3/common/vec_env/base_vec_env.py b/stable_baselines3/common/vec_env/base_vec_env.py
index 6b5a421..6338a30 100644
--- a/stable_baselines3/common/vec_env/base_vec_env.py
+++ b/stable_baselines3/common/vec_env/base_vec_env.py
@@ -1,5 +1,4 @@
 import inspect
-import pickle
 from abc import ABC, abstractmethod
 from typing import List, Optional, Sequence, Union
 
@@ -349,7 +348,7 @@ class VecEnvWrapper(VecEnv):
         return shadowed_wrapper_class
 
 
-class CloudpickleWrapper(object):
+class CloudpickleWrapper:
     def __init__(self, var):
         """
         Uses cloudpickle to serialize contents (otherwise multiprocessing tries to use pickle)
@@ -362,4 +361,4 @@ class CloudpickleWrapper(object):
         return cloudpickle.dumps(self.var)
 
     def __setstate__(self, obs):
-        self.var = pickle.loads(obs)
+        self.var = cloudpickle.loads(obs)