diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
index 9d4a1c2..be9a7f3 100644
--- a/docs/misc/changelog.rst
+++ b/docs/misc/changelog.rst
@@ -31,6 +31,7 @@ Bug Fixes:
 - Fixed a bug in the ``close()`` method of ``SubprocVecEnv``, causing wrappers further down in the wrapper stack to not be closed. (@NeoExtended)
 - Fix target for updating q values in SAC: the entropy term was not conditioned by terminals states
 - Use ``cloudpickle.load`` instead of ``pickle.load`` in ``CloudpickleWrapper``. (@shwang)
+- Fixed a bug with orthogonal initialization when `bias=False` in custom policy (@rk37)
 
 Deprecations:
 ^^^^^^^^^^^^^
@@ -356,4 +357,4 @@ And all the contributors:
 @Miffyli @dwiel @miguelrass @qxcv @jaberkow @eavelardev @ruifeng96150 @pedrohbtp @srivatsankrishnan @evilsocket
 @MarvineGothic @jdossgollin @SyllogismRXS @rusu24edward @jbulow @Antymon @seheevic @justinkterry @edbeeching
 @flodorner @KuKuXia @NeoExtended @PartiallyTyped @mmcenta @richardwu @kinalmehta @rolandgvc @tkelestemur @mloo3
-@tirafesi @blurLake @koulakis @joeljosephjin @shwang
+@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37
diff --git a/stable_baselines3/common/policies.py b/stable_baselines3/common/policies.py
index acadd88..01b788f 100644
--- a/stable_baselines3/common/policies.py
+++ b/stable_baselines3/common/policies.py
@@ -188,7 +188,8 @@ class BasePolicy(BaseModel):
         """
         if isinstance(module, (nn.Linear, nn.Conv2d)):
             nn.init.orthogonal_(module.weight, gain=gain)
-            module.bias.data.fill_(0.0)
+            if module.bias is not None:
+                module.bias.data.fill_(0.0)
 
     @abstractmethod
     def _predict(self, observation: th.Tensor, deterministic: bool = False) -> th.Tensor: