diff --git a/stable_baselines3/common/buffers.py b/stable_baselines3/common/buffers.py
index 64cba78..0b51f0d 100644
--- a/stable_baselines3/common/buffers.py
+++ b/stable_baselines3/common/buffers.py
@@ -457,7 +457,7 @@ class NstepReplayBuffer(ReplayBuffer):
             # Compute entropy term
             # TODO: convert to pytorch tensor on the correct device
             with th.no_grad():
-                obs = th.as_tensor(self.observations[unique_indices, :]).to(self.actor.device)
+                obs = th.as_tensor(self.observations[unique_indices, 0, :]).to(self.actor.device)
                 _, log_prob = self.actor.action_log_prob(obs)
 
             # Memory inneficient version but fast computation