diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
index ed985fa..052f2c2 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -1,5 +1,5 @@
 name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us improve Stable-Baselines3
+description: If you encounter an unexpected behavior, software crash, or other bug.
 title: "[Bug]: bug title"
 labels: ["bug"]
 body:
@@ -61,6 +61,8 @@ body:
     attributes:
       label: Checklist
       options:
+        - label: My issue does not relate to a custom gym environment. (Use the custom gym env template instead)
+          required: true
         - label: I have checked that there is no similar [issue](https://github.com/DLR-RM/stable-baselines3/issues) in the repo
           required: true
         - label: I have read the [documentation](https://stable-baselines3.readthedocs.io/en/master/)
diff --git a/.github/ISSUE_TEMPLATE/custom_env.yml b/.github/ISSUE_TEMPLATE/custom_env.yml
index 5fa448a..4040d9d 100644
--- a/.github/ISSUE_TEMPLATE/custom_env.yml
+++ b/.github/ISSUE_TEMPLATE/custom_env.yml
@@ -1,6 +1,6 @@
 name: "\U0001F916 Custom Gym Environment Issue"
-description: How to report an issue when using a custom Gym environment
-labels: ["question", "custom gym env"]
+description: If your problem involves a custom gym environment.
+labels: ["custom gym env"]
 body:
   - type: markdown
     attributes:
diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml
index 025e2d3..7647090 100644
--- a/.github/ISSUE_TEMPLATE/documentation.yml
+++ b/.github/ISSUE_TEMPLATE/documentation.yml
@@ -1,5 +1,5 @@
 name: "\U0001F4DA Documentation"
-description: Report an issue related to Stable-Baselines3 documentation
+description: If you want to improve the documentation by reporting errors, inconsistencies, or missing information.
 labels: ["documentation"]
 body:
   - type: markdown
diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
index 1d598a1..9e72949 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -1,5 +1,5 @@
 name: "\U0001F680 Feature Request"
-description: How to create an issue for requesting a feature
+description: If you have an idea for a new feature or an improvement.
 title: "[Feature Request] request title"
 labels: ["enhancement"]
 body:
diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml
index b2fb2f5..2942068 100644
--- a/.github/ISSUE_TEMPLATE/question.yml
+++ b/.github/ISSUE_TEMPLATE/question.yml
@@ -1,5 +1,5 @@
 name: "❓ Question"
-description: How to ask a question regarding Stable-Baselines3
+description: If you have a general question about Stable-Baselines3.
 title: "[Question] question title"
 labels: ["question"]
 body:
@@ -12,7 +12,9 @@ body:
     id: question
     attributes:
       label: ❓ Question
-      description: Your question. This can be e.g. questions regarding confusing or unclear behaviour of functions or a question if X can be done using stable-baselines3. Make sure to check out the documentation first.
+      description: |
+        Your question. This can be e.g. questions regarding confusing or unclear behaviour of functions or a question if X can be done using stable-baselines3. Make sure to check out the documentation first.
+        **Important Note: If your question is anything like "Why is my code generating this error?", you must [submit a bug report](https://github.com/DLR-RM/stable-baselines3/issues/new?assignees=&labels=bug&projects=&template=bug_report.yml&title=%5BBug%5D%3A+bug+title) instead.**
     validations:
       required: true
   - type: checkboxes
diff --git a/setup.py b/setup.py
index b170e6e..de2b031 100644
--- a/setup.py
+++ b/setup.py
@@ -131,7 +131,7 @@ setup(
             "black",
         ],
         "docs": [
-            "sphinx",
+            "sphinx>=5.3,<7.0",
             "sphinx-autobuild",
             "sphinx-rtd-theme",
             # For spelling
diff --git a/stable_baselines3/common/envs/bit_flipping_env.py b/stable_baselines3/common/envs/bit_flipping_env.py
index 869b463..1089797 100644
--- a/stable_baselines3/common/envs/bit_flipping_env.py
+++ b/stable_baselines3/common/envs/bit_flipping_env.py
@@ -179,7 +179,7 @@ class BitFlippingEnv(Env):
         else:
             self.state[action] = 1 - self.state[action]
         obs = self._get_obs()
-        reward = float(self.compute_reward(obs["achieved_goal"], obs["desired_goal"], None))
+        reward = float(self.compute_reward(obs["achieved_goal"], obs["desired_goal"], None).item())
         terminated = reward == 0
         self.current_step += 1
         # Episode terminate when we reached the goal or the max number of steps
diff --git a/stable_baselines3/common/logger.py b/stable_baselines3/common/logger.py
index 8a6facf..4d0d346 100644
--- a/stable_baselines3/common/logger.py
+++ b/stable_baselines3/common/logger.py
@@ -297,7 +297,7 @@ class JSONOutputFormat(KVWriter):
             if hasattr(value, "dtype"):
                 if value.shape == () or len(value) == 1:
                     # if value is a dimensionless numpy array or of length 1, serialize as a float
-                    return float(value)
+                    return float(value.item())
                 else:
                     # otherwise, a value is a numpy array, serialize as a list or nested lists
                     return value.tolist()