diff --git a/onnxruntime/__init__.py b/onnxruntime/__init__.py
index c46534ade7..5d6490b343 100644
--- a/onnxruntime/__init__.py
+++ b/onnxruntime/__init__.py
@@ -10,6 +10,17 @@ or the `Github project <https://github.com/microsoft/onnxruntime/>`_.
 __version__ = "1.5.2"
 __author__ = "Microsoft"
 
+import os
+import platform
+import sys
+
+# Python 3.8 (and later) on Windows doesn't search system PATH when loading DLLs,
+# so CUDA location needs to be specified explicitly. This needs to be done before importing
+# onnxruntime.capi._pybind_state
+if "CUDA_PATH" in os.environ and platform.system() == "Windows" and sys.version_info >= (3, 8):
+    cuda_bin_dir = os.path.join(os.environ["CUDA_PATH"], "bin")
+    os.add_dll_directory(cuda_bin_dir)
+
 from onnxruntime.capi._pybind_state import get_all_providers, get_available_providers, get_device, set_seed, \
     RunOptions, SessionOptions, set_default_logger_severity, enable_telemetry_events, disable_telemetry_events, \
     NodeArg, ModelMetadata, GraphOptimizationLevel, ExecutionMode, ExecutionOrder, OrtDevice, SessionIOBinding, \
diff --git a/onnxruntime/python/_pybind_state.py b/onnxruntime/python/_pybind_state.py
index 6a398dd52d..86a24f1895 100644
--- a/onnxruntime/python/_pybind_state.py
+++ b/onnxruntime/python/_pybind_state.py
@@ -5,23 +5,9 @@
 
 import os
 import platform
-import sys
 import warnings
 import onnxruntime.capi._ld_preload  # noqa: F401
 
-# Python 3.8 (and later) on Windows doesn't search system PATH when loading DLLs,
-# so CUDA location needs to be specified explicitly.
-if platform.system() == "Windows" and sys.version_info >= (3, 8):
-    CUDA_VERSION = "10.2"
-    CUDNN_VERSION = "8"
-    cuda_env_variable = "CUDA_PATH_V" + CUDA_VERSION.replace(".", "_")
-    if cuda_env_variable not in os.environ:
-        raise ImportError("CUDA Toolkit %s not installed on the machine." % CUDA_VERSION)
-    cuda_bin_dir = os.path.join(os.environ[cuda_env_variable], "bin")
-    if not os.path.isfile(os.path.join(cuda_bin_dir, "cudnn64_%s.dll" % CUDNN_VERSION)):
-        raise ImportError("cuDNN %s not installed on the machine." % CUDNN_VERSION)
-    os.add_dll_directory(cuda_bin_dir)
-
 try:
     from onnxruntime.capi.onnxruntime_pybind11_state import *  # noqa
 except ImportError as e: