diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc
index c269070ea1..f53a4e5eb2 100644
--- a/onnxruntime/core/framework/provider_bridge_ort.cc
+++ b/onnxruntime/core/framework/provider_bridge_ort.cc
@@ -509,22 +509,6 @@ struct ProviderLibrary {
     if (!handle_)
       return;
 
-#if defined(_WIN32) && !defined(_OPENMP)
-    {
-      // We crash when unloading DNNL on Windows when OpenMP also unloads (As there are threads
-      // still running code inside the openmp runtime DLL if OMP_WAIT_POLICY is set to ACTIVE).
-      // To avoid this, we pin the OpenMP DLL so that it unloads as late as possible.
-      HMODULE handle{};
-#ifdef _DEBUG
-      constexpr const char* dll_name = "vcomp140d.dll";
-#else
-      constexpr const char* dll_name = "vcomp140.dll";
-#endif
-      ::GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_PIN, dll_name, &handle);
-      assert(handle);  // It should exist
-    }
-#endif
-
     Provider* (*PGetProvider)();
     Env::Default().GetSymbolFromLibrary(handle_, "GetProvider", (void**)&PGetProvider);
 
diff --git a/onnxruntime/core/providers/dnnl/dnnl_provider_factory.cc b/onnxruntime/core/providers/dnnl/dnnl_provider_factory.cc
index 00a1ba53f9..59c3bc934a 100644
--- a/onnxruntime/core/providers/dnnl/dnnl_provider_factory.cc
+++ b/onnxruntime/core/providers/dnnl/dnnl_provider_factory.cc
@@ -4,6 +4,7 @@
 #include "core/providers/shared_library/provider_api.h"
 #include "core/providers/dnnl/dnnl_provider_factory.h"
 #include <atomic>
+#include <cassert>
 #include "dnnl_execution_provider.h"
 
 using namespace onnxruntime;
@@ -32,6 +33,21 @@ std::unique_ptr<Provider_IExecutionProvider> DnnlProviderFactory::CreateProvider
 
 struct Dnnl_Provider : Provider {
   std::shared_ptr<Provider_IExecutionProviderFactory> CreateExecutionProviderFactory(int use_arena) override {
+#if defined(_WIN32) && !defined(_OPENMP)
+    {
+      // We crash when unloading DNNL on Windows when OpenMP also unloads (As there are threads
+      // still running code inside the openmp runtime DLL if OMP_WAIT_POLICY is set to ACTIVE).
+      // To avoid this, we pin the OpenMP DLL so that it unloads as late as possible.
+      HMODULE handle{};
+#ifdef _DEBUG
+      constexpr const char* dll_name = "vcomp140d.dll";
+#else
+      constexpr const char* dll_name = "vcomp140.dll";
+#endif
+      ::GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_PIN, dll_name, &handle);
+      assert(handle);  // It should exist
+    }
+#endif
     return std::make_shared<DnnlProviderFactory>(use_arena != 0);
   }