diff --git a/onnxruntime/core/framework/provider_bridge_ort.cc b/onnxruntime/core/framework/provider_bridge_ort.cc
index 9c5c26dea0..bfb89f2aff 100644
--- a/onnxruntime/core/framework/provider_bridge_ort.cc
+++ b/onnxruntime/core/framework/provider_bridge_ort.cc
@@ -504,12 +504,27 @@ struct ProviderHostImpl : ProviderHost {
 
 struct ProviderLibrary {
   ProviderLibrary(const char* filename) {
-
     std::string full_path = Env::Default().GetRuntimePath() + std::string(filename);
     Env::Default().LoadDynamicLibrary(full_path, &handle_);
     if (!handle_)
       return;
 
+#if defined(_WIN32) && !defined(_OPENMP)
+    {
+      // We crash when unloading DNNL on Windows when OpenMP also unloads (As there are threads
+      // still running code inside the openmp runtime DLL if OMP_WAIT_POLICY is set to ACTIVE).
+      // To avoid this, we pin the OpenMP DLL so that it unloads as late as possible.
+      HMODULE handle{};
+#ifdef _DEBUG
+      constexpr const char* dll_name = "vcomp140d.dll";
+#else
+      constexpr const char* dll_name = "vcomp140.dll";
+#endif
+      ::GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_PIN, dll_name, &handle);
+      assert(handle);  // It should exist
+    }
+#endif
+
     Provider* (*PGetProvider)();
     Env::Default().GetSymbolFromLibrary(handle_, "GetProvider", (void**)&PGetProvider);
 
diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc
index 336a36da5a..4aa4c0e251 100644
--- a/onnxruntime/test/onnx/main.cc
+++ b/onnxruntime/test/onnx/main.cc
@@ -24,8 +24,6 @@
 
 using namespace onnxruntime;
 
-static bool g_exit_fast = false;
-
 namespace {
 void usage() {
   printf(
@@ -323,7 +321,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
     if (enable_dnnl) {
 #ifdef USE_DNNL
       Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Dnnl(sf, enable_cpu_mem_arena ? 1 : 0));
-      g_exit_fast = true;
 #else
       fprintf(stderr, "DNNL is not supported in this build");
       return -1;
@@ -498,19 +495,19 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
       {"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"},
       {"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"},
       {"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398"},
-      {"training_dropout", "result differs", {}},               // Temporary, subsequent PR will remove this.
-      {"training_dropout_default", "result differs", {}},       // Temporary, subsequent PR will remove this.
-      {"training_dropout_default_mask", "result differs", {}},  // Temporary, subsequent PR will remove this.
-      {"training_dropout_mask", "result differs", {}},          // Temporary, subsequent PR will remove this.
-      {"adagrad", "not a registered function/op", {}},          // Op not registered.
-      {"adagrad_multiple", "not a registered function/op", {}}, // Op not registered.
-      {"adam", "not a registered function/op", {}},  // Op not registered.
-      {"adam_multiple", "not a registered function/op", {}}, // Op not registered.
-      {"gradient_of_add", "not a registered function/op", {}}, // Op not registered.
-      {"gradient_of_add_and_mul", "not a registered function/op", {}}, // Op not registered.
-      {"momentum", "not a registered function/op", {}},  // Op not registered.
-      {"momentum_multiple", "not a registered function/op", {}}, // Op not registered.
-      {"nesterov_momentum", "not a registered function/op", {}}, // Op not registered.
+      {"training_dropout", "result differs", {}},                       // Temporary, subsequent PR will remove this.
+      {"training_dropout_default", "result differs", {}},               // Temporary, subsequent PR will remove this.
+      {"training_dropout_default_mask", "result differs", {}},          // Temporary, subsequent PR will remove this.
+      {"training_dropout_mask", "result differs", {}},                  // Temporary, subsequent PR will remove this.
+      {"adagrad", "not a registered function/op", {}},                  // Op not registered.
+      {"adagrad_multiple", "not a registered function/op", {}},         // Op not registered.
+      {"adam", "not a registered function/op", {}},                     // Op not registered.
+      {"adam_multiple", "not a registered function/op", {}},            // Op not registered.
+      {"gradient_of_add", "not a registered function/op", {}},          // Op not registered.
+      {"gradient_of_add_and_mul", "not a registered function/op", {}},  // Op not registered.
+      {"momentum", "not a registered function/op", {}},                 // Op not registered.
+      {"momentum_multiple", "not a registered function/op", {}},        // Op not registered.
+      {"nesterov_momentum", "not a registered function/op", {}},        // Op not registered.
   };
 
   if (enable_ngraph) {
@@ -848,7 +845,5 @@ int main(int argc, char* argv[]) {
     retval = -1;
   }
   ::google::protobuf::ShutdownProtobufLibrary();
-  if (g_exit_fast)
-    std::_Exit(retval);
   return retval;
 }