Pass SessionOptions to XnnpackProviderFactoryCreator. (#13318)

### Description
To pass session_options to Xnnpack EP via
`XnnpackProviderFactoryCreator` for Initializing xnnpack's threadpool.

If you want to use different threadpool size or even disable xnnpack's
threadpool, just setting intra_threadpool to 1 by xnnpack EP's
provider_options.


### Motivation and Context

Co-authored-by: Guangyun Han <guangyunhan@microsoft.com>
Co-authored-by: Jicheng Wen <jicwen@microsoft.com>

include/onnxruntime/core/optimizer/graph_transformer_level.h

@@ -3,8 +3,6 @@
 
 #pragma once
 
-#include "core/common/common.h"
-
 namespace onnxruntime {
 
 // Graph transformer level

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -3500,6 +3500,7 @@ struct OrtApi {
    *
    * XNNPACK supported keys:
    *   "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider.
+   *      default value is 0, which means to use the session thread-pool size.
    *
    * \since Version 1.12.
    */

onnxruntime/core/framework/config_options.cc

@@ -2,6 +2,8 @@
 // Licensed under the MIT License.
 
 #include "core/framework/config_options.h"
+#include "core/common/common.h"
+#include "core/common/logging/logging.h"
 
 namespace onnxruntime {
 

onnxruntime/core/framework/config_options.h

@@ -5,9 +5,8 @@
 
 #include <string>
 #include <unordered_map>
-#include "core/common/common.h"
+#include "core/common/status.h"
 #include "core/common/optional.h"
-#include "core/common/logging/logging.h"
 
 namespace onnxruntime {
 

onnxruntime/core/providers/shared_library/provider_api.h

@@ -165,6 +165,7 @@ enum class Mode : int;
 struct EinsumComputePreprocessor;
 template <typename T>
 struct EinsumTypedComputeProcessor;
+struct SessionOptions;
 
 namespace contrib {
 class ATen;

onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc

@@ -14,6 +14,7 @@
 #include "core/framework/compute_capability.h"
 #include "core/framework/kernel_registry.h"
 #include "core/providers/shared/node_unit/node_unit.h"
+#include "core/session/onnxruntime_session_options_config_keys.h"
 
 #include "xnnpack_init.h"
 
@@ -123,10 +124,28 @@ using namespace xnnpack;
 
 XnnpackExecutionProvider::XnnpackExecutionProvider(const XnnpackExecutionProviderInfo& info)
     : IExecutionProvider{kXnnpackExecutionProvider, true} {
-  if (info.xnn_thread_pool_size > 1) {
-    // pthreadpool is independent of ort-threadpoool, so we have to disable cpu spinning for ort-threadpool.
-    // otherwise, the pthreadpool will be starved and harm performance a lot.
-    xnnpack_thread_pool_ = pthreadpool_create(static_cast<size_t>(info.xnn_thread_pool_size));
+  int xnn_thread_pool_size = info.xnn_thread_pool_size;
+  int ort_thread_pool_size = info.session_options ? info.session_options->intra_op_param.thread_pool_size : 1;
+  bool allow_intra_op_spinning = (info.session_options == nullptr) ||
+                                 (info.session_options &&
+                                  info.session_options->config_options.GetConfigOrDefault(
+                                      kOrtSessionOptionsConfigAllowIntraOpSpinning, "1") == "1");
+  if (xnn_thread_pool_size > 1 && allow_intra_op_spinning && ort_thread_pool_size > 1) {
+    LOGS_DEFAULT(WARNING)
+        << "The XNNPACK EP utilizes an internal pthread-based thread pool for multi-threading."
+           "If ORT's thread pool size is > 1 and spinning is enabled, "
+           "there will be contention between the two thread pools, and performance will suffer."
+           "Please set either intra_op_param.allow_spinning to 0 in the SessionOption config params,"
+           "or the ORT intra-op threadpool size to 1.";
+  }
+
+  if (xnn_thread_pool_size == 0) {
+    xnn_thread_pool_size = ort_thread_pool_size;
+  }
+
+  if (xnn_thread_pool_size > 1) {
+    // pthreadpool is independent of ort-threadpoool, so we had better disable cpu spinning for ort-threadpool.
+    xnnpack_thread_pool_ = pthreadpool_create(static_cast<size_t>(xnn_thread_pool_size));
   }
 }
 

onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.h

@@ -8,15 +8,18 @@
 #include "core/framework/execution_provider.h"
 #include "core/graph/constants.h"
 #include "core/providers/providers.h"
+#include "core/framework/session_options.h"
 
 struct pthreadpool;
 namespace onnxruntime {
 // placeholder for future use. no options currently
 struct XnnpackExecutionProviderInfo {
   int xnn_thread_pool_size{0};
+  const SessionOptions* session_options{nullptr};
   XnnpackExecutionProviderInfo() = default;
 
-  XnnpackExecutionProviderInfo(const ProviderOptions& po) {
+  XnnpackExecutionProviderInfo(const ProviderOptions& po, const SessionOptions* sess_option)
+      : session_options(sess_option) {
     if (auto it = po.find("intra_op_num_threads"); it != po.end()) {
       xnn_thread_pool_size = std::stoi(it->second);
     }

onnxruntime/core/providers/xnnpack/xnnpack_provider_factory.cc

@@ -10,8 +10,8 @@
 namespace onnxruntime {
 
 struct XnnpackProviderFactory : IExecutionProviderFactory {
-  XnnpackProviderFactory(const ProviderOptions& provider_options)
-      : info_{provider_options} {
+  XnnpackProviderFactory(const ProviderOptions& provider_options, const SessionOptions* session_options)
+      : info_{provider_options, session_options}{
   }
 
   std::unique_ptr<IExecutionProvider> CreateProvider() override {
@@ -23,8 +23,8 @@ struct XnnpackProviderFactory : IExecutionProviderFactory {
 };
 
 std::shared_ptr<IExecutionProviderFactory> XnnpackProviderFactoryCreator::Create(
-    const ProviderOptions& provider_options) {
-  return std::make_shared<XnnpackProviderFactory>(provider_options);
+    const ProviderOptions& provider_options, const SessionOptions* session_options) {
+  return std::make_shared<XnnpackProviderFactory>(provider_options, session_options);
 }
 
 }  // namespace onnxruntime

onnxruntime/core/providers/xnnpack/xnnpack_provider_factory_creator.h

@@ -9,9 +9,11 @@
 #include "core/providers/providers.h"
 
 namespace onnxruntime {
+struct SessionOptions;
 
 struct XnnpackProviderFactoryCreator {
-  static std::shared_ptr<IExecutionProviderFactory> Create(const ProviderOptions& provider_options);
+  static std::shared_ptr<IExecutionProviderFactory> Create(const ProviderOptions& provider_options,
+                                                           const SessionOptions* session_options);
 };
 
 }  // namespace onnxruntime

onnxruntime/core/session/provider_registration.cc

@@ -74,7 +74,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider,
 #endif
   } else if (strcmp(provider_name, "XNNPACK") == 0) {
 #if defined(USE_XNNPACK)
-    options->provider_factories.push_back(XnnpackProviderFactoryCreator::Create(provider_options));
+    options->provider_factories.push_back(XnnpackProviderFactoryCreator::Create(provider_options, &(options->value)));
 #else
     status = create_not_supported_status();
 #endif

onnxruntime/python/onnxruntime_pybind_schema.cc

@@ -74,7 +74,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) {
             onnxruntime::CoreMLProviderFactoryCreator::Create(0),
 #endif
 #ifdef USE_XNNPACK
-            onnxruntime::XnnpackProviderFactoryCreator::Create(ProviderOptions{}),
+            onnxruntime::XnnpackProviderFactoryCreator::Create(ProviderOptions{}, nullptr),
 #endif
 #ifdef USE_CANN
             []() {
@@ -84,7 +84,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) {
 #endif
         };
 
-      for (const auto& f : factories) {
+        for (const auto& f : factories) {
           auto kernel_registry = f->CreateProvider()->GetKernelRegistry();
           for (const auto& m : kernel_registry->GetKernelCreateMap()) {
             result.emplace_back(*(m.second.kernel_def));

onnxruntime/python/onnxruntime_pybind_state.cc

@@ -774,7 +774,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
 #if defined(USE_XNNPACK)
     auto cit = provider_options_map.find(type);
     return onnxruntime::XnnpackProviderFactoryCreator::Create(
-               cit == provider_options_map.end() ? ProviderOptions{} : cit->second)
+               cit == provider_options_map.end() ? ProviderOptions{} : cit->second, &session_options)
         ->CreateProvider();
 #endif
   } else if (type == kCannExecutionProvider) {

onnxruntime/test/util/default_providers.cc

@@ -200,7 +200,7 @@ std::unique_ptr<IExecutionProvider> DefaultSnpeExecutionProvider() {
 
 std::unique_ptr<IExecutionProvider> DefaultXnnpackExecutionProvider() {
 #ifdef USE_XNNPACK
-  return XnnpackProviderFactoryCreator::Create(ProviderOptions())->CreateProvider();
+  return XnnpackProviderFactoryCreator::Create(ProviderOptions(), nullptr)->CreateProvider();
 #else
   return nullptr;
 #endif