From 22fa62152a14520086ff4088eed0b446954c5e8e Mon Sep 17 00:00:00 2001 From: JiCheng Date: Sat, 10 Dec 2022 14:23:46 +0800 Subject: [PATCH] Pass SessionOptions to XnnpackProviderFactoryCreator. (#13318) ### Description To pass session_options to Xnnpack EP via `XnnpackProviderFactoryCreator` for Initializing xnnpack's threadpool. If you want to use different threadpool size or even disable xnnpack's threadpool, just setting intra_threadpool to 1 by xnnpack EP's provider_options. ### Motivation and Context Co-authored-by: Guangyun Han Co-authored-by: Jicheng Wen --- .../core/optimizer/graph_transformer_level.h | 2 -- .../core/session/onnxruntime_c_api.h | 1 + onnxruntime/core/framework/config_options.cc | 2 ++ onnxruntime/core/framework/config_options.h | 3 +-- .../providers/shared_library/provider_api.h | 1 + .../xnnpack/xnnpack_execution_provider.cc | 27 ++++++++++++++++--- .../xnnpack/xnnpack_execution_provider.h | 5 +++- .../xnnpack/xnnpack_provider_factory.cc | 8 +++--- .../xnnpack_provider_factory_creator.h | 4 ++- .../core/session/provider_registration.cc | 2 +- .../python/onnxruntime_pybind_schema.cc | 4 +-- .../python/onnxruntime_pybind_state.cc | 2 +- onnxruntime/test/util/default_providers.cc | 2 +- 13 files changed, 44 insertions(+), 19 deletions(-) diff --git a/include/onnxruntime/core/optimizer/graph_transformer_level.h b/include/onnxruntime/core/optimizer/graph_transformer_level.h index cc05b219af..111f38f9cc 100644 --- a/include/onnxruntime/core/optimizer/graph_transformer_level.h +++ b/include/onnxruntime/core/optimizer/graph_transformer_level.h @@ -3,8 +3,6 @@ #pragma once -#include "core/common/common.h" - namespace onnxruntime { // Graph transformer level diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 0d41c6b3b6..d5d7920472 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3500,6 +3500,7 @@ struct OrtApi { * * XNNPACK supported keys: * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. + * default value is 0, which means to use the session thread-pool size. * * \since Version 1.12. */ diff --git a/onnxruntime/core/framework/config_options.cc b/onnxruntime/core/framework/config_options.cc index d74989bb47..3a2caf242b 100644 --- a/onnxruntime/core/framework/config_options.cc +++ b/onnxruntime/core/framework/config_options.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License. #include "core/framework/config_options.h" +#include "core/common/common.h" +#include "core/common/logging/logging.h" namespace onnxruntime { diff --git a/onnxruntime/core/framework/config_options.h b/onnxruntime/core/framework/config_options.h index e70261797f..4297819bed 100644 --- a/onnxruntime/core/framework/config_options.h +++ b/onnxruntime/core/framework/config_options.h @@ -5,9 +5,8 @@ #include #include -#include "core/common/common.h" +#include "core/common/status.h" #include "core/common/optional.h" -#include "core/common/logging/logging.h" namespace onnxruntime { diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 5df59de207..7198b1c3a7 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -165,6 +165,7 @@ enum class Mode : int; struct EinsumComputePreprocessor; template struct EinsumTypedComputeProcessor; +struct SessionOptions; namespace contrib { class ATen; diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc index bef78fab41..e1338f94d8 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc @@ -14,6 +14,7 @@ #include "core/framework/compute_capability.h" #include "core/framework/kernel_registry.h" #include "core/providers/shared/node_unit/node_unit.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "xnnpack_init.h" @@ -123,10 +124,28 @@ using namespace xnnpack; XnnpackExecutionProvider::XnnpackExecutionProvider(const XnnpackExecutionProviderInfo& info) : IExecutionProvider{kXnnpackExecutionProvider, true} { - if (info.xnn_thread_pool_size > 1) { - // pthreadpool is independent of ort-threadpoool, so we have to disable cpu spinning for ort-threadpool. - // otherwise, the pthreadpool will be starved and harm performance a lot. - xnnpack_thread_pool_ = pthreadpool_create(static_cast(info.xnn_thread_pool_size)); + int xnn_thread_pool_size = info.xnn_thread_pool_size; + int ort_thread_pool_size = info.session_options ? info.session_options->intra_op_param.thread_pool_size : 1; + bool allow_intra_op_spinning = (info.session_options == nullptr) || + (info.session_options && + info.session_options->config_options.GetConfigOrDefault( + kOrtSessionOptionsConfigAllowIntraOpSpinning, "1") == "1"); + if (xnn_thread_pool_size > 1 && allow_intra_op_spinning && ort_thread_pool_size > 1) { + LOGS_DEFAULT(WARNING) + << "The XNNPACK EP utilizes an internal pthread-based thread pool for multi-threading." + "If ORT's thread pool size is > 1 and spinning is enabled, " + "there will be contention between the two thread pools, and performance will suffer." + "Please set either intra_op_param.allow_spinning to 0 in the SessionOption config params," + "or the ORT intra-op threadpool size to 1."; + } + + if (xnn_thread_pool_size == 0) { + xnn_thread_pool_size = ort_thread_pool_size; + } + + if (xnn_thread_pool_size > 1) { + // pthreadpool is independent of ort-threadpoool, so we had better disable cpu spinning for ort-threadpool. + xnnpack_thread_pool_ = pthreadpool_create(static_cast(xnn_thread_pool_size)); } } diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.h b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.h index 31a5de3428..d4eefdd759 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.h +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.h @@ -8,15 +8,18 @@ #include "core/framework/execution_provider.h" #include "core/graph/constants.h" #include "core/providers/providers.h" +#include "core/framework/session_options.h" struct pthreadpool; namespace onnxruntime { // placeholder for future use. no options currently struct XnnpackExecutionProviderInfo { int xnn_thread_pool_size{0}; + const SessionOptions* session_options{nullptr}; XnnpackExecutionProviderInfo() = default; - XnnpackExecutionProviderInfo(const ProviderOptions& po) { + XnnpackExecutionProviderInfo(const ProviderOptions& po, const SessionOptions* sess_option) + : session_options(sess_option) { if (auto it = po.find("intra_op_num_threads"); it != po.end()) { xnn_thread_pool_size = std::stoi(it->second); } diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory.cc b/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory.cc index ee58858f15..26fd7179bc 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory.cc @@ -10,8 +10,8 @@ namespace onnxruntime { struct XnnpackProviderFactory : IExecutionProviderFactory { - XnnpackProviderFactory(const ProviderOptions& provider_options) - : info_{provider_options} { + XnnpackProviderFactory(const ProviderOptions& provider_options, const SessionOptions* session_options) + : info_{provider_options, session_options}{ } std::unique_ptr CreateProvider() override { @@ -23,8 +23,8 @@ struct XnnpackProviderFactory : IExecutionProviderFactory { }; std::shared_ptr XnnpackProviderFactoryCreator::Create( - const ProviderOptions& provider_options) { - return std::make_shared(provider_options); + const ProviderOptions& provider_options, const SessionOptions* session_options) { + return std::make_shared(provider_options, session_options); } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory_creator.h b/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory_creator.h index 43b8c546e5..88920ca858 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory_creator.h +++ b/onnxruntime/core/providers/xnnpack/xnnpack_provider_factory_creator.h @@ -9,9 +9,11 @@ #include "core/providers/providers.h" namespace onnxruntime { +struct SessionOptions; struct XnnpackProviderFactoryCreator { - static std::shared_ptr Create(const ProviderOptions& provider_options); + static std::shared_ptr Create(const ProviderOptions& provider_options, + const SessionOptions* session_options); }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 4d13a3ef54..608ca4d3ff 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -74,7 +74,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, #endif } else if (strcmp(provider_name, "XNNPACK") == 0) { #if defined(USE_XNNPACK) - options->provider_factories.push_back(XnnpackProviderFactoryCreator::Create(provider_options)); + options->provider_factories.push_back(XnnpackProviderFactoryCreator::Create(provider_options, &(options->value))); #else status = create_not_supported_status(); #endif diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index f594b5712a..a46bbe51f0 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -74,7 +74,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) { onnxruntime::CoreMLProviderFactoryCreator::Create(0), #endif #ifdef USE_XNNPACK - onnxruntime::XnnpackProviderFactoryCreator::Create(ProviderOptions{}), + onnxruntime::XnnpackProviderFactoryCreator::Create(ProviderOptions{}, nullptr), #endif #ifdef USE_CANN []() { @@ -84,7 +84,7 @@ void addGlobalSchemaFunctions(pybind11::module& m) { #endif }; - for (const auto& f : factories) { + for (const auto& f : factories) { auto kernel_registry = f->CreateProvider()->GetKernelRegistry(); for (const auto& m : kernel_registry->GetKernelCreateMap()) { result.emplace_back(*(m.second.kernel_def)); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index fb6a138389..c04311f2fc 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -774,7 +774,7 @@ std::unique_ptr CreateExecutionProviderInstance( #if defined(USE_XNNPACK) auto cit = provider_options_map.find(type); return onnxruntime::XnnpackProviderFactoryCreator::Create( - cit == provider_options_map.end() ? ProviderOptions{} : cit->second) + cit == provider_options_map.end() ? ProviderOptions{} : cit->second, &session_options) ->CreateProvider(); #endif } else if (type == kCannExecutionProvider) { diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 43d4152317..044c037365 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -200,7 +200,7 @@ std::unique_ptr DefaultSnpeExecutionProvider() { std::unique_ptr DefaultXnnpackExecutionProvider() { #ifdef USE_XNNPACK - return XnnpackProviderFactoryCreator::Create(ProviderOptions())->CreateProvider(); + return XnnpackProviderFactoryCreator::Create(ProviderOptions(), nullptr)->CreateProvider(); #else return nullptr; #endif