diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index ec8102dff7..a438c1440b 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -113,8 +113,8 @@ common::Status AllocateHelper(const IExecutionProvider& execution_provider, cons } std::unique_ptr p_tensor = onnxruntime::make_unique(fetched_tensor.DataType(), - fetched_tensor.Shape(), - allocator); + fetched_tensor.Shape(), + allocator); output_mlvalue.Init(p_tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); @@ -439,7 +439,13 @@ static common::Status ExecuteGraphImpl(const SessionState& session_state, if (sequential_execution) { p_exec = std::unique_ptr(new SequentialExecutor(terminate_flag)); } else { - p_exec = std::unique_ptr(new ParallelExecutor(session_state, terminate_flag)); + auto* p_inter_op_thread_pool = session_state.GetInterOpThreadPool(); + if (!p_inter_op_thread_pool) { + LOGS(logger, WARNING) << "Only one thread was configured for parallel execution. Hence will use sequential execution."; + p_exec = std::unique_ptr(new SequentialExecutor(terminate_flag)); + } else { + p_exec = std::unique_ptr(new ParallelExecutor(session_state, terminate_flag)); + } } const auto& feeds_fetches_info = feeds_fetches_manager.GetFeedsFetchesInfo(); diff --git a/onnxruntime/test/framework/parallel_executor_test.cc b/onnxruntime/test/framework/parallel_executor_test.cc index 2c25008219..5d49e50b55 100644 --- a/onnxruntime/test/framework/parallel_executor_test.cc +++ b/onnxruntime/test/framework/parallel_executor_test.cc @@ -5,6 +5,7 @@ #include "core/framework/op_kernel.h" #include "test/providers/provider_test_utils.h" #include "test_utils.h" +#include "core/session/inference_session.h" #include "gtest/gtest.h" @@ -76,7 +77,7 @@ struct TestOp { }; // test that the status from TestOp is correctly returned from InferenceSession::Run -TEST(ParallelExecutor, DISABLED_TestStatusPropagation) { +TEST(ParallelExecutor, TestStatusPropagation) { auto registry = std::make_shared(); std::vector schemas{TestOp::OpSchema()}; Status status; @@ -113,5 +114,28 @@ TEST(ParallelExecutor, DISABLED_TestStatusPropagation) { tester.Run(OpTester::ExpectResult::kExpectFailure, "Throwing as action was 2", {kTensorrtExecutionProvider}, nullptr, nullptr, false); } } + +TEST(ParallelExecutor, TestNullInterOpThreadPool) { + auto registry = std::make_shared(); + std::vector schemas{TestOp::OpSchema()}; + Status status; + ASSERT_TRUE((status = registry->RegisterOpSet(schemas, TestOp::OpDomain, 10, 11)).IsOK()) << status; + KernelCreateFn kernel_create_fn = [](const OpKernelInfo& info) { return new typename TestOp::OpKernelImpl(info); }; + auto kernel_def = TestOp::KernelDef(); + ASSERT_TRUE((status = registry->RegisterCustomKernel(kernel_def, kernel_create_fn)).IsOK()) << status; + + OpTester tester{"TestOp", 10, TestOp::OpDomain}; + tester.AddCustomOpRegistry(registry); + + tester.AddInput("action", {1}, {/*success*/ 0}); + tester.AddOutput("action_out", {1}, {0}); + // TensorRT doesn't handle a custom op. Possibly it should, but that would be a separate PR + onnxruntime::SessionOptions so; + so.session_logid = "TestOp"; + so.session_log_verbosity_level = 1; + so.enable_sequential_execution = false; + so.inter_op_num_threads = 1; + tester.Run(so, OpTester::ExpectResult::kExpectSuccess, {}, {kTensorrtExecutionProvider}, nullptr, nullptr); +} } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index c1260c7d1b..cc37ec8620 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -214,8 +214,8 @@ void CheckDispatch(MLDataType type, const OpTester::Data& expected_data, OrtValu void Check(const OpTester::Data& expected_data, OrtValue& ort_value, const std::string& provider_type) { #ifdef MICROSOFT_AUTOML - CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, - provider_type); + CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, + provider_type); #else CheckDispatch(expected_data.data_.Type(), expected_data, ort_value, provider_type); @@ -333,7 +333,7 @@ std::unique_ptr OpTester::BuildGraph() { std::unordered_map domain_to_version; domain_to_version[domain_] = opset_version_; auto p_model = onnxruntime::make_unique("test", false, ModelMetaData(), - custom_schema_registries_, domain_to_version); + custom_schema_registries_, domain_to_version); onnxruntime::Graph& graph = p_model->MainGraph(); AddNodes(graph, node_input_defs, output_defs, add_attribute_funcs_); @@ -445,6 +445,19 @@ void OpTester::Run(ExpectResult expect_result, const RunOptions* run_options, std::vector>* execution_providers, bool sequential_execution) { + SessionOptions so; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + so.enable_sequential_execution = sequential_execution; + Run(so, expect_result, expected_failure_string, excluded_provider_types, run_options, execution_providers); +} + +void OpTester::Run(const SessionOptions& so, + ExpectResult expect_result, + const std::string& expected_failure_string, + const std::unordered_set& excluded_provider_types, + const RunOptions* run_options, + std::vector>* execution_providers) { try { #ifndef NDEBUG run_called_ = true; @@ -483,11 +496,6 @@ void OpTester::Run(ExpectResult expect_result, std::vector output_names; FillFeedsAndOutputNames(feeds, output_names); // Run the model - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - so.enable_sequential_execution = sequential_execution; - static const std::string all_provider_types[] = { kCpuExecutionProvider, kCudaExecutionProvider, diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 2892150673..8f965c86e6 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -24,6 +24,7 @@ namespace onnxruntime { class InferenceSession; +struct SessionOptions; namespace test { // unfortunately std::optional is in C++17 so use a miniversion of it @@ -313,13 +314,19 @@ class OpTester { std::vector>* execution_providers = nullptr, bool sequential_execution = true); + void Run(const SessionOptions& session_options, + ExpectResult expect_result = ExpectResult::kExpectSuccess, + const std::string& expected_failure_string = "", + const std::unordered_set& excluded_provider_types = {}, + const RunOptions* run_options = nullptr, + std::vector>* execution_providers = nullptr); + struct Data { onnxruntime::NodeArg def_; OrtValue data_; optional relative_error_; optional absolute_error_; - Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional&& rel, optional&& abs) : - def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {} + Data(onnxruntime::NodeArg&& def, OrtValue&& data, optional&& rel, optional&& abs) : def_(std::move(def)), data_(std::move(data)), relative_error_(std::move(rel)), absolute_error_(abs) {} Data(Data&&) = default; Data& operator=(Data&&) = default; };