diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 08043dc1d4..8d2cf16a6b 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -950,6 +950,62 @@ std::vector OpTester::ExecuteModel( return fetches; } +bool SetEpsForAllNodes( + Graph& graph, + const std::vector>& execution_providers, + const std::vector>* custom_registries) { + const OpSchemaKernelTypeStrResolver kernel_type_str_resolver{}; + for (auto& node : graph.Nodes()) { + if (node.OpType() == kConstant) + continue; + + bool found = false; + + for (const auto& ep : execution_providers) { + auto provider_type = ep->Type(); + + node.SetExecutionProviderType(provider_type); + if (provider_type == onnxruntime::kOpenVINOExecutionProvider || + provider_type == onnxruntime::kTensorrtExecutionProvider || + // provider_type == onnxruntime::kTvmExecutionProvider || + provider_type == onnxruntime::kNnapiExecutionProvider || + provider_type == onnxruntime::kCoreMLExecutionProvider || + provider_type == onnxruntime::kDnnlExecutionProvider || + provider_type == onnxruntime::kSnpeExecutionProvider) { + found = true; + break; + } + + // Check the EP has an impl for the node from builtin registry. + if (KernelRegistry::HasImplementationOf(*ep->GetKernelRegistry(), node, ep->Type(), kernel_type_str_resolver)) { + found = true; + break; + } + + // Check the EP has an impl for the node from custom_registries + if (custom_registries != nullptr && + std::any_of(custom_registries->cbegin(), custom_registries->cend(), + [&](auto reg) { return KernelRegistry::HasImplementationOf( + *reg->GetKernelRegistry(), + node, ep->Type(), + kernel_type_str_resolver); })) { + found = true; + break; + } + } + + // We will reach here: + // - either we could not find an impl in all possible kernel registries + // - or we skip the registry search and blindly assign the node to the EP due to impl details. + if (!found) { + return false; + } + } + + // all nodes have been assigned an EP + return true; +} + void OpTester::Run( ExpectResult expect_result, const std::string& expected_failure_string, const std::unordered_set& excluded_provider_types, @@ -1057,99 +1113,50 @@ void OpTester::Run( std::unordered_map feeds; std::vector output_names; FillFeedsAndOutputNames(feeds, output_names); + // Run the model + if (execution_providers) { + ExecuteModelForEps( + std::move(*execution_providers), *p_model, so, + expect_result, expected_failure_string, + run_options, feeds, output_names, + /*custom_registries=*/nullptr, + /*assign_ep_for_nodes=*/false, + allow_released_onnx_opset_only, + number_of_pre_packed_weights_counter, + number_of_shared_pre_packed_weights_counter); + } else { #ifdef USE_TENSORRT - // only run trt ep to reduce test time - static const std::string all_provider_types[] = { - kTensorrtExecutionProvider, - }; + // only run trt ep to reduce test time + static const std::string all_provider_types[] = { + kTensorrtExecutionProvider, + }; #else - static const std::string all_provider_types[] = { - kCpuExecutionProvider, - kCudaExecutionProvider, - kDnnlExecutionProvider, - kTensorrtExecutionProvider, - kOpenVINOExecutionProvider, - kDmlExecutionProvider, - kAclExecutionProvider, - kArmNNExecutionProvider, - kNnapiExecutionProvider, - kRocmExecutionProvider, - kCoreMLExecutionProvider, - kSnpeExecutionProvider, - kXnnpackExecutionProvider, - }; + static const std::string all_provider_types[] = { + kCpuExecutionProvider, + kCudaExecutionProvider, + kDnnlExecutionProvider, + kTensorrtExecutionProvider, + kOpenVINOExecutionProvider, + kDmlExecutionProvider, + kAclExecutionProvider, + kArmNNExecutionProvider, + kNnapiExecutionProvider, + kRocmExecutionProvider, + kCoreMLExecutionProvider, + kSnpeExecutionProvider, + kXnnpackExecutionProvider, + }; #endif - bool has_run = false; + bool has_run = false; - if (execution_providers) { - for (auto& entry : *execution_providers) { - // Be noted, entry in execution providers passed in OpTester will be std::moved in the first OpTester::Run(), - // To make the error more obvious to debug (instead of a segment fault), we do check explicitly here. - ASSERT_TRUE(entry) << "Execution provider entry invalid."; - - if (entry->Type() == kDmlExecutionProvider) { - so.enable_mem_pattern = false; - so.execution_mode = ExecutionMode::ORT_SEQUENTIAL; - break; - } - } - - InferenceSession session_object{so, GetEnvironment()}; - - if (add_prepacked_shared_container_to_sessions_) { - ASSERT_STATUS_OK(session_object.AddPrePackedWeightsContainer(&prepacked_weights_container_)); - } - - ASSERT_TRUE(!execution_providers->empty()) - << "Empty execution providers vector."; - std::string provider_types; - - for (auto& entry : *execution_providers) { - provider_types += entry->Type() + ":"; - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::move(entry))); - } - - fetches_ = ExecuteModel( - *p_model, session_object, expect_result, expected_failure_string, - run_options, feeds, output_names, provider_types, allow_released_onnx_opset_only); - - // After the model has initialized (happens in ExecuteModel), - // we should be able to tell how many constant initializers were pre-packed - // and out of these pre-packed ones how many of them used a "cached" version - // from the shared container. - // Populate these value if the user has requested this information. - if (number_of_pre_packed_weights_counter) { - *number_of_pre_packed_weights_counter = - session_object.GetSessionState().GetNumberOfPrepacksCounter(); - } - - if (number_of_shared_pre_packed_weights_counter) { - *number_of_shared_pre_packed_weights_counter = - session_object.GetSessionState().GetUsedSharedPrePackedWeightCounter(); - } - - } else { for (const std::string& provider_type : all_provider_types) { if (excluded_provider_types.count(provider_type) > 0) continue; cur_provider = provider_type; - if (provider_type == kDmlExecutionProvider) { - so.enable_mem_pattern = false; - so.execution_mode = ExecutionMode::ORT_SEQUENTIAL; - } - InferenceSession session_object{so, GetEnvironment()}; - - if (add_prepacked_shared_container_to_sessions_) { - ASSERT_STATUS_OK(session_object.AddPrePackedWeightsContainer(&prepacked_weights_container_)); - } - - for (auto& custom_session_registry : custom_session_registries_) - ASSERT_PROVIDER_STATUS_OK(session_object.RegisterCustomRegistry(custom_session_registry)); - std::unique_ptr execution_provider; if (provider_type == onnxruntime::kCpuExecutionProvider) execution_provider = DefaultCpuExecutionProvider(); @@ -1182,71 +1189,22 @@ void OpTester::Run( if (execution_provider == nullptr) continue; - bool valid = true; - const OpSchemaKernelTypeStrResolver kernel_type_str_resolver{}; - - // set execution provider for all nodes in the graph - for (auto& node : graph.Nodes()) { - if (node.OpType() == kConstant) - continue; - - // if node is not registered for the provider, skip - node.SetExecutionProviderType(provider_type); - if (provider_type == onnxruntime::kOpenVINOExecutionProvider || - provider_type == onnxruntime::kTensorrtExecutionProvider || - // provider_type == onnxruntime::kTvmExecutionProvider || - provider_type == onnxruntime::kNnapiExecutionProvider || - provider_type == onnxruntime::kCoreMLExecutionProvider || - provider_type == onnxruntime::kDnnlExecutionProvider || - provider_type == onnxruntime::kSnpeExecutionProvider) - continue; - auto reg = execution_provider->GetKernelRegistry(); - if (!KernelRegistry::HasImplementationOf(*reg, node, execution_provider->Type(), - kernel_type_str_resolver)) { - valid = false; - for (auto& custom_session_registry : custom_session_registries_) { - if (KernelRegistry::HasImplementationOf(*custom_session_registry->GetKernelRegistry(), - node, execution_provider->Type(), - kernel_type_str_resolver)) { - valid = true; - break; - } - } - - if (!valid) { - break; - } - } - } - - if (!valid) - continue; - - for (auto& custom_session_registry : custom_session_registries_) - ASSERT_PROVIDER_STATUS_OK(session_object.RegisterCustomRegistry(custom_session_registry)); + ExecuteModelForEps( + [&execution_provider]() { + std::vector> ret; + ret.emplace_back(std::move(execution_provider)); + return ret; + }(), + *p_model, so, + expect_result, expected_failure_string, + run_options, feeds, output_names, + &custom_session_registries_, + /*try_assign_ep_for_nodes=*/true, + allow_released_onnx_opset_only, + number_of_pre_packed_weights_counter, + number_of_shared_pre_packed_weights_counter); has_run = true; - - ASSERT_PROVIDER_STATUS_OK(session_object.RegisterExecutionProvider(std::move(execution_provider))); - fetches_ = ExecuteModel( - *p_model, session_object, expect_result, expected_failure_string, - run_options, feeds, output_names, provider_type, allow_released_onnx_opset_only); - - // After the model has initialized (happens in ExecuteModel), - // we should be able to tell how many constant initializers were pre-packed - // and out of these pre-packed ones how many of them used a "cached" version - // from the shared container. - // Populate these value if the user has requested this information. - if (number_of_pre_packed_weights_counter) { - *number_of_pre_packed_weights_counter = - session_object.GetSessionState().GetNumberOfPrepacksCounter(); - } - - if (number_of_shared_pre_packed_weights_counter) { - *number_of_shared_pre_packed_weights_counter = - session_object.GetSessionState().GetUsedSharedPrePackedWeightCounter(); - } - cur_provider = "not set"; } @@ -1269,6 +1227,83 @@ void OpTester::Run( } } +void OpTester::ExecuteModelForEps( + std::vector>&& execution_providers, + onnxruntime::Model& model, + SessionOptions sess_options, // session options is copied to avoid the side effect in this function + onnxruntime::test::OpTester::ExpectResult expect_result, + const std::string& expected_failure_string, + const onnxruntime::RunOptions* run_options, + const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector>* custom_registries, + bool try_assign_ep_for_nodes, + bool allow_released_onnx_opset_only, + size_t* number_of_pre_packed_weights_counter, + size_t* number_of_shared_pre_packed_weights_counter) { + for (auto& entry : execution_providers) { + // Be noted, entry in execution providers passed in OpTester will be std::moved in the first OpTester::Run(), + // To make the error more obvious to debug (instead of a segment fault), we do check explicitly here. + ASSERT_TRUE(entry) << "Execution provider entry invalid."; + + if (entry->Type() == kDmlExecutionProvider) { + sess_options.enable_mem_pattern = false; + sess_options.execution_mode = ExecutionMode::ORT_SEQUENTIAL; + break; + } + } + + InferenceSession session_object{sess_options, GetEnvironment()}; + + if (add_prepacked_shared_container_to_sessions_) { + ASSERT_STATUS_OK(session_object.AddPrePackedWeightsContainer(&prepacked_weights_container_)); + } + ASSERT_TRUE(!execution_providers.empty()) << "Empty execution providers vector."; + if (try_assign_ep_for_nodes && !SetEpsForAllNodes(model.MainGraph(), execution_providers, custom_registries)) { + std::string providers; + for (const auto& ep: execution_providers) { + providers.append(ep->Type() + " "); + } + LOGS_DEFAULT(WARNING) << "registered execution providers " << providers << "were unable to run the model."; + return; + } + + std::string provider_type; + for (auto&& ep : execution_providers) { + provider_type += ep->Type() + ":"; + } + provider_type.resize(provider_type.size() - 1); // remove the trailing ':' + + if (custom_registries != nullptr) { + for (const auto& reg : *custom_registries) { + ASSERT_PROVIDER_STATUS_OK(session_object.RegisterCustomRegistry(reg)); + } + } + + for (auto&& entry : execution_providers) { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::move(entry))); + } + + fetches_ = ExecuteModel( + model, session_object, expect_result, expected_failure_string, + run_options, feeds, output_names, provider_type, allow_released_onnx_opset_only); + + // After the model has initialized (happens in ExecuteModel), + // we should be able to tell how many constant initializers were pre-packed + // and out of these pre-packed ones how many of them used a "cached" version + // from the shared container. + // Populate these value if the user has requested this information. + if (number_of_pre_packed_weights_counter != nullptr) { + *number_of_pre_packed_weights_counter = + session_object.GetSessionState().GetNumberOfPrepacksCounter(); + } + + if (number_of_shared_pre_packed_weights_counter != nullptr) { + *number_of_shared_pre_packed_weights_counter = + session_object.GetSessionState().GetUsedSharedPrePackedWeightCounter(); + } +}; + void OpTester::AddReferenceOutputs(const std::string& model_path, float abs_error) { SessionOptions so; so.session_logid = op_; diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 8b27bf8797..8391a5389c 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -946,6 +946,22 @@ class OpTester { } private: + // Execute the model for a single execution providers combination + void ExecuteModelForEps( + std::vector>&& execution_providers, + onnxruntime::Model& model, + SessionOptions sess_options, + onnxruntime::test::OpTester::ExpectResult expect_result, + const std::string& expected_failure_string, + const onnxruntime::RunOptions* run_options, + const std::unordered_map& feeds, + const std::vector& output_names, + const std::vector>* custom_registries, + bool try_assign_ep_for_nodes, + bool allow_released_onnx_opset_only, + size_t* number_of_pre_packed_weights_counter, + size_t* number_of_shared_pre_packed_weights_counter); + template void AddSeqData(std::vector& data, const char* name, const SeqTensors* seq_tensors,