From 6baaaf516538f9059da3558b2cd22128a9e42c07 Mon Sep 17 00:00:00 2001 From: Preetha Veeramalai Date: Fri, 28 Jun 2024 08:31:02 -0700 Subject: [PATCH] OVEP options to disable CPU fallback at compile time (#21166) ### Description Provide user level options to control the fallback on CPU for models not supported on Intel's NPU hardware. ### Motivation and Context - Current workflow of OVEP allows safe fallback from OV NPU to OV CPU on compilation failures. Also supports MLAS CPU fallback in presence of unsupported custom ops. - The PR provides a build-time option to disable fallback from OV NPU to OV CPU. - The session Option "kOrtSessionOptionsDisableCPUEPFallback" disables OV CPU and MLAS CPU fallback. - Also has bug fix for proto creation. --------- Co-authored-by: jatinwadhwa921 Co-authored-by: ankitm3k --- cmake/CMakeLists.txt | 4 + onnxruntime/core/framework/config_options.cc | 4 +- .../core/graph/graph_proto_serializer.cc | 16 +- .../providers/openvino/backend_manager.cc | 18 +- .../openvino/backends/basic_backend.cc | 7 +- .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_execution_provider.cc | 1 + .../openvino/openvino_execution_provider.h | 6 +- .../openvino/openvino_provider_factory.cc | 24 +- .../openvino_provider_factory_creator.h | 4 +- .../core/session/provider_bridge_ort.cc | 16 +- .../core/session/provider_registration.cc | 3 +- .../python/onnxruntime_pybind_schema.cc | 3 +- .../python/onnxruntime_pybind_state.cc | 2 +- .../python/onnxruntime_pybind_state_common.h | 4 + onnxruntime/test/perftest/ort_test_session.cc | 294 +++++++++--------- onnxruntime/test/util/default_providers.cc | 3 +- tools/ci_build/build.py | 2 + 18 files changed, 247 insertions(+), 165 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6ba0db7899..4483e4d5cb 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1341,6 +1341,10 @@ if (onnxruntime_USE_OPENVINO) add_definitions(-DUSE_OPENVINO=1) + if(onnxruntime_NPU_NO_FALLBACK) + add_definitions(-DOPENVINO_DISABLE_NPU_FALLBACK=1) + endif() + if (onnxruntime_USE_OPENVINO_GPU) add_definitions(-DOPENVINO_CONFIG_GPU=1) endif() diff --git a/onnxruntime/core/framework/config_options.cc b/onnxruntime/core/framework/config_options.cc index 1a4acb6dab..9fe5beafd6 100644 --- a/onnxruntime/core/framework/config_options.cc +++ b/onnxruntime/core/framework/config_options.cc @@ -30,11 +30,11 @@ std::string ConfigOptions::GetConfigOrDefault(const std::string& config_key, } Status ConfigOptions::AddConfigEntry(const char* config_key, const char* config_value) noexcept { - std::string key(config_key); + std::string key = config_key; if (key.empty() || key.length() > 128) return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Config key is empty or longer than maximum length 128"); - std::string val(config_value); + std::string val = config_value; if (val.length() > onnxruntime::kMaxStrLen) return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Config value is longer than maximum length: ", diff --git a/onnxruntime/core/graph/graph_proto_serializer.cc b/onnxruntime/core/graph/graph_proto_serializer.cc index aefad28eb3..eb0fb22346 100644 --- a/onnxruntime/core/graph/graph_proto_serializer.cc +++ b/onnxruntime/core/graph/graph_proto_serializer.cc @@ -21,7 +21,21 @@ void GraphViewerToProto(const GraphViewer& graph_view, *(graph_proto.mutable_output()->Add()) = output_arg->ToProto(); } - for (const auto* value_info : graph_view.GetValueInfo()) { + std::unordered_set value_info_ = graph_view.GetValueInfo(); + + // Reserve memory for the vector to avoid reallocations + std::vector value_info_sorted; + value_info_sorted.reserve(value_info_.size()); + + value_info_sorted.assign(value_info_.begin(), value_info_.end()); + auto sort_predicate = [](const NodeArg* v1, const NodeArg* v2) { + return v1->Name() < v2->Name(); + }; + + // This ensures consistent ordering of value_info entries in the output graph + std::sort(value_info_sorted.begin(), value_info_sorted.end(), sort_predicate); + + for (const auto* value_info : value_info_sorted) { *(graph_proto.mutable_value_info()->Add()) = value_info->ToProto(); } diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index d0ef447a46..1c027e39fa 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -105,7 +105,11 @@ BackendManager::BackendManager(const GlobalContext& global_context, subgraph_context_, ep_ctx_handle_); } catch (const OnnxRuntimeException& ex) { - if (device_type.find("NPU") != std::string::npos) { +#if defined(OPENVINO_DISABLE_NPU_FALLBACK) + ORT_THROW(ex.what()); +#else + if (device_type.find("NPU") != std::string::npos && + !GetGlobalContext().disable_cpu_fallback) { LOGS_DEFAULT(WARNING) << ex.what(); LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." << "Falling back to OV CPU for execution"; @@ -122,6 +126,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, } else { ORT_THROW(ex.what()); } +#endif } } } @@ -419,7 +424,13 @@ void BackendManager::Compute(OrtKernelContext* context) { subgraph_context_, ep_ctx_handle_); } catch (const OnnxRuntimeException& ex) { - if (GetGlobalContext().device_type.find("NPU") != std::string::npos) { + // Build option disables fallback to CPU on compilation failures with NPU. +#if defined(OPENVINO_DISABLE_NPU_FALLBACK) + LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."; + ORT_THROW(ex.what()); +#else + if (GetGlobalContext().device_type.find("NPU") != std::string::npos && + !GetGlobalContext().disable_cpu_fallback) { LOGS_DEFAULT(WARNING) << ex.what(); LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." << "Falling back to OV CPU for execution"; @@ -434,7 +445,10 @@ void BackendManager::Compute(OrtKernelContext* context) { } catch (std::string const& msg) { ORT_THROW(msg); } + } else { + ORT_THROW(ex.what()); } +#endif } backend_map_.insert({key, dynamic_backend}); } else { diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 9da6e5945a..f8046bcb3a 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -545,6 +545,11 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { std::cout << "Inference successful" << std::endl; } + // Create a duplicate infer_request_ shared ptr on the stack in the current local scope, + // as the infer_request gets freed in the next stage the reference count for the infer_request decrements & + // thus we dont have any dangling ptr leading to seg faults in the debug mode subsequent execution call + OVInferRequestPtr infer_request_ = infer_request; + // Once the inference is completed, the infer_request becomes free and is placed back into pool of infer_requests_ inferRequestsQueue_->putIdleRequest(std::move(infer_request)); #ifndef NDEBUG @@ -552,7 +557,7 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { if (openvino_ep::backend_utils::IsDebugEnabled()) { inferRequestsQueue_->printstatus(); // Printing the elements of infer_requests_ vector pool only in debug mode std::string& hw_target = global_context_.device_type; - printPerformanceCounts(infer_request, std::cout, hw_target); + printPerformanceCounts(std::move(infer_request_), std::cout, hw_target); } #endif #endif diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 6e11cbf4a6..598e985676 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -21,6 +21,7 @@ struct GlobalContext { bool ep_context_embed_mode = true; bool export_ep_ctx_blob = false; bool enable_qdq_optimizer = false; + bool disable_cpu_fallback = false; size_t num_of_threads; std::string device_type; std::string precision_str; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 6f7e1fb607..040c56926a 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -33,6 +33,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv global_context_->OpenVINO_Version = {OPENVINO_VERSION_MAJOR, OPENVINO_VERSION_MINOR}; global_context_->export_ep_ctx_blob = info.export_ep_ctx_blob_; global_context_->enable_qdq_optimizer = info.enable_qdq_optimizer_; + global_context_->disable_cpu_fallback = info.disable_cpu_fallback_; // to check if target device is available // using ie_core capability GetAvailableDevices to fetch list of devices plugged in diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index d950255c77..050fb91c51 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -74,6 +74,7 @@ struct OpenVINOExecutionProviderInfo { bool disable_dynamic_shapes_{false}; bool export_ep_ctx_blob_{false}; bool enable_qdq_optimizer_{false}; + bool disable_cpu_fallback_{false}; OpenVINOExecutionProviderInfo() = delete; @@ -81,7 +82,7 @@ struct OpenVINOExecutionProviderInfo { size_t num_of_threads, std::string cache_dir, std::string model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, - bool enable_qdq_optimizer) + bool enable_qdq_optimizer, bool disable_cpu_fallback) : precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), @@ -92,7 +93,8 @@ struct OpenVINOExecutionProviderInfo { enable_opencl_throttling_(enable_opencl_throttling), disable_dynamic_shapes_(disable_dynamic_shapes), export_ep_ctx_blob_(export_ep_ctx_blob), - enable_qdq_optimizer_(enable_qdq_optimizer) { + enable_qdq_optimizer_(enable_qdq_optimizer), + disable_cpu_fallback_(disable_cpu_fallback) { std::set ov_supported_device_types = {"CPU", "GPU", "GPU.0", "GPU.1", "NPU"}; if (dev_type == "") { diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index a45c1fd236..45bba43174 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -13,7 +13,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { const char* cache_dir, const char* model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, - bool export_ep_ctx_blob, bool enable_qdq_optimizer) + bool export_ep_ctx_blob, bool enable_qdq_optimizer, + bool disable_cpu_fallback) : precision_(precision), enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), @@ -23,7 +24,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { enable_opencl_throttling_(enable_opencl_throttling), disable_dynamic_shapes_(disable_dynamic_shapes), export_ep_ctx_blob_(export_ep_ctx_blob), - enable_qdq_optimizer_(enable_qdq_optimizer) { + enable_qdq_optimizer_(enable_qdq_optimizer), + disable_cpu_fallback_(disable_cpu_fallback) { device_type_ = (device_type == nullptr) ? "" : device_type; cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; } @@ -45,12 +47,14 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { bool disable_dynamic_shapes_; bool export_ep_ctx_blob_; bool enable_qdq_optimizer_; + bool disable_cpu_fallback_; }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, - disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_); + disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_, + disable_cpu_fallback_); return std::make_unique(info); } @@ -99,6 +103,8 @@ struct OpenVINO_Provider : Provider { bool enable_qdq_optimizer = false; + bool disable_cpu_fallback = false; + if (provider_options_map.find("device_type") != provider_options_map.end()) { device_type = provider_options_map.at("device_type").c_str(); @@ -256,6 +262,15 @@ struct OpenVINO_Provider : Provider { export_ep_ctx_blob = false; bool_flag = ""; } + + if (provider_options_map.find("disable_cpu_fallback") != provider_options_map.end()) { + bool_flag = provider_options_map.at("disable_cpu_fallback"); + if (bool_flag == "true" || bool_flag == "True") + disable_cpu_fallback = true; + else if (bool_flag == "false" || bool_flag == "False") + disable_cpu_fallback = false; + bool_flag = ""; + } return std::make_shared(const_cast(device_type.c_str()), const_cast(precision.c_str()), enable_npu_fast_compile, @@ -267,7 +282,8 @@ struct OpenVINO_Provider : Provider { enable_opencl_throttling, disable_dynamic_shapes, export_ep_ctx_blob, - enable_qdq_optimizer); + enable_qdq_optimizer, + disable_cpu_fallback); } void Initialize() override { diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h b/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h index 4df653b022..bff70a90b6 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h @@ -11,9 +11,11 @@ struct OrtOpenVINOProviderOptions; namespace onnxruntime { +struct SessionOptions; // defined in provider_bridge_ort.cc struct OpenVINOProviderFactoryCreator { - static std::shared_ptr Create(const ProviderOptions* provider_options_map); + static std::shared_ptr Create(ProviderOptions* provider_options_map, + const SessionOptions* session_options); static std::shared_ptr Create(const OrtOpenVINOProviderOptions* provider_options); }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 7f7ed5e436..d4c6e3d506 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -35,6 +35,7 @@ #include "core/framework/model_metadef_id_generator.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" #include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/string_helper.h" @@ -1800,7 +1801,18 @@ std::shared_ptr OpenVINOProviderFactoryCreator::Creat return s_library_openvino.Get().CreateExecutionProviderFactory(&ov_options_converted_map); } -std::shared_ptr OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map) { +void ORTSessionOptionsToOrtOpenVINOProviderOptions(ProviderOptions& ov_options, + const SessionOptions* session_options) { + bool disable_cpu_fallback = session_options->config_options.GetConfigOrDefault( + kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; + if (disable_cpu_fallback) + ov_options["disable_cpu_fallback"] = "true"; +} + +std::shared_ptr OpenVINOProviderFactoryCreator::Create(ProviderOptions* provider_options_map, + const SessionOptions* session_options) { + if (session_options) + onnxruntime::ORTSessionOptionsToOrtOpenVINOProviderOptions(*provider_options_map, session_options); return s_library_openvino.Get().CreateExecutionProviderFactory(provider_options_map); } @@ -2075,7 +2087,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO_V2, provider_options[provider_options_keys[i]] = provider_options_values[i]; } - auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options); + auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options, &(options->value)); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_OpenVINO_V2: Failed to load shared library"); } diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 05408db988..688ee76c59 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -108,11 +108,10 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, #endif } else if (strcmp(provider_name, "OpenVINO") == 0) { #if defined(USE_OPENVINO) - options->provider_factories.push_back(OpenVINOProviderFactoryCreator::Create(&provider_options)); + options->provider_factories.push_back(OpenVINOProviderFactoryCreator::Create(&provider_options, &(options->value))); #else status = create_not_supported_status(); #endif - } else if (strcmp(provider_name, "SNPE") == 0) { #if defined(USE_SNPE) options->provider_factories.push_back(SNPEProviderFactoryCreator::Create(provider_options)); diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index 4da25eac32..218b59688b 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -40,7 +40,8 @@ void addGlobalSchemaFunctions(pybind11::module& m) { #ifdef USE_OPENVINO []() { ProviderOptions provider_options_map; - return onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options_map); + SessionOptions session_options; + return onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options_map, &session_options); }(), #endif #ifdef USE_TENSORRT diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index fa4c906dd0..e539614fd6 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1084,7 +1084,7 @@ std::unique_ptr CreateExecutionProviderInstance( } } if (std::shared_ptr openvino_provider_factory = onnxruntime::OpenVINOProviderFactoryCreator::Create( - &OV_provider_options_map)) { + &OV_provider_options_map, &session_options)) { auto p = openvino_provider_factory->CreateProvider(); // Reset global variables config to avoid it being accidentally passed on to the next session openvino_device_type.clear(); diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index dc9394a83a..4d6e411def 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -65,7 +65,11 @@ struct OrtStatus { #elif OPENVINO_CONFIG_HETERO #define BACKEND_OPENVINO "-OPENVINO_HETERO" + +#elif OPENVINO_DISABLE_NPU_FALLBACK +#define BACKEND_OPENVINO "-OPENVINO_DISABLE_NPU_FALLBACK" #endif + #else #define BACKEND_OPENVINO "" #endif diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 81053bf400..1485a4456d 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -47,6 +47,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device const TestModelInfo& m) : rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) { Ort::SessionOptions session_options; + provider_name_ = performance_test_config.machine_config.provider_type_name; if (provider_name_ == onnxruntime::kDnnlExecutionProvider) { #ifdef USE_DNNL @@ -221,150 +222,6 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device session_options.AppendExecutionProvider_CUDA(cuda_options); #else ORT_THROW("TensorRT is not supported in this build\n"); -#endif - } else if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) { -#ifdef USE_OPENVINO -#ifdef _MSC_VER - std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string); -#else - std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; -#endif - std::unordered_map ov_options; - std::istringstream ss(ov_string); - std::string token; - while (ss >> token) { - if (token == "") { - continue; - } - auto pos = token.find("|"); - if (pos == std::string::npos || pos == 0 || pos == token.length()) { - ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); - } - - auto key = token.substr(0, pos); - auto value = token.substr(pos + 1); - - if (key == "device_type") { - std::set ov_supported_device_types = {"CPU", "GPU", - "GPU.0", "GPU.1", "NPU"}; - std::set deprecated_device_types = {"CPU_FP32", "GPU_FP32", - "GPU.0_FP32", "GPU.1_FP32", "GPU_FP16", - "GPU.0_FP16", "GPU.1_FP16"}; - if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) { - ov_options[key] = value; - } else if (deprecated_device_types.find(value) != deprecated_device_types.end()) { - ov_options[key] = value; - } else if (value.find("HETERO:") == 0) { - ov_options[key] = value; - } else if (value.find("MULTI:") == 0) { - ov_options[key] = value; - } else if (value.find("AUTO:") == 0) { - ov_options[key] = value; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. " - "Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from" - " HETERO/MULTI/AUTO options available. \n"); - } - } else if (key == "device_id") { - if (value == "CPU" || value == "GPU" || value == "NPU") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options."); - } - } else if (key == "precision") { - auto device_type = ov_options["device_type"]; - if (device_type.find("GPU") != std::string::npos) { - if (value == "") { - ov_options[key] = "FP16"; - continue; - } else if (value == "ACCURACY" || value == "FP16" || value == "FP32") { - ov_options[key] = value; - continue; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] Unsupported inference precision is selected. " - "GPU only supported FP32 / FP16. \n"); - } - } else if (device_type.find("NPU") != std::string::npos) { - if (value == "" || value == "ACCURACY" || value == "FP16") { - ov_options[key] = "FP16"; - continue; - } else { - ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n"); - } - } else if (device_type.find("CPU") != std::string::npos) { - if (value == "" || value == "ACCURACY" || value == "FP32") { - ov_options[key] = "FP32"; - continue; - } else { - ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n"); - } - } - } else if (key == "enable_npu_fast_compile") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n"); - } - } else if (key == "enable_opencl_throttling") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n"); - } - } else if (key == "enable_qdq_optimizer") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_qdq_optimizer' should be a boolean i.e. true or false. Default value is false.\n"); - } - } else if (key == "disable_dynamic_shapes") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' " - "should be a boolean i.e. true or false. Default value is false.\n"); - } - } else if (key == "num_of_threads") { - if (std::stoi(value) <= 0) { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n"); - } else { - ov_options[key] = value; - } - } else if (key == "model_priority") { - ov_options[key] = value; - } else if (key == "cache_dir") { - ov_options[key] = value; - } else if (key == "context") { - ov_options[key] = value; - } else if (key == "num_streams") { - if (std::stoi(value) <= 0 && std::stoi(value) > 8) { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n"); - } else { - ov_options[key] = value; - } - } else if (key == "export_ep_ctx_blob") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " - "should be a boolean i.e. true or false. Default value is false.\n"); - } - } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); - } - } - session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); -#else - ORT_THROW("OpenVINO is not supported in this build\n"); #endif } else if (provider_name_ == onnxruntime::kQnnExecutionProvider) { #ifdef USE_QNN @@ -716,7 +573,9 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else ORT_THROW("VitisAI is not supported in this build\n"); #endif - } else if (!provider_name_.empty() && provider_name_ != onnxruntime::kCpuExecutionProvider) { + } else if (!provider_name_.empty() && + provider_name_ != onnxruntime::kCpuExecutionProvider && + provider_name_ != onnxruntime::kOpenVINOExecutionProvider) { ORT_THROW("This backend is not included in perf test runner.\n"); } @@ -805,6 +664,151 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } } } + if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) { +#ifdef USE_OPENVINO +#ifdef _MSC_VER + std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string); +#else + std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; +#endif + std::unordered_map ov_options; + std::istringstream ss(ov_string); + std::string token; + while (ss >> token) { + if (token == "") { + continue; + } + auto pos = token.find("|"); + if (pos == std::string::npos || pos == 0 || pos == token.length()) { + ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option you are trying to use.\n"); + } + + auto key = token.substr(0, pos); + auto value = token.substr(pos + 1); + + if (key == "device_type") { + std::set ov_supported_device_types = {"CPU", "GPU", + "GPU.0", "GPU.1", "NPU"}; + std::set deprecated_device_types = {"CPU_FP32", "GPU_FP32", + "GPU.0_FP32", "GPU.1_FP32", "GPU_FP16", + "GPU.0_FP16", "GPU.1_FP16"}; + if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) { + ov_options[key] = value; + } else if (deprecated_device_types.find(value) != deprecated_device_types.end()) { + ov_options[key] = value; + } else if (value.find("HETERO:") == 0) { + ov_options[key] = value; + } else if (value.find("MULTI:") == 0) { + ov_options[key] = value; + } else if (value.find("AUTO:") == 0) { + ov_options[key] = value; + } else { + ORT_THROW( + "[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. " + "Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from" + " HETERO/MULTI/AUTO options available. \n"); + } + } else if (key == "device_id") { + if (value == "CPU" || value == "GPU" || value == "NPU") { + ov_options[key] = value; + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options."); + } + } else if (key == "precision") { + auto device_type = ov_options["device_type"]; + if (device_type.find("GPU") != std::string::npos) { + if (value == "") { + ov_options[key] = "FP16"; + continue; + } else if (value == "ACCURACY" || value == "FP16" || value == "FP32") { + ov_options[key] = value; + continue; + } else { + ORT_THROW( + "[ERROR] [OpenVINO] Unsupported inference precision is selected. " + "GPU only supported FP32 / FP16. \n"); + } + } else if (device_type.find("NPU") != std::string::npos) { + if (value == "" || value == "ACCURACY" || value == "FP16") { + ov_options[key] = "FP16"; + continue; + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n"); + } + } else if (device_type.find("CPU") != std::string::npos) { + if (value == "" || value == "ACCURACY" || value == "FP32") { + ov_options[key] = "FP32"; + continue; + } else { + ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n"); + } + } + } else if (key == "enable_npu_fast_compile") { + if (value == "true" || value == "True" || + value == "false" || value == "False") { + ov_options[key] = value; + } else { + ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "enable_opencl_throttling") { + if (value == "true" || value == "True" || + value == "false" || value == "False") { + ov_options[key] = value; + } else { + ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "enable_qdq_optimizer") { + if (value == "true" || value == "True" || + value == "false" || value == "False") { + ov_options[key] = value; + } else { + ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_qdq_optimizer' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "disable_dynamic_shapes") { + if (value == "true" || value == "True" || + value == "false" || value == "False") { + ov_options[key] = value; + } else { + ORT_THROW( + "[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' " + "should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "num_of_threads") { + if (std::stoi(value) <= 0) { + ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n"); + } else { + ov_options[key] = value; + } + } else if (key == "model_priority") { + ov_options[key] = value; + } else if (key == "cache_dir") { + ov_options[key] = value; + } else if (key == "context") { + ov_options[key] = value; + } else if (key == "num_streams") { + if (std::stoi(value) <= 0 && std::stoi(value) > 8) { + ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n"); + } else { + ov_options[key] = value; + } + } else if (key == "export_ep_ctx_blob") { + if (value == "true" || value == "True" || + value == "false" || value == "False") { + ov_options[key] = value; + } else { + ORT_THROW( + "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " + "should be a boolean i.e. true or false. Default value is false.\n"); + } + } else { + ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); + } + } + session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); +#else + ORT_THROW("OpenVINO is not supported in this build\n"); +#endif + } session_ = Ort::Session(env, performance_test_config.model_info.model_file_path.c_str(), session_options); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index 6f07385729..f15ac100f4 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -109,7 +109,8 @@ std::unique_ptr OpenVINOExecutionProviderWithOptions(const O std::unique_ptr DefaultOpenVINOExecutionProvider() { #ifdef USE_OPENVINO ProviderOptions provider_options_map; - return OpenVINOProviderFactoryCreator::Create(&provider_options_map)->CreateProvider(); + SessionOptions session_options; + return OpenVINOProviderFactoryCreator::Create(&provider_options_map, &session_options)->CreateProvider(); #else return nullptr; #endif diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 3e587e9b56..f431f471c4 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -79,6 +79,7 @@ def _openvino_verify_device_type(device_read): "CPU_NO_PARTITION", "GPU_NO_PARTITION", "NPU_NO_PARTITION", + "NPU_NO_CPU_FALLBACK", ] status_hetero = True res = False @@ -1227,6 +1228,7 @@ def generate_build_tree( if args.use_openvino: cmake_args += [ "-Donnxruntime_USE_OPENVINO=ON", + "-Donnxruntime_NPU_NO_FALLBACK=" + ("ON" if args.use_openvino == "NPU_NO_CPU_FALLBACK" else "OFF"), "-Donnxruntime_USE_OPENVINO_GPU=" + ("ON" if args.use_openvino == "GPU" else "OFF"), "-Donnxruntime_USE_OPENVINO_CPU=" + ("ON" if args.use_openvino == "CPU" else "OFF"), "-Donnxruntime_USE_OPENVINO_NPU=" + ("ON" if args.use_openvino == "NPU" else "OFF"),