From 0723dcb8b591a559db60885ff2cad610fd989ad4 Mon Sep 17 00:00:00 2001 From: Suryaprakash Shanmugam Date: Thu, 14 Dec 2023 05:26:43 +0530 Subject: [PATCH] OpenVINO Execution Provider with 2023.2 support (#18596) - Add support for OpenVINO 2023.2 - num_of_threads provider option is mapped to the CPU device property inference_num_threads of the CPU plugin, so users can control the #threads used for inference by the CPU - Logging in Debug mode now includes the runtime properties set for devices - Fix issue in using external weights through OpenVINO --------- Co-authored-by: Preetha Veeramalai --- cmake/CMakeLists.txt | 15 +++--- .../providers/openvino/backend_manager.cc | 24 +++++---- .../core/providers/openvino/backend_utils.cc | 4 +- .../openvino/backends/basic_backend.cc | 40 +++++++++------ .../openvino/backends/basic_backend.h | 1 + .../core/providers/openvino/contexts.h | 2 +- .../openvino/openvino_execution_provider.cc | 26 +++------- .../openvino/openvino_execution_provider.h | 6 +-- .../openvino/openvino_provider_factory.cc | 22 ++++---- .../core/providers/openvino/ov_interface.cc | 50 +++++++++++++++++-- .../core/providers/openvino/ov_interface.h | 7 +-- .../openvino/ov_versions/capability.cc | 10 ++-- .../openvino/ov_versions/data_ops.cc | 8 +-- .../providers/openvino/ov_versions/data_ops.h | 1 + .../core/session/provider_bridge_ort.cc | 8 ++- .../core/session/provider_registration.cc | 1 + .../python/onnxruntime_pybind_state.cc | 4 +- onnxruntime/test/perftest/ort_test_session.cc | 4 +- 18 files changed, 140 insertions(+), 93 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 7c5cfee611..7494035e47 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1258,13 +1258,7 @@ if (onnxruntime_USE_OPENVINO) endif() # Check OpenVINO version for support - if (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1") - set(OPENVINO_VERSION "2022.1") - add_definitions(-DOPENVINO_2022_1=1) - elseif (${VER} MATCHES "2022.2" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.2") - set(OPENVINO_VERSION "2022.2") - add_definitions(-DOPENVINO_2022_2=1) - elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3") + if ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3") set(OPENVINO_VERSION "2022.3") add_definitions(-DOPENVINO_2022_3=1) elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.0") @@ -1273,9 +1267,12 @@ if (onnxruntime_USE_OPENVINO) elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.1") set(OPENVINO_VERSION "2023.1") add_definitions(-DOPENVINO_2023_1=1) - elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino") - set(OPENVINO_VERSION "2023.1") + elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.2") + set(OPENVINO_VERSION "2023.2") add_definitions(-DOPENVINO_2023_1=1) + elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino") + set(OPENVINO_VERSION "2023.2") + add_definitions(-DOPENVINO_2023_2=1) else() message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}") endif() diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 7e4c0dc8d7..b2a7028f49 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -74,17 +74,19 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has symbolic input dims"; if (GetGlobalContext().device_type.find("CPU") != std::string::npos || GetGlobalContext().device_type.find("GPU") != std::string::npos) { - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. " - << "Creating backend Dynamic Shapes"; - try { - concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, - GetGlobalContext(), - subgraph_context_); - } catch (std::string const& msg) { - throw msg; + if (!GetGlobalContext().disable_dynamic_shapes) { + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. " + << "Creating backend Dynamic Shapes"; + try { + concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, + GetGlobalContext(), + subgraph_context_); + } catch (std::string const& msg) { + throw msg; + } + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " + << "Backend created for graph " << subgraph_context_.subgraph_name; } - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " - << "Backend created for graph " << subgraph_context_.subgraph_name; } } else { LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has concrete input dims. " @@ -260,7 +262,7 @@ void BackendManager::Compute(OrtKernelContext* context) { } #endif bool use_dynamic_backend = true; - if (subgraph_context_.has_dynamic_input_shape && + if (!GetGlobalContext().disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape && (GetGlobalContext().device_type.find("CPU") != std::string::npos || GetGlobalContext().device_type.find("GPU") != std::string::npos)) { concrete_backend_->Infer(context); diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index d47c91dd46..5092fffcfc 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -54,7 +54,7 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext } const std::string model = model_proto.SerializeAsString(); try { - auto cnn_network = global_context.ie_core.ReadModel(model); + auto cnn_network = global_context.ie_core.ReadModel(model, global_context.onnx_model_path_name); if ((subgraph_context.precision == "FP16") && (global_context.device_type.find("NPU") == std::string::npos)) { // FP16 transformations @@ -95,7 +95,7 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext } } #ifndef NDEBUG -#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) +#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2) if (IsDebugEnabled()) { std::string name = cnn_network->get_friendly_name(); ov::pass::Serialize serializer(name + ".xml", name + ".bin"); diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 09e1322ff5..2280d853e3 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -40,6 +40,9 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, // Enable streams; default=1 unless ovverriden by user config EnableStreams(); + // Set the inference_num_threads property of the CPU + SetNumThreads(device_config); + #ifndef NDEBUG if (IsDebugEnabled()) { std::string file_name = subgraph_context.subgraph_name + "_static.onnx"; @@ -67,8 +70,8 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; } #else -#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) - if (!subgraph_context_.has_dynamic_input_shape && dev_prec != "CPU_FP16") { +#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2) + if (global_context_.disable_dynamic_shapes && dev_prec != "CPU_FP16") { const std::string model = model_proto.SerializeAsString(); exe_network_ = global_context_.ie_core.LoadNetwork( model, hw_target, device_config, subgraph_context_.subgraph_name); @@ -96,16 +99,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, throw(msg); } - // The infer_requests_ pool will be intialized with a default value of 8 infer_request's - // The nireq value can also be configured to any num_of_threads during runtime - size_t nireq = global_context_.num_of_threads; - LOGS_DEFAULT(INFO) << log_tag << "The value of nireq being used is: " << nireq; -#ifndef NDEBUG - if (openvino_ep::backend_utils::IsDebugEnabled()) { - std::cout << "The value of nireq being used is: " << nireq << std::endl; - } -#endif - inferRequestsQueue_ = std::unique_ptr(new InferRequestsQueue(exe_network_, nireq)); + inferRequestsQueue_ = std::unique_ptr(new InferRequestsQueue(exe_network_, 1)); } bool BasicBackend::ValidateSubgraph(std::map>& const_outputs_map) { @@ -132,7 +126,7 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { device_config.emplace(ov::enable_profiling(true)); } #endif -#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) +#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVION_2023_2) if (global_context_.device_type.find("NPU") != std::string::npos) { std::pair device_property; device_property = std::make_pair("NPU_COMPILER_TYPE", "DRIVER"); @@ -168,7 +162,24 @@ void BasicBackend::EnableGPUThrottling(ov::AnyMap& device_config) { } void BasicBackend::EnableStreams() { - global_context_.ie_core.SetStreams(global_context_.device_type, global_context_.num_streams); + // Streams can be set only if the device is not one of AUTO, MULTI, or HETERO + // Throw an exception if the user tries to set num_streams for these devices + if ((global_context_.device_type.find("MULTI") != std::string::npos) || + (global_context_.device_type.find("HETERO") != std::string::npos) || + (global_context_.device_type.find("AUTO") != std::string::npos)) { + if (global_context_.num_streams != 1) { + throw(log_tag + "Cannot set NUM_STREAMS to " + std::to_string(global_context_.num_streams) + " for device " + global_context_.device_type); + } + // Do nothing + } else { + global_context_.ie_core.SetStreams(global_context_.device_type, global_context_.num_streams); + } +} + +void BasicBackend::SetNumThreads(ov::AnyMap& device_config) { + // inference_num_threads is applicable only for the CPU device + if (global_context_.device_type.find("CPU") != std::string::npos) + device_config.emplace(ov::inference_num_threads(global_context_.num_of_threads)); } // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on @@ -199,6 +210,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque } size_t batch_slice_idx = 0; if (subgraph_context_.has_dynamic_input_shape && + !global_context_.disable_dynamic_shapes && (global_context_.device_type.find("CPU") != std::string::npos || global_context_.device_type.find("GPU") != std::string::npos)) { auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 6eda641451..aa96dadbf0 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -37,6 +37,7 @@ class BasicBackend : public IBackend { void EnableCaching(); void EnableGPUThrottling(ov::AnyMap& device_config); void EnableStreams(); + void SetNumThreads(ov::AnyMap& device_config); void StartAsyncInference(Ort::KernelContext& context, std::shared_ptr infer_request); #ifdef IO_BUFFER_ENABLED diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 29233e72c3..5f19c71683 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -17,7 +17,7 @@ struct GlobalContext { bool is_wholly_supported_graph = false; bool enable_npu_fast_compile = false; bool enable_opencl_throttling = false; - bool enable_dynamic_shapes = false; + bool disable_dynamic_shapes = false; size_t num_of_threads; std::string device_type; std::string precision_str; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index a4c6b0f851..aa389f6297 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -22,17 +22,9 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv openvino_ep::BackendManager::GetGlobalContext().num_streams = info.num_streams_; openvino_ep::BackendManager::GetGlobalContext().context = info.context_; openvino_ep::BackendManager::GetGlobalContext().enable_opencl_throttling = info.enable_opencl_throttling_; - openvino_ep::BackendManager::GetGlobalContext().enable_dynamic_shapes = info.enable_dynamic_shapes_; + openvino_ep::BackendManager::GetGlobalContext().disable_dynamic_shapes = info.disable_dynamic_shapes_; + openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_; - if (static_cast(info.num_of_threads_) <= 0) { - openvino_ep::BackendManager::GetGlobalContext().num_of_threads = 8; - } else if (static_cast(info.num_of_threads_) > 8) { - std::string err_msg = std::string("\n [ERROR] num_of_threads configured during runtime is: ") + - std::to_string(info.num_of_threads_) + "\nnum_of_threads configured should be >0 and <=8.\n"; - ORT_THROW(err_msg); - } else { - openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_; - } // to check if target device is available // using ie_core capability GetAvailableDevices to fetch list of devices plugged in if (info.cache_dir_.empty()) { @@ -120,15 +112,7 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, openvino_ep::BackendManager::GetGlobalContext().onnx_opset_version = graph_viewer.DomainToVersionMap().at(kOnnxDomain); -#if defined(OPENVINO_2022_1) - openvino_ep::GetCapability obj(graph_viewer, - openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_1"); - result = obj.Execute(); -#elif defined(OPENVINO_2022_2) - openvino_ep::GetCapability obj(graph_viewer, - openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_2"); - result = obj.Execute(); -#elif defined(OPENVINO_2022_3) +#if defined(OPENVINO_2022_3) openvino_ep::GetCapability obj(graph_viewer, openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_3"); result = obj.Execute(); @@ -140,6 +124,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, openvino_ep::GetCapability obj(graph_viewer, openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2023_1"); result = obj.Execute(); +#elif defined(OPENVINO_2023_2) + openvino_ep::GetCapability obj(graph_viewer, + openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2023_2"); + result = obj.Execute(); #endif return result; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 3b56b54410..7cc2fb9b1e 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -69,12 +69,12 @@ struct OpenVINOExecutionProviderInfo { int num_streams_; void* context_; bool enable_opencl_throttling_; - bool enable_dynamic_shapes_; + bool disable_dynamic_shapes_; explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_npu_fast_compile, std::string dev_id, size_t num_of_threads, std::string cache_dir, int num_streams, void* context, bool enable_opencl_throttling, - bool enable_dynamic_shapes) + bool disable_dynamic_shapes) : enable_npu_fast_compile_(enable_npu_fast_compile), device_id_(dev_id), num_of_threads_(num_of_threads), @@ -82,7 +82,7 @@ struct OpenVINOExecutionProviderInfo { num_streams_(num_streams), context_(context), enable_opencl_throttling_(enable_opencl_throttling), - enable_dynamic_shapes_(enable_dynamic_shapes) { + disable_dynamic_shapes_(disable_dynamic_shapes) { if (dev_type == "") { LOGS_DEFAULT(INFO) << "[OpenVINO-EP]" << "No runtime device selection option provided."; diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index fbb89710c8..749907da18 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -11,13 +11,13 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const char* device_type, bool enable_npu_fast_compile, const char* device_id, size_t num_of_threads, const char* cache_dir, int num_streams, void* context, - bool enable_opencl_throttling, bool enable_dynamic_shapes) + bool enable_opencl_throttling, bool disable_dynamic_shapes) : enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), num_streams_(num_streams), context_(context), enable_opencl_throttling_(enable_opencl_throttling), - enable_dynamic_shapes_(enable_dynamic_shapes) { + disable_dynamic_shapes_(disable_dynamic_shapes) { device_type_ = (device_type == nullptr) ? "" : device_type; device_id_ = (device_id == nullptr) ? "" : device_id; cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; @@ -36,13 +36,13 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { int num_streams_; void* context_; bool enable_opencl_throttling_; - bool enable_dynamic_shapes_; + bool disable_dynamic_shapes_; }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { OpenVINOExecutionProviderInfo info(device_type_, enable_npu_fast_compile_, device_id_, num_of_threads_, cache_dir_, num_streams_, context_, enable_opencl_throttling_, - enable_dynamic_shapes_); + disable_dynamic_shapes_); return std::make_unique(info); } @@ -67,7 +67,7 @@ struct OpenVINO_Provider : Provider { bool enable_npu_fast_compile = false; // [enable_npu_fast_compile]: Fast-compile may be optionally enabled to // speeds up the model's compilation to NPU device specific format. const char* device_id = ""; // [device_id]: Selects a particular hardware device for inference. - int num_of_threads = 8; // [num_of_threads]: Overrides the accelerator default value of number of + int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of // threads with this value at runtime. const char* cache_dir = ""; // [cache_dir]: specify the path to // dump and load the blobs for the model caching/kernel caching (GPU) @@ -78,7 +78,7 @@ struct OpenVINO_Provider : Provider { // with this value at runtime. bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU // device (Reduces CPU Utilization when using GPU) - bool enable_dynamic_shapes = false; // [enable_dynamic_shapes]: Enables Dynamic Shapes feature for CPU device) + bool disable_dynamic_shapes = false; // [disable_dynamic_shapes]: Execute model with default static shape for optimal performance. void* context = nullptr; if (provider_options_map.find("device_type") != provider_options_map.end()) { @@ -147,12 +147,12 @@ struct OpenVINO_Provider : Provider { bool_flag = ""; } - if (provider_options_map.find("enable_dynamic_shapes") != provider_options_map.end()) { - bool_flag = provider_options_map.at("enable_dynamic_shapes"); + if (provider_options_map.find("disable_dynamic_shapes") != provider_options_map.end()) { + bool_flag = provider_options_map.at("disable_dynamic_shapes"); if (bool_flag == "true" || bool_flag == "True") - enable_dynamic_shapes = true; + disable_dynamic_shapes = true; else if (bool_flag == "false" || bool_flag == "False") - enable_dynamic_shapes = false; + disable_dynamic_shapes = false; } return std::make_shared(const_cast(device_type.c_str()), enable_npu_fast_compile, @@ -162,7 +162,7 @@ struct OpenVINO_Provider : Provider { num_streams, context, enable_opencl_throttling, - enable_dynamic_shapes); + disable_dynamic_shapes); } void Initialize() override { diff --git a/onnxruntime/core/providers/openvino/ov_interface.cc b/onnxruntime/core/providers/openvino/ov_interface.cc index d2ce378c97..31952e5b15 100644 --- a/onnxruntime/core/providers/openvino/ov_interface.cc +++ b/onnxruntime/core/providers/openvino/ov_interface.cc @@ -6,6 +6,7 @@ #define ORT_API_MANUAL_INIT #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/shared_library/provider_api.h" +#include "backend_utils.h" #if defined(OV_API_20) using Exception = ov::Exception; @@ -18,10 +19,22 @@ namespace onnxruntime { namespace openvino_ep { const std::string log_tag = "[OpenVINO-EP] "; -std::shared_ptr OVCore::ReadModel(const std::string& model) const { +std::shared_ptr OVCore::ReadModel(const std::string& model, const std::string& model_path) const { try { - OVTensor weights; - return oe.read_model(model, weights); + std::istringstream modelStringStream(model); + std::istream& modelStream = modelStringStream; + // Try to load with FrontEndManager + ov::frontend::FrontEndManager manager; + ov::frontend::FrontEnd::Ptr FE; + ov::frontend::InputModel::Ptr inputModel; + + ov::AnyVector params{&modelStream, model_path}; + + FE = manager.load_by_model(params); + if (FE) { + inputModel = FE->load(params); + } + return FE->convert(inputModel); } catch (const Exception& e) { throw std::string(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what())); } catch (...) { @@ -36,6 +49,35 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& ie_cnn_network, ov::CompiledModel obj; try { obj = oe.compile_model(ie_cnn_network, hw_target, device_config); + +#ifndef NDEBUG + if (onnxruntime::openvino_ep::backend_utils::IsDebugEnabled()) { + // output of the actual settings that the device selected + auto supported_properties = obj.get_property(ov::supported_properties); + std::cout << "Model:" << std::endl; + for (const auto& cfg : supported_properties) { + if (cfg == ov::supported_properties) + continue; + auto prop = obj.get_property(cfg); + if (cfg == ov::device::properties) { + auto devices_properties = prop.as(); + for (auto& item : devices_properties) { + std::cout << " " << item.first << ": " << std::endl; + for (auto& item2 : item.second.as()) { + OPENVINO_SUPPRESS_DEPRECATED_START + if (item2.first == ov::supported_properties || item2.first == "SUPPORTED_CONFIG_KEYS)" || + item2.first == "SUPPORTED_METRICS") + continue; + OPENVINO_SUPPRESS_DEPRECATED_END + std::cout << " " << item2.first << ": " << item2.second.as() << std::endl; + } + } + } else { + std::cout << " " << cfg << ": " << prop.as() << std::endl; + } + } + } +#endif OVExeNetwork exe(obj); return exe; } catch (const Exception& e) { @@ -45,7 +87,7 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& ie_cnn_network, } } -#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) +#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2) OVExeNetwork OVCore::LoadNetwork(const std::string& model, std::string& hw_target, ov::AnyMap& device_config, diff --git a/onnxruntime/core/providers/openvino/ov_interface.h b/onnxruntime/core/providers/openvino/ov_interface.h index 935ac8f684..690e91742b 100644 --- a/onnxruntime/core/providers/openvino/ov_interface.h +++ b/onnxruntime/core/providers/openvino/ov_interface.h @@ -6,10 +6,11 @@ #include #include -#if defined(OPENVINO_2022_1) || (OPENVINO_2022_2) || (OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) +#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2) #define OV_API_20 #include "openvino/openvino.hpp" #include "openvino/pass/convert_fp32_to_fp16.hpp" +#include "openvino/frontend/manager.hpp" #else #include #endif @@ -43,12 +44,12 @@ class OVCore { ov::Core oe; public: - std::shared_ptr ReadModel(const std::string& model_stream) const; + std::shared_ptr ReadModel(const std::string& model_stream, const std::string& model_path) const; OVExeNetwork LoadNetwork(std::shared_ptr& ie_cnn_network, std::string& hw_target, ov::AnyMap& device_config, std::string name); -#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) +#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2) OVExeNetwork LoadNetwork(const std::string& model_stream, std::string& hw_target, ov::AnyMap& device_config, diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc index 454f3dd5eb..4494bb8ab2 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc @@ -26,18 +26,16 @@ namespace openvino_ep { GetCapability::GetCapability(const GraphViewer& graph_viewer_param, std::string device_type_param, const std::string version_param) : graph_viewer_(graph_viewer_param), device_type_(device_type_param) { - if (version_param == "V_2022_1") { - data_ops_ = new DataOps(graph_viewer_, V_2022_1, device_type_); - } else if (version_param == "V_2022_2") { - data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_); - } else if (version_param == "V_2022_3") { + if (version_param == "V_2022_3") { data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_); } else if (version_param == "V_2023_0") { data_ops_ = new DataOps(graph_viewer_, V_2023_0, device_type_); } else if (version_param == "V_2023_1") { data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_); + } else if (version_param == "V_2023_2") { + data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_); } else { - data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_); + data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_); } } diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index a5a0faa3a8..8749885660 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -146,7 +146,7 @@ std::vector supported_op_mode = { {"Dropout", V_2023_0, {"NPU"}}, {"Elu", V_2020_4, {"CPU", "GPU"}}, {"Elu", V_2023_0, {"NPU"}}, - // {"Einsum", V_2023_0, {"CPU", "GPU"}}, + {"Einsum", V_2023_1, {"CPU", "GPU"}}, {"Equal", V_2020_4, {"CPU", "GPU"}}, {"Equal", V_2023_0, {"NPU"}}, // Added for whisper decoder model. {"Erf", V_2020_4, {"CPU", "GPU"}}, @@ -705,7 +705,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"PRelu", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2}, [this](const Node* node, const InitializedTensorSet&) { const auto& input_arg = node->InputDefs()[1]; auto shape = input_arg->Shape(); @@ -820,7 +820,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Squeeze", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2}, [this](const Node* node, const InitializedTensorSet&) { // If the operator is unsqueeze // If axes is an input, then we cannot produce a static graph. @@ -835,7 +835,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Unsqueeze", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2}, [this](const Node* node, const InitializedTensorSet&) { // check for attributes auto& upsample_attr = node->GetAttributes(); diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h index a5aa3f8256..f6ad2dd5c9 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h @@ -25,6 +25,7 @@ enum versionNum { V_2022_3, V_2023_0, V_2023_1, + V_2023_2 }; using VersionNum = enum versionNum; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index df4dd55417..e3b8dea90a 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1449,8 +1449,12 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O ov_options_converted_map["context"] = context_string.str(); ov_options_converted_map["enable_opencl_throttling"] = legacy_ov_options->enable_opencl_throttling; - ov_options_converted_map["enable_dynamic_shapes"] = legacy_ov_options->enable_dynamic_shapes; - + std::string enable_dynamic_shapes = reinterpret_cast(legacy_ov_options->enable_dynamic_shapes); + if (enable_dynamic_shapes == "true" || enable_dynamic_shapes == "True") { + ov_options_converted_map["disable_dynamic_shapes"] = "false"; + } else if (enable_dynamic_shapes == "false" || enable_dynamic_shapes == "False") { + ov_options_converted_map["disable_dynamic_shapes"] = "true"; + } // Add new provider option below ov_options_converted_map["num_streams"] = "1"; return ov_options_converted_map; diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 81e58c9dd0..2e9af9f1f9 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -104,6 +104,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, #else status = create_not_supported_status(); #endif + } else if (strcmp(provider_name, "SNPE") == 0) { #if defined(USE_SNPE) options->provider_factories.push_back(SNPEProviderFactoryCreator::Create(provider_options)); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 27fbf19084..6f383d733e 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -903,10 +903,10 @@ std::unique_ptr CreateExecutionProviderInstance( ORT_THROW("Invalid value passed for enable_opencl_throttling: ", option.second); } OV_provider_options_map[option.first] = option.second; - } else if (option.first == "enable_dynamic_shapes") { + } else if (option.first == "disable_dynamic_shapes") { if (!(option.second == "True" || option.second == "true" || option.second == "False" || option.second == "false")) { - ORT_THROW("Invalid value passed for enable_dynamic_shapes: ", option.second); + ORT_THROW("Invalid value passed for disable_dynamic_shapes: ", option.second); } OV_provider_options_map[option.first] = option.second; } else if (option.first == "device_id") { diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index eb2a77c07f..6a99d6a0b0 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -272,7 +272,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device } else { ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n"); } - } else if (key == "enable_dynamic_shapes") { + } else if (key == "disable_dynamic_shapes") { if (value == "true" || value == "True" || value == "false" || value == "False") { ov_options[key] = value; @@ -298,7 +298,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device ov_options[key] = value; } } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling|true'] \n"); + ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); } } session_options.AppendExecutionProvider("OpenVINO", ov_options);