mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
OpenVINO Execution Provider with 2023.2 support (#18596)
- Add support for OpenVINO 2023.2 - num_of_threads provider option is mapped to the CPU device property inference_num_threads of the CPU plugin, so users can control the #threads used for inference by the CPU - Logging in Debug mode now includes the runtime properties set for devices - Fix issue in using external weights through OpenVINO --------- Co-authored-by: Preetha Veeramalai <preetha.veeramalai@intel.com>
This commit is contained in:
Suryaprakash Shanmugam
GitHub
parent
f3fa045681
commit
0723dcb8b5
18 changed files with 140 additions and 93 deletions
|
|
@ -1258,13 +1258,7 @@ if (onnxruntime_USE_OPENVINO)
|
|||
endif()
|
||||
|
||||
# Check OpenVINO version for support
|
||||
if (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1")
|
||||
set(OPENVINO_VERSION "2022.1")
|
||||
add_definitions(-DOPENVINO_2022_1=1)
|
||||
elseif (${VER} MATCHES "2022.2" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.2")
|
||||
set(OPENVINO_VERSION "2022.2")
|
||||
add_definitions(-DOPENVINO_2022_2=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3")
|
||||
if ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3")
|
||||
set(OPENVINO_VERSION "2022.3")
|
||||
add_definitions(-DOPENVINO_2022_3=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.0")
|
||||
|
|
@ -1273,9 +1267,12 @@ if (onnxruntime_USE_OPENVINO)
|
|||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.1")
|
||||
set(OPENVINO_VERSION "2023.1")
|
||||
add_definitions(-DOPENVINO_2023_1=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino")
|
||||
set(OPENVINO_VERSION "2023.1")
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.2")
|
||||
set(OPENVINO_VERSION "2023.2")
|
||||
add_definitions(-DOPENVINO_2023_1=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino")
|
||||
set(OPENVINO_VERSION "2023.2")
|
||||
add_definitions(-DOPENVINO_2023_2=1)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}")
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -74,17 +74,19 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has symbolic input dims";
|
||||
if (GetGlobalContext().device_type.find("CPU") != std::string::npos ||
|
||||
GetGlobalContext().device_type.find("GPU") != std::string::npos) {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
|
||||
<< "Creating backend Dynamic Shapes";
|
||||
try {
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const& msg) {
|
||||
throw msg;
|
||||
if (!GetGlobalContext().disable_dynamic_shapes) {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
|
||||
<< "Creating backend Dynamic Shapes";
|
||||
try {
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const& msg) {
|
||||
throw msg;
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
|
||||
<< "Backend created for graph " << subgraph_context_.subgraph_name;
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
|
||||
<< "Backend created for graph " << subgraph_context_.subgraph_name;
|
||||
}
|
||||
} else {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has concrete input dims. "
|
||||
|
|
@ -260,7 +262,7 @@ void BackendManager::Compute(OrtKernelContext* context) {
|
|||
}
|
||||
#endif
|
||||
bool use_dynamic_backend = true;
|
||||
if (subgraph_context_.has_dynamic_input_shape &&
|
||||
if (!GetGlobalContext().disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape &&
|
||||
(GetGlobalContext().device_type.find("CPU") != std::string::npos ||
|
||||
GetGlobalContext().device_type.find("GPU") != std::string::npos)) {
|
||||
concrete_backend_->Infer(context);
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext
|
|||
}
|
||||
const std::string model = model_proto.SerializeAsString();
|
||||
try {
|
||||
auto cnn_network = global_context.ie_core.ReadModel(model);
|
||||
auto cnn_network = global_context.ie_core.ReadModel(model, global_context.onnx_model_path_name);
|
||||
if ((subgraph_context.precision == "FP16") &&
|
||||
(global_context.device_type.find("NPU") == std::string::npos)) {
|
||||
// FP16 transformations
|
||||
|
|
@ -95,7 +95,7 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext
|
|||
}
|
||||
}
|
||||
#ifndef NDEBUG
|
||||
#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2)
|
||||
if (IsDebugEnabled()) {
|
||||
std::string name = cnn_network->get_friendly_name();
|
||||
ov::pass::Serialize serializer(name + ".xml", name + ".bin");
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
// Enable streams; default=1 unless ovverriden by user config
|
||||
EnableStreams();
|
||||
|
||||
// Set the inference_num_threads property of the CPU
|
||||
SetNumThreads(device_config);
|
||||
|
||||
#ifndef NDEBUG
|
||||
if (IsDebugEnabled()) {
|
||||
std::string file_name = subgraph_context.subgraph_name + "_static.onnx";
|
||||
|
|
@ -67,8 +70,8 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
|
||||
}
|
||||
#else
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
if (!subgraph_context_.has_dynamic_input_shape && dev_prec != "CPU_FP16") {
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2)
|
||||
if (global_context_.disable_dynamic_shapes && dev_prec != "CPU_FP16") {
|
||||
const std::string model = model_proto.SerializeAsString();
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(
|
||||
model, hw_target, device_config, subgraph_context_.subgraph_name);
|
||||
|
|
@ -96,16 +99,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
throw(msg);
|
||||
}
|
||||
|
||||
// The infer_requests_ pool will be intialized with a default value of 8 infer_request's
|
||||
// The nireq value can also be configured to any num_of_threads during runtime
|
||||
size_t nireq = global_context_.num_of_threads;
|
||||
LOGS_DEFAULT(INFO) << log_tag << "The value of nireq being used is: " << nireq;
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
std::cout << "The value of nireq being used is: " << nireq << std::endl;
|
||||
}
|
||||
#endif
|
||||
inferRequestsQueue_ = std::unique_ptr<InferRequestsQueue>(new InferRequestsQueue(exe_network_, nireq));
|
||||
inferRequestsQueue_ = std::unique_ptr<InferRequestsQueue>(new InferRequestsQueue(exe_network_, 1));
|
||||
}
|
||||
|
||||
bool BasicBackend::ValidateSubgraph(std::map<std::string, std::shared_ptr<ov::Node>>& const_outputs_map) {
|
||||
|
|
@ -132,7 +126,7 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) {
|
|||
device_config.emplace(ov::enable_profiling(true));
|
||||
}
|
||||
#endif
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVION_2023_2)
|
||||
if (global_context_.device_type.find("NPU") != std::string::npos) {
|
||||
std::pair<std::string, ov::Any> device_property;
|
||||
device_property = std::make_pair("NPU_COMPILER_TYPE", "DRIVER");
|
||||
|
|
@ -168,7 +162,24 @@ void BasicBackend::EnableGPUThrottling(ov::AnyMap& device_config) {
|
|||
}
|
||||
|
||||
void BasicBackend::EnableStreams() {
|
||||
global_context_.ie_core.SetStreams(global_context_.device_type, global_context_.num_streams);
|
||||
// Streams can be set only if the device is not one of AUTO, MULTI, or HETERO
|
||||
// Throw an exception if the user tries to set num_streams for these devices
|
||||
if ((global_context_.device_type.find("MULTI") != std::string::npos) ||
|
||||
(global_context_.device_type.find("HETERO") != std::string::npos) ||
|
||||
(global_context_.device_type.find("AUTO") != std::string::npos)) {
|
||||
if (global_context_.num_streams != 1) {
|
||||
throw(log_tag + "Cannot set NUM_STREAMS to " + std::to_string(global_context_.num_streams) + " for device " + global_context_.device_type);
|
||||
}
|
||||
// Do nothing
|
||||
} else {
|
||||
global_context_.ie_core.SetStreams(global_context_.device_type, global_context_.num_streams);
|
||||
}
|
||||
}
|
||||
|
||||
void BasicBackend::SetNumThreads(ov::AnyMap& device_config) {
|
||||
// inference_num_threads is applicable only for the CPU device
|
||||
if (global_context_.device_type.find("CPU") != std::string::npos)
|
||||
device_config.emplace(ov::inference_num_threads(global_context_.num_of_threads));
|
||||
}
|
||||
|
||||
// Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on
|
||||
|
|
@ -199,6 +210,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque
|
|||
}
|
||||
size_t batch_slice_idx = 0;
|
||||
if (subgraph_context_.has_dynamic_input_shape &&
|
||||
!global_context_.disable_dynamic_shapes &&
|
||||
(global_context_.device_type.find("CPU") != std::string::npos ||
|
||||
global_context_.device_type.find("GPU") != std::string::npos)) {
|
||||
auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name));
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ class BasicBackend : public IBackend {
|
|||
void EnableCaching();
|
||||
void EnableGPUThrottling(ov::AnyMap& device_config);
|
||||
void EnableStreams();
|
||||
void SetNumThreads(ov::AnyMap& device_config);
|
||||
void StartAsyncInference(Ort::KernelContext& context, std::shared_ptr<OVInferRequest> infer_request);
|
||||
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ struct GlobalContext {
|
|||
bool is_wholly_supported_graph = false;
|
||||
bool enable_npu_fast_compile = false;
|
||||
bool enable_opencl_throttling = false;
|
||||
bool enable_dynamic_shapes = false;
|
||||
bool disable_dynamic_shapes = false;
|
||||
size_t num_of_threads;
|
||||
std::string device_type;
|
||||
std::string precision_str;
|
||||
|
|
|
|||
|
|
@ -22,17 +22,9 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
|
|||
openvino_ep::BackendManager::GetGlobalContext().num_streams = info.num_streams_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().context = info.context_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().enable_opencl_throttling = info.enable_opencl_throttling_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().enable_dynamic_shapes = info.enable_dynamic_shapes_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().disable_dynamic_shapes = info.disable_dynamic_shapes_;
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_;
|
||||
|
||||
if (static_cast<int>(info.num_of_threads_) <= 0) {
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = 8;
|
||||
} else if (static_cast<int>(info.num_of_threads_) > 8) {
|
||||
std::string err_msg = std::string("\n [ERROR] num_of_threads configured during runtime is: ") +
|
||||
std::to_string(info.num_of_threads_) + "\nnum_of_threads configured should be >0 and <=8.\n";
|
||||
ORT_THROW(err_msg);
|
||||
} else {
|
||||
openvino_ep::BackendManager::GetGlobalContext().num_of_threads = info.num_of_threads_;
|
||||
}
|
||||
// to check if target device is available
|
||||
// using ie_core capability GetAvailableDevices to fetch list of devices plugged in
|
||||
if (info.cache_dir_.empty()) {
|
||||
|
|
@ -120,15 +112,7 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
|
|||
openvino_ep::BackendManager::GetGlobalContext().onnx_opset_version =
|
||||
graph_viewer.DomainToVersionMap().at(kOnnxDomain);
|
||||
|
||||
#if defined(OPENVINO_2022_1)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_1");
|
||||
result = obj.Execute();
|
||||
#elif defined(OPENVINO_2022_2)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_2");
|
||||
result = obj.Execute();
|
||||
#elif defined(OPENVINO_2022_3)
|
||||
#if defined(OPENVINO_2022_3)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_3");
|
||||
result = obj.Execute();
|
||||
|
|
@ -140,6 +124,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
|
|||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2023_1");
|
||||
result = obj.Execute();
|
||||
#elif defined(OPENVINO_2023_2)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2023_2");
|
||||
result = obj.Execute();
|
||||
#endif
|
||||
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -69,12 +69,12 @@ struct OpenVINOExecutionProviderInfo {
|
|||
int num_streams_;
|
||||
void* context_;
|
||||
bool enable_opencl_throttling_;
|
||||
bool enable_dynamic_shapes_;
|
||||
bool disable_dynamic_shapes_;
|
||||
|
||||
explicit OpenVINOExecutionProviderInfo(std::string dev_type, bool enable_npu_fast_compile, std::string dev_id,
|
||||
size_t num_of_threads, std::string cache_dir, int num_streams,
|
||||
void* context, bool enable_opencl_throttling,
|
||||
bool enable_dynamic_shapes)
|
||||
bool disable_dynamic_shapes)
|
||||
: enable_npu_fast_compile_(enable_npu_fast_compile),
|
||||
device_id_(dev_id),
|
||||
num_of_threads_(num_of_threads),
|
||||
|
|
@ -82,7 +82,7 @@ struct OpenVINOExecutionProviderInfo {
|
|||
num_streams_(num_streams),
|
||||
context_(context),
|
||||
enable_opencl_throttling_(enable_opencl_throttling),
|
||||
enable_dynamic_shapes_(enable_dynamic_shapes) {
|
||||
disable_dynamic_shapes_(disable_dynamic_shapes) {
|
||||
if (dev_type == "") {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
|
||||
<< "No runtime device selection option provided.";
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
|
|||
OpenVINOProviderFactory(const char* device_type, bool enable_npu_fast_compile,
|
||||
const char* device_id, size_t num_of_threads,
|
||||
const char* cache_dir, int num_streams, void* context,
|
||||
bool enable_opencl_throttling, bool enable_dynamic_shapes)
|
||||
bool enable_opencl_throttling, bool disable_dynamic_shapes)
|
||||
: enable_npu_fast_compile_(enable_npu_fast_compile),
|
||||
num_of_threads_(num_of_threads),
|
||||
num_streams_(num_streams),
|
||||
context_(context),
|
||||
enable_opencl_throttling_(enable_opencl_throttling),
|
||||
enable_dynamic_shapes_(enable_dynamic_shapes) {
|
||||
disable_dynamic_shapes_(disable_dynamic_shapes) {
|
||||
device_type_ = (device_type == nullptr) ? "" : device_type;
|
||||
device_id_ = (device_id == nullptr) ? "" : device_id;
|
||||
cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir;
|
||||
|
|
@ -36,13 +36,13 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
|
|||
int num_streams_;
|
||||
void* context_;
|
||||
bool enable_opencl_throttling_;
|
||||
bool enable_dynamic_shapes_;
|
||||
bool disable_dynamic_shapes_;
|
||||
};
|
||||
|
||||
std::unique_ptr<IExecutionProvider> OpenVINOProviderFactory::CreateProvider() {
|
||||
OpenVINOExecutionProviderInfo info(device_type_, enable_npu_fast_compile_, device_id_, num_of_threads_,
|
||||
cache_dir_, num_streams_, context_, enable_opencl_throttling_,
|
||||
enable_dynamic_shapes_);
|
||||
disable_dynamic_shapes_);
|
||||
return std::make_unique<OpenVINOExecutionProvider>(info);
|
||||
}
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ struct OpenVINO_Provider : Provider {
|
|||
bool enable_npu_fast_compile = false; // [enable_npu_fast_compile]: Fast-compile may be optionally enabled to
|
||||
// speeds up the model's compilation to NPU device specific format.
|
||||
const char* device_id = ""; // [device_id]: Selects a particular hardware device for inference.
|
||||
int num_of_threads = 8; // [num_of_threads]: Overrides the accelerator default value of number of
|
||||
int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of
|
||||
// threads with this value at runtime.
|
||||
const char* cache_dir = ""; // [cache_dir]: specify the path to
|
||||
// dump and load the blobs for the model caching/kernel caching (GPU)
|
||||
|
|
@ -78,7 +78,7 @@ struct OpenVINO_Provider : Provider {
|
|||
// with this value at runtime.
|
||||
bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU
|
||||
// device (Reduces CPU Utilization when using GPU)
|
||||
bool enable_dynamic_shapes = false; // [enable_dynamic_shapes]: Enables Dynamic Shapes feature for CPU device)
|
||||
bool disable_dynamic_shapes = false; // [disable_dynamic_shapes]: Execute model with default static shape for optimal performance.
|
||||
void* context = nullptr;
|
||||
|
||||
if (provider_options_map.find("device_type") != provider_options_map.end()) {
|
||||
|
|
@ -147,12 +147,12 @@ struct OpenVINO_Provider : Provider {
|
|||
bool_flag = "";
|
||||
}
|
||||
|
||||
if (provider_options_map.find("enable_dynamic_shapes") != provider_options_map.end()) {
|
||||
bool_flag = provider_options_map.at("enable_dynamic_shapes");
|
||||
if (provider_options_map.find("disable_dynamic_shapes") != provider_options_map.end()) {
|
||||
bool_flag = provider_options_map.at("disable_dynamic_shapes");
|
||||
if (bool_flag == "true" || bool_flag == "True")
|
||||
enable_dynamic_shapes = true;
|
||||
disable_dynamic_shapes = true;
|
||||
else if (bool_flag == "false" || bool_flag == "False")
|
||||
enable_dynamic_shapes = false;
|
||||
disable_dynamic_shapes = false;
|
||||
}
|
||||
return std::make_shared<OpenVINOProviderFactory>(const_cast<char*>(device_type.c_str()),
|
||||
enable_npu_fast_compile,
|
||||
|
|
@ -162,7 +162,7 @@ struct OpenVINO_Provider : Provider {
|
|||
num_streams,
|
||||
context,
|
||||
enable_opencl_throttling,
|
||||
enable_dynamic_shapes);
|
||||
disable_dynamic_shapes);
|
||||
}
|
||||
|
||||
void Initialize() override {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#define ORT_API_MANUAL_INIT
|
||||
#include "core/session/onnxruntime_cxx_api.h"
|
||||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "backend_utils.h"
|
||||
|
||||
#if defined(OV_API_20)
|
||||
using Exception = ov::Exception;
|
||||
|
|
@ -18,10 +19,22 @@ namespace onnxruntime {
|
|||
namespace openvino_ep {
|
||||
|
||||
const std::string log_tag = "[OpenVINO-EP] ";
|
||||
std::shared_ptr<OVNetwork> OVCore::ReadModel(const std::string& model) const {
|
||||
std::shared_ptr<OVNetwork> OVCore::ReadModel(const std::string& model, const std::string& model_path) const {
|
||||
try {
|
||||
OVTensor weights;
|
||||
return oe.read_model(model, weights);
|
||||
std::istringstream modelStringStream(model);
|
||||
std::istream& modelStream = modelStringStream;
|
||||
// Try to load with FrontEndManager
|
||||
ov::frontend::FrontEndManager manager;
|
||||
ov::frontend::FrontEnd::Ptr FE;
|
||||
ov::frontend::InputModel::Ptr inputModel;
|
||||
|
||||
ov::AnyVector params{&modelStream, model_path};
|
||||
|
||||
FE = manager.load_by_model(params);
|
||||
if (FE) {
|
||||
inputModel = FE->load(params);
|
||||
}
|
||||
return FE->convert(inputModel);
|
||||
} catch (const Exception& e) {
|
||||
throw std::string(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what()));
|
||||
} catch (...) {
|
||||
|
|
@ -36,6 +49,35 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network,
|
|||
ov::CompiledModel obj;
|
||||
try {
|
||||
obj = oe.compile_model(ie_cnn_network, hw_target, device_config);
|
||||
|
||||
#ifndef NDEBUG
|
||||
if (onnxruntime::openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
// output of the actual settings that the device selected
|
||||
auto supported_properties = obj.get_property(ov::supported_properties);
|
||||
std::cout << "Model:" << std::endl;
|
||||
for (const auto& cfg : supported_properties) {
|
||||
if (cfg == ov::supported_properties)
|
||||
continue;
|
||||
auto prop = obj.get_property(cfg);
|
||||
if (cfg == ov::device::properties) {
|
||||
auto devices_properties = prop.as<ov::AnyMap>();
|
||||
for (auto& item : devices_properties) {
|
||||
std::cout << " " << item.first << ": " << std::endl;
|
||||
for (auto& item2 : item.second.as<ov::AnyMap>()) {
|
||||
OPENVINO_SUPPRESS_DEPRECATED_START
|
||||
if (item2.first == ov::supported_properties || item2.first == "SUPPORTED_CONFIG_KEYS)" ||
|
||||
item2.first == "SUPPORTED_METRICS")
|
||||
continue;
|
||||
OPENVINO_SUPPRESS_DEPRECATED_END
|
||||
std::cout << " " << item2.first << ": " << item2.second.as<std::string>() << std::endl;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
std::cout << " " << cfg << ": " << prop.as<std::string>() << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
OVExeNetwork exe(obj);
|
||||
return exe;
|
||||
} catch (const Exception& e) {
|
||||
|
|
@ -45,7 +87,7 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network,
|
|||
}
|
||||
}
|
||||
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2)
|
||||
OVExeNetwork OVCore::LoadNetwork(const std::string& model,
|
||||
std::string& hw_target,
|
||||
ov::AnyMap& device_config,
|
||||
|
|
|
|||
|
|
@ -6,10 +6,11 @@
|
|||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
#if defined(OPENVINO_2022_1) || (OPENVINO_2022_2) || (OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
#if defined(OPENVINO_2022_3) || (OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2)
|
||||
#define OV_API_20
|
||||
#include "openvino/openvino.hpp"
|
||||
#include "openvino/pass/convert_fp32_to_fp16.hpp"
|
||||
#include "openvino/frontend/manager.hpp"
|
||||
#else
|
||||
#include <inference_engine.hpp>
|
||||
#endif
|
||||
|
|
@ -43,12 +44,12 @@ class OVCore {
|
|||
ov::Core oe;
|
||||
|
||||
public:
|
||||
std::shared_ptr<OVNetwork> ReadModel(const std::string& model_stream) const;
|
||||
std::shared_ptr<OVNetwork> ReadModel(const std::string& model_stream, const std::string& model_path) const;
|
||||
OVExeNetwork LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network,
|
||||
std::string& hw_target,
|
||||
ov::AnyMap& device_config,
|
||||
std::string name);
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1)
|
||||
#if defined(OPENVINO_2023_0) || (OPENVINO_2023_1) || (OPENVINO_2023_2)
|
||||
OVExeNetwork LoadNetwork(const std::string& model_stream,
|
||||
std::string& hw_target,
|
||||
ov::AnyMap& device_config,
|
||||
|
|
|
|||
|
|
@ -26,18 +26,16 @@ namespace openvino_ep {
|
|||
GetCapability::GetCapability(const GraphViewer& graph_viewer_param, std::string device_type_param,
|
||||
const std::string version_param)
|
||||
: graph_viewer_(graph_viewer_param), device_type_(device_type_param) {
|
||||
if (version_param == "V_2022_1") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_1, device_type_);
|
||||
} else if (version_param == "V_2022_2") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_);
|
||||
} else if (version_param == "V_2022_3") {
|
||||
if (version_param == "V_2022_3") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_);
|
||||
} else if (version_param == "V_2023_0") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2023_0, device_type_);
|
||||
} else if (version_param == "V_2023_1") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_);
|
||||
} else if (version_param == "V_2023_2") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_);
|
||||
} else {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_);
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ std::vector<SupportedOp> supported_op_mode = {
|
|||
{"Dropout", V_2023_0, {"NPU"}},
|
||||
{"Elu", V_2020_4, {"CPU", "GPU"}},
|
||||
{"Elu", V_2023_0, {"NPU"}},
|
||||
// {"Einsum", V_2023_0, {"CPU", "GPU"}},
|
||||
{"Einsum", V_2023_1, {"CPU", "GPU"}},
|
||||
{"Equal", V_2020_4, {"CPU", "GPU"}},
|
||||
{"Equal", V_2023_0, {"NPU"}}, // Added for whisper decoder model.
|
||||
{"Erf", V_2020_4, {"CPU", "GPU"}},
|
||||
|
|
@ -705,7 +705,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"PRelu", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& input_arg = node->InputDefs()[1];
|
||||
auto shape = input_arg->Shape();
|
||||
|
|
@ -820,7 +820,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Squeeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
// If the operator is unsqueeze
|
||||
// If axes is an input, then we cannot produce a static graph.
|
||||
|
|
@ -835,7 +835,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Unsqueeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3, V_2023_0, V_2023_1, V_2023_2},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
// check for attributes
|
||||
auto& upsample_attr = node->GetAttributes();
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ enum versionNum {
|
|||
V_2022_3,
|
||||
V_2023_0,
|
||||
V_2023_1,
|
||||
V_2023_2
|
||||
};
|
||||
|
||||
using VersionNum = enum versionNum;
|
||||
|
|
|
|||
|
|
@ -1449,8 +1449,12 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O
|
|||
ov_options_converted_map["context"] = context_string.str();
|
||||
|
||||
ov_options_converted_map["enable_opencl_throttling"] = legacy_ov_options->enable_opencl_throttling;
|
||||
ov_options_converted_map["enable_dynamic_shapes"] = legacy_ov_options->enable_dynamic_shapes;
|
||||
|
||||
std::string enable_dynamic_shapes = reinterpret_cast<const char*>(legacy_ov_options->enable_dynamic_shapes);
|
||||
if (enable_dynamic_shapes == "true" || enable_dynamic_shapes == "True") {
|
||||
ov_options_converted_map["disable_dynamic_shapes"] = "false";
|
||||
} else if (enable_dynamic_shapes == "false" || enable_dynamic_shapes == "False") {
|
||||
ov_options_converted_map["disable_dynamic_shapes"] = "true";
|
||||
}
|
||||
// Add new provider option below
|
||||
ov_options_converted_map["num_streams"] = "1";
|
||||
return ov_options_converted_map;
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider,
|
|||
#else
|
||||
status = create_not_supported_status();
|
||||
#endif
|
||||
|
||||
} else if (strcmp(provider_name, "SNPE") == 0) {
|
||||
#if defined(USE_SNPE)
|
||||
options->provider_factories.push_back(SNPEProviderFactoryCreator::Create(provider_options));
|
||||
|
|
|
|||
|
|
@ -903,10 +903,10 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
|
|||
ORT_THROW("Invalid value passed for enable_opencl_throttling: ", option.second);
|
||||
}
|
||||
OV_provider_options_map[option.first] = option.second;
|
||||
} else if (option.first == "enable_dynamic_shapes") {
|
||||
} else if (option.first == "disable_dynamic_shapes") {
|
||||
if (!(option.second == "True" || option.second == "true" ||
|
||||
option.second == "False" || option.second == "false")) {
|
||||
ORT_THROW("Invalid value passed for enable_dynamic_shapes: ", option.second);
|
||||
ORT_THROW("Invalid value passed for disable_dynamic_shapes: ", option.second);
|
||||
}
|
||||
OV_provider_options_map[option.first] = option.second;
|
||||
} else if (option.first == "device_id") {
|
||||
|
|
|
|||
|
|
@ -272,7 +272,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
} else {
|
||||
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n");
|
||||
}
|
||||
} else if (key == "enable_dynamic_shapes") {
|
||||
} else if (key == "disable_dynamic_shapes") {
|
||||
if (value == "true" || value == "True" ||
|
||||
value == "false" || value == "False") {
|
||||
ov_options[key] = value;
|
||||
|
|
@ -298,7 +298,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
ov_options[key] = value;
|
||||
}
|
||||
} else {
|
||||
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling|true'] \n");
|
||||
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
|
||||
}
|
||||
}
|
||||
session_options.AppendExecutionProvider("OpenVINO", ov_options);
|
||||
|
|
|
|||
Loading…
Reference in a new issue