saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

OVEP options to disable CPU fallback at compile time (#21166)

Browse source 
### Description
Provide user level options to control the fallback on CPU for models not
supported on Intel's NPU hardware.


### Motivation and Context
- Current workflow of OVEP allows safe fallback from OV NPU to OV CPU on
compilation failures. Also supports MLAS CPU fallback in presence of
unsupported custom ops.
- The PR provides a build-time option to disable fallback from OV NPU to
OV CPU.
- The session Option "kOrtSessionOptionsDisableCPUEPFallback" disables
OV CPU and MLAS CPU fallback.
- Also has bug fix for proto creation.

---------

Co-authored-by: jatinwadhwa921 <jatin.wadhwa@intel.com>
Co-authored-by: ankitm3k <ankit.maheshkar@intel.com>
This commit is contained in:
Preetha Veeramalai 2024-06-28 08:31:02 -07:00 committed by GitHub
parent 21ad004237
commit 6baaaf5165
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 247 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmake/CMakeLists.txt
View file

@ -1341,6 +1341,10 @@ if (onnxruntime_USE_OPENVINO)
add_definitions(-DUSE_OPENVINO=1)
if(onnxruntime_NPU_NO_FALLBACK)
add_definitions(-DOPENVINO_DISABLE_NPU_FALLBACK=1)
endif()
if (onnxruntime_USE_OPENVINO_GPU)
add_definitions(-DOPENVINO_CONFIG_GPU=1)
endif()

4
onnxruntime/core/framework/config_options.cc
View file

@ -30,11 +30,11 @@ std::string ConfigOptions::GetConfigOrDefault(const std::string& config_key,
}
Status ConfigOptions::AddConfigEntry(const char* config_key, const char* config_value) noexcept {
std::string key(config_key);
std::string key = config_key;
if (key.empty() || key.length() > 128)
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Config key is empty or longer than maximum length 128");
std::string val(config_value);
std::string val = config_value;
if (val.length() > onnxruntime::kMaxStrLen)
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Config value is longer than maximum length: ",

16
onnxruntime/core/graph/graph_proto_serializer.cc
View file

@ -21,7 +21,21 @@ void GraphViewerToProto(const GraphViewer& graph_view,
*(graph_proto.mutable_output()->Add()) = output_arg->ToProto();
}
for (const auto* value_info : graph_view.GetValueInfo()) {
std::unordered_set<const onnxruntime::NodeArg*> value_info_ = graph_view.GetValueInfo();
// Reserve memory for the vector to avoid reallocations
std::vector<const NodeArg*> value_info_sorted;
value_info_sorted.reserve(value_info_.size());
value_info_sorted.assign(value_info_.begin(), value_info_.end());
auto sort_predicate = [](const NodeArg* v1, const NodeArg* v2) {
return v1->Name() < v2->Name();
};
// This ensures consistent ordering of value_info entries in the output graph
std::sort(value_info_sorted.begin(), value_info_sorted.end(), sort_predicate);
for (const auto* value_info : value_info_sorted) {
*(graph_proto.mutable_value_info()->Add()) = value_info->ToProto();
}

18
onnxruntime/core/providers/openvino/backend_manager.cc
View file

@ -105,7 +105,11 @@ BackendManager::BackendManager(const GlobalContext& global_context,
subgraph_context_,
ep_ctx_handle_);
} catch (const OnnxRuntimeException& ex) {
if (device_type.find("NPU") != std::string::npos) {
#if defined(OPENVINO_DISABLE_NPU_FALLBACK)
ORT_THROW(ex.what());
#else
if (device_type.find("NPU") != std::string::npos &&
!GetGlobalContext().disable_cpu_fallback) {
LOGS_DEFAULT(WARNING) << ex.what();
LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."
<< "Falling back to OV CPU for execution";
@ -122,6 +126,7 @@ BackendManager::BackendManager(const GlobalContext& global_context,
} else {
ORT_THROW(ex.what());
}
#endif
}
}
}
@ -419,7 +424,13 @@ void BackendManager::Compute(OrtKernelContext* context) {
subgraph_context_,
ep_ctx_handle_);
} catch (const OnnxRuntimeException& ex) {
if (GetGlobalContext().device_type.find("NPU") != std::string::npos) {
// Build option disables fallback to CPU on compilation failures with NPU.
#if defined(OPENVINO_DISABLE_NPU_FALLBACK)
LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU.";
ORT_THROW(ex.what());
#else
if (GetGlobalContext().device_type.find("NPU") != std::string::npos &&
!GetGlobalContext().disable_cpu_fallback) {
LOGS_DEFAULT(WARNING) << ex.what();
LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."
<< "Falling back to OV CPU for execution";
@ -434,7 +445,10 @@ void BackendManager::Compute(OrtKernelContext* context) {
} catch (std::string const& msg) {
ORT_THROW(msg);
}
} else {
ORT_THROW(ex.what());
}
#endif
}
backend_map_.insert({key, dynamic_backend});
} else {

7
onnxruntime/core/providers/openvino/backends/basic_backend.cc
View file

@ -545,6 +545,11 @@ void BasicBackend::Infer(OrtKernelContext* ctx) {
std::cout << "Inference successful" << std::endl;
}
// Create a duplicate infer_request_ shared ptr on the stack in the current local scope,
// as the infer_request gets freed in the next stage the reference count for the infer_request decrements &
// thus we dont have any dangling ptr leading to seg faults in the debug mode subsequent execution call
OVInferRequestPtr infer_request_ = infer_request;
// Once the inference is completed, the infer_request becomes free and is placed back into pool of infer_requests_
inferRequestsQueue_->putIdleRequest(std::move(infer_request));
#ifndef NDEBUG
@ -552,7 +557,7 @@ void BasicBackend::Infer(OrtKernelContext* ctx) {
if (openvino_ep::backend_utils::IsDebugEnabled()) {
inferRequestsQueue_->printstatus(); // Printing the elements of infer_requests_ vector pool only in debug mode
std::string& hw_target = global_context_.device_type;
printPerformanceCounts(infer_request, std::cout, hw_target);
printPerformanceCounts(std::move(infer_request_), std::cout, hw_target);
}
#endif
#endif

1
onnxruntime/core/providers/openvino/contexts.h
View file

@ -21,6 +21,7 @@ struct GlobalContext {
bool ep_context_embed_mode = true;
bool export_ep_ctx_blob = false;
bool enable_qdq_optimizer = false;
bool disable_cpu_fallback = false;
size_t num_of_threads;
std::string device_type;
std::string precision_str;

1
onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file

@ -33,6 +33,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
global_context_->OpenVINO_Version = {OPENVINO_VERSION_MAJOR, OPENVINO_VERSION_MINOR};
global_context_->export_ep_ctx_blob = info.export_ep_ctx_blob_;
global_context_->enable_qdq_optimizer = info.enable_qdq_optimizer_;
global_context_->disable_cpu_fallback = info.disable_cpu_fallback_;
// to check if target device is available
// using ie_core capability GetAvailableDevices to fetch list of devices plugged in

6
onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file

@ -74,6 +74,7 @@ struct OpenVINOExecutionProviderInfo {
bool disable_dynamic_shapes_{false};
bool export_ep_ctx_blob_{false};
bool enable_qdq_optimizer_{false};
bool disable_cpu_fallback_{false};
OpenVINOExecutionProviderInfo() = delete;
@ -81,7 +82,7 @@ struct OpenVINOExecutionProviderInfo {
size_t num_of_threads, std::string cache_dir, std::string model_priority,
int num_streams, void* context, bool enable_opencl_throttling,
bool disable_dynamic_shapes, bool export_ep_ctx_blob,
bool enable_qdq_optimizer)
bool enable_qdq_optimizer, bool disable_cpu_fallback)
: precision_(precision),
enable_npu_fast_compile_(enable_npu_fast_compile),
num_of_threads_(num_of_threads),
@ -92,7 +93,8 @@ struct OpenVINOExecutionProviderInfo {
enable_opencl_throttling_(enable_opencl_throttling),
disable_dynamic_shapes_(disable_dynamic_shapes),
export_ep_ctx_blob_(export_ep_ctx_blob),
enable_qdq_optimizer_(enable_qdq_optimizer) {
enable_qdq_optimizer_(enable_qdq_optimizer),
disable_cpu_fallback_(disable_cpu_fallback) {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};
if (dev_type == "") {

24
onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file

@ -13,7 +13,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
const char* cache_dir, const char* model_priority,
int num_streams, void* context,
bool enable_opencl_throttling, bool disable_dynamic_shapes,
bool export_ep_ctx_blob, bool enable_qdq_optimizer)
bool export_ep_ctx_blob, bool enable_qdq_optimizer,
bool disable_cpu_fallback)
: precision_(precision),
enable_npu_fast_compile_(enable_npu_fast_compile),
num_of_threads_(num_of_threads),
@ -23,7 +24,8 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
enable_opencl_throttling_(enable_opencl_throttling),
disable_dynamic_shapes_(disable_dynamic_shapes),
export_ep_ctx_blob_(export_ep_ctx_blob),
enable_qdq_optimizer_(enable_qdq_optimizer) {
enable_qdq_optimizer_(enable_qdq_optimizer),
disable_cpu_fallback_(disable_cpu_fallback) {
device_type_ = (device_type == nullptr) ? "" : device_type;
cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir;
}
@ -45,12 +47,14 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
bool disable_dynamic_shapes_;
bool export_ep_ctx_blob_;
bool enable_qdq_optimizer_;
bool disable_cpu_fallback_;
};
std::unique_ptr<IExecutionProvider> OpenVINOProviderFactory::CreateProvider() {
OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_,
cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_,
disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_);
disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_,
disable_cpu_fallback_);
return std::make_unique<OpenVINOExecutionProvider>(info);
}
@ -99,6 +103,8 @@ struct OpenVINO_Provider : Provider {
bool enable_qdq_optimizer = false;
bool disable_cpu_fallback = false;
if (provider_options_map.find("device_type") != provider_options_map.end()) {
device_type = provider_options_map.at("device_type").c_str();
@ -256,6 +262,15 @@ struct OpenVINO_Provider : Provider {
export_ep_ctx_blob = false;
bool_flag = "";
}
if (provider_options_map.find("disable_cpu_fallback") != provider_options_map.end()) {
bool_flag = provider_options_map.at("disable_cpu_fallback");
if (bool_flag == "true" || bool_flag == "True")
disable_cpu_fallback = true;
else if (bool_flag == "false" || bool_flag == "False")
disable_cpu_fallback = false;
bool_flag = "";
}
return std::make_shared<OpenVINOProviderFactory>(const_cast<char*>(device_type.c_str()),
const_cast<char*>(precision.c_str()),
enable_npu_fast_compile,
@ -267,7 +282,8 @@ struct OpenVINO_Provider : Provider {
enable_opencl_throttling,
disable_dynamic_shapes,
export_ep_ctx_blob,
enable_qdq_optimizer);
enable_qdq_optimizer,
disable_cpu_fallback);
}
void Initialize() override {

4
onnxruntime/core/providers/openvino/openvino_provider_factory_creator.h
View file

@ -11,9 +11,11 @@
struct OrtOpenVINOProviderOptions;
namespace onnxruntime {
struct SessionOptions;
// defined in provider_bridge_ort.cc
struct OpenVINOProviderFactoryCreator {
static std::shared_ptr<IExecutionProviderFactory> Create(const ProviderOptions* provider_options_map);
static std::shared_ptr<IExecutionProviderFactory> Create(ProviderOptions* provider_options_map,
const SessionOptions* session_options);
static std::shared_ptr<IExecutionProviderFactory> Create(const OrtOpenVINOProviderOptions* provider_options);
};
} // namespace onnxruntime

16
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -35,6 +35,7 @@
#include "core/framework/model_metadef_id_generator.h"
#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h"
#include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/session/onnxruntime_c_api.h"
#include "core/common/string_helper.h"
@ -1800,7 +1801,18 @@ std::shared_ptr<IExecutionProviderFactory> OpenVINOProviderFactoryCreator::Creat
return s_library_openvino.Get().CreateExecutionProviderFactory(&ov_options_converted_map);
}
std::shared_ptr<IExecutionProviderFactory> OpenVINOProviderFactoryCreator::Create(const ProviderOptions* provider_options_map) {
void ORTSessionOptionsToOrtOpenVINOProviderOptions(ProviderOptions& ov_options,
const SessionOptions* session_options) {
bool disable_cpu_fallback = session_options->config_options.GetConfigOrDefault(
kOrtSessionOptionsDisableCPUEPFallback, "0") == "1";
if (disable_cpu_fallback)
ov_options["disable_cpu_fallback"] = "true";
}
std::shared_ptr<IExecutionProviderFactory> OpenVINOProviderFactoryCreator::Create(ProviderOptions* provider_options_map,
const SessionOptions* session_options) {
if (session_options)
onnxruntime::ORTSessionOptionsToOrtOpenVINOProviderOptions(*provider_options_map, session_options);
return s_library_openvino.Get().CreateExecutionProviderFactory(provider_options_map);
}
@ -2075,7 +2087,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_OpenVINO_V2,
provider_options[provider_options_keys[i]] = provider_options_values[i];
}
auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options);
auto factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options, &(options->value));
if (!factory) {
return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_OpenVINO_V2: Failed to load shared library");
}

3
onnxruntime/core/session/provider_registration.cc
View file

@ -108,11 +108,10 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider,
#endif
} else if (strcmp(provider_name, "OpenVINO") == 0) {
#if defined(USE_OPENVINO)
options->provider_factories.push_back(OpenVINOProviderFactoryCreator::Create(&provider_options));
options->provider_factories.push_back(OpenVINOProviderFactoryCreator::Create(&provider_options, &(options->value)));
#else
status = create_not_supported_status();
#endif
} else if (strcmp(provider_name, "SNPE") == 0) {
#if defined(USE_SNPE)
options->provider_factories.push_back(SNPEProviderFactoryCreator::Create(provider_options));

3
onnxruntime/python/onnxruntime_pybind_schema.cc
View file

@ -40,7 +40,8 @@ void addGlobalSchemaFunctions(pybind11::module& m) {
#ifdef USE_OPENVINO
[]() {
ProviderOptions provider_options_map;
return onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options_map);
SessionOptions session_options;
return onnxruntime::OpenVINOProviderFactoryCreator::Create(&provider_options_map, &session_options);
}(),
#endif
#ifdef USE_TENSORRT

2
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -1084,7 +1084,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
}
}
if (std::shared_ptr<IExecutionProviderFactory> openvino_provider_factory = onnxruntime::OpenVINOProviderFactoryCreator::Create(
&OV_provider_options_map)) {
&OV_provider_options_map, &session_options)) {
auto p = openvino_provider_factory->CreateProvider();
// Reset global variables config to avoid it being accidentally passed on to the next session
openvino_device_type.clear();

4
onnxruntime/python/onnxruntime_pybind_state_common.h
View file

@ -65,7 +65,11 @@ struct OrtStatus {
#elif OPENVINO_CONFIG_HETERO
#define BACKEND_OPENVINO "-OPENVINO_HETERO"
#elif OPENVINO_DISABLE_NPU_FALLBACK
#define BACKEND_OPENVINO "-OPENVINO_DISABLE_NPU_FALLBACK"
#endif
#else
#define BACKEND_OPENVINO ""
#endif

294
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -47,6 +47,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
const TestModelInfo& m)
: rand_engine_(rd()), input_names_(m.GetInputCount()), input_names_str_(m.GetInputCount()), input_length_(m.GetInputCount()) {
Ort::SessionOptions session_options;
provider_name_ = performance_test_config.machine_config.provider_type_name;
if (provider_name_ == onnxruntime::kDnnlExecutionProvider) {
#ifdef USE_DNNL
@ -221,150 +222,6 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
session_options.AppendExecutionProvider_CUDA(cuda_options);
#else
ORT_THROW("TensorRT is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) {
#ifdef USE_OPENVINO
#ifdef _MSC_VER
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
#else
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
#endif
std::unordered_map<std::string, std::string> ov_options;
std::istringstream ss(ov_string);
std::string token;
while (ss >> token) {
if (token == "") {
continue;
}
auto pos = token.find("|");
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option you are trying to use.\n");
}
auto key = token.substr(0, pos);
auto value = token.substr(pos + 1);
if (key == "device_type") {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) {
ov_options[key] = value;
} else if (deprecated_device_types.find(value) != deprecated_device_types.end()) {
ov_options[key] = value;
} else if (value.find("HETERO:") == 0) {
ov_options[key] = value;
} else if (value.find("MULTI:") == 0) {
ov_options[key] = value;
} else if (value.find("AUTO:") == 0) {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
" HETERO/MULTI/AUTO options available. \n");
}
} else if (key == "device_id") {
if (value == "CPU" || value == "GPU" || value == "NPU") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options.");
}
} else if (key == "precision") {
auto device_type = ov_options["device_type"];
if (device_type.find("GPU") != std::string::npos) {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
continue;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] Unsupported inference precision is selected. "
"GPU only supported FP32 / FP16. \n");
}
} else if (device_type.find("NPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP16") {
ov_options[key] = "FP16";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type.find("CPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}
} else if (key == "enable_npu_fast_compile") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_opencl_throttling") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_qdq_optimizer") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_qdq_optimizer' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "disable_dynamic_shapes") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "num_of_threads") {
if (std::stoi(value) <= 0) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n");
} else {
ov_options[key] = value;
}
} else if (key == "model_priority") {
ov_options[key] = value;
} else if (key == "cache_dir") {
ov_options[key] = value;
} else if (key == "context") {
ov_options[key] = value;
} else if (key == "num_streams") {
if (std::stoi(value) <= 0 && std::stoi(value) > 8) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n");
} else {
ov_options[key] = value;
}
} else if (key == "export_ep_ctx_blob") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
}
}
session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);
#else
ORT_THROW("OpenVINO is not supported in this build\n");
#endif
} else if (provider_name_ == onnxruntime::kQnnExecutionProvider) {
#ifdef USE_QNN
@ -716,7 +573,9 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
#else
ORT_THROW("VitisAI is not supported in this build\n");
#endif
} else if (!provider_name_.empty() && provider_name_ != onnxruntime::kCpuExecutionProvider) {
} else if (!provider_name_.empty() &&
provider_name_ != onnxruntime::kCpuExecutionProvider &&
provider_name_ != onnxruntime::kOpenVINOExecutionProvider) {
ORT_THROW("This backend is not included in perf test runner.\n");
}
@ -805,6 +664,151 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
}
}
}
if (provider_name_ == onnxruntime::kOpenVINOExecutionProvider) {
#ifdef USE_OPENVINO
#ifdef _MSC_VER
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
#else
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
#endif
std::unordered_map<std::string, std::string> ov_options;
std::istringstream ss(ov_string);
std::string token;
while (ss >> token) {
if (token == "") {
continue;
}
auto pos = token.find("|");
if (pos == std::string::npos || pos == 0 || pos == token.length()) {
ORT_THROW("[ERROR] [OpenVINO] Use a '|' to separate the key and value for the run-time option you are trying to use.\n");
}
auto key = token.substr(0, pos);
auto value = token.substr(pos + 1);
if (key == "device_type") {
std::set<std::string> ov_supported_device_types = {"CPU", "GPU",
"GPU.0", "GPU.1", "NPU"};
std::set<std::string> deprecated_device_types = {"CPU_FP32", "GPU_FP32",
"GPU.0_FP32", "GPU.1_FP32", "GPU_FP16",
"GPU.0_FP16", "GPU.1_FP16"};
if (ov_supported_device_types.find(value) != ov_supported_device_types.end()) {
ov_options[key] = value;
} else if (deprecated_device_types.find(value) != deprecated_device_types.end()) {
ov_options[key] = value;
} else if (value.find("HETERO:") == 0) {
ov_options[key] = value;
} else if (value.find("MULTI:") == 0) {
ov_options[key] = value;
} else if (value.find("AUTO:") == 0) {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. "
"Select from 'CPU', 'GPU', 'GPU.0', 'GPU.1', 'NPU' or from"
" HETERO/MULTI/AUTO options available. \n");
}
} else if (key == "device_id") {
if (value == "CPU" || value == "GPU" || value == "NPU") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported device_id is selected. Select from available options.");
}
} else if (key == "precision") {
auto device_type = ov_options["device_type"];
if (device_type.find("GPU") != std::string::npos) {
if (value == "") {
ov_options[key] = "FP16";
continue;
} else if (value == "ACCURACY" || value == "FP16" || value == "FP32") {
ov_options[key] = value;
continue;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] Unsupported inference precision is selected. "
"GPU only supported FP32 / FP16. \n");
}
} else if (device_type.find("NPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP16") {
ov_options[key] = "FP16";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. NPU only supported FP16. \n");
}
} else if (device_type.find("CPU") != std::string::npos) {
if (value == "" || value == "ACCURACY" || value == "FP32") {
ov_options[key] = "FP32";
continue;
} else {
ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
}
}
} else if (key == "enable_npu_fast_compile") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_opencl_throttling") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_opencl_throttling' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "enable_qdq_optimizer") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_qdq_optimizer' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "disable_dynamic_shapes") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'enable_dynamic_shapes' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "num_of_threads") {
if (std::stoi(value) <= 0) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_of_threads' should be greater than 0\n");
} else {
ov_options[key] = value;
}
} else if (key == "model_priority") {
ov_options[key] = value;
} else if (key == "cache_dir") {
ov_options[key] = value;
} else if (key == "context") {
ov_options[key] = value;
} else if (key == "num_streams") {
if (std::stoi(value) <= 0 && std::stoi(value) > 8) {
ORT_THROW("[ERROR] [OpenVINO] The value for the key 'num_streams' should be in the range of 1-8 \n");
} else {
ov_options[key] = value;
}
} else if (key == "export_ep_ctx_blob") {
if (value == "true" || value == "True" ||
value == "false" || value == "False") {
ov_options[key] = value;
} else {
ORT_THROW(
"[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' "
"should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
}
}
session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);
#else
ORT_THROW("OpenVINO is not supported in this build\n");
#endif
}
session_ = Ort::Session(env, performance_test_config.model_info.model_file_path.c_str(), session_options);

3
onnxruntime/test/util/default_providers.cc
View file

@ -109,7 +109,8 @@ std::unique_ptr<IExecutionProvider> OpenVINOExecutionProviderWithOptions(const O
std::unique_ptr<IExecutionProvider> DefaultOpenVINOExecutionProvider() {
#ifdef USE_OPENVINO
ProviderOptions provider_options_map;
return OpenVINOProviderFactoryCreator::Create(&provider_options_map)->CreateProvider();
SessionOptions session_options;
return OpenVINOProviderFactoryCreator::Create(&provider_options_map, &session_options)->CreateProvider();
#else
return nullptr;
#endif

2
tools/ci_build/build.py
View file

@ -79,6 +79,7 @@ def _openvino_verify_device_type(device_read):
"CPU_NO_PARTITION",
"GPU_NO_PARTITION",
"NPU_NO_PARTITION",
"NPU_NO_CPU_FALLBACK",
]
status_hetero = True
res = False
@ -1227,6 +1228,7 @@ def generate_build_tree(
if args.use_openvino:
cmake_args += [
"-Donnxruntime_USE_OPENVINO=ON",
"-Donnxruntime_NPU_NO_FALLBACK=" + ("ON" if args.use_openvino == "NPU_NO_CPU_FALLBACK" else "OFF"),
"-Donnxruntime_USE_OPENVINO_GPU=" + ("ON" if args.use_openvino == "GPU" else "OFF"),
"-Donnxruntime_USE_OPENVINO_CPU=" + ("ON" if args.use_openvino == "CPU" else "OFF"),
"-Donnxruntime_USE_OPENVINO_NPU=" + ("ON" if args.use_openvino == "NPU" else "OFF"),