saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-13 18:08:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Memory Optimization for Compilation in OVEP (#21872)

Browse source 
Calling Split API Calls Read+Model in lieu of unified Compile Model call
for export compile flow to ensure memory optimization. Freeing up model
proto and serialized string and read model ov ir later to free up memory
for the ahead pipeline
Optimization during EpCtxt flow
All the Graph related operations require all the Node Attributes to be
set while dealing with model instances internally with them, in the
existing implementation these attributes make a copy when constructing a
Graph dynamically during runtime.
Propose to use these attributes in place without creating a copy to
avoid memory allocation / copy while calling these Graph related
functions.
Changes to ensure the bug fixes related to openvino version and epctxt
file path.
Moving Compiler version to C++20 for getting r-value mem optimizations
benefit

### Motivation and Context
This change is required because memory optimization during Compilation
flow is too high.

---------

Co-authored-by: saurabhkale17 <saurabh1.kale@intel.com>
Co-authored-by: Preetha Veeramalai <preetha.veeramalai@intel.com>
Co-authored-by: Vishnudas Thaniel S <vishnudas.thaniel.s@intel.com>
Co-authored-by: Javier E. Martinez <javier.e.martinez@intel.com>
Co-authored-by: jatinwadhwa921 <110383850+jatinwadhwa921@users.noreply.github.com>
Co-authored-by: ankitm3k <ankit.maheshkar@intel.com>
Co-authored-by: jatinwadhwa921 <jatin.wadhwa@intel.com>
This commit is contained in:
sfatimar 2024-09-04 02:22:31 +05:30 committed by GitHub
parent 4962252c8f
commit 8dba8e3e24
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 248 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmake/onnxruntime_providers_openvino.cmake
View file

@ -36,6 +36,7 @@
onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx)
install(FILES ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino/openvino_provider_factory.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/)
set_target_properties(onnxruntime_providers_openvino PROPERTIES CXX_STANDARD 20)
set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX)
set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime")
if(NOT MSVC)

14
include/onnxruntime/core/graph/graph.h
View file

@ -571,6 +571,13 @@ class Node {
gsl::span<NodeArg* const> output_args,
const NodeAttributes* attributes,
std::string_view domain);
void Init(std::string_view name,
std::string_view op_type,
std::string_view description,
gsl::span<NodeArg* const> input_args,
gsl::span<NodeArg* const> output_args,
NodeAttributes&& attributes,
std::string_view domain);
#endif
#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@ -952,6 +959,13 @@ class Graph { // NOLINT(clang-analyzer-optin.performance.Padding): preserve exi
const NodeAttributes* attributes = nullptr,
const std::string& domain = kOnnxDomain);
Node& AddNode(const std::string& name,
const std::string& op_type,
const std::string& description,
gsl::span<NodeArg* const> input_args,
gsl::span<NodeArg* const> output_args,
NodeAttributes&& attributes,
const std::string& domain = kOnnxDomain);
Node& AddNode(const std::string& name,
const std::string& op_type,
const std::string& description,

65
onnxruntime/core/graph/graph.cc
View file

@ -915,6 +915,42 @@ void Node::Init(std::string_view name,
}
}
}
void Node::Init(std::string_view name,
std::string_view op_type,
std::string_view description,
gsl::span<NodeArg* const> input_args,
gsl::span<NodeArg* const> output_args,
NodeAttributes&& attributes,
std::string_view domain) {
name_ = name;
op_type_ = op_type;
description_ = description;
definitions_.input_defs.assign(input_args.begin(), input_args.end());
definitions_.output_defs.assign(output_args.begin(), output_args.end());
domain_ = domain;
can_be_saved_ = true;
priority_ = 0;
if (kOnnxDomainAlias == domain_) {
domain_ = kOnnxDomain;
}
// Set each arg count as 1 by default.
// It could be adjusted when resolving the node with its operator
// information.
definitions_.input_arg_count.assign(input_args.size(), 1);
attributes_ = std::move(attributes);
for (auto& name_to_attr : attributes_) {
if (utils::HasGraph(name_to_attr.second)) {
#if !defined(ORT_MINIMAL_BUILD)
CreateSubgraph(name_to_attr.first);
#else
ORT_THROW("Creating node with a subgraph via AddNode is not supported in this build.");
#endif
}
}
}
#endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@ -3923,6 +3959,35 @@ Node& Graph::AddNode(const std::string& name,
return *node;
}
Node& Graph::AddNode(const std::string& name,
const std::string& op_type,
const std::string& description,
gsl::span<NodeArg* const> input_args,
gsl::span<NodeArg* const> output_args,
NodeAttributes&& attributes,
const std::string& domain) {
InlinedVector<NodeArg*> inputs;
InlinedVector<NodeArg*> outputs;
inputs.resize(input_args.size());
outputs.resize(output_args.size());
int i = 0;
for (auto input_arg : input_args) {
inputs[i++] = &GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto());
}
i = 0;
for (auto output_arg : output_args) {
outputs[i++] = &GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto());
}
const gsl::not_null<Node*> node = AllocateNode();
node->Init(name, op_type, description, inputs, outputs, std::move(attributes), domain);
if (0 != op_type.compare(kNoOp)) {
GraphProtoSyncNeeded(true);
}
return *node;
}
bool Graph::RemoveNode(NodeIndex p_index) {
auto node = GetNode(p_index);
if (nullptr == node) {

77
onnxruntime/core/providers/openvino/backend_manager.cc
View file

@ -64,7 +64,7 @@ BackendManager::BackendManager(const GlobalContext& global_context,
i++;
}
subgraph_context_.subgraph_name = fused_node.Name();
model_proto_ = GetModelProtoFromFusedNode(fused_node, subgraph, logger);
auto model_proto = GetModelProtoFromFusedNode(fused_node, subgraph, logger);
std::string device_type = openvino_ep::BackendManager::GetGlobalContext().device_type;
if (ModelHasSymbolicInputDims(subgraph)) {
@ -73,22 +73,25 @@ BackendManager::BackendManager(const GlobalContext& global_context,
ORT_ENFORCE(!global_context_.enable_qdq_optimizer,
"QDQ stripping should not be enabled for models with dynamic input shapes. "
"Set enable_qdq_optimizer to False");
if (GetGlobalContext().device_type.find("CPU") != std::string::npos ||
GetGlobalContext().device_type.find("GPU") != std::string::npos) {
if (!GetGlobalContext().disable_dynamic_shapes) {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
<< "Creating backend Dynamic Shapes";
try {
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);
} catch (std::string const& msg) {
ORT_THROW(msg);
}
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
<< "Backend created for graph " << subgraph_context_.subgraph_name;
if ((GetGlobalContext().device_type.find("CPU") != std::string::npos ||
GetGlobalContext().device_type.find("GPU") != std::string::npos) &&
!GetGlobalContext().disable_dynamic_shapes) {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
<< "Creating backend Dynamic Shapes";
try {
concrete_backend_ = BackendFactory::MakeBackend(model_proto,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);
} catch (std::string const& msg) {
ORT_THROW(msg);
}
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
<< "Backend created for graph " << subgraph_context_.subgraph_name;
} else {
// Only cache model_proto in global to rewrite the model with input shapes at runtime.
// For dynamic backend creation
model_proto_ = std::move(model_proto);
}
} else {
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has concrete input dims. "
@ -99,7 +102,7 @@ BackendManager::BackendManager(const GlobalContext& global_context,
// OV NPU plugin is supported with fallback to OV CPU upon compilation failures.
try {
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
concrete_backend_ = BackendFactory::MakeBackend(model_proto,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);
@ -115,7 +118,7 @@ BackendManager::BackendManager(const GlobalContext& global_context,
GetGlobalContext().device_type = "CPU";
GetGlobalContext().precision_str = "FP32";
try {
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
concrete_backend_ = BackendFactory::MakeBackend(model_proto,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);
@ -155,32 +158,28 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
auto compiled_model = concrete_backend_->GetOVCompiledModel();
std::string graph_name = "";
// Epctx file path from SO is mapped to cache_dir variable for OVEP for readability
if (global_context_.cache_dir != "") {
if (!global_context_.cache_dir.empty()) {
graph_name = global_context_.cache_dir;
} else {
graph_name = global_context_.onnx_model_path_name;
// Remove extension so we can append suffix to form the complete name of output graph
graph_name = [&]() {
size_t dot = graph_name.find_last_of(".");
if (dot == std::string::npos) return graph_name;
return graph_name.substr(0, dot);
}();
graph_name = graph_name + "_ctx.onnx";
size_t dot = global_context_.onnx_model_path_name.find_last_of(".");
graph_name = graph_name.substr(0, dot);
if (dot != std::string::npos) graph_name += "_ctx.onnx";
}
// If embed_mode, then pass on the serialized blob
// If not embed_mode, dump the blob here and only pass on the path to the blob
if (global_context_.ep_context_embed_mode) {
std::ostringstream model_blob_stream;
compiled_model.export_model(model_blob_stream);
model_blob_str = model_blob_stream.str();
ORT_ENFORCE(model_blob_str.size() != 0);
model_blob_str = std::move(model_blob_stream).str();
if (model_blob_str.empty()) {
ORT_THROW("Model blob stream is empty after exporting the compiled model.");
}
} else {
// Remove extension so we can append suffix to form the complete name of output graph
auto blob_name = [&]() {
size_t dot = graph_name.find_last_of(".");
if (dot == std::string::npos) return graph_name;
return graph_name.substr(0, dot);
}();
auto blob_name = graph_name.substr(0, graph_name.find_last_of("."));
std::ofstream blob_file(blob_name + ".blob",
std::ios::out | std::ios::trunc | std::ios::binary);
if (!blob_file) {
@ -194,7 +193,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
graph_name,
logger,
global_context_.ep_context_embed_mode,
model_blob_str,
std::move(model_blob_str),
openvino_sdk_version_));
return Status::OK();
@ -365,10 +364,10 @@ std::string MakeMapKeyString(const std::vector<std::vector<int64_t>>& shapes,
return key;
}
std::shared_ptr<ONNX_NAMESPACE::ModelProto>
std::unique_ptr<ONNX_NAMESPACE::ModelProto>
BackendManager::ReWriteInputShapeInfo(const ONNX_NAMESPACE::ModelProto& model_proto,
const std::vector<std::vector<int64_t>>& input_shapes) {
auto model_copy = std::shared_ptr<ONNX_NAMESPACE::ModelProto>(ONNX_NAMESPACE::ModelProto::Create());
auto model_copy = ONNX_NAMESPACE::ModelProto::Create();
std::string proto_str;
model_proto.SerializeToString(proto_str);
model_copy->ParseFromString(proto_str);
@ -422,14 +421,12 @@ void BackendManager::Compute(OrtKernelContext* context) {
// if disable_dynamic_shapes is set to true then execution of dynamic model is done
// by rewriting the model to static shaped model at runtime based on input shape.
// disable_dynamic_shapes is always set to true for OV NPU plugin.
bool use_dynamic_backend = true;
if (subgraph_context_.has_dynamic_input_shape &&
!GetGlobalContext().disable_dynamic_shapes &&
(GetGlobalContext().device_type.find("CPU") != std::string::npos ||
GetGlobalContext().device_type.find("GPU") != std::string::npos)) {
concrete_backend_->Infer(context);
use_dynamic_backend = false;
} else if (use_dynamic_backend && subgraph_context_.has_dynamic_input_shape) {
} else if (subgraph_context_.has_dynamic_input_shape) {
std::vector<std::vector<int64_t>> tensor_shapes = GetInputTensorShapes(ctx);
auto key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type);
std::shared_ptr<IBackend> dynamic_backend;
@ -441,7 +438,7 @@ void BackendManager::Compute(OrtKernelContext* context) {
<< "Backend created for graph " << subgraph_context_.subgraph_name;
auto modelproto_with_concrete_shapes = ReWriteInputShapeInfo(*model_proto_, tensor_shapes);
try {
dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes,
dynamic_backend = BackendFactory::MakeBackend(modelproto_with_concrete_shapes,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);
@ -460,7 +457,7 @@ void BackendManager::Compute(OrtKernelContext* context) {
GetGlobalContext().precision_str = "FP32";
key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type);
try {
dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes,
dynamic_backend = BackendFactory::MakeBackend(modelproto_with_concrete_shapes,
GetGlobalContext(),
subgraph_context_,
ep_ctx_handle_);

2
onnxruntime/core/providers/openvino/backend_manager.h
View file

@ -43,7 +43,7 @@ class BackendManager {
std::shared_ptr<ONNX_NAMESPACE::ModelProto>
ReWriteBatchDimWithOne(const ONNX_NAMESPACE::ModelProto& model_proto);
std::shared_ptr<ONNX_NAMESPACE::ModelProto>
std::unique_ptr<ONNX_NAMESPACE::ModelProto>
ReWriteInputShapeInfo(const ONNX_NAMESPACE::ModelProto& model_proto,
const std::vector<std::vector<int64_t>>& input_shapes);

2
onnxruntime/core/providers/openvino/backends/backend_factory.cc
View file

@ -11,7 +11,7 @@ namespace onnxruntime {
namespace openvino_ep {
std::shared_ptr<IBackend>
BackendFactory::MakeBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
BackendFactory::MakeBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
GlobalContext& global_context,
const SubGraphContext& subgraph_context,
EPCtxHandler& ep_ctx_handle) {

40
onnxruntime/core/providers/openvino/backends/basic_backend.cc
View file

@ -20,7 +20,7 @@ namespace openvino_ep {
using namespace backend_utils;
BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
BasicBackend::BasicBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
GlobalContext& global_context,
const SubGraphContext& subgraph_context,
EPCtxHandler& ep_ctx_handle)
@ -52,7 +52,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
if (IsDebugEnabled()) {
std::string file_name = subgraph_context.subgraph_name + "_static.onnx";
std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary);
model_proto.SerializeToOstream(outfile);
model_proto->SerializeToOstream(outfile);
}
#endif
@ -66,7 +66,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
exe_network_ = global_context_.ie_core.ImportModel(model_stream,
remote_context_,
subgraph_context_.subgraph_name);
ie_cnn_network_ = exe_network_.Get().get_runtime_model();
} else if ((global_context.device_type.find("GPU") != std::string::npos) &&
(global_context_.context != nullptr)) {
LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled";
@ -75,7 +74,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
exe_network_ = global_context_.ie_core.CompileModel(
ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name);
ie_cnn_network_ = exe_network_.Get().get_runtime_model();
} else {
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
exe_network_ = global_context_.ie_core.CompileModel(
@ -91,26 +89,36 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
device_config,
global_context_.ep_context_embed_mode,
subgraph_context_.subgraph_name);
ie_cnn_network_ = exe_network_.Get().get_runtime_model();
// ie_cnn_network_ = exe_network_.Get().get_runtime_model();
} else if (global_context_.export_ep_ctx_blob &&
hw_target.find("NPU") != std::string::npos) {
std::shared_ptr<ov::Model> ov_model;
{
const std::string model = model_proto->SerializeAsString();
if (!subgraph_context.has_dynamic_input_shape) {
delete model_proto.release();
}
ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor());
}
exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config));
} else if ((!subgraph_context_.has_dynamic_input_shape) &&
((hw_target.find("AUTO") == std::string::npos) ||
(global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) {
// Optimized OV compile_model API is supported with AUTO from version 2024.3 and above
// Inputs with static dimenstions
const std::string model = model_proto.SerializeAsString();
const std::string model = model_proto->SerializeAsString();
exe_network_ = global_context_.ie_core.CompileModel(model,
hw_target,
device_config,
subgraph_context_.subgraph_name);
ie_cnn_network_ = exe_network_.Get().get_runtime_model();
} else { // For all other types use ov::Model Type
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_);
ie_cnn_network_ = CreateOVModel(*model_proto, global_context_, const_outputs_map_);
exe_network_ = global_context_.ie_core.CompileModel(
ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name);
}
#endif
} else { // Full graph is not supported
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_);
ie_cnn_network_ = CreateOVModel(*model_proto, global_context_, const_outputs_map_);
exe_network_ = global_context_.ie_core.CompileModel(
ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name);
}
@ -270,14 +278,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque
input_tensor_shape[tensor_iter] = *i;
tensor_iter += 1;
}
auto input = ie_cnn_network_->get_parameters().at(input_idx);
auto input = graph_input_info.at(input_idx);
OVTensorPtr tensor_ptr;
// avoid input copies on the CPU device
if (global_context_.device_type.find("CPU") != std::string::npos) {
tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape,
tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape,
(void*)tensor_data);
} else {
tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape);
tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape);
FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_);
}
@ -341,9 +349,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
const void* tensor_data = tensor.GetTensorRawData();
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
// Create an Input Remote Blob
auto input = ie_cnn_network_->get_parameters().at(0);
auto input = graph_input_info.at(0);
auto remote_blob = remote_context_->create_tensor(
input->get_element_type(), input->get_shape(), *shared_buffer_const);
input.get_element_type(), input.get_shape(), *shared_buffer_const);
ov::Tensor tensor_remote = static_cast<ov::Tensor>(remote_blob);
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_remote);
infer_request->SetTensor(input_name, tensor_ptr);
@ -392,9 +400,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
const void* tensor_data = tensor.GetTensorRawData();
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
// Create a shared Blob, set the Infer Request Output Blob
auto output = ie_cnn_network_->get_results().at(0);
auto output = graph_output_info.at(0);
auto remote_tensor =
remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const);
remote_context_->create_tensor(output.get_element_type(), output.get_shape(), *shared_buffer_const);
ov::Tensor tensor_t = static_cast<ov::Tensor>(remote_tensor);
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_t);
try {

2
onnxruntime/core/providers/openvino/backends/basic_backend.h
View file

@ -23,7 +23,7 @@ namespace openvino_ep {
class InferRequestsQueue;
class BasicBackend : public IBackend {
public:
BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
BasicBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
GlobalContext& global_context,
const SubGraphContext& subgraph_context,
EPCtxHandler& ep_ctx_handle);

2
onnxruntime/core/providers/openvino/ibackend.h
View file

@ -20,7 +20,7 @@ class IBackend {
class BackendFactory {
public:
static std::shared_ptr<IBackend>
MakeBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
MakeBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_proto,
GlobalContext& global_context,
const SubGraphContext& subgraph_context,
EPCtxHandler& ctx_handle);

106
onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
View file

@ -4,6 +4,7 @@
#include <string>
#include <fstream>
#include <vector>
#include <algorithm>
#include "core/providers/openvino/onnx_ctx_model_helper.h"
@ -18,71 +19,76 @@ Status EPCtxHandler::ExportEPCtxModel(const GraphViewer& graph_viewer,
const std::string& graph_name,
const logging::Logger& logger,
const bool& ep_context_embed_mode,
const std::string& model_blob_str,
std::string&& model_blob_str,
const std::string& openvino_sdk_version) const {
auto model_build = graph_viewer.CreateModel(logger);
auto& graph_build = model_build->MainGraph();
// Get graph inputs and outputs
std::vector<onnxruntime::NodeArg*> inputs, outputs;
for (auto input : graph_viewer.GetInputs()) {
auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
inputs.push_back(&n_input);
}
for (auto output : graph_viewer.GetOutputs()) {
auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
outputs.push_back(&n_output);
}
const auto& viewer_inputs = graph_viewer.GetInputs();
const auto& viewer_outputs = graph_viewer.GetOutputs();
std::vector<onnxruntime::NodeArg*> inputs(viewer_inputs.size()), outputs(viewer_outputs.size());
auto transform_f = [&](const onnxruntime::NodeArg* iter) { return &graph_build.GetOrCreateNodeArg(iter->Name(), iter->TypeAsProto()); };
auto fill_vectors = [transform_f](auto& src, auto& dst) {
std::transform(src.begin(), src.end(), dst.begin(), transform_f);
};
fill_vectors(viewer_inputs, inputs);
fill_vectors(viewer_outputs, outputs);
// Create EP context node attributes
auto attr_0 = ONNX_NAMESPACE::AttributeProto::Create();
auto attr_1 = ONNX_NAMESPACE::AttributeProto::Create();
auto attr_2 = ONNX_NAMESPACE::AttributeProto::Create();
auto attr_3 = ONNX_NAMESPACE::AttributeProto::Create();
// embed mode
attr_0->set_name(EMBED_MODE);
attr_0->set_type(onnx::AttributeProto_AttributeType_INT);
attr_0->set_i(ep_context_embed_mode);
// ep context
attr_1->set_name(EP_CACHE_CONTEXT);
attr_1->set_type(onnx::AttributeProto_AttributeType_STRING);
attr_1->set_s(model_blob_str);
// sdk version
attr_2->set_name(EP_SDK_VER);
attr_2->set_type(onnx::AttributeProto_AttributeType_STRING);
attr_2->set_s(openvino_sdk_version);
// source
attr_3->set_name(SOURCE);
attr_3->set_type(onnx::AttributeProto_AttributeType_STRING);
attr_3->set_s(kOpenVINOExecutionProvider);
auto node_attributes = ONNX_NAMESPACE::NodeAttributes::Create();
node_attributes->reserve(4);
node_attributes->emplace(EMBED_MODE, *attr_0);
node_attributes->emplace(EP_CACHE_CONTEXT, *attr_1);
node_attributes->emplace(EP_SDK_VER, *attr_2);
node_attributes->emplace(SOURCE, *attr_3);
{
// Create EP context node attributes
// embed mode
auto embed_mode_attr = ONNX_NAMESPACE::AttributeProto::Create();
embed_mode_attr->set_name(EMBED_MODE);
embed_mode_attr->set_type(onnx::AttributeProto_AttributeType_INT);
embed_mode_attr->set_i(ep_context_embed_mode);
node_attributes->emplace(EMBED_MODE, std::move(*embed_mode_attr));
// ep context
auto ep_cache_context_attr = ONNX_NAMESPACE::AttributeProto::Create();
ep_cache_context_attr->set_name(EP_CACHE_CONTEXT);
ep_cache_context_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
ep_cache_context_attr->set_s(std::move(model_blob_str));
node_attributes->emplace(EP_CACHE_CONTEXT, std::move(*ep_cache_context_attr));
// sdk version
auto sdk_version_attr = ONNX_NAMESPACE::AttributeProto::Create();
sdk_version_attr->set_name(EP_SDK_VER);
sdk_version_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
sdk_version_attr->set_s(openvino_sdk_version);
node_attributes->emplace(EP_SDK_VER, std::move(*sdk_version_attr));
// source
auto source_attr = ONNX_NAMESPACE::AttributeProto::Create();
source_attr->set_name(SOURCE);
source_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
source_attr->set_s(kOpenVINOExecutionProvider);
node_attributes->emplace(SOURCE, std::move(*source_attr));
}
// Create EP context node
graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, node_attributes.get(), kMSDomain);
graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, std::move(*node_attributes), kMSDomain);
ORT_ENFORCE(graph_build.Resolve().IsOK());
// Serialize modelproto to string
auto new_graph_viewer = graph_build.CreateGraphViewer();
auto model = new_graph_viewer->CreateModel(logger);
auto model_proto = model->ToProto();
new_graph_viewer->ToProto(*model_proto->mutable_graph(), true, true);
model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
{
// Serialize modelproto to string
auto model_proto = model_build->ToProto();
model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
// Finally, dump the model
std::ofstream epctx_onnx_model(graph_name,
std::ios::out | std::ios::trunc | std::ios::binary);
if (!epctx_onnx_model) {
ORT_THROW("Unable to create epctx onnx model file ");
// Finally, dump the model
std::ofstream epctx_onnx_model(graph_name,
std::ios::out | std::ios::trunc | std::ios::binary);
if (!epctx_onnx_model) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unable to create epctx onnx model file");
}
if (!model_proto->SerializeToOstream(epctx_onnx_model)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to serialize model to file");
}
}
model_proto->SerializeToOstream(epctx_onnx_model);
LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Export blob as EPContext Node";
return Status::OK();

2
onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h
View file

@ -28,7 +28,7 @@ class EPCtxHandler {
const std::string& graph_name,
const logging::Logger& logger,
const bool& ep_context_embed_mode,
const std::string& model_blob_str,
std::string&& model_blob_str,
const std::string& openvino_sdk_version) const;
Status ImportBlobFromEPCtxModel(const GraphViewer& graph_viewer);
bool CheckForOVEPCtxNode(const GraphViewer& graph_viewer, std::string openvino_sdk_version) const;

40
onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file

@ -30,10 +30,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
so_epctx_embed_mode_(so_epctx_embed_mode) {
device_type_ = (device_type == nullptr) ? "" : device_type;
cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir;
if (cache_dir != nullptr) {
free(const_cast<void*>(static_cast<const void*>(cache_dir)));
cache_dir = nullptr;
}
}
~OpenVINOProviderFactory() override {
@ -94,7 +90,7 @@ struct OpenVINO_Provider : Provider {
// speeds up the model's compilation to NPU device specific format.
int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of
// threads with this value at runtime.
const char* cache_dir = nullptr; // [cache_dir]: specify the path to
std::string cache_dir = ""; // [cache_dir]: specify the path to
// dump and load the blobs for the model caching/kernel caching (GPU)
// feature. If blob files are already present, it will be directly loaded.
const char* model_priority = "DEFAULT"; // High-level OpenVINO model priority hint
@ -186,7 +182,7 @@ struct OpenVINO_Provider : Provider {
}
if (provider_options_map.find("cache_dir") != provider_options_map.end()) {
cache_dir = provider_options_map.at("cache_dir").c_str();
cache_dir = provider_options_map.at("cache_dir");
}
if (provider_options_map.find("context") != provider_options_map.end()) {
@ -305,26 +301,20 @@ struct OpenVINO_Provider : Provider {
if (provider_options_map.find("so_epctx_path") != provider_options_map.end()) {
// The path to dump epctx model is valid only when epctx is enabled.
// Overrides the cache_dir option to dump model cache files from OV.
if (export_ep_ctx_blob) {
auto ep_context_file_path_ = provider_options_map.at("so_epctx_path");
auto file_path = std::filesystem::path(ep_context_file_path_);
if (export_ep_ctx_blob &&
!provider_options_map.at("so_epctx_path").empty()) {
cache_dir = provider_options_map.at("so_epctx_path");
auto file_path = std::filesystem::path(cache_dir);
// ep_context_file_path_ file extension must be .onnx
if (!ep_context_file_path_.empty()) {
if (file_path.extension().generic_string() == ".onnx") {
// ep_context_file_path_ must be provided as a directory, create it if doesn't exist
auto parent_path = file_path.parent_path();
if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) &&
!std::filesystem::create_directory(parent_path)) {
ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n");
}
#ifdef _WIN32
cache_dir = _strdup(ep_context_file_path_.c_str());
#else
cache_dir = strdup(ep_context_file_path_.c_str());
#endif
} else {
ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + ep_context_file_path_ + " \n");
if (file_path.extension().generic_string() == ".onnx") {
// ep_context_file_path_ must be provided as a directory, create it if doesn't exist
auto parent_path = file_path.parent_path();
if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) &&
!std::filesystem::create_directory(parent_path)) {
ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n");
}
} else {
ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir + " \n");
}
}
}
@ -333,7 +323,7 @@ struct OpenVINO_Provider : Provider {
const_cast<char*>(precision.c_str()),
enable_npu_fast_compile,
num_of_threads,
cache_dir,
const_cast<char*>(cache_dir.c_str()),
model_priority,
num_streams,
context,

3
onnxruntime/core/providers/shared_library/provider_interfaces.h
View file

@ -381,6 +381,7 @@ struct ProviderHost {
virtual float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) = 0;
virtual const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) = 0;
virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) = 0;
virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) = 0;
virtual void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) = 0;
virtual void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) = 0;
virtual void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0;
@ -858,6 +859,7 @@ struct ProviderHost {
virtual std::unique_ptr<NodeAttributes_Iterator> NodeAttributes__find(const NodeAttributes* p, const std::string& key) = 0;
virtual void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) = 0;
virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0;
virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) = 0;
virtual void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0;
virtual void NodeAttributes__reserve(NodeAttributes* p, size_t size) = 0;
@ -909,6 +911,7 @@ struct ProviderHost {
virtual Status Graph__Resolve(Graph* p) = 0;
virtual void Graph__AddInitializedTensor(Graph* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0;
virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, const NodeAttributes* attributes, const std::string& domain) = 0;
virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, NodeAttributes&& attributes, const std::string& domain) = 0;
virtual Node& Graph__AddNode(Graph* p, const Node& other) = 0;
virtual const std::vector<const NodeArg*>& Graph__GetOutputs(const Graph* p) noexcept = 0;

3
onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
View file

@ -114,6 +114,7 @@ struct AttributeProto final {
float f() const { return g_host->AttributeProto__f(this); }
const ONNX_NAMESPACE::TensorProto& t() const { return g_host->AttributeProto__t(this); }
void set_s(const ::std::string& value) { return g_host->AttributeProto__set_s(this, value); }
void set_s(::std::string&& value) { return g_host->AttributeProto__set_s(this, ::std::move(value)); }
void set_f(const float& value) { return g_host->AttributeProto__set_f(this, value); }
void set_i(int64_t value) { return g_host->AttributeProto__set_i(this, value); }
void set_t(const TensorProto& value) { return g_host->AttributeProto__set_t(this, value); }
@ -872,6 +873,7 @@ struct NodeAttributes final {
IteratorHolder<NodeAttributes_Iterator, std::pair<const std::string, ONNX_NAMESPACE::AttributeProto>> find(const std::string& key) const { return g_host->NodeAttributes__find(this, key); }
void insert(const NodeAttributes& v) { return g_host->NodeAttributes__insert(this, v); }
void emplace(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__emplace(this, k, v); }
void emplace(const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) { g_host->NodeAttributes__emplace(this, k, std::move(v)); }
void insert_or_assign(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__insert_or_assign(this, k, v); }
void reserve(size_t size) { g_host->NodeAttributes__reserve(this, size); }
@ -957,6 +959,7 @@ struct Graph final {
Status Resolve() { return g_host->Graph__Resolve(this); }
void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor) { return g_host->Graph__AddInitializedTensor(this, tensor); }
Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span<NodeArg* const> input_args, gsl::span<NodeArg* const> output_args, const NodeAttributes* attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, attributes, domain); }
Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span<NodeArg* const> input_args, gsl::span<NodeArg* const> output_args, NodeAttributes&& attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, std::move(attributes), domain); }
Node& AddNode(const Node& other) { return g_host->Graph__AddNode(this, other); }
const std::vector<const NodeArg*>& GetOutputs() const noexcept { return g_host->Graph__GetOutputs(this); }

5
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -487,6 +487,7 @@ struct ProviderHostImpl : ProviderHost {
float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) override { return p->f(); }
const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) override { return p->t(); }
void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) override { return p->set_s(value); }
void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) override { return p->set_s(::std::move(value)); }
void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) override { return p->set_f(value); }
void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) override { return p->set_i(value); }
void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& value) override { *p->mutable_t() = value; }
@ -999,6 +1000,7 @@ struct ProviderHostImpl : ProviderHost {
}
void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) override { return p->insert(v.begin(), v.end()); }
void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->emplace(k, v); }
void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) override { p->emplace(k, std::move(v)); }
void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->insert_or_assign(k, v); }
void NodeAttributes__reserve(NodeAttributes* p, size_t size) override { p->reserve(size); }
@ -1076,6 +1078,9 @@ struct ProviderHostImpl : ProviderHost {
Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, const NodeAttributes* attributes, const std::string& domain) override {
return p->AddNode(name, op_type, description, input_args, output_args, attributes, domain);
}
Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, NodeAttributes&& attributes, const std::string& domain) override {
return p->AddNode(name, op_type, description, input_args, output_args, ::std::move(attributes), domain);
}
Node& Graph__AddNode(Graph* p, const Node& other) override {
return p->AddNode(other);
}

16
onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc
View file

@ -151,7 +151,8 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul2D_U8U8) {
{168, 115, 255,
1, 66, 151});
test.Run();
// Skip OpenVINOP EP for now as there are Accuracy Mismatches
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider});
};
run_test(false);
@ -242,7 +243,11 @@ static void QLinearMatMul2DTest(bool only_t1_not_initializer) {
test_non_empty.AddInput<float>("y_scale", {1}, {0.0107f}, only_t1_not_initializer);
test_non_empty.AddInput<uint8_t>("y_zero_point", {1}, {118}, only_t1_not_initializer);
test_non_empty.AddOutput<uint8_t>("T3", {2, 3}, {168, 115, 255, 1, 66, 151});
test_non_empty.Run();
if (only_t1_not_initializer == true) {
test_non_empty.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider});
} else {
test_non_empty.Run();
}
// Test with an empty input
OpTester test_empty("QLinearMatMul", 10);
@ -257,7 +262,12 @@ static void QLinearMatMul2DTest(bool only_t1_not_initializer) {
test_empty.AddOutput<uint8_t>("T3", {0, 3}, {});
// Skip NNAPI as it doesn't support empty output for now
test_empty.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider});
// Skip OpenVINO EP as there are accuracy mismatches for OpenVINO
if (only_t1_not_initializer == true) {
test_empty.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kNnapiExecutionProvider});
} else {
test_empty.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider});
}
}
TEST(QuantizeLinearMatmulOpTest, QLinearMatMul) {

3
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -1694,8 +1694,9 @@ class TestInferenceSession(unittest.TestCase):
available_eps = C.get_available_providers()
# skip amd gpu build
if "ROCMExecutionProvider" in available_eps:
if "RocmExecutionProvider" in available_eps:
return
if sys.platform.startswith("win"):
shared_library = "test_execution_provider.dll"

2
tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml
View file

@ -33,5 +33,5 @@ jobs:
parameters:
AgentPool : 'Linux-CPU-2019'
JobName: 'Linux_CI_Dev'
RunDockerBuildArgs: '-o ubuntu22.04 -p 3.10 -d openvino -v 2024.0.0 -x "--use_openvino CPU --build_wheel"'
RunDockerBuildArgs: '-o ubuntu22.04 -p 3.10 -d openvino -v 2024.3.0 -x "--use_openvino CPU --build_wheel"'
TimeoutInMinutes: 120

8
tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
View file

@ -1,7 +1,7 @@
ARG UBUNTU_VERSION=22.04
FROM ubuntu:${UBUNTU_VERSION}
ARG OPENVINO_VERSION=2024.0.0
ARG OPENVINO_VERSION=2024.3.0
ARG PYTHON_VERSION=3.10
ADD scripts /tmp/scripts
@ -19,9 +19,9 @@ ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64
ENV DEBIAN_FRONTEND=noninteractive
RUN cd /opt && mkdir -p intel && cd intel && \
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/linux/l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \
tar xzf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \
mv l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64 openvino_2024.0.0 && \
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \
tar xzf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \
mv l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64 openvino_2024.3.0 && \
cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y
WORKDIR /root