saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added OpenVINO 2021.4 support (#7470)

Browse source 
* Added OpenVINO 2021.4 support

* Added OPENVINO_2021_4 handling
This commit is contained in:
Ilya Lavrenov 2021-04-29 22:25:04 +03:00 committed by GitHub
parent 7b003967b1
commit 6358e96b63
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 81 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmake/CMakeLists.txt
View file

@ -1280,6 +1280,9 @@ if(onnxruntime_USE_OPENVINO)
elseif (${VER} MATCHES "2021.3" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2021.3")
set(OPENVINO_VERSION "2021.3")
add_definitions(-DOPENVINO_2021_3=1)
elseif (${VER} MATCHES "2021.4" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2021.4")
set(OPENVINO_VERSION "2021.4")
add_definitions(-DOPENVINO_2021_4=1)
else()
message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}")
endif()

23
onnxruntime/core/providers/openvino/backend_utils.cc
View file

@ -9,6 +9,12 @@
#include <inference_engine.hpp>
#ifdef OPENVINO_2021_4
using Exception = InferenceEngine::Exception;
#else
using Exception = InferenceEngine::details::InferenceEngineException;
#endif
#include <ngraph/frontend/onnx_import/onnx.hpp>
#include <ngraph/pass/convert_fp32_to_fp16.hpp>
#include <ngraph/pass/constant_folding.hpp>
@ -122,7 +128,8 @@ CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalCont
ng_function->validate_nodes_and_infer_types();
}
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || (defined OPENVINO_2021_3)
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || \
(defined OPENVINO_2021_3) || (defined OPENVINO_2021_4)
if (!global_context.is_wholly_supported_graph) {
std::map<std::string, std::string> result_to_output;
for (auto& result : ng_function->get_results()) {
@ -144,7 +151,7 @@ CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalCont
try {
return std::make_shared<InferenceEngine::CNNNetwork>(ng_function);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception thrown while making IE::CNNNetwork: " + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception thrown while making IE::CNNNetwork");
@ -203,7 +210,8 @@ void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto,
auto outputInfo = network->getOutputsInfo();
for (auto iter = outputInfo.begin(); iter != outputInfo.end(); ++iter) {
auto output_name = iter->first;
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || (defined OPENVINO_2021_3)
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || \
(defined OPENVINO_2021_3) || (defined OPENVINO_2021_4)
auto it = const_outputs_map.find(output_name);
//Output is constant and don't need to set precision
if (it != const_outputs_map.end())
@ -247,7 +255,8 @@ GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_s
return output_tensor;
}
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || (defined OPENVINO_2021_3)
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || \
(defined OPENVINO_2021_3) || (defined OPENVINO_2021_4)
OrtValue*
GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context,
std::string output_name,
@ -295,7 +304,8 @@ int GetFirstAvailableDevice(GlobalContext& global_context) {
return i;
}
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || (defined OPENVINO_2021_3)
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || \
(defined OPENVINO_2021_3) || (defined OPENVINO_2021_4)
void FillOutputsWithConstantData(Ort::CustomOpApi& ort, std::shared_ptr<ngraph::Node> node, OrtValue* out_tensor) {
switch (node->get_element_type()) {
case ngraph::element::Type_t::f32: {
@ -320,7 +330,8 @@ void FillOutputsWithConstantData(Ort::CustomOpApi& ort, std::shared_ptr<ngraph::
}
#endif
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || (defined OPENVINO_2021_3)
#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) || (defined OPENVINO_2021_2) || \
(defined OPENVINO_2021_3) || (defined OPENVINO_2021_4)
template <typename T>
void FillOutputHelper(Ort::CustomOpApi& ort, OrtValue* out_tensor, std::shared_ptr<ngraph::Node> node) {
auto const_node = std::dynamic_pointer_cast<ngraph::op::Constant>(node);

3
onnxruntime/core/providers/openvino/backend_utils.h
View file

@ -48,7 +48,8 @@ CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalCont
int GetFirstAvailableDevice(GlobalContext& global_context);
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
void FillOutputsWithConstantData(Ort::CustomOpApi& ort, std::shared_ptr<ngraph::Node> node, OrtValue* out_tensor);
template <typename T>

37
onnxruntime/core/providers/openvino/backends/basic_backend.cc
View file

@ -9,6 +9,14 @@
#include <inference_engine.hpp>
#ifdef OPENVINO_2021_4
using Exception = InferenceEngine::Exception;
using WaitMode = InferenceEngine::InferRequest::WaitMode;
#else
using Exception = InferenceEngine::details::InferenceEngineException;
using WaitMode = InferenceEngine::IInferRequest::WaitMode;
#endif
#include "core/providers/shared_library/provider_api.h"
#include "../backend_utils.h"
@ -70,7 +78,8 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
if(!openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph) {
ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_);
SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type);
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
if (const_outputs_map_.size() == subgraph_context_.output_names.size())
subgraph_context_.is_constant = true;
#endif
@ -95,7 +104,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
throw std::runtime_error("The compiled blob path is not set");
exe_network_ = global_context_.ie_core.ImportNetwork(compiled_blob_path, hw_target, {});
}
} catch (InferenceEngine::details::InferenceEngineException &e) {
} catch (Exception &e) {
ORT_THROW(log_tag + " Exception while Importing Network for graph: " + subgraph_context_.subgraph_name + ": " + e.what());
} catch(...) {
ORT_THROW(log_tag + " Exception while Importing Network for graph: " + subgraph_context_.subgraph_name);
@ -108,7 +117,8 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
if(!openvino_ep::backend_utils::UseCompiledNetwork()) {
ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_);
SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type);
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
if (const_outputs_map_.size() == subgraph_context_.output_names.size())
subgraph_context_.is_constant = true;
#endif
@ -123,7 +133,8 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
}
#endif
if (global_context_.device_type.find("MYRIAD") != std::string::npos) {
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
if (subgraph_context_.set_vpu_config) {
config["MYRIAD_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO);
}
@ -146,7 +157,7 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
}
try {
exe_network_ = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, hw_target, config);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + ": " + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
@ -185,7 +196,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
try {
graph_input_blob = infer_request->GetBlob(input_name);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name);
@ -197,7 +208,7 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
// Start Async inference
try {
infer_request->StartAsync();
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
} catch (...) {
ORT_THROW(log_tag + " Couldn't start Inference");
@ -209,8 +220,8 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext*
void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, std::shared_ptr<InferenceEngine::InferRequest> infer_request) {
// Wait for Async inference completion
try {
infer_request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
infer_request->Wait(WaitMode::RESULT_READY);
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception with completing Inference" + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception with completing Inference");
@ -224,7 +235,7 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex
auto output_name = output_info_iter->first;
try {
graph_output_blob = infer_request->GetBlob(output_name);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name);
@ -236,7 +247,8 @@ void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContex
size_t batch_slice = 0;
FillOutputBlob(graph_output_blob, output_tensor, ort, precision, batch_slice);
}
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
if (!const_outputs_map_.empty()) {
for (auto item : const_outputs_map_) {
auto out_name = item.first;
@ -256,7 +268,8 @@ void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) {
LOGS_DEFAULT(INFO) << log_tag << "In Infer";
if (subgraph_context_.is_constant) {
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
for (auto item : const_outputs_map_) {
auto out_name = item.first;
auto node = item.second;

3
onnxruntime/core/providers/openvino/backends/basic_backend.h
View file

@ -12,6 +12,7 @@
#include "core/providers/openvino/ibackend.h"
#include <vector>
#include <iostream>
#include <string>
#include <condition_variable>
#include <mutex>
@ -47,7 +48,7 @@ class InferRequestsQueue {
InferRequestsQueue(InferenceEngine::ExecutableNetwork& net, size_t nireq) {
InferenceEngine::InferRequest::Ptr infer_request;
for (size_t id = 0; id < nireq; id++) {
infer_request = net.CreateInferRequestPtr();
infer_request = std::make_shared<InferenceEngine::InferRequest>(net.CreateInferRequest());
infer_requests_.push_back(infer_request);
}
}

42
onnxruntime/core/providers/openvino/backends/vadm_backend.cc
View file

@ -9,12 +9,21 @@
#include <inference_engine.hpp>
#ifdef OPENVINO_2021_4
using Exception = InferenceEngine::Exception;
using WaitMode = InferenceEngine::InferRequest::WaitMode;
#else
using Exception = InferenceEngine::details::InferenceEngineException;
using WaitMode = InferenceEngine::IInferRequest::WaitMode;
#endif
#include "core/providers/shared_library/provider_api.h"
#include "../contexts.h"
#include "../backend_utils.h"
#include "vadm_backend.h"
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
#include <vpu/hddl_config.hpp>
#else
#include <vpu/hddl_plugin_config.hpp>
@ -68,14 +77,15 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
if (global_context_.is_wholly_supported_graph && subgraph_context_.enable_batching) {
for (int j = 0; j < 8; j++) {
InferenceEngine::ExecutableNetwork exe_network;
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[j];
#else
config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[j];
#endif
try {
exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, hw_target, config);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
@ -86,8 +96,8 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
for (size_t j = 0; j < num_inf_reqs_; j++) {
InferenceEngine::InferRequest::Ptr infRequest;
try {
infRequest = exe_networks[j].CreateInferRequestPtr();
} catch (const InferenceEngine::details::InferenceEngineException& e) {
infRequest = std::make_shared<InferenceEngine::InferRequest>(exe_networks[j].CreateInferRequest());
} catch (const Exception& e) {
ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what());
} catch (...) {
ORT_THROW(log_tag + "Exception while creating InferRequest object.");
@ -101,7 +111,8 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
else {
i = GetFirstAvailableDevice(global_context);
LOGS_DEFAULT(INFO) << log_tag << "Device Tag is: " << i;
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i];
#else
config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[i];
@ -109,7 +120,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
InferenceEngine::ExecutableNetwork exe_network;
try {
exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, hw_target, config);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + subgraph_context_.subgraph_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception while Loading Network for graph " + subgraph_context_.subgraph_name);
@ -117,8 +128,8 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
InferenceEngine::InferRequest::Ptr infRequest;
try {
infRequest = exe_network.CreateInferRequestPtr();
} catch (const InferenceEngine::details::InferenceEngineException& e) {
infRequest = std::make_shared<InferenceEngine::InferRequest>(exe_network.CreateInferRequest());
} catch (const Exception& e) {
ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what());
} catch (...) {
ORT_THROW(log_tag + "Exception while creating InferRequest object.");
@ -143,7 +154,7 @@ void VADMBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* c
std::string input_name = input_info_iter->first;
try {
graph_input_blob = infer_request->GetBlob(input_name);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name);
@ -155,7 +166,7 @@ void VADMBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* c
// Start Async inference
try {
infer_request->StartAsync();
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
} catch (...) {
ORT_THROW(log_tag + " Couldn't start Inference");
@ -171,8 +182,8 @@ void VADMBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext
// Wait for Async inference completion
try {
infer_request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
infer_request->Wait(WaitMode::RESULT_READY);
} catch (const Exception& e) {
ORT_THROW(log_tag + " Exception with completing Inference: " + e.what());
} catch (...) {
ORT_THROW(log_tag + " Exception with completing Inference");
@ -186,7 +197,7 @@ void VADMBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext
auto output_name = output_info_iter->first;
try {
graph_output_blob = infer_request->GetBlob(output_name);
} catch (const InferenceEngine::details::InferenceEngineException& e) {
} catch (const Exception& e) {
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what());
} catch (...) {
ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name);
@ -246,7 +257,8 @@ void VADMBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) {
size_t remainder_parallel_runs = batch_size % num_inf_reqs_;
if (subgraph_context_.is_constant) {
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || defined(OPENVINO_2021_3)
#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) || defined(OPENVINO_2021_2) || \
defined(OPENVINO_2021_3) || defined(OPENVINO_2021_4)
for (auto item : const_outputs_map_) {
auto out_name = item.first;
auto node = item.second;

4
onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file

@ -86,6 +86,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, const
openvino_ep::GetCapability obj(graph_viewer,
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2021_3");
result = obj.Execute();
#elif defined (OPENVINO_2021_4)
openvino_ep::GetCapability obj(graph_viewer,
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2021_3");
result = obj.Execute();
#endif
return result;

1
onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file

@ -6,6 +6,7 @@
#include "backend_manager.h"
#include <map>
#include <algorithm>
#include <iostream>
namespace onnxruntime {