From 7654cd50e864e75ee477982aa919992e636fdcaf Mon Sep 17 00:00:00 2001 From: sfatimar Date: Thu, 12 Jan 2023 06:01:26 +0530 Subject: [PATCH] Openvino ep 2022.3 v4.3 (#14210) ### Description Changes to incorporate OpenVINO EP 2022.3 ### Motivation and Context This change is required to incorportate OpenVINO EP 2022.3 - If it fixes an open issue, please link to the issue here. --> Co-authored-by: mohsinmx Co-authored-by: Preetha Veeramalai Co-authored-by: Aravind Co-authored-by: mayavijx Co-authored-by: flexci --- cmake/CMakeLists.txt | 12 +- dockerfiles/Dockerfile.openvino | 2 +- dockerfiles/Dockerfile.openvino-csharp | 2 +- dockerfiles/Dockerfile.openvino-rhel8 | 25 +- dockerfiles/README.md | 12 +- docs/python/ReadMeOV.rst | 16 +- .../providers/openvino/backend_manager.cc | 67 +- .../core/providers/openvino/backend_utils.cc | 366 ++--------- .../core/providers/openvino/backend_utils.h | 35 +- .../openvino/backends/backend_factory.cc | 13 +- .../openvino/backends/basic_backend.cc | 604 ++++++++---------- .../openvino/backends/basic_backend.h | 10 +- .../openvino/backends/vadm_backend.cc | 77 +-- .../openvino/openvino_execution_provider.cc | 14 +- .../openvino/openvino_execution_provider.h | 12 +- .../core/providers/openvino/ov_interface.cc | 139 +--- .../core/providers/openvino/ov_interface.h | 66 +- .../openvino/ov_versions/capability.cc | 8 +- .../openvino/ov_versions/data_ops.cc | 550 +++------------- .../providers/openvino/ov_versions/data_ops.h | 3 +- .../providers/openvino/ov_versions/utils.cc | 6 +- .../test/python/onnx_backend_test_series.py | 3 + .../onnx_backend_test_series_filters.jsonc | 14 + setup.py | 38 +- .../linux-openvino-ci-pipeline.yml | 2 +- .../linux/docker/Dockerfile.ubuntu_openvino | 8 +- ...kerfile_manylinux2014_openvino_multipython | 13 +- .../nuget/generate_nuspec_for_native_nuget.py | 18 +- 28 files changed, 668 insertions(+), 1467 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index baa92afb9b..89d827b5d8 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1105,18 +1105,18 @@ if (onnxruntime_USE_OPENVINO) endif() # Check OpenVINO version for support - if (${VER} MATCHES "2021.4" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2021.4") - set(OPENVINO_VERSION "2021.4") - add_definitions(-DOPENVINO_2021_4=1) - elseif (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1") + if (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1") set(OPENVINO_VERSION "2022.1") add_definitions(-DOPENVINO_2022_1=1) elseif (${VER} MATCHES "2022.2" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.2") set(OPENVINO_VERSION "2022.2") add_definitions(-DOPENVINO_2022_2=1) + elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3") + set(OPENVINO_VERSION "2022.3") + add_definitions(-DOPENVINO_2022_3=1) elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino") - set(OPENVINO_VERSION "2022.2") - add_definitions(-DOPENVINO_2022_2=1) + set(OPENVINO_VERSION "2022.3") + add_definitions(-DOPENVINO_2022_3=1) else() message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}") endif() diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino index 65000d650d..911fc982c8 100644 --- a/dockerfiles/Dockerfile.openvino +++ b/dockerfiles/Dockerfile.openvino @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT #-------------------------------------------------------------------------- -ARG OPENVINO_VERSION=2022.2.0 +ARG OPENVINO_VERSION=2022.3.0 # Build stage diff --git a/dockerfiles/Dockerfile.openvino-csharp b/dockerfiles/Dockerfile.openvino-csharp index 752457ae82..1edddaf201 100644 --- a/dockerfiles/Dockerfile.openvino-csharp +++ b/dockerfiles/Dockerfile.openvino-csharp @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT #-------------------------------------------------------------------------- -ARG OPENVINO_VERSION=2022.2.0 +ARG OPENVINO_VERSION=2022.3.0 # Build stage FROM openvino/ubuntu20_runtime:${OPENVINO_VERSION} AS base diff --git a/dockerfiles/Dockerfile.openvino-rhel8 b/dockerfiles/Dockerfile.openvino-rhel8 index 3a13633342..1a5be8b256 100644 --- a/dockerfiles/Dockerfile.openvino-rhel8 +++ b/dockerfiles/Dockerfile.openvino-rhel8 @@ -8,7 +8,7 @@ ARG DEVICE=CPU_FP32 ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime ARG ONNXRUNTIME_BRANCH=main -ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2022.2.0.7713 +ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2022.3.0 ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/runtime/lib/intel64/ @@ -17,6 +17,7 @@ ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/runtime/3rdparty/tbb/lib/:${IE_PLUGINS ENV OpenCV_DIR=${INTEL_OPENVINO_DIR}/extras/opencv/cmake ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/extras/opencv/lib:${LD_LIBRARY_PATH} ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64:/lib64:${LD_LIBRARY_PATH} +ENV PATH=${MY_ROOT}/cmake-dir/bin:$PATH # Install packages RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-devel libffi-devel openssl-devel git make gcc && \ @@ -25,7 +26,7 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve cd $MY_ROOT && \ wget https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tgz && tar xvf Python-3.8.9.tgz && rm -rf Python-3.8.9.tgz && \ cd Python-3.8*/ && ./configure && make && make install && \ - cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.9/ /usr/bin/Python38 && \ + cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.9/ /usr/bin/Python38 && ln -s /usr/bin/pip3 /usr/bin/pip && \ # libusb1.0.22 cd /opt/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \ unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /opt/libusb-1.0.22 && \ @@ -42,15 +43,16 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve /bin/mkdir -p '/usr/local/lib/pkgconfig' && \ # Install openvino cd /opt/ && mkdir intel/ && cd intel && \ - wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \ - tar xvf l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \ - rm -rf l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \ - mv l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64 openvino_2022.2.0.7713 && \ + wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ + tar xvf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ + rm -rf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ + mv l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 && \ cd ${INTEL_OPENVINO_DIR}/install_dependencies/ && ./install_openvino_dependencies.sh -y && ./install_NEO_OCL_driver.sh -y && \ - printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2022.2.0.7713/setupvars.sh && \ + printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2022.3.0/setupvars.sh && \ cd /opt/libusb-1.0.22 && \ /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \ - cp /opt/intel/openvino_2022.2.0.7713/install_dependencies/97-myriad-usbboot.rules /etc/udev/rules.d/ && \ + # MYRIAD plugins are not available for openvino 2022.3.0 release + #cp /opt/intel/openvino_2022.3.0/install_dependencies/97-myriad-usbboot.rules /etc/udev/rules.d/ && \ ldconfig && \ #Install protobuf cd $MY_ROOT && \ @@ -64,10 +66,11 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve make install && \ # Build onnxruntime cd $MY_ROOT && \ - pip3 install numpy wheel setuptools cython && \ + pip3 install numpy wheel setuptools cython onnx && \ git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} && \ - pip3 install onnx && \ - source /opt/intel/openvino_2022.2.0.7713/setupvars.sh && \ + bash onnxruntime/dockerfiles/scripts/install_common_deps.sh && \ + ln -s cmake-* cmake-dir && \ + source /opt/intel/openvino_2022.3.0/setupvars.sh && \ cd /code/onnxruntime && ./build.sh --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel && \ pip3 install /code/onnxruntime/build/Linux/Release/dist/*-linux_x86_64.whl && \ # Clean up diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 3d4d4ffa84..fd418aa189 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -84,11 +84,12 @@ git submodule update --init ## OpenVINO *Public Preview* -**Ubuntu 18.04, Python & C# Bindings** +**Ubuntu 20.04, Python & C# Bindings** +**RHEL 8.4, Python Binding** ### **1. Using pre-built container images for Python API** -The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu18) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code. +The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu20) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code. If the `device_type` runtime config option is not explicitly specified, CPU will be chosen as the hardware target execution. ### **2. Building from Dockerfile** @@ -96,7 +97,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will Retrieve your docker image in one of the following ways. - - Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as for building an OpenVINO 2021.3 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is main. + - Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as for building latest OpenVINO based Docker image for Ubuntu20.04 and Dockerfile.openvino-rhel for Python API for RHEL 8.4. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is main. ``` docker build --rm -t onnxruntime --build-arg DEVICE=$DEVICE -f . ``` @@ -107,6 +108,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will | Device Option | Target Device | | --------- | -------- | | CPU_FP32 | Intel CPUs | + | CPU_FP16 | Intel CPUs | | GPU_FP32 |Intel Integrated Graphics | | GPU_FP16 | Intel Integrated Graphics | | MYRIAD_FP16 | Intel MovidiusTM USB sticks | @@ -173,7 +175,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will ### OpenVINO on VAD-M Accelerator Version -1. Download OpenVINO **Full package** for version **2021.4** for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html) +1. Download OpenVINO **Full package** for latest version for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html) 2. Install the drivers on the host machine according to the reference in [here](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux_ivad_vpu.html) @@ -184,7 +186,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will 4. Run hddldaemon on the host in a separate terminal session using the following steps: - Initialize the OpenVINO environment. ``` - source /bin/setupvars.sh + source /setupvars.sh ``` - Edit the hddl_service.config file from $HDDL_INSTALL_DIR/config/hddl_service.config and change the field “bypass_device_number” to 8. - Restart the hddl daemon for the changes to take effect. diff --git a/docs/python/ReadMeOV.rst b/docs/python/ReadMeOV.rst index 3a96ced1c8..d4c944af56 100644 --- a/docs/python/ReadMeOV.rst +++ b/docs/python/ReadMeOV.rst @@ -25,12 +25,12 @@ This package supports: Please Note for VAD-M use Docker installation / Build from Source for Linux. -``pip3 install onnxruntime-openvino==1.12.0`` +``pip3 install onnxruntime-openvino==1.13.1`` Please install OpenVINO™ PyPi Package separately for Windows. For installation instructions on Windows please refer to `OpenVINO™ Execution Provider for ONNX Runtime for Windows `_. -**OpenVINO™ Execution Provider for ONNX Runtime** Linux Wheels comes with pre-built libraries of OpenVINO™ version 2022.1.0 eliminating the need to install OpenVINO™ separately. The OpenVINO™ libraries are prebuilt with CXX11_ABI flag set to 0. +**OpenVINO™ Execution Provider for ONNX Runtime** Linux Wheels comes with pre-built libraries of OpenVINO™ version 2022.2.0 eliminating the need to install OpenVINO™ separately. The OpenVINO™ libraries are prebuilt with CXX11_ABI flag set to 0. The package also includes module that is used by torch-ort-inference to accelerate inference for PyTorch models with OpenVINO Execution Provider. See `torch-ort-inference `_ for more details. @@ -50,18 +50,6 @@ Samples To see what you can do with **OpenVINO™ Execution Provider for ONNX Runtime**, explore the demos located in the `Examples `_. -Docker Support -^^^^^^^^^^^^^^ - -The latest OpenVINO™ EP docker image can be downloaded from DockerHub. -For more details see `Docker ReadMe `_. - - -Prebuilt Images -^^^^^^^^^^^^^^^^ - -- Please find prebuilt docker images for Intel® CPU and Intel® iGPU on OpenVINO™ Execution Provider `Release Page `_. - License ^^^^^^^^ diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 80d633dc0f..b466138a31 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -31,8 +31,8 @@ void BackendManager::ReleaseGlobalContext() { g_global_context.reset(); } -BackendManager::BackendManager(const onnxruntime::Node& fused_node, - const onnxruntime::GraphViewer& subgraph, +BackendManager::BackendManager(const onnxruntime::Node& fused_node, + const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger) { auto prec_str = GetGlobalContext().precision_str; if (prec_str == "FP32") { @@ -40,7 +40,7 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, } else if (prec_str == "FP16") { subgraph_context_.precision = InferenceEngine::Precision::FP16; } else { - ORT_THROW("Invalid OpenVINO Precision type: " + prec_str); + throw std::string("Invalid OpenVINO Precision type: " + prec_str); } // Save the indexes of graph inputs among fused_node's inputDefs @@ -64,7 +64,7 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, } auto it = subgraph_context_.input_names.find(input->Name()); if (it == subgraph_context_.input_names.end()) { - ORT_THROW("Input not found in the input defs list"); + throw std::string("Input not found in the input defs list"); } int index = it->second; subgraph_context_.input_indexes.push_back(index); @@ -85,7 +85,13 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, subgraph_context_.enable_batching = true; LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model can be Batch inferenced \n"; auto model_copy = ReWriteBatchDimWithOne(*model_proto_); - concrete_backend_ = BackendFactory::MakeBackend(*model_copy, GetGlobalContext(), subgraph_context_); + try { + concrete_backend_ = BackendFactory::MakeBackend(*model_copy, + GetGlobalContext(), + subgraph_context_); + } catch (std::string const & msg) { + throw msg; + } subgraph_context_.has_dynamic_input_shape = false; } else if (ModelHasSymbolicInputDims(subgraph)) { @@ -97,15 +103,19 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, if (GetGlobalContext().device_type.find("CPU") != std::string::npos) { LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has symbolic input dims and " << "device_type is CPU."; - #if (defined OV_API_20) - if (GetGlobalContext().enable_dynamic_shapes) { - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. " - << "Creating backend Dynamic Shapes"; - concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, GetGlobalContext(), subgraph_context_); - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " - << "Backend created for graph " << subgraph_context_.subgraph_name; - } - #endif + if (GetGlobalContext().enable_dynamic_shapes) { + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. " + << "Creating backend Dynamic Shapes"; + try { + concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, + GetGlobalContext(), + subgraph_context_); + } catch (std::string const & msg) { + throw msg; + } + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " + << "Backend created for graph " << subgraph_context_.subgraph_name; + } } } else if (ModelHasSymbolicInputDims(subgraph) && GetGlobalContext().device_type.find("GPU") != std::string::npos) { @@ -115,7 +125,13 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node, LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has concreate input dims. Initializing backend for graph " << subgraph_context_.subgraph_name; subgraph_context_.has_dynamic_input_shape = false; - concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, GetGlobalContext(), subgraph_context_); + try { + concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, + GetGlobalContext(), + subgraph_context_); + } catch (std::string const & msg) { + throw msg; + } } } @@ -172,8 +188,8 @@ bool BackendManager::ModelHasSymbolicInputDims(const onnxruntime::GraphViewer& s } std::unique_ptr -BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, - const onnxruntime::GraphViewer& subgraph, +BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, + const onnxruntime::GraphViewer& subgraph, const logging::Logger& logger) const { auto model = subgraph.CreateModel(logger); @@ -195,7 +211,7 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node, } std::vector> GetInputTensorShapes(Ort::KernelContext& context) { - + const auto input_count = context.GetInputCount(); std::vector> input_shapes; input_shapes.reserve(input_count); @@ -273,10 +289,8 @@ void BackendManager::Compute(OrtKernelContext* context) { bool use_dynamic_backend = true; if (GetGlobalContext().enable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape && GetGlobalContext().device_type.find("CPU") != std::string::npos) { - #if (defined OV_API_20) - concrete_backend_->Infer(context); - use_dynamic_backend = false; - #endif + concrete_backend_->Infer(context); + use_dynamic_backend = false; } else if (use_dynamic_backend && subgraph_context_.has_dynamic_input_shape) { std::vector> tensor_shapes = GetInputTensorShapes(ctx); @@ -297,8 +311,13 @@ void BackendManager::Compute(OrtKernelContext* context) { LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " << "Backend created for graph " << subgraph_context_.subgraph_name; auto modelproto_with_concrete_shapes = ReWriteInputShapeInfo(*model_proto_, tensor_shapes); - dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes, - GetGlobalContext(), subgraph_context_); + try { + dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes, + GetGlobalContext(), + subgraph_context_); + } catch (std::string const & msg) { + throw msg; + } backend_map_.insert({key, dynamic_backend}); } else { dynamic_backend = search->second; diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index c213ccd4bb..7a8c7f36aa 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -13,6 +13,16 @@ #include "core/providers/shared_library/provider_api.h" #include "backend_utils.h" +#if defined (OV_API_20) +using Exception = ov::Exception; +#elif defined (OPENVINO_2021_4) +using Exception = InferenceEngine::Exception; +using WaitMode = InferenceEngine::InferRequest::WaitMode; +#else +using Exception = InferenceEngine::details::InferenceEngineException; +using WaitMode = InferenceEngine::IInferRequest::WaitMode; +#endif + namespace onnxruntime { namespace openvino_ep { namespace backend_utils { @@ -25,11 +35,6 @@ bool IsDebugEnabled() { } return false; } -void DumpOnnxModelProto(const ONNX_NAMESPACE::ModelProto& model_proto, std::string file_name) { - std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary); - model_proto.SerializeToOstream(outfile); -} - #endif bool IsCILogEnabled() { @@ -71,145 +76,74 @@ bool IsDirExists(const std::string& pathname) { return false; } -void CreateDirectory(const std::string& ov_compiled_blobs_dir) { - LOGS_DEFAULT(INFO) << log_tag << "'ov_compiled_blobs' directory doesn't exist at the executable path, so creating one"; -#if defined(_WIN32) - if (_mkdir(ov_compiled_blobs_dir.c_str()) == 0) { // Creating a directory - LOGS_DEFAULT(INFO) << log_tag << "created a directory named 'ov_compiled_blobs' at the executable path"; - } else { - LOGS_DEFAULT(INFO) << log_tag << "Error creating a directory named 'ov_compiled_blobs' at the executable path"; - throw std::runtime_error("Could not create the directory"); - } -#else - if (mkdir(ov_compiled_blobs_dir.c_str(), 0777) == 0) { // Creating a directory - LOGS_DEFAULT(INFO) << log_tag << "created a directory named 'ov_compiled_blobs' at the executable path"; - } else { - LOGS_DEFAULT(INFO) << log_tag << "Error creating a directory named 'ov_compiled_blobs' at the executable path"; - throw std::runtime_error("Could not create the directory"); - } -#endif -} - struct static_cast_int64 { template // T1 models type statically convertible to T int64_t operator()(const T1& x) const { return static_cast(x); } }; -std::shared_ptr -CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map>& const_outputs_map) { +std::shared_ptr +CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, + const SubGraphContext& subgraph_context, + std::map>& const_outputs_map) { if(IsCILogEnabled()) { std::cout << "CreateNgraphFunc" << std::endl; } - -#ifndef NDEBUG - if (IsDebugEnabled()) { - DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx"); - } -#endif - - std::shared_ptr ng_function; - #if defined (OPENVINO_2021_4) - const std::string model = model_proto.SerializeAsString(); + const std::string model = model_proto.SerializeAsString(); + try { auto cnn_network = global_context.ie_core.ReadModel(model); - ng_function = cnn_network->getFunction(); - #else - ORT_UNUSED_PARAMETER(model_proto); - #endif + if ((subgraph_context.precision == InferenceEngine::Precision::FP16) && + (global_context.device_type.find("MYRIAD") == std::string::npos)) { + //FP16 transformations + ov::pass::ConvertFP32ToFP16 pass_obj; + pass_obj.run_on_model(cnn_network); + cnn_network->validate_nodes_and_infer_types(); - if (global_context.device_type.find("GPU") != std::string::npos && - subgraph_context.precision == InferenceEngine::Precision::FP16) { - //FP16 transformations - ngraph::pass::ConvertFP32ToFP16().run_on_function(ng_function); - ng_function->validate_nodes_and_infer_types(); - } - - if (!global_context.is_wholly_supported_graph) { - std::map result_to_output; - for (auto& result : ng_function->get_results()) { - result_to_output[result->get_friendly_name()] = result->input_value(0).get_node_shared_ptr()->get_friendly_name(); - } - ngraph::pass::ConstantFolding().run_on_function(ng_function); - auto& results = const_cast<::ngraph::ResultVector&>(ng_function->get_results()); - size_t index = results.size() - 1; - #if defined (OV_API_20) - for (auto it = results.rbegin(); it != results.rend(); ++it) { - if (auto const_node = std::dynamic_pointer_cast((*it)->input_value(0).get_node_shared_ptr())) { - const_outputs_map[(*it)->get_friendly_name()] = const_node; - results.erase(results.begin() + index); + auto proc = ov::preprocess::PrePostProcessor(cnn_network); + for (size_t i=0; i < cnn_network->inputs().size(); i++) { + if(cnn_network->inputs()[i].get_element_type() == ov::element::f16) { + proc.input(i).tensor().set_element_type(ov::element::f32); + proc.input(i).preprocess().convert_element_type(ov::element::f16); + } } - --index; + + for (size_t i=0; i < cnn_network->outputs().size(); i++) { + if(cnn_network->outputs()[i].get_element_type() == ov::element::f16) { + proc.output(i).postprocess().convert_element_type(ov::element::f32); + } + } + cnn_network = proc.build(); } - #else + + //Check for Constant Folding + if (!global_context.is_wholly_supported_graph) { + ov::pass::ConstantFolding pass_const_obj; + pass_const_obj.run_on_model(cnn_network); + auto& results = const_cast(cnn_network.get()->get_results()); + size_t index = results.size() - 1; + for (auto it = results.rbegin(); it != results.rend(); ++it) { if (auto const_node = std::dynamic_pointer_cast((*it)->input_value(0).get_node_shared_ptr())) { - const_outputs_map[result_to_output.at((*it)->get_friendly_name())] = const_node; + const_outputs_map[(*it)->get_friendly_name()] = const_node; results.erase(results.begin() + index); } --index; } - #endif - } - - return std::make_shared(ng_function); -}; - -#if defined (OV_API_20) -std::shared_ptr -CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map>& const_outputs_map) { - if(IsCILogEnabled()) { - std::cout << "CreateNgraphFunc" << std::endl; - } - + } #ifndef NDEBUG - if (IsDebugEnabled()) { - DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx"); - } +#if defined OPENVINO_2022_3 + if (IsDebugEnabled()) { + std::string name = cnn_network->get_friendly_name(); + ov::pass::Serialize serializer(name + ".xml", name + ".bin"); + serializer.run_on_model(cnn_network); + ngraph::plot_graph(cnn_network, name+"_executable" + ".dot"); + } #endif - - const std::string model = model_proto.SerializeAsString(); - auto cnn_network = global_context.ie_core.ReadModel(model); - - if ((subgraph_context.precision == InferenceEngine::Precision::FP16) && - (global_context.device_type.find("MYRIAD") == std::string::npos)) { - //FP16 transformations - ov::pass::ConvertFP32ToFP16 pass_obj; - pass_obj.run_on_model(cnn_network); - cnn_network->validate_nodes_and_infer_types(); - - auto proc = ov::preprocess::PrePostProcessor(cnn_network); - for (size_t i=0; i < cnn_network->inputs().size(); i++) { - if(cnn_network->inputs()[i].get_element_type() == ov::element::f16) { - proc.input(i).tensor().set_element_type(ov::element::f32); - proc.input(i).preprocess().convert_element_type(ov::element::f16); - } - } - - for (size_t i=0; i < cnn_network->outputs().size(); i++) { - if(cnn_network->outputs()[i].get_element_type() == ov::element::f16) { - proc.output(i).postprocess().convert_element_type(ov::element::f32); - } - } - cnn_network = proc.build(); +#endif + return cnn_network; + }catch (std::string const & msg) { + throw msg; } - - //Check for Constant Folding - if (!global_context.is_wholly_supported_graph) { - ov::pass::ConstantFolding pass_const_obj; - pass_const_obj.run_on_model(cnn_network); - auto& results = const_cast(cnn_network.get()->get_results()); - size_t index = results.size() - 1; - - for (auto it = results.rbegin(); it != results.rend(); ++it) { - if (auto const_node = std::dynamic_pointer_cast((*it)->input_value(0).get_node_shared_ptr())) { - const_outputs_map[(*it)->get_friendly_name()] = const_node; - results.erase(results.begin() + index); - } - --index; - } - } - return cnn_network; } -#endif InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type, std::string device) { ONNX_NAMESPACE::DataType type_string = ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(onnx_type); @@ -236,40 +170,7 @@ InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE:: } else if (*type_string == "int64" || *type_string == "tensor(int64)") { return InferenceEngine::Precision::I32; } else { - ORT_THROW(log_tag + "Unsupported Data type"); - } -} - -void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, - std::shared_ptr network, - std::unordered_map output_names, - std::map>& const_outputs_map, - std::string device) { - // Configure input & output - // Prepare input blobs - - auto inputInfo = network->getInputsInfo(); - int input_idx = 0; - for (auto iter = inputInfo.begin(); iter != inputInfo.end(); ++iter, ++input_idx) { - // Get the onnx index for the corresponding input (ignoring initializers) - auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().input(input_idx).type(), device); - iter->second->setPrecision(precision); - } - - // Prepare output blobs - auto outputInfo = network->getOutputsInfo(); - for (auto iter = outputInfo.begin(); iter != outputInfo.end(); ++iter) { - auto output_name = iter->first; - auto it = const_outputs_map.find(output_name); - //Output is constant and don't need to set precision - if (it != const_outputs_map.end()) - break; - auto itr = output_names.find(output_name); - if (itr == output_names.end()) { - ORT_THROW(log_tag + "Output Names Mismatch: " + output_name + " doesn't exist"); - } - auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().output(itr->second).type(), device); - iter->second->setPrecision(precision); + throw std::string(log_tag + "Unsupported Data type"); } } @@ -281,11 +182,7 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size, auto graph_output_blob = infer_request->GetTensor(output_name); - #if defined (OV_API_20) auto graph_output_dims = graph_output_blob->get_shape(); - #else - auto graph_output_dims = graph_output_blob->getTensorDesc().getDims(); - #endif if (batch_size > 1) { // Add the batch size as dim 0. @@ -298,7 +195,7 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size, } auto it = output_names.find(output_name); if (it == output_names.end()) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); + throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX"); } int index = it->second; return context.GetOutput(index, output_shape.get(), num_dims); @@ -310,16 +207,14 @@ GetOutputTensor(Ort::KernelContext& context, std::unordered_map output_names, std::shared_ptr node) { - #if (defined OV_API_20) - // Find position of '/' in the output_name - int pos = output_name.find("/"); - // Copy the substring from start to pos - output_name = output_name.substr(0 , pos); - #endif + // Find position of '/' in the output_name + int pos = output_name.find("/"); + // Copy the substring from start to pos + output_name = output_name.substr(0 , pos); auto it = output_names.find(output_name); if (it == output_names.end()) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); + throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX"); } int index = it->second; auto shape = node->get_shape(); @@ -378,7 +273,7 @@ void FillOutputsWithConstantData(std::shared_ptr node, Ort::Unowne break; } default: - ORT_THROW(log_tag + "Unsupported output data type"); + throw std::string(log_tag + "Unsupported output data type"); } } @@ -390,81 +285,6 @@ void FillOutputHelper(Ort::UnownedValue& out_tensor, std::shared_ptr(inputBlob); - auto minputHolder = minput->wmap(); - - auto input_data = minputHolder.as::value_type*>(); - size_t input_data_size = inputBlob->byteSize(); - - auto tensor = context.GetInput(subgraph_context.input_names.at(input_name)); - auto mem_info = tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - ORT_THROW(log_tag + "IO Buffering is not enabled, Please enable Input on CPU"); - } - auto tensor_shape = tensor.GetTensorTypeAndShapeInfo(); - auto elem_type = tensor_shape.GetElementType(); - - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - const int64_t* tensor_data_64 = tensor.GetTensorData(); - auto data_len = (input_data_size * 2) / sizeof(int64_t); - const int64_t* batch_memory_offset = tensor_data_64 + data_len * batch_slice_idx; - - std::copy(batch_memory_offset, batch_memory_offset + data_len, (uint32_t*)input_data); - } else { - // Copy input data into OpenVINO's input buffer - const char* tensor_data = tensor.GetTensorData(); - const char* batch_memory_offset = tensor_data + input_data_size * batch_slice_idx; - std::memcpy(input_data, batch_memory_offset, input_data_size); - } -} - -void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, Ort::UnownedValue& output_tensor, - InferenceEngine::Precision precision, size_t batch_slice_idx) { - auto moutput = InferenceEngine::as(outputBlob); - - auto moutputHolder = moutput->rmap(); - - const auto output_data = moutputHolder.as::value_type*>(); - - size_t output_data_size = outputBlob->byteSize(); - auto tensor_shape = output_tensor.GetTensorTypeAndShapeInfo(); - const auto elem_type = tensor_shape.GetElementType(); - - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - int64_t* tensor_data = output_tensor.GetTensorMutableData(); - auto data_len = output_data_size / sizeof(int32_t); - int64_t* batch_memory_offset = tensor_data + data_len * batch_slice_idx; - - std::transform((int32_t*)output_data, ((int32_t*)output_data) + data_len, batch_memory_offset, static_cast_int64()); - - } else { - char* tensor_data = output_tensor.GetTensorMutableData(); - char* batch_memory_offset = tensor_data + output_data_size * batch_slice_idx; - - std::memcpy(batch_memory_offset, output_data, output_data_size); - } -} - -std::vector> -perfCountersSorted(std::map perfMap) { - using perfItem = std::pair; - std::vector sorted; - for (auto& kvp : perfMap) sorted.push_back(kvp); - - std::stable_sort(sorted.begin(), sorted.end(), - [](const perfItem& l, const perfItem& r) { - return l.second.execution_index < r.second.execution_index; - }); - - return sorted; -} - -#if defined (OV_API_20) void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx, std::string input_name, Ort::KernelContext& context, const SubGraphContext& subgraph_context) { @@ -474,7 +294,7 @@ void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx, auto tensor = context.GetInput(subgraph_context.input_names.at(input_name)); auto mem_info = tensor.GetTensorMemoryInfo(); if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - ORT_THROW(log_tag + "IO Buffering is not enabled, Please enable Input on CPU"); + throw std::string(log_tag + "IO Buffering is not enabled, Please enable Input on CPU"); } // Copy input data into OpenVINO's input buffer const char* tensor_data = tensor.GetTensorData(); @@ -527,65 +347,15 @@ void printPerformanceCounts(const std::vector& performanceMap, totalTime += it.real_time.count(); } } - stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; - std::cout << std::endl; - std::cout << "Full device name: " << deviceName << std::endl; - std::cout << std::endl; -} -#endif - -void printPerformanceCounts(const std::map& performanceMap, - std::ostream& stream, std::string deviceName) { - long long totalTime = 0; - // Print performance counts - stream << std::endl - << "performance counts:" << std::endl - << std::endl; - - auto performanceMapSorted = perfCountersSorted(performanceMap); - - for (const auto& it : performanceMapSorted) { - std::string toPrint(it.first); - const int maxLayerName = 30; - - if (it.first.length() >= maxLayerName) { - toPrint = it.first.substr(0, maxLayerName - 4); - toPrint += "..."; - } - stream << std::setw(maxLayerName) << std::left << toPrint; - switch (it.second.status) { - case InferenceEngine::InferenceEngineProfileInfo::EXECUTED: - stream << std::setw(15) << std::left << "EXECUTED"; - break; - case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN: - stream << std::setw(15) << std::left << "NOT_RUN"; - break; - case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT: - stream << std::setw(15) << std::left << "OPTIMIZED_OUT"; - break; - } - stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " "; - stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec); - stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_uSec); - stream << " execType: " << it.second.exec_type << std::endl; - if (it.second.realTime_uSec > 0) { - totalTime += it.second.realTime_uSec; - } - } - stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; + stream << std::setw(20) << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl; std::cout << std::endl; std::cout << "Full device name: " << deviceName << std::endl; std::cout << std::endl; } void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName) { - #if defined (OV_API_20) - auto performanceMap = request->GetNewObj().get_profiling_info(); - printPerformanceCounts(performanceMap, stream, deviceName); - #else - auto performanceMap = request->GetObj().GetPerformanceCounts(); - printPerformanceCounts(performanceMap, stream, deviceName); - #endif + auto performanceMap = request->GetNewObj().get_profiling_info(); + printPerformanceCounts(performanceMap, stream, deviceName); } } // namespace backend_utils diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h index b6aee97ee8..c40030cded 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.h +++ b/onnxruntime/core/providers/openvino/backend_utils.h @@ -27,25 +27,16 @@ const std::string log_tag = "[OpenVINO-EP] "; bool IsDebugEnabled(); #endif -//Internal diagnostic function. +// Internal diagnostic function. bool IsCILogEnabled(); bool UseCompiledNetwork(); -std::string GetCurrentWorkingDir(); +// std::string GetCurrentWorkingDir(); -bool IsDirExists(const std::string& pathname); +// bool IsDirExists(const std::string& pathname); -void CreateDirectory(const std::string& ov_compiled_blobs_dir); - -void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, - std::shared_ptr network, - std::unordered_map output_names, - std::map>& const_outputs_map, - std::string device); - -std::shared_ptr -CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map>& const_outputs_map); +// void CreateDirectory(const std::string& ov_compiled_blobs_dir); int GetFirstAvailableDevice(GlobalContext& global_context); @@ -69,7 +60,6 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size, std::string output_name, std::unordered_map output_names); -#if defined (OV_API_20) void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx, std::string input_name, Ort::KernelContext& context, const SubGraphContext& subgraph_context); @@ -78,24 +68,11 @@ void FillOutputBlob(OVTensorPtr outputBlob, Ort::UnownedValue& output_tensor, size_t batch_slice_idx); std::shared_ptr -CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map>& const_outputs_map); +CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, + std::map>& const_outputs_map); void printPerformanceCounts(const std::vector& performanceMap, std::ostream& stream, std::string deviceName); -#endif - -void printPerformanceCounts(const std::map& performanceMap, - std::ostream& stream, std::string deviceName); - -std::vector> -perfCountersSorted(std::map perfMap); - -void FillInputBlob(InferenceEngine::Blob::Ptr& inputBlob, size_t batch_slice_idx, - std::string input_name, Ort::KernelContext& context, - InferenceEngine::Precision precision, const SubGraphContext& subgraph_context); - -void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, Ort::UnownedValue& output_tensor, - InferenceEngine::Precision precision, size_t batch_slice_idx); void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName); diff --git a/onnxruntime/core/providers/openvino/backends/backend_factory.cc b/onnxruntime/core/providers/openvino/backends/backend_factory.cc index 5c1a2eadfd..57cbdb4c59 100644 --- a/onnxruntime/core/providers/openvino/backends/backend_factory.cc +++ b/onnxruntime/core/providers/openvino/backends/backend_factory.cc @@ -21,11 +21,16 @@ BackendFactory::MakeBackend(const ONNX_NAMESPACE::ModelProto& model_proto, } else if (type == "CPU" || type.find("GPU") != std::string::npos || type == "MYRIAD" || type.find("HETERO") != std::string::npos || type.find("MULTI") != std::string::npos || type.find("AUTO") != std::string::npos) { - return std::make_shared(model_proto, global_context, subgraph_context); + std::shared_ptr concrete_backend_; + try { + concrete_backend_=std::make_shared(model_proto, global_context, subgraph_context); + } catch (std::string const & msg) { + throw msg; + } + return concrete_backend_; } else { - ORT_THROW("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type); + throw std::string("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type); } } - -} // namespace openvino_ep +} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index acebe55eb4..ff7eba0d3e 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -24,56 +24,61 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, const SubGraphContext& subgraph_context) : global_context_(global_context), subgraph_context_(subgraph_context) { std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type; - bool vpu_status = false; - std::string model_blob_name; - std::string ov_compiled_blobs_dir = ""; - - if (hw_target == "MYRIAD") - vpu_status = true; - if (!ImportBlob(hw_target, vpu_status)) { - #if defined (OPENVINO_2021_4) - ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_); - SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, - global_context_.device_type); - #else + try{ + #ifndef NDEBUG + if (IsDebugEnabled()) { + std::string file_name = subgraph_context.subgraph_name + "_static.onnx"; + std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary); + model_proto.SerializeToOstream(outfile); + // DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx"); + } + #endif ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); - #endif - + } catch (std::string const & msg) { + throw msg; + } if (ValidateSubgraph(const_outputs_map_)) return; // OV Config OVConfig config; - PopulateConfigValue(config); + ov::AnyMap device_config; + PopulateConfigValue(config, device_config); //Enable caching EnableCaching(); //Setting OpenCL queue throttling for GPU - #if defined (OV_API_20) - EnableGPUThrottling(config); - #endif + EnableGPUThrottling(device_config); #if defined(IO_BUFFER_ENABLED) - if ((global_context.device_type.find("GPU") != std::string::npos) && - (global_context_.context != nullptr) && - (openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) { - LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled"; - cl_context ctx = static_cast(global_context_.context); - #ifdef OV_API_20 - remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_.ie_core.Get(), ctx); - #else - remote_context_ = InferenceEngine::gpu::make_shared_context(global_context_.ie_core.Get(), hw_target, ctx); - #endif - exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name); - } else { - exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name); - } + try { + if ((global_context.device_type.find("GPU") != std::string::npos) && + (global_context_.context != nullptr) && + (openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) { + LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled"; + cl_context ctx = static_cast(global_context_.context); + #ifdef OV_API_20 + remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_.ie_core.Get(), ctx); + #else + remote_context_ = InferenceEngine::gpu::make_shared_context(global_context_.ie_core.Get(), hw_target, ctx); + #endif + exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name); + } else { + exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name); + } + }catch (const char* msg) { + throw(msg); + } #else - exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name); + try{ + exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name); + } catch (const char* msg) { + throw(msg); + } #endif LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; - } + //The infer_requests_ pool will be intialized with a default value of 8 infer_request's //The nireq value can also be configured to any num_of_threads during runtime @@ -97,78 +102,46 @@ bool BasicBackend::ValidateSubgraph(std::mapget_names(); - std::string onnx_input_name; - std::string input_name; - // use names retrieved from original ONNX model to assign the right onnx input name for the graph - for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) { - if (it->second == input_idx) { - onnx_input_name = it->first; - break; + try{ + auto graph_input_info = exe_network_.Get().inputs(); + int input_idx = 0; + for (auto input_info_iter = graph_input_info.begin(); + input_info_iter != graph_input_info.end(); ++input_info_iter) { + auto input_names = input_info_iter->get_names(); + std::string onnx_input_name; + std::string input_name; + // use names retrieved from original ONNX model to assign the right onnx input name for the graph + for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) { + if (it->second == input_idx) { + onnx_input_name = it->first; + break; + } } - } - // using the input name retrieved from ONNX original to match with the input names returned by OV tensors - if (input_names.find(onnx_input_name) != input_names.end()) { - input_name = onnx_input_name; - } else { - ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); - } - size_t batch_slice_idx = 0; - if (subgraph_context_.has_dynamic_input_shape && - global_context_.enable_dynamic_shapes == true && - global_context_.device_type.find("CPU") != std::string::npos) { - auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); - auto tensor_shape = tensor_info.GetShape(); - auto tensor_size = tensor_shape.size(); - auto tensor_iter = 0; - ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0); - for (auto i = tensor_shape.begin(); i != tensor_shape.end(); ++i) { - input_tensor_shape[tensor_iter] = *i; - tensor_iter+=1; + // using the input name retrieved from ONNX original to match with the input names returned by OV tensors + if (input_names.find(onnx_input_name) != input_names.end()) { + input_name = onnx_input_name; + } else { + throw(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); } - auto input = ie_cnn_network_->get_parameters().at(0); - OVTensorPtr tensor_ptr = std::make_shared(input->get_element_type(), input_tensor_shape); - FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_); - infer_request->SetTensor(input_name, tensor_ptr); - } else { - OVTensorPtr graph_input_blob; - graph_input_blob = infer_request->GetTensor(input_name); - FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_); + size_t batch_slice_idx = 0; + if (subgraph_context_.has_dynamic_input_shape && + global_context_.enable_dynamic_shapes == true && + global_context_.device_type.find("CPU") != std::string::npos) { + auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + auto tensor_info = tensor.GetTensorTypeAndShapeInfo(); + auto tensor_shape = tensor_info.GetShape(); + auto tensor_size = tensor_shape.size(); + auto tensor_iter = 0; + ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0); + for (auto i = tensor_shape.begin(); i != tensor_shape.end(); ++i) { + input_tensor_shape[tensor_iter] = *i; + tensor_iter+=1; + } + auto input = ie_cnn_network_->get_parameters().at(0); + OVTensorPtr tensor_ptr = std::make_shared(input->get_element_type(), input_tensor_shape); + FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_); + try{ + infer_request->SetTensor(input_name, tensor_ptr); + } catch (const char* msg) { + throw(msg); + } + } else { + OVTensorPtr graph_input_blob; + try{ + graph_input_blob = infer_request->GetTensor(input_name); + } catch (const char* msg) { + throw(msg); + } + FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_); + } + input_idx++; + } + // Start Async inference + infer_request->StartAsync(); + }catch (const char* msg) { + throw(msg); } - input_idx++; - } - #else - auto graph_input_info = exe_network_.Get().GetInputsInfo(); - for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter) { - // Get OpenVINO's input buffer - OVTensorPtr graph_input_blob; - std::string input_name = input_info_iter->first; - graph_input_blob = infer_request->GetTensor(input_name); - auto precision = input_info_iter->second->getPrecision(); - size_t batch_slice = 0; - FillInputBlob(graph_input_blob, batch_slice, input_name, context, precision, subgraph_context_); - } - #endif - // Start Async inference - infer_request->StartAsync(); } #ifdef IO_BUFFER_ENABLED //Wait for Remote Aynchronous inference completion void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) { - #if defined (OV_API_20) - auto graph_input_info = exe_network_.Get().inputs(); - int input_idx = 0; - for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter) { - auto input_names = input_info_iter->get_names(); - std::string onnx_input_name; - std::string input_name; - // use names retrieved from original ONNX model to assign the right onnx input name for the graph - for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) { - if (it->second == input_idx) { - onnx_input_name = it->first; - break; + try{ + auto graph_input_info = exe_network_.Get().inputs(); + int input_idx = 0; + for (auto input_info_iter = graph_input_info.begin(); + input_info_iter != graph_input_info.end(); ++input_info_iter) { + auto input_names = input_info_iter->get_names(); + std::string onnx_input_name; + std::string input_name; + // use names retrieved from original ONNX model to assign the right onnx input name for the graph + for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) { + if (it->second == input_idx) { + onnx_input_name = it->first; + break; + } } - } - // using the input name retrieved from ONNX original to match with the input names returned by OV tensors - if (input_names.find(onnx_input_name) != input_names.end()) { - input_name = onnx_input_name; - } else { - ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); - } - input_idx++; - #else - auto graph_input_info = exe_network_.Get().GetInputsInfo(); - for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter) { - std::string input_name = input_info_iter->first; - #endif - // Kernel Context Input Buffer - const auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); - // If the ORTValue wraps a device pointer - auto mem_info = tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - //Get the shared buffer pointer - const void *tensor_data = tensor.GetTensorRawData(); - const cl::Buffer* shared_buffer_const = static_cast(tensor_data); - //Create an Input Remote Blob - #if defined(OV_API_20) + // using the input name retrieved from ONNX original to match with the input names returned by OV tensors + if (input_names.find(onnx_input_name) != input_names.end()) { + input_name = onnx_input_name; + } else { + throw(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); + } + input_idx++; + // Kernel Context Input Buffer + const auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name)); + // If the ORTValue wraps a device pointer + auto mem_info = tensor.GetTensorMemoryInfo(); + if (mem_info.GetAllocatorName() == OpenVINO_GPU) { + //Get the shared buffer pointer + const void *tensor_data = tensor.GetTensorRawData(); + const cl::Buffer* shared_buffer_const = static_cast(tensor_data); + //Create an Input Remote Blob auto input = ie_cnn_network_->get_parameters().at(0); auto remote_blob = remote_context_->create_tensor(input->get_element_type(), input->get_shape(), *shared_buffer_const); ov::Tensor tensor = static_cast(remote_blob); OVTensorPtr tensor_ptr = std::make_shared(tensor); - #else - cl::Buffer* shared_buffer = const_cast(shared_buffer_const); - OVTensorPtr tensor_ptr = InferenceEngine::gpu::make_shared_blob(input_info_iter->second->getTensorDesc(), remote_context_, *shared_buffer); - #endif - infer_request->SetTensor(input_name, tensor_ptr); - } else { - OVTensorPtr graph_input_blob; - graph_input_blob = infer_request->GetTensor(input_name); - size_t batch_slice_idx = 0; - #if defined (OV_API_20) + infer_request->SetTensor(input_name, tensor_ptr); + } else { + OVTensorPtr graph_input_blob; + graph_input_blob = infer_request->GetTensor(input_name); + size_t batch_slice_idx = 0; FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_); - #else - auto precision = input_info_iter->second->getPrecision(); - FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, precision, subgraph_context_); - #endif - } - } - - //Set the output blob as remote blob - #if defined (OV_API_20) - auto graph_output_info = exe_network_.Get().outputs(); - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - auto output_names = output_info_iter->get_names(); - std::string onnx_output_name; - std::string output_name; - bool output_name_found = false; - // using the output name retrieved from ONNX original to match with the output names returned by OV tensors - for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { - onnx_output_name = it->first; - if (output_names.find(onnx_output_name) != output_names.end()) { - //Assigning the output_name - output_name = it->first; - output_name_found = true; - break; } } - if(!output_name_found) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names"); - } - #else - auto graph_output_info = exe_network_.Get().GetOutputsInfo(); - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - // Get Ort Output Tensor - auto output_name = output_info_iter->first; - #endif - size_t batch_size = 1; - auto tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); - auto mem_info = tensor.GetTensorMemoryInfo(); - // Check if ORT Value wraps a device pointer - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - const void *tensor_data = tensor.GetTensorRawData(); - const cl::Buffer* shared_buffer_const = static_cast(tensor_data); - // Create a shared Blob, set the Infer Request Output Blob - #if defined(OV_API_20) + //Set the output blob as remote blob + auto graph_output_info = exe_network_.Get().outputs(); + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + bool output_name_found = false; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + //Assigning the output_name + output_name = it->first; + output_name_found = true; + break; + } + } + if(!output_name_found) { + throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names"); + } + + size_t batch_size = 1; + auto tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); + auto mem_info = tensor.GetTensorMemoryInfo(); + // Check if ORT Value wraps a device pointer + if (mem_info.GetAllocatorName() == OpenVINO_GPU) { + const void *tensor_data = tensor.GetTensorRawData(); + const cl::Buffer* shared_buffer_const = static_cast(tensor_data); + // Create a shared Blob, set the Infer Request Output Blob auto output = ie_cnn_network_->get_results().at(0); auto remote_tensor = remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const); ov::Tensor tensor = static_cast(remote_tensor); OVTensorPtr tensor_ptr = std::make_shared(tensor); - #else - cl::Buffer* shared_buffer = const_cast(shared_buffer_const); - OVTensorPtr tensor_ptr = InferenceEngine::gpu::make_shared_blob(output_info_iter->second->getTensorDesc(), remote_context_, *shared_buffer); - #endif - infer_request->SetTensor(output_name, tensor_ptr); + try { + infer_request->SetTensor(output_name, tensor_ptr); + } catch (const char* msg) { + throw(msg); + } + } } - } - // Start Async inference - infer_request->StartAsync(); + // Start Async inference + infer_request->StartAsync(); + }catch (const char* msg) { + throw(msg); + } } #endif @@ -378,76 +325,62 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe // and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) { // Wait for Async inference completion - - infer_request->WaitRequest(); - #if defined (OV_API_20) - auto graph_output_info = exe_network_.Get().outputs(); - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - OVTensorPtr graph_output_blob; - auto output_names = output_info_iter->get_names(); - std::string onnx_output_name; - std::string output_name; - bool output_name_found = false; - // using the output name retrieved from ONNX original to match with the output names returned by OV tensors - for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { - onnx_output_name = it->first; - if (output_names.find(onnx_output_name) != output_names.end()) { - // Assigning the output_name - output_name = it->first; - output_name_found = true; - break; + try{ + infer_request->WaitRequest(); + auto graph_output_info = exe_network_.Get().outputs(); + for (auto output_info_iter = graph_output_info.begin(); + output_info_iter != graph_output_info.end(); ++output_info_iter) { + OVTensorPtr graph_output_blob; + auto output_names = output_info_iter->get_names(); + std::string onnx_output_name; + std::string output_name; + bool output_name_found = false; + // using the output name retrieved from ONNX original to match with the output names returned by OV tensors + for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) { + onnx_output_name = it->first; + if (output_names.find(onnx_output_name) != output_names.end()) { + // Assigning the output_name + output_name = it->first; + output_name_found = true; + break; + } } - } - if (!output_name_found) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX. " - "[ONNX Output: ] " + onnx_output_name + " doesn't exist in the " - "list of OpenVINO output tensor names"); - } - graph_output_blob = infer_request->GetTensor(output_name); - size_t batch_size = 1; - auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); - auto mem_info = output_tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - return; - } else { - size_t batch_slice = 0; - FillOutputBlob(graph_output_blob, output_tensor, batch_slice); - } - } - #else - auto graph_output_info = exe_network_.Get().GetOutputsInfo(); - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - // Get OpenVINO's output blob - OVTensorPtr graph_output_blob; - auto output_name = output_info_iter->first; - graph_output_blob = infer_request->GetTensor(output_name); - size_t batch_size = 1; - auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); - auto mem_info = output_tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - return; - } else { - auto precision = output_info_iter->second->getPrecision(); - size_t batch_slice = 0; - FillOutputBlob(graph_output_blob, output_tensor, precision, batch_slice); - } - } - #endif - - if (!const_outputs_map_.empty()) { - for (auto item : const_outputs_map_) { - auto out_name = item.first; - auto node = item.second; - auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); + if (!output_name_found) { + throw(log_tag + "Output names mismatch between OpenVINO and ONNX. " + "[ONNX Output: ] " + onnx_output_name + " doesn't exist in the " + "list of OpenVINO output tensor names"); + } + try{ + graph_output_blob = infer_request->GetTensor(output_name); + } catch (const char* msg) { + throw(msg); + } + size_t batch_size = 1; + auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); auto mem_info = output_tensor.GetTensorMemoryInfo(); if (mem_info.GetAllocatorName() == OpenVINO_GPU) { - ORT_THROW(log_tag + "IO Buffering is not supported for constant subgraphs"); + return; } else { - FillOutputsWithConstantData(node, output_tensor); + size_t batch_slice = 0; + FillOutputBlob(graph_output_blob, output_tensor, batch_slice); } } + + if (!const_outputs_map_.empty()) { + for (auto item : const_outputs_map_) { + auto out_name = item.first; + auto node = item.second; + auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); + auto mem_info = output_tensor.GetTensorMemoryInfo(); + if (mem_info.GetAllocatorName() == OpenVINO_GPU) { + throw(log_tag + "IO Buffering is not supported for constant subgraphs"); + } else { + FillOutputsWithConstantData(node, output_tensor); + } + } + } + }catch (const char* msg) { + throw(msg); } } @@ -463,8 +396,12 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { for (auto item : const_outputs_map_) { auto out_name = item.first; auto node = item.second; - auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); - FillOutputsWithConstantData(node, output_tensor); + try { + auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); + FillOutputsWithConstantData(node, output_tensor); + } catch (std::string const & msg) { + throw msg; + } } // Get Output tensors LOGS_DEFAULT(INFO) << log_tag << "Inference successful"; @@ -482,15 +419,30 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { if ((global_context_.device_type.find("GPU") != std::string::npos) && (global_context_.context != nullptr) && (openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) { - StartRemoteAsyncInference(context, infer_request); + try { + StartRemoteAsyncInference(context, infer_request); + } catch (std::string const & msg) { + throw msg; + } } else { - StartAsyncInference(context, infer_request); + try{ + StartAsyncInference(context, infer_request); + } catch(std::string const & msg) { + throw msg; + } } #else - StartAsyncInference(context, infer_request); + try{ + StartAsyncInference(context, infer_request); + } catch(std::string const & msg) { + throw msg; + } #endif - - CompleteAsyncInference(context, infer_request); + try{ + CompleteAsyncInference(context, infer_request); + }catch(std::string const & msg) { + throw msg; + } // Get Output tensors LOGS_DEFAULT(INFO) << log_tag << "Inference successful"; diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index da1eaa8f26..f156900538 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -32,11 +32,9 @@ class BasicBackend : public IBackend { bool ImportBlob(std::string hw_target, bool vpu_status); void PopulateCompiledDirectory(std::string, std::string&, std::string&, bool&); bool ValidateSubgraph(std::map>& const_outputs_map); - void PopulateConfigValue(OVConfig& config); + void PopulateConfigValue(OVConfig& config, ov::AnyMap& device_config); void EnableCaching(); - #if defined(OV_API_20) - void EnableGPUThrottling(OVConfig& config); - #endif + void EnableGPUThrottling(ov::AnyMap& device_config); void StartAsyncInference(Ort::KernelContext& context, std::shared_ptr infer_request); #ifdef IO_BUFFER_ENABLED @@ -78,11 +76,7 @@ class InferRequestsQueue { void printstatus() { std::cout << "printing elements of the vector (infer_requests_): " << std::endl; for (auto i = infer_requests_.begin(); i != infer_requests_.end(); ++i) { - #if defined (OV_API_20) i->get()->QueryStatus(); - #else - std::cout << *i << "\n"; - #endif } std::cout << '\n'; } diff --git a/onnxruntime/core/providers/openvino/backends/vadm_backend.cc b/onnxruntime/core/providers/openvino/backends/vadm_backend.cc index dd6a12c454..f8ee006ea2 100644 --- a/onnxruntime/core/providers/openvino/backends/vadm_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/vadm_backend.cc @@ -35,19 +35,16 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, // of Infer Requests only if the VAD-M accelerator is being used. // sets number of maximum parallel inferences num_inf_reqs_ = 8; - - #if defined(OV_API_20) - ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); - #else - ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_); - SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type); + #ifndef NDEBUG + if (IsDebugEnabled()) { + std::string file_name = subgraph_context.subgraph_name + "_static.onnx"; + std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary); + model_proto.SerializeToOstream(outfile); + // DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx"); + } #endif - OVConfig config; -#ifndef NDEBUG - if (openvino_ep::backend_utils::IsDebugEnabled()) { - config["PERF_COUNT"] = CONFIG_VALUE(YES); - } -#endif + ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); + if (const_outputs_map_.size() == subgraph_context_.output_names.size()) subgraph_context_.is_constant = true; @@ -56,6 +53,14 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, if (subgraph_context_.is_constant) return; std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type; + OVConfig config; +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + config["PERF_COUNT"] = CONFIG_VALUE(YES); + } +#endif + ov::AnyMap device_config; + // Loading model to the plugin //If graph is fully supported and batching is enabled, load the network onto all VPU's and infer std::vector exe_networks; @@ -63,7 +68,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, for (int j = 0; j < 8; j++) { OVExeNetwork exe_network; config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[j]; - exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name); + exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name); exe_networks.push_back(exe_network); } LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; @@ -79,9 +84,9 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, else { i = GetFirstAvailableDevice(global_context); LOGS_DEFAULT(INFO) << log_tag << "Device Tag is: " << i; - config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i]; OVExeNetwork exe_network; - exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name); + config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i]; + exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name); LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; OVInferRequestPtr infRequest; infRequest = std::make_shared(exe_network.CreateInferRequest()); @@ -95,8 +100,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, void VADMBackend::StartAsyncInference(Ort::KernelContext& context, size_t batch_slice_idx, size_t infer_req_idx) { auto infer_request = infer_requests_[infer_req_idx]; - - #if defined (OV_API_20) + auto graph_input_info = ie_cnn_network_->inputs(); int input_idx = 0; for (auto input_info_iter = graph_input_info.begin(); @@ -111,32 +115,20 @@ void VADMBackend::StartAsyncInference(Ort::KernelContext& context, break; } } - // using the input name retrieved from ONNX original to match with the input names returned by OV tensors + // using the input name retrieved from ONNX original to match with the input names returned by OV tensors if (input_names.find(onnx_input_name) != input_names.end()) { input_name = onnx_input_name; } else { ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names"); } - OVTensorPtr graph_input_blob; + OVTensorPtr graph_input_blob; graph_input_blob = infer_request->GetTensor(input_name); FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_); input_idx++; } - #else - auto graph_input_info = ie_cnn_network_->getInputsInfo(); - for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter) { - // Get OpenVINO's input buffer - std::string input_name = input_info_iter->first; - auto precision = input_info_iter->second->getPrecision(); - auto graph_input_blob = infer_request->GetTensor(input_name); - FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, precision, subgraph_context_); - } - #endif // Start Async inference infer_request->StartAsync(); - } // Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx @@ -148,8 +140,7 @@ void VADMBackend::CompleteAsyncInference(Ort::KernelContext& context, // Wait for Async inference completion infer_request->WaitRequest(); - - #if defined (OV_API_20) + auto graph_output_info = ie_cnn_network_->outputs(); for (auto output_info_iter = graph_output_info.begin(); output_info_iter != graph_output_info.end(); ++output_info_iter) { @@ -175,19 +166,7 @@ void VADMBackend::CompleteAsyncInference(Ort::KernelContext& context, auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); FillOutputBlob(graph_output_blob, output_tensor, batch_slice_idx); } - #else - auto graph_output_info = ie_cnn_network_->getOutputsInfo(); - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter) { - // Get OpenVINO's output blob - OVTensorPtr graph_output_blob; - auto output_name = output_info_iter->first; - graph_output_blob = infer_request->GetTensor(output_name); - auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names); - auto precision = output_info_iter->second->getPrecision(); - FillOutputBlob(graph_output_blob, output_tensor, precision, batch_slice_idx); - } - #endif + if (!const_outputs_map_.empty()) { for (auto item : const_outputs_map_) { auto out_name = item.first; @@ -220,7 +199,7 @@ void VADMBackend::Infer(OrtKernelContext* context) { LOGS_DEFAULT(INFO) << log_tag << "Running graph " << subgraph_context_.subgraph_name; LOGS_DEFAULT(INFO) << log_tag << "In Infer"; Ort::KernelContext ctx(context); - + std::lock_guard lock(compute_lock_); size_t batch_size = 1; @@ -228,11 +207,7 @@ void VADMBackend::Infer(OrtKernelContext* context) { if (subgraph_context_.enable_batching) { // Calculate the batch_size from the input tensor shape. auto tensor = ctx.GetInput(subgraph_context_.input_indexes[0]); - #if defined (OV_API_20) batch_size = DeduceBatchSize(tensor, ie_cnn_network_->get_result()->get_shape()); - #else - batch_size = DeduceBatchSize(tensor, ie_cnn_network_->getInputsInfo().begin()->second->getTensorDesc().getDims()); - #endif } size_t full_parallel_runs = batch_size / num_inf_reqs_; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 9b1dc7e695..24d1849f43 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -122,11 +122,7 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, #endif openvino_ep::BackendManager::GetGlobalContext().onnx_opset_version = graph_viewer.DomainToVersionMap().at(kOnnxDomain); -#if defined (OPENVINO_2021_4) - openvino_ep::GetCapability obj(graph_viewer, - openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2021_4"); - result = obj.Execute(); -#elif defined (OPENVINO_2022_1) +#if defined (OPENVINO_2022_1) openvino_ep::GetCapability obj(graph_viewer, openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_1"); result = obj.Execute(); @@ -134,6 +130,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, openvino_ep::GetCapability obj(graph_viewer, openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_2"); result = obj.Execute(); +#elif defined (OPENVINO_2022_3) + openvino_ep::GetCapability obj(graph_viewer, + openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_3"); + result = obj.Execute(); #endif return result; @@ -148,11 +148,7 @@ common::Status OpenVINOExecutionProvider::Compile( NodeComputeInfo compute_info; -#if defined(OV_API_20) openvino_ep::BackendManager::GetGlobalContext().use_api_2 = true; -#else - openvino_ep::BackendManager::GetGlobalContext().use_api_2 = false; -#endif std::shared_ptr backend_manager = std::make_shared(fused_node, graph_body_viewer, *GetLogger()); diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index e32142c955..bdab48790d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -77,7 +77,7 @@ struct OpenVINOExecutionProviderInfo { precision_ = "FP32"; #elif defined OPENVINO_CONFIG_CPU_FP16 device_type_ = "CPU"; - precision_ = "FP16"; + precision_ = "FP16"; #elif defined OPENVINO_CONFIG_GPU_FP32 device_type_ = "GPU"; precision_ = "FP32"; @@ -112,7 +112,7 @@ struct OpenVINOExecutionProviderInfo { precision_ = "FP32"; } else if (dev_type == "CPU_FP16") { device_type_ = "CPU"; - precision_ = "FP16"; + precision_ = "FP16"; } else if (dev_type == "GPU_FP32") { device_type_ = "GPU"; precision_ = "FP32"; @@ -140,14 +140,18 @@ struct OpenVINOExecutionProviderInfo { } else if (dev_type == "VAD-F_FP32") { device_type_ = "HETERO:FPGA,CPU"; precision_ = "FP32"; - } else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) { + } else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0) { std::vector devices = parseDevices(dev_type); precision_ = "FP16"; if (devices[0] == "CPU") { precision_ = "FP32"; } device_type_ = dev_type; - } else { + } else if ( dev_type.find("AUTO") == 0){ + std::vector devices = parseDevices(dev_type); + precision_ = "FP32"; + device_type_ = dev_type; + }else { ORT_THROW("Invalid device string: " + dev_type); } LOGS_DEFAULT(INFO) << "[OpenVINO-EP]" diff --git a/onnxruntime/core/providers/openvino/ov_interface.cc b/onnxruntime/core/providers/openvino/ov_interface.cc index 782e11eaef..ab60f0686e 100644 --- a/onnxruntime/core/providers/openvino/ov_interface.cc +++ b/onnxruntime/core/providers/openvino/ov_interface.cc @@ -9,9 +9,6 @@ #if defined (OV_API_20) using Exception = ov::Exception; -#elif defined (OPENVINO_2021_4) -using Exception = InferenceEngine::Exception; -using WaitMode = InferenceEngine::InferRequest::WaitMode; #else using Exception = InferenceEngine::details::InferenceEngineException; using WaitMode = InferenceEngine::IInferRequest::WaitMode; @@ -23,176 +20,110 @@ namespace onnxruntime { const std::string log_tag = "[OpenVINO-EP] "; std::shared_ptr OVCore::ReadModel(const std::string& model) const { try { - #if defined (OV_API_20) OVTensor weights; return oe.read_model(model, weights); - #else - OVTensorPtr blob = {nullptr}; - auto network = oe.ReadNetwork(model, blob); - auto obj = std::make_shared(network); - return obj; - #endif } catch (const Exception& e) { - ORT_THROW(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what())); + throw std::string(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what())); } catch (...) { - ORT_THROW(log_tag + "[OpenVINO-EP] Unknown exception while Reading network"); + throw std::string(log_tag + "[OpenVINO-EP] Unknown exception while Reading network"); } } - - OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& ie_cnn_network, std::string& hw_target, OVConfig config, std::string name) { - try { - #if defined (OV_API_20) - auto obj = oe.compile_model(ie_cnn_network, hw_target, config); - OVExeNetwork exe(obj); - return exe; - #else - auto obj = oe.LoadNetwork(*ie_cnn_network, hw_target, config); - OVExeNetwork exe(obj); - return exe; - #endif - } catch (const Exception& e) { - ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what()); - } catch (...) { - ORT_THROW(log_tag + " Exception while Loading Network for graph " + name); - } - } - OVExeNetwork OVCore::ImportModel(const std::string& compiled_blob, std::string hw_target, std::string name) { + OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& ie_cnn_network, std::string& hw_target, OVConfig& config, ov::AnyMap& device_config, std::string name) { + ov::CompiledModel obj; try { - #if defined (OV_API_20) - std::ifstream blob_stream_obj(compiled_blob); - auto obj = oe.import_model(blob_stream_obj, hw_target, {}); - return OVExeNetwork(obj); - #else - auto obj = oe.ImportNetwork(compiled_blob, hw_target, {}); - return OVExeNetwork(obj); - #endif - } catch (Exception &e) { - ORT_THROW(log_tag + " Exception while Importing Network for graph: " + name + ": " + e.what()); - } catch(...) { - ORT_THROW(log_tag + " Exception while Importing Network for graph: " + name); + if (hw_target.find("MYRIAD")==std::string::npos || hw_target.find("VAD-M_FP16")==std::string::npos){ + obj = oe.compile_model(ie_cnn_network, hw_target, device_config); + } else { + obj = oe.compile_model(ie_cnn_network, hw_target, config); + } + OVExeNetwork exe(obj); + return exe; + } catch (const Exception& e) { + throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what()); + } catch (...) { + throw std::string(log_tag + " Exception while Loading Network for graph " + name); } } void OVCore::SetCache(std::string cache_dir_path) { - #if defined(OV_API_20) oe.set_property(ov::cache_dir(cache_dir_path)); - #else - oe.SetConfig({{CONFIG_KEY(CACHE_DIR), cache_dir_path}}); - #endif } #ifdef IO_BUFFER_ENABLED OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& model, OVRemoteContextPtr context, std::string& name) { try { - #if defined(OV_API_20) auto obj = oe.compile_model(model, *context); return OVExeNetwork(obj); - #else - auto obj = oe.LoadNetwork(*model, context); - return OVExeNetwork(obj); - #endif } catch (const Exception& e) { - ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what()); + throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what()); } catch (...) { - ORT_THROW(log_tag + " Exception while Loading Network for graph " + name); - } + throw std::string(log_tag + " Exception while Loading Network for graph " + name); + } } #endif std::vector OVCore::GetAvailableDevices() { - #if defined (OV_API_20) auto obj = oe.get_available_devices(); return obj; - #else - auto obj = oe.GetAvailableDevices(); - return obj; - #endif } - + OVInferRequest OVExeNetwork::CreateInferRequest() { try { - #if defined (OV_API_20) - auto infReq = obj.create_infer_request(); - OVInferRequest inf_obj(infReq); - return inf_obj; - #else - auto infReq = obj.CreateInferRequest(); - OVInferRequest inf_obj(infReq); - return inf_obj; - #endif + auto infReq = obj.create_infer_request(); + OVInferRequest inf_obj(infReq); + return inf_obj; } catch (const Exception& e) { - ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what()); + throw std::string(log_tag + "Exception while creating InferRequest object: " + e.what()); } catch (...) { - ORT_THROW(log_tag + "Exception while creating InferRequest object."); + throw std::string(log_tag + "Exception while creating InferRequest object."); } } - + OVTensorPtr OVInferRequest::GetTensor(const std::string& input_name) { try { - #if defined (OV_API_20) auto tobj = ovInfReq.get_tensor(input_name); OVTensorPtr blob = std::make_shared(tobj); return blob; - #else - auto blob = infReq.GetBlob(input_name); - return blob; - #endif } catch (const Exception& e) { - ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); + throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); } catch (...) { - ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name); + throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name); } } void OVInferRequest::SetTensor(const std::string& name, OVTensorPtr& blob) { try { - #if defined(OV_API_20) ovInfReq.set_tensor(name, *(blob.get())); - #else - infReq.SetBlob(name, blob); - #endif } catch (const Exception& e) { - ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name + e.what()); + throw std::string(log_tag + " Cannot set Remote Blob for output: " + name + e.what()); } catch (...) { - ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name); + throw std::string(log_tag + " Cannot set Remote Blob for output: " + name); } } void OVInferRequest::StartAsync() { try { - #if defined (OV_API_20) ovInfReq.start_async(); - #else - infReq.StartAsync(); - #endif } catch (const Exception& e) { - ORT_THROW(log_tag + " Couldn't start Inference: " + e.what()); + throw std::string(log_tag + " Couldn't start Inference: " + e.what()); } catch (...) { - ORT_THROW(log_tag + " In Error Couldn't start Inference"); + throw std::string(log_tag + " In Error Couldn't start Inference"); } } void OVInferRequest::WaitRequest() { try { - #if defined (OV_API_20) - ovInfReq.wait(); - #else - infReq.Wait(WaitMode::RESULT_READY); - #endif + ovInfReq.wait(); } catch (const Exception& e) { - ORT_THROW(log_tag + " Wait Model Failed: " + e.what()); + throw std::string(log_tag + " Wait Model Failed: " + e.what()); } catch (...) { - ORT_THROW(log_tag + " Wait Mode Failed"); - } + throw std::string(log_tag + " Wait Mode Failed"); + } } void OVInferRequest::QueryStatus() { - #if defined (OV_API_20) std::cout << "ovInfReq.query_state()" << " "; - #else - std::cout << &infReq << " "; - #endif } } } diff --git a/onnxruntime/core/providers/openvino/ov_interface.h b/onnxruntime/core/providers/openvino/ov_interface.h index 58ff27b064..fadbddd45c 100644 --- a/onnxruntime/core/providers/openvino/ov_interface.h +++ b/onnxruntime/core/providers/openvino/ov_interface.h @@ -6,7 +6,7 @@ #include #include -#if defined (OPENVINO_2022_1) || (OPENVINO_2022_2) +#if defined (OPENVINO_2022_1) || (OPENVINO_2022_2) || (OPENVINO_2022_3) #define OV_API_20 #include "openvino/openvino.hpp" #include "openvino/pass/convert_fp32_to_fp16.hpp" @@ -15,10 +15,8 @@ #ifdef IO_BUFFER_ENABLED #include #include -#if defined (OV_API_20) #include #endif -#endif #include @@ -28,102 +26,56 @@ class OVCore; class OVInferRequest; class OVExeNetwork; -#if defined (OV_API_20) - typedef InferenceEngine::Precision OVPrecision; - typedef ov::Tensor OVTensor; - typedef ov::ProfilingInfo OVProfilingInfo; - typedef ov::AnyMap OVConfig; - typedef ov::Model OVNetwork; -#else - typedef InferenceEngine::Precision OVPrecision; - typedef InferenceEngine::Blob OVTensor; - typedef InferenceEngine::InferenceEngineProfileInfo OVProfilingInfo; - typedef std::map OVConfig; - typedef InferenceEngine::CNNNetwork OVNetwork; -#endif - typedef std::shared_ptr OVInferRequestPtr; - typedef std::shared_ptr OVTensorPtr; +typedef InferenceEngine::Precision OVPrecision; +typedef ov::Tensor OVTensor; +typedef ov::ProfilingInfo OVProfilingInfo; +typedef ov::AnyMap OVConfig; +typedef ov::Model OVNetwork; +typedef std::shared_ptr OVInferRequestPtr; +typedef std::shared_ptr OVTensorPtr; #ifdef IO_BUFFER_ENABLED - #ifdef OV_API_20 typedef ov::intel_gpu::ocl::ClContext* OVRemoteContextPtr; typedef ov::RemoteContext OVRemoteContext; - #else - typedef InferenceEngine::RemoteContext::Ptr OVRemoteContextPtr; - typedef InferenceEngine::RemoteContext OVRemoteContext; - #endif #endif class OVCore { - #if defined (OV_API_20) ov::Core oe; - #else - InferenceEngine::Core oe; - #endif public: std::shared_ptr ReadModel(const std::string& model_stream) const; - OVExeNetwork LoadNetwork(std::shared_ptr& ie_cnn_network, std::string& hw_target, OVConfig config, std::string name); - OVExeNetwork ImportModel(const std::string& compiled_blob, std::string hw_target, std::string name); + OVExeNetwork LoadNetwork(std::shared_ptr& ie_cnn_network, std::string& hw_target, OVConfig& config, ov::AnyMap& device_config, std::string name); void SetCache(std::string cache_dir_path); #ifdef IO_BUFFER_ENABLED OVExeNetwork LoadNetwork(std::shared_ptr& model, OVRemoteContextPtr context, std::string& name); #endif std::vector GetAvailableDevices(); - #if defined (OV_API_20) ov::Core& Get() { return oe; } - #else - InferenceEngine::Core& Get() { - return oe; - } - #endif }; class OVExeNetwork { - #if (defined OV_API_20) ov::CompiledModel obj; - #else - InferenceEngine::ExecutableNetwork obj; - #endif public: - #if defined (OV_API_20) OVExeNetwork(ov::CompiledModel md) { obj = md; } OVExeNetwork() { obj = ov::CompiledModel(); } ov::CompiledModel& Get() { return obj; } - #else - OVExeNetwork(InferenceEngine::ExecutableNetwork md) { obj = md; } - OVExeNetwork() { obj = InferenceEngine::ExecutableNetwork(); } - InferenceEngine::ExecutableNetwork& Get() { return obj ; } - #endif OVInferRequest CreateInferRequest(); }; class OVInferRequest { - #if defined (OV_API_20) ov::InferRequest ovInfReq; - #else - InferenceEngine::InferRequest infReq; - #endif public: OVTensorPtr GetTensor(const std::string& name); void SetTensor(const std::string& name, OVTensorPtr& blob); void StartAsync(); void WaitRequest(); void QueryStatus(); - #if defined (OV_API_20) explicit OVInferRequest(ov::InferRequest obj) { ovInfReq = obj; } OVInferRequest() { ovInfReq = ov::InferRequest(); } ov::InferRequest& GetNewObj() { return ovInfReq; } - #else - explicit OVInferRequest(InferenceEngine::InferRequest obj) { infReq = obj; } - OVInferRequest() { infReq = InferenceEngine::InferRequest(); } - InferenceEngine::InferRequest& GetObj() { - return infReq; - } - #endif }; } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc index 98379b3174..72828faa4d 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc @@ -26,14 +26,14 @@ namespace openvino_ep { GetCapability::GetCapability(const GraphViewer& graph_viewer_param, std::string device_type_param, const std::string version_param): graph_viewer_(graph_viewer_param), device_type_(device_type_param){ - if (version_param == "V_2021_4") { - data_ops_ = new DataOps(graph_viewer_, V_2021_4, device_type_); - } else if (version_param == "V_2022_1") { + if (version_param == "V_2022_1") { data_ops_ = new DataOps(graph_viewer_, V_2022_1, device_type_); } else if (version_param == "V_2022_2") { data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_); + } else if (version_param == "V_2022_3") { + data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_); } else { - data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_); + data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_); } } diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 277a8dfda3..8efd0609ee 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -128,7 +128,9 @@ std::vector supported_op_mode = { {"GlobalMaxPool", V_2022_1, {"CPU", "GPU"}}, {"Greater", V_2020_4, {"All"}}, {"GreaterOrEqual", V_2022_1, {"All"}}, + {"GridSample", V_2022_3, {"CPU"}}, {"Identity", V_2020_4, {"All"}}, + {"If", V_2022_3, {"All"}}, {"ImageScaler", V_2022_1, {"All"}}, {"InstanceNormalization", V_2020_4, {"All"}}, {"HardSigmoid", V_2020_4, {"CPU", "GPU"}}, @@ -161,6 +163,7 @@ std::vector supported_op_mode = { {"Pad", V_2020_4, {"All"}}, {"Pow", V_2020_4, {"All"}}, {"PRelu", V_2020_4, {"All"}}, + {"QLinearMatMul", V_2022_3, {"CPU"}}, {"QuantizeLinear", V_2021_4, {"CPU", "GPU"}}, {"Range", V_2021_2, {"MYRIAD"}}, {"Range", V_2022_1, {"All"}}, @@ -317,18 +320,7 @@ void DataOps::populate_op_mode_supported() { //populate unsupportedmode_t { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - for (size_t i = 0; i < node->InputDefs().size(); i++) { - if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) - return true; - } - return false; - }}; - op_list_.insert({"Abs", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //Abs is not supproted with INT8 or INT32 as input data type on GPU if (device_id_.find("GPU") != std::string::npos) { @@ -343,7 +335,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Abs", obj}); } { - UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //tensor type does not support select last index auto& attributes = node->GetAttributes(); @@ -359,34 +351,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"ArgMin", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - //auto pad null value is not supported - const auto& attributes = node->GetAttributes(); - auto auto_attr = attributes.find("auto_pad"); - if (auto_attr->second().s() == "") { - return true; - } - // default value of ceil_mode (0) is supported. - auto ceil_attr = attributes.find("ceil_mode"); - if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true; - return (!dimension_unsupported(node)); - }}; - op_list_.insert({"AveragePool", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - //Only float 16, float and double data types are supported - const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - const bool data_is_float16 = node->InputDefs()[0]->Type()->find("float16") != std::string::npos; - const bool data_is_double = node->InputDefs()[0]->Type()->find("double") != std::string::npos; - return !(data_is_float || data_is_float16 || data_is_double); - }}; - op_list_.insert({"Clip", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("GPU") != std::string::npos) { //int64 data type is not supported on GPU @@ -405,46 +370,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Clip", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& ) { - if (device_id_.find("MYRIAD") != std::string::npos) { - const auto& attributes = node->GetAttributes(); - auto conv_filter = attributes.find("kernel_shape"); - auto& ints = conv_filter->second().ints(); - //If the kernel size is not 2D, the op is rejected in case of MYRIAD - if(ints.size() !=2) { - return true; - } - } - //If the device is GPU - if (device_id_.find("GPU") != std::string::npos) { - bool if_bias = false; - const auto& attributes = node->GetAttributes(); - auto conv_filter = attributes.find("kernel_shape"); - if (conv_filter != attributes.end()) { - auto& ints = conv_filter->second().ints(); - //check if the Input for the op has bias - if(node->InputDefs().size() > 2) { - if(node->InputDefs()[2]->Name() == "B") - if_bias = true; - } - //If the kernel size is 1D and the input has bias and the output precision is FP32, the op is rejected. - if(ints.size() == 1 && if_bias) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) - return true; - } - //If the kernel size is 3D and the input doesnot have bias, the op is rejected in case of GPU - if(ints.size() == 3 && !if_bias) - return true; - } - } - return false; - }}; - op_list_.insert({"Conv", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet& ) { if (device_id_.find("MYRIAD") != std::string::npos) { const auto& attributes = node->GetAttributes(); @@ -477,74 +403,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Conv", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - auto& attributes = node->GetAttributes(); - if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") { - return true; - } - return false; - }}; - op_list_.insert({"Conv", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - if (device_id_.find("MYRIAD") != std::string::npos) { - if (GetInputCount(node, initializers) > 1) - return true; - } - bool if_bias = false; - const auto& attributes = node->GetAttributes(); - auto out_shape_attr = attributes.find("output_shape"); - - // If the device is GPU - if (device_id_.find("GPU") != std::string::npos) { - auto conv_filter = attributes.find("kernel_shape"); - if (conv_filter != attributes.end()) { - auto& kernel_size = conv_filter->second().ints(); - - //If 3D convolution, reject the op - if(kernel_size.size() == 3) - return true; - //In 1D conv, if the pads are asymmetric, then the op is rejected - if(kernel_size.size() == 1) { - if (attributes.count("pads") > 0) { - auto& pads_attr = attributes.at("pads"); - auto int_size = pads_attr.ints_size(); - if (int_size > 1 && (pads_attr.ints(0) != pads_attr.ints(1))) - return true; - } - } - //check if the Input for the op has bias - if(node->InputDefs().size() > 2) { - if(node->InputDefs()[2]->Name() == "B") - if_bias = true; - } - //If the kernel size is 2D, the input has no bias, the padding is 0 and the op has dilations, the op is rejected - if(kernel_size.size() == 2 && !if_bias) { - if (attributes.count("pads") > 0) { - auto& pads_attr = attributes.at("pads"); - auto int_size = pads_attr.ints_size(); - // comparing if all the 4 values in the padding are equal to 1 - if (int_size == 4 && (pads_attr.ints(0) == pads_attr.ints(1) == pads_attr.ints(2) == pads_attr.ints(3))) { - if(pads_attr.ints(0) == 1) - return false; - } - // If the op has dilations as an attribute after the above checks, then the op is rejected. - auto dilation_attr = attributes.find("dilations"); - if (dilation_attr != attributes.end()) - return true; - } - } - } - } - return false; - }}; - op_list_.insert({"ConvTranspose", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet& initializers) { if (device_id_.find("MYRIAD") != std::string::npos) { if (GetInputCount(node, initializers) > 1) @@ -576,69 +435,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"ConvTranspose", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - auto& attributes = node->GetAttributes(); - if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") { - return true; - } - return false; - }}; - op_list_.insert({"ConvTranspose", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - // all ConvInteger zero points need to be constants - if (node->InputDefs().size() == 3) { - return (initializers.find(node->InputDefs()[2]->Name()) == initializers.end()); - } else if (node->InputDefs().size() == 4) { - return initializers.find(node->InputDefs()[2]->Name()) == initializers.end() || - initializers.find(node->InputDefs()[3]->Name()) == initializers.end(); - } - return false; - }}; - op_list_.insert({"ConvInteger", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - if (device_id_.find("GPU") != std::string::npos) { - const auto& input = node->InputDefs()[0]; - auto graph_inputs = graph_viewer_.GetInputs(); - auto it = find(graph_inputs.begin(), graph_inputs.end(), input); - if (it != graph_inputs.end()) { - const auto& indices_arg = node->InputDefs()[1]; - if (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64) - return true; - } - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - //If the output of Gather op is INT8 or UINT8, it is rejected for GPU. - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 || - output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8) - return true; - } - return false; - }}; - op_list_.insert({"Gather", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - const auto& indices_arg = node->InputDefs()[0]; - const auto& output_arg = node->OutputDefs()[0]; - if (indices_arg->TypeAsProto()->tensor_type().elem_type() != output_arg->TypeAsProto()->tensor_type().elem_type()) - return true; - if ((indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) || - (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)) { - return false; - } - return true; - }}; - op_list_.insert({"GatherElements", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("MYRIAD") != std::string::npos) { const auto& indices_arg = node->InputDefs()[0]; @@ -656,32 +453,23 @@ void DataOps::populate_op_mode_supported() { }}; op_list_.insert({"GatherElements", obj}); } - { - UnsupportedOpMode obj = {{V_2021_4}, + { + UnsupportedOpMode obj = {{V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { - const auto& input = node->InputDefs()[0]; - const auto& output = node->OutputDefs()[0]; - auto graph_inputs = this->graph_viewer_.GetInputs(); - auto graph_outputs = this->graph_viewer_.GetOutputs(); - auto input_it = find(graph_inputs.begin(), graph_inputs.end(), input); - auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); - if (input_it != graph_inputs.end() && output_it != graph_outputs.end()) - return true; - return false; + if (device_id_.find("GPU") != std::string::npos && node->OpType()=="If"){ + // Only Equal op is supported as input for IF op in GPU + for (auto nit = node->InputNodesBegin(); nit != node->InputNodesEnd(); ++nit){ + if (nit->OpType() == "Equal"){ + return false; + } + } + } + return true; }}; - op_list_.insert({"Identity", obj}); + op_list_.insert({"If", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - //Loop has to be initializer - const auto& cond = node->InputDefs()[1]; - return (initializers.find(cond->Name()) == initializers.end()); - }}; - op_list_.insert({"Loop", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //If the Input size of LSTM is greater than 3, it is rejected. if (device_id_.find("MYRIAD") != std::string::npos) { @@ -693,35 +481,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"LSTM", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - //MaxPool "indices" output is not currently supported. - //if (node->OutputDefs().size() > 1) - // return true; - const auto& attributes = node->GetAttributes(); - /* default value of ceil_mode (0) is supported. - auto ceil_attr = attributes.find("ceil_mode"); - if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) - return true;*/ - auto auto_attr = attributes.find("auto_pad"); - //auto pad null value is not supported - if (auto_attr->second().s() == "") - return true; - // dilations attrs are not supported in nGraph - if (attributes.find("dilations") != attributes.end()) - return true; - // uint8 as output type for maxpool op is not supported on GPU - if (device_id_.find("GPU") != std::string::npos) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8) - return true; - } - return (!this->dimension_unsupported(node)); - }}; - op_list_.insert({"MaxPool", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { const auto& attributes = node->GetAttributes(); // dilations attrs are not supported yet for Maxpool @@ -732,47 +492,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"MaxPool", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - if (device_id_.find("MYRIAD") == std::string::npos) { - if (GetInputCount(node, initializers) == 1) - return true; - } - return false; - }}; - op_list_.insert({"Max", obj}); - op_list_.insert({"Min", obj}); - op_list_.insert({"Mean", obj}); - op_list_.insert({"Sum", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - //All matmuls except float have computation missmatch - const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; - const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; - return (A_is_float && B_is_float) ? false : true; - }}; - op_list_.insert({"MatMul", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - // all MatMulInteger zero points need to be constants - if (node->InputDefs().size() == 3) { - // not found in initializers -> not const - return initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); - } else if (node->InputDefs().size() == 4) { - // not found in initializers -> not const - return ((initializers.find(node->InputDefs()[2]->Name()) == initializers.end()) || - (initializers.find(node->InputDefs()[2]->Name()) == initializers.end())); - } - return false; - }}; - op_list_.insert({"MatMulInteger", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("GPU") != std::string::npos) { auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); @@ -788,32 +508,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Mod", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - if (device_id_.find("GPU") != std::string::npos) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - //If the output of Neg op is INT8, it is rejected for GPU. - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) - return true; - } - return false; - }}; - op_list_.insert({"Neg", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - auto graph_outputs = graph_viewer_.GetOutputs(); - const auto& output = node->OutputDefs()[0]; - auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); - if (output_it != graph_outputs.end()) - return true; - return false; - }}; - op_list_.insert({"NonMaxSuppression", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("GPU") != std::string::npos) { auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); @@ -832,7 +527,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Pow", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { // Max op with one input is not supporting for GPU_FP16 if (device_id_.find("GPU") != std::string::npos) { @@ -848,7 +543,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Max", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { // Min op with one input is not supporting for GPU_FP16 if (device_id_.find("GPU") != std::string::npos) { @@ -864,7 +559,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Min", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { // Sum op with one input is not supporting for GPU_FP16 if (device_id_.find("GPU") != std::string::npos) { @@ -880,24 +575,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Sum", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - auto slope = node->InputDefs()[1]; - //PRelu slope has to be an initializer or needs to come from a constant node - if (initializers.count(slope->Name())) - return false; - else { - for (auto input_node = node->InputNodesBegin(); input_node != node->InputNodesEnd(); ++input_node) { - if (GetInputCount(this->graph_viewer_.GetNode((*input_node).Index()), initializers) == 0) - return false; - } - } - return true; - }}; - op_list_.insert({"PRelu", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet& initializers) { if (device_id_.find("GPU") != std::string::npos) { auto slope = node->InputDefs()[1]; @@ -916,20 +594,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"PRelu", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet& initializers) { - bool non_const_zero_point = false; - // check if any of the zero points is NOT in the initializers list - non_const_zero_point |= initializers.find(node->InputDefs()[2]->Name()) == initializers.end(); - non_const_zero_point |= initializers.find(node->InputDefs()[5]->Name()) == initializers.end(); - non_const_zero_point |= initializers.find(node->InputDefs()[7]->Name()) == initializers.end(); - // QLinearMatMul is not supported if any of the zero points is a dynamic input - return non_const_zero_point; - }}; - op_list_.insert({"QLinearMatMul", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { const auto& input_arg = node->InputDefs()[1]; auto shape = input_arg->Shape(); @@ -945,7 +610,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Reshape", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1}, [this](const Node* node, const InitializedTensorSet&) { auto& attributes = node->GetAttributes(); if (attributes.count("mode") ==1 && attributes.at("mode").s() == "linear") { @@ -958,21 +623,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Resize", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - if (device_id_.find("GPU") != std::string::npos) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - //If the output of ReduceMax op is INT8 or UINT8, it is rejected for GPU. - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 || - output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8) - return true; - } - return false; - }}; - op_list_.insert({"ReduceMax", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("GPU") != std::string::npos || device_id_.find("MYRIAD") != std::string::npos) { //INT32 dataype is not supported as input @@ -986,35 +637,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"ReduceLogSumExp", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - const auto& attributes = node->GetAttributes(); - auto axis_attr = attributes.find("axis"); - //Negative axis is not supported - if (axis_attr->second().i() < 0) - return true; - if (device_id_.find("MYRIAD") != std::string::npos) { - const auto& input_arg = node->InputDefs()[2]; - auto updates_shape = input_arg->Shape(); - const auto& output_arg = node->OutputDefs()[0]; - auto out_shape = output_arg->Shape(); - //If updates attribute dim value greater than output_shape dim value, we reject - if(node->InputDefs()[2]->Name() == "updates") - { - size_t updates_size = updates_shape->dim_size(); - if(updates_size == 2) { - if(updates_shape->dim(1).dim_value() > out_shape->dim(1).dim_value()) - return true; - } - } - } - return false; - }}; - op_list_.insert({"Scatter", obj}); - op_list_.insert({"ScatterElements", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("MYRIAD") != std::string::npos) { const auto& attributes = node->GetAttributes(); @@ -1043,7 +666,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"ScatterElements", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { if (device_id_.find("GPU") != std::string::npos) { auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); @@ -1058,7 +681,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Scatter", obj}); } { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //If the Input of Shrink op is UINT8, it is rejected (Due to output mismatch) for (size_t i = 0; i < node->InputDefs().size(); i++) { @@ -1070,7 +693,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Shrink", obj}); } { - UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet& initializers) { //start, end, axes need to be a initializer bool cond_for_slice = false; @@ -1095,17 +718,6 @@ void DataOps::populate_op_mode_supported() { }}; op_list_.insert({"Slice", obj}); } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - //Shape can't have empty axes attribute - const auto& attributes = node->GetAttributes(); - if (attributes.count("axes") == 0) - return true; - return false; - }}; - op_list_.insert({"Squeeze", obj}); - } { UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, [this](const Node* node, const InitializedTensorSet&) { @@ -1121,27 +733,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Squeeze", obj}); } { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - if (device_id_.find("GPU") != std::string::npos) { - auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - //If the output of Transpose op is INT8, it is rejected for GPU. - if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) - return true; - } - return false; - }}; - op_list_.insert({"Transpose", obj}); - } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - return (!this->dimension_unsupported(node)); - }}; - op_list_.insert({"Unsqueeze", obj}); - } - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //If the operator is unsqueeze //If axes is an input, then we cannot produce a static graph. Conversion fails in convert_function_to_cnn_network. @@ -1155,7 +747,7 @@ void DataOps::populate_op_mode_supported() { op_list_.insert({"Unsqueeze", obj}); } { - UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2}, + UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, [this](const Node* node, const InitializedTensorSet&) { //check for attributes auto& upsample_attr = node->GetAttributes(); @@ -1187,17 +779,13 @@ void DataOps::populate_op_mode_supported() { }}; op_list_.insert({"Upsample", obj}); } - { - UnsupportedOpMode obj = {{V_2021_4}, - [this](const Node* node, const InitializedTensorSet&) { - return (!this->dimension_unsupported(node)); - }}; - op_list_.insert({"ReduceSum", obj}); - } } bool DataOps::op_is_supported(std::string name, std::vector& op_list) { + bool auto_support = false; + bool multi_support = false; for (size_t i = 0; i < op_list.size(); i++) { + if (op_list[i].optype == name) { if (op_list[i].version <= version_id_) { auto it = op_list[i].device_type.begin(); @@ -1205,46 +793,63 @@ bool DataOps::op_is_supported(std::string name, std::vector& op_lis //status variable is set to True if it's Hetero/Multi/Auto device type bool status = false; - //if device supported is all then we support it - if (*it == "All") { - return true; - } - //The operator to be marked true, it should be supported by either of the devices specified with HETERO if (device_id_.find("HETERO") == 0) { status = true; - if (device_id_.find(*it) != std::string::npos) { + if (device_id_.find(*it) != std::string::npos || (*it == "All")) { return true; } } //The operator to be marked true, it should be supported by all the devices specified with MULTI/AUTO - if (device_id_.find("MULTI") == 0 || device_id_.find("AUTO") == 0) { + if (device_id_.find("MULTI") == 0) { status = true; - if (device_id_.find(*it) == std::string::npos) { - return false; + if ((*it == "All") || device_id_.find(*it) != std::string::npos) { + multi_support = true; } } - + //The operator to be marked true, it should be supported by atleast CPU device specified with AUTO + if (device_id_.find("AUTO") == 0) { + if (std::string(*it).find("CPU")==std::string::npos){ + auto_support = false; + } + else if((*it == "All") || (device_id_.find(*it) != std::string::npos)){ + auto_support = true; + } + } + //if device supported is all then we support it + if (*it == "All") { + return true; + } //check for device supported if (status == false) { if (device_id_.find(*it) != std::string::npos) { return true; } } - it++; } + } } } - + if (device_id_.find("AUTO") == 0 && auto_support == true) { + return true; + } + if (device_id_.find("MULTI") == 0 && multi_support == true) { + return true; + } return false; } bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { const auto* type_proto = node_arg->TypeAsProto(); if (!type_proto) { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Node is not a proto " << std::endl; + } +#endif return false; } @@ -1297,9 +902,6 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { return false; } else if (device_id_ == "GPU") { - auto prec_str = openvino_ep::BackendManager::GetGlobalContext().precision_str; - if (prec_str == "FP32" && dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) - return false; for (auto const& var : supported_types_gpu_) { if ((var.first <= version_id_) && (var.second == dtype)) { @@ -1417,6 +1019,11 @@ bool DataOps::node_is_supported(const std::map& } } else if (node->OpType() == "MaxPool" && device_id_.find("MYRIAD") != std::string::npos) { auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); - if (output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT || - output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) { + if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT || + output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) { + return false; + } + else { return true; } } diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h index adc00b8fdf..46c485092a 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h @@ -16,7 +16,8 @@ enum versionNum { V_2021_3, V_2021_4, V_2022_1, - V_2022_2 + V_2022_2, + V_2022_3 }; using VersionNum = enum versionNum; diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.cc b/onnxruntime/core/providers/openvino/ov_versions/utils.cc index 940ac801ac..24f7c60dee 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/utils.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/utils.cc @@ -115,8 +115,10 @@ GetPartitionedClusters(const std::vector& topological_order, const st if (!this_cluster.empty()) { ng_clusters.push_back(std::move(this_cluster)); } - // Point prev to node idx past this unsuported node. - prev = ++it; + if (it != topological_order.end()) { + // Point prev to node idx past this unsuported node. + prev = ++it; + } } //Tail diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 721372c3d1..3fa6954a7c 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -140,6 +140,9 @@ def create_backend_test(test_name=None): if backend.supports_device("OPENVINO_CPU_FP32"): current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP32") + if backend.supports_device("OPENVINO_CPU_FP16"): + current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP16") + if backend.supports_device("MIGRAPHX"): current_failing_tests += apply_filters(filters, "current_failing_tests_MIGRAPHX") diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index 6d49d33e94..674c2d5cfa 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -261,6 +261,8 @@ "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. "^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op. "^test_operator_permute2", + "^test_leakyrelu", + "^test_leakyrelu_example", "^test_operator_repeat", "^test_operator_repeat_dim_overflow", "^test_add_bcast.*", @@ -279,7 +281,10 @@ "^test_unique_not_sorted_without_axis", "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. "^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op. + "^test_scatter_elements_with_negative_indices", "^test_sce.*", + "^test_squeeze", + "^test_squeeze_negative_axes", "^test_nllloss.*", "^test_gather_negative_indices.*", "^test_reduce_sum_do_not_keepdims*", // Does not support axes as input @@ -303,6 +308,14 @@ "^test_maxpool_with_argmax_2d_precomputed_strides", //Disabled as it throws segfault "^test_maxpool_with_argmax_2d_precomputed_pads" //Disabled as it throws segfault ], + "current_failing_tests_OPENVINO_CPU_FP16": [ + "^test_gridsample*", + "^test_reduce_sum_do_not_keepdims_example", + "^test_reduce_sum_do_not_keepdims_random", + "^test_scatter_elements_with_negative_indices", + "^test_squeeze", + "^test_squeeze_negative_axes" + ], "current_failing_tests_OPENVINO_CPU_FP32": [ "^test_operator_permute2", "^test_operator_repeat", @@ -312,6 +325,7 @@ "^test_max_float64.*", "^test_min_float64.*", "^test_gather_negative_indices.*", + "^test_gridsample*", "^test_sce.*", "^test_nllloss.*", "^test_upsample_nearest.*", diff --git a/setup.py b/setup.py index 6ad931159c..b7cd7c753a 100644 --- a/setup.py +++ b/setup.py @@ -567,25 +567,25 @@ if enable_training: # To support the package consisting of both openvino and training modules part of it package_name = "onnxruntime-training" - disable_local_version = environ.get("ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION", "0") - disable_local_version = ( - disable_local_version == "1" - or disable_local_version.lower() == "true" - or disable_local_version.lower() == "yes" - ) - # local version should be disabled for internal feeds. - if not disable_local_version: - # we want put default training packages to pypi. pypi does not accept package with a local version. - if not default_training_package_device or nightly_build: - if cuda_version: - # removing '.' to make Cuda version number in the same form as Pytorch. - local_version = "+cu" + cuda_version.replace(".", "") - elif rocm_version: - # removing '.' to make Rocm version number in the same form as Pytorch. - local_version = "+rocm" + rocm_version.replace(".", "") - else: - # cpu version for documentation - local_version = "+cpu" + disable_local_version = environ.get("ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION", "0") + disable_local_version = ( + disable_local_version == "1" + or disable_local_version.lower() == "true" + or disable_local_version.lower() == "yes" + ) + # local version should be disabled for internal feeds. + if not disable_local_version: + # we want put default training packages to pypi. pypi does not accept package with a local version. + if not default_training_package_device or nightly_build: + if cuda_version: + # removing '.' to make Cuda version number in the same form as Pytorch. + local_version = "+cu" + cuda_version.replace(".", "") + elif rocm_version: + # removing '.' to make Rocm version number in the same form as Pytorch. + local_version = "+rocm" + rocm_version.replace(".", "") + else: + # cpu version for documentation + local_version = "+cpu" if package_name == "onnxruntime-tvm": packages += ["onnxruntime.providers.tvm"] diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 27e11be8bb..a1bbec12d1 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU-2019' JobName: 'Linux_CI_Dev' - RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2022.2.0 -x "--use_openvino CPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2022.3.0 -x "--use_openvino CPU_FP32 --build_wheel"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino index f0143f8c10..a91f7100ab 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino @@ -1,7 +1,7 @@ ARG UBUNTU_VERSION=20.04 FROM ubuntu:${UBUNTU_VERSION} -ARG OPENVINO_VERSION=2022.2.0 +ARG OPENVINO_VERSION=2022.3.0 ARG PYTHON_VERSION=3.8 ADD scripts /tmp/scripts @@ -21,9 +21,9 @@ ENV DEBIAN_FRONTEND=noninteractive ENV PATH /usr/local/gradle/bin:$PATH RUN cd /opt && mkdir -p intel && cd intel && \ - wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \ - tar xzf l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \ - mv l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64 openvino_2022.2.0 && \ + wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ + tar xzf l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ + mv l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 && \ cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y WORKDIR /root diff --git a/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython b/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython index a38d8bbe3b..d9aa07c035 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython +++ b/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython @@ -4,13 +4,14 @@ ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbi ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts -ARG PYTHON_VER_PATH="cp36-cp36m" -ARG PYTHON_VERSION="3.6" +ARG PYTHON_VER_PATH="cp38-cp38" +ARG PYTHON_VERSION="3.8" ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev ARG OV_DEVICE_PRECISION="CPU_FP32" ARG ENABLE_TRAINING=true -ARG ORT_BRANCH="main" +ARG ORT_BRANCH="rel-1.13.1" +ARG OV_VERSION="2022.2.0" RUN adduser --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER ENV PYTHON_EXE="/opt/python/$PYTHON_VER_PATH/bin/python$PYTHON_VERSION" @@ -36,15 +37,15 @@ RUN ${PYTHON_EXE} -m pip install onnx numpy wheel USER $BUILD_USER RUN cd $WORKDIR && git clone https://github.com/openvinotoolkit/openvino.git && \ cd openvino && \ - git checkout 2022.1.0 && \ + git checkout $OV_VERSION && \ git submodule init && \ git submodule update --recursive RUN cd $WORKDIR && cd openvino && mkdir build && cd build && \ - cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DCMAKE_INSTALL_PREFIX=/home/onnxruntimedev/openvino_2022.1.0 && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=$PYTHON_EXE -DCMAKE_INSTALL_PREFIX=/home/onnxruntimedev/openvino_$OV_VERSION && \ make -j8 && make install -ENV INTEL_OPENVINO_DIR /home/onnxruntimedev/openvino_2022.1.0 +ENV INTEL_OPENVINO_DIR /home/onnxruntimedev/openvino_$OV_VERSION ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64:$INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH ENV TBB_LIBS $INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib ENV InferenceEngine_DIR $INTEL_OPENVINO_DIR/runtime/cmake diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index 532ed8d1ef..ca82de2451 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -708,14 +708,16 @@ def generate_files(line_list, args): + '\\native" />' ) # usb-ma2x8x.mvcmd - files_list.append( - "' - ) + # OpenVINO 2022.3 doesn't have usb-ma2x8x.mvcmd + if "2022.3" not in openvino_path: + files_list.append( + "' + ) for tbb_element in os.listdir(tbb_list_path): if tbb_element.endswith("dll"): files_list.append(