mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-25 19:48:11 +00:00
Openvino ep 2022.3 v4.3 (#14210)
### Description Changes to incorporate OpenVINO EP 2022.3 ### Motivation and Context This change is required to incorportate OpenVINO EP 2022.3 - If it fixes an open issue, please link to the issue here. --> Co-authored-by: mohsinmx <mohsinx.mohammad@intel.com> Co-authored-by: Preetha Veeramalai <preetha.veeramalai@intel.com> Co-authored-by: Aravind <aravindx.gunda@intel.com> Co-authored-by: mayavijx <mayax.vijayan@intel.com> Co-authored-by: flexci <mohsinmx>
This commit is contained in:
sfatimar
GitHub
parent
26a6b40554
commit
7654cd50e8
28 changed files with 668 additions and 1467 deletions
|
|
@ -1105,18 +1105,18 @@ if (onnxruntime_USE_OPENVINO)
|
|||
endif()
|
||||
|
||||
# Check OpenVINO version for support
|
||||
if (${VER} MATCHES "2021.4" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2021.4")
|
||||
set(OPENVINO_VERSION "2021.4")
|
||||
add_definitions(-DOPENVINO_2021_4=1)
|
||||
elseif (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1")
|
||||
if (${VER} MATCHES "2022.1" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.1")
|
||||
set(OPENVINO_VERSION "2022.1")
|
||||
add_definitions(-DOPENVINO_2022_1=1)
|
||||
elseif (${VER} MATCHES "2022.2" OR $ENV{INTEL_OPENVINO_DIR} MATCHES "2022.2")
|
||||
set(OPENVINO_VERSION "2022.2")
|
||||
add_definitions(-DOPENVINO_2022_2=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2022.3")
|
||||
set(OPENVINO_VERSION "2022.3")
|
||||
add_definitions(-DOPENVINO_2022_3=1)
|
||||
elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino")
|
||||
set(OPENVINO_VERSION "2022.2")
|
||||
add_definitions(-DOPENVINO_2022_2=1)
|
||||
set(OPENVINO_VERSION "2022.3")
|
||||
add_definitions(-DOPENVINO_2022_3=1)
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}")
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
#--------------------------------------------------------------------------
|
||||
|
||||
ARG OPENVINO_VERSION=2022.2.0
|
||||
ARG OPENVINO_VERSION=2022.3.0
|
||||
|
||||
|
||||
# Build stage
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
#--------------------------------------------------------------------------
|
||||
|
||||
ARG OPENVINO_VERSION=2022.2.0
|
||||
ARG OPENVINO_VERSION=2022.3.0
|
||||
|
||||
# Build stage
|
||||
FROM openvino/ubuntu20_runtime:${OPENVINO_VERSION} AS base
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ ARG DEVICE=CPU_FP32
|
|||
ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime
|
||||
ARG ONNXRUNTIME_BRANCH=main
|
||||
|
||||
ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2022.2.0.7713
|
||||
ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2022.3.0
|
||||
|
||||
ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake
|
||||
ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/runtime/lib/intel64/
|
||||
|
|
@ -17,6 +17,7 @@ ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/runtime/3rdparty/tbb/lib/:${IE_PLUGINS
|
|||
ENV OpenCV_DIR=${INTEL_OPENVINO_DIR}/extras/opencv/cmake
|
||||
ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/extras/opencv/lib:${LD_LIBRARY_PATH}
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64:/lib64:${LD_LIBRARY_PATH}
|
||||
ENV PATH=${MY_ROOT}/cmake-dir/bin:$PATH
|
||||
|
||||
# Install packages
|
||||
RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-devel libffi-devel openssl-devel git make gcc && \
|
||||
|
|
@ -25,7 +26,7 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve
|
|||
cd $MY_ROOT && \
|
||||
wget https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tgz && tar xvf Python-3.8.9.tgz && rm -rf Python-3.8.9.tgz && \
|
||||
cd Python-3.8*/ && ./configure && make && make install && \
|
||||
cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.9/ /usr/bin/Python38 && \
|
||||
cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.9/ /usr/bin/Python38 && ln -s /usr/bin/pip3 /usr/bin/pip && \
|
||||
# libusb1.0.22
|
||||
cd /opt/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \
|
||||
unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /opt/libusb-1.0.22 && \
|
||||
|
|
@ -42,15 +43,16 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve
|
|||
/bin/mkdir -p '/usr/local/lib/pkgconfig' && \
|
||||
# Install openvino
|
||||
cd /opt/ && mkdir intel/ && cd intel && \
|
||||
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \
|
||||
tar xvf l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \
|
||||
rm -rf l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \
|
||||
mv l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64 openvino_2022.2.0.7713 && \
|
||||
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \
|
||||
tar xvf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \
|
||||
rm -rf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \
|
||||
mv l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 && \
|
||||
cd ${INTEL_OPENVINO_DIR}/install_dependencies/ && ./install_openvino_dependencies.sh -y && ./install_NEO_OCL_driver.sh -y && \
|
||||
printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2022.2.0.7713/setupvars.sh && \
|
||||
printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2022.3.0/setupvars.sh && \
|
||||
cd /opt/libusb-1.0.22 && \
|
||||
/usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \
|
||||
cp /opt/intel/openvino_2022.2.0.7713/install_dependencies/97-myriad-usbboot.rules /etc/udev/rules.d/ && \
|
||||
# MYRIAD plugins are not available for openvino 2022.3.0 release
|
||||
#cp /opt/intel/openvino_2022.3.0/install_dependencies/97-myriad-usbboot.rules /etc/udev/rules.d/ && \
|
||||
ldconfig && \
|
||||
#Install protobuf
|
||||
cd $MY_ROOT && \
|
||||
|
|
@ -64,10 +66,11 @@ RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-deve
|
|||
make install && \
|
||||
# Build onnxruntime
|
||||
cd $MY_ROOT && \
|
||||
pip3 install numpy wheel setuptools cython && \
|
||||
pip3 install numpy wheel setuptools cython onnx && \
|
||||
git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} && \
|
||||
pip3 install onnx && \
|
||||
source /opt/intel/openvino_2022.2.0.7713/setupvars.sh && \
|
||||
bash onnxruntime/dockerfiles/scripts/install_common_deps.sh && \
|
||||
ln -s cmake-* cmake-dir && \
|
||||
source /opt/intel/openvino_2022.3.0/setupvars.sh && \
|
||||
cd /code/onnxruntime && ./build.sh --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel && \
|
||||
pip3 install /code/onnxruntime/build/Linux/Release/dist/*-linux_x86_64.whl && \
|
||||
# Clean up
|
||||
|
|
|
|||
|
|
@ -84,11 +84,12 @@ git submodule update --init
|
|||
## OpenVINO
|
||||
*Public Preview*
|
||||
|
||||
**Ubuntu 18.04, Python & C# Bindings**
|
||||
**Ubuntu 20.04, Python & C# Bindings**
|
||||
**RHEL 8.4, Python Binding**
|
||||
|
||||
### **1. Using pre-built container images for Python API**
|
||||
|
||||
The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu18) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code.
|
||||
The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu20) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code.
|
||||
If the `device_type` runtime config option is not explicitly specified, CPU will be chosen as the hardware target execution.
|
||||
### **2. Building from Dockerfile**
|
||||
|
||||
|
|
@ -96,7 +97,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will
|
|||
|
||||
Retrieve your docker image in one of the following ways.
|
||||
|
||||
- Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as <Dockerfile> for building an OpenVINO 2021.3 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is main.
|
||||
- Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as <Dockerfile> for building latest OpenVINO based Docker image for Ubuntu20.04 and Dockerfile.openvino-rhel for Python API for RHEL 8.4. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is main.
|
||||
```
|
||||
docker build --rm -t onnxruntime --build-arg DEVICE=$DEVICE -f <Dockerfile> .
|
||||
```
|
||||
|
|
@ -107,6 +108,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will
|
|||
| Device Option | Target Device |
|
||||
| --------- | -------- |
|
||||
| <code>CPU_FP32</code> | Intel<sup></sup> CPUs |
|
||||
| <code>CPU_FP16</code> | Intel<sup></sup> CPUs |
|
||||
| <code>GPU_FP32</code> |Intel<sup></sup> Integrated Graphics |
|
||||
| <code>GPU_FP16</code> | Intel<sup></sup> Integrated Graphics |
|
||||
| <code>MYRIAD_FP16</code> | Intel<sup></sup> Movidius<sup>TM</sup> USB sticks |
|
||||
|
|
@ -173,7 +175,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will
|
|||
|
||||
### OpenVINO on VAD-M Accelerator Version
|
||||
|
||||
1. Download OpenVINO **Full package** for version **2021.4** for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html)
|
||||
1. Download OpenVINO **Full package** for latest version for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html)
|
||||
|
||||
2. Install the drivers on the host machine according to the reference in [here](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux_ivad_vpu.html)
|
||||
|
||||
|
|
@ -184,7 +186,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will
|
|||
4. Run hddldaemon on the host in a separate terminal session using the following steps:
|
||||
- Initialize the OpenVINO environment.
|
||||
```
|
||||
source <openvino_install_directory>/bin/setupvars.sh
|
||||
source <openvino_install_directory>/setupvars.sh
|
||||
```
|
||||
- Edit the hddl_service.config file from $HDDL_INSTALL_DIR/config/hddl_service.config and change the field “bypass_device_number” to 8.
|
||||
- Restart the hddl daemon for the changes to take effect.
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ This package supports:
|
|||
|
||||
Please Note for VAD-M use Docker installation / Build from Source for Linux.
|
||||
|
||||
``pip3 install onnxruntime-openvino==1.12.0``
|
||||
``pip3 install onnxruntime-openvino==1.13.1``
|
||||
|
||||
Please install OpenVINO™ PyPi Package separately for Windows.
|
||||
For installation instructions on Windows please refer to `OpenVINO™ Execution Provider for ONNX Runtime for Windows <https://github.com/intel/onnxruntime/releases/>`_.
|
||||
|
||||
**OpenVINO™ Execution Provider for ONNX Runtime** Linux Wheels comes with pre-built libraries of OpenVINO™ version 2022.1.0 eliminating the need to install OpenVINO™ separately. The OpenVINO™ libraries are prebuilt with CXX11_ABI flag set to 0.
|
||||
**OpenVINO™ Execution Provider for ONNX Runtime** Linux Wheels comes with pre-built libraries of OpenVINO™ version 2022.2.0 eliminating the need to install OpenVINO™ separately. The OpenVINO™ libraries are prebuilt with CXX11_ABI flag set to 0.
|
||||
|
||||
The package also includes module that is used by torch-ort-inference to accelerate inference for PyTorch models with OpenVINO Execution Provider.
|
||||
See `torch-ort-inference <https://github.com/pytorch/ort#accelerate-inference-for-pytorch-models-with-onnx-runtime-preview>`_ for more details.
|
||||
|
|
@ -50,18 +50,6 @@ Samples
|
|||
|
||||
To see what you can do with **OpenVINO™ Execution Provider for ONNX Runtime**, explore the demos located in the `Examples <https://github.com/microsoft/onnxruntime-inference-examples/tree/main/python/OpenVINO_EP>`_.
|
||||
|
||||
Docker Support
|
||||
^^^^^^^^^^^^^^
|
||||
|
||||
The latest OpenVINO™ EP docker image can be downloaded from DockerHub.
|
||||
For more details see `Docker ReadMe <https://hub.docker.com/r/openvino/onnxruntime_ep_ubuntu18>`_.
|
||||
|
||||
|
||||
Prebuilt Images
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
- Please find prebuilt docker images for Intel® CPU and Intel® iGPU on OpenVINO™ Execution Provider `Release Page <https://github.com/intel/onnxruntime/releases/>`_.
|
||||
|
||||
License
|
||||
^^^^^^^^
|
||||
|
||||
|
|
|
|||
|
|
@ -31,8 +31,8 @@ void BackendManager::ReleaseGlobalContext() {
|
|||
g_global_context.reset();
|
||||
}
|
||||
|
||||
BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
||||
const onnxruntime::GraphViewer& subgraph,
|
||||
BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
||||
const onnxruntime::GraphViewer& subgraph,
|
||||
const logging::Logger& logger) {
|
||||
auto prec_str = GetGlobalContext().precision_str;
|
||||
if (prec_str == "FP32") {
|
||||
|
|
@ -40,7 +40,7 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
} else if (prec_str == "FP16") {
|
||||
subgraph_context_.precision = InferenceEngine::Precision::FP16;
|
||||
} else {
|
||||
ORT_THROW("Invalid OpenVINO Precision type: " + prec_str);
|
||||
throw std::string("Invalid OpenVINO Precision type: " + prec_str);
|
||||
}
|
||||
|
||||
// Save the indexes of graph inputs among fused_node's inputDefs
|
||||
|
|
@ -64,7 +64,7 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
}
|
||||
auto it = subgraph_context_.input_names.find(input->Name());
|
||||
if (it == subgraph_context_.input_names.end()) {
|
||||
ORT_THROW("Input not found in the input defs list");
|
||||
throw std::string("Input not found in the input defs list");
|
||||
}
|
||||
int index = it->second;
|
||||
subgraph_context_.input_indexes.push_back(index);
|
||||
|
|
@ -85,7 +85,13 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
subgraph_context_.enable_batching = true;
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model can be Batch inferenced \n";
|
||||
auto model_copy = ReWriteBatchDimWithOne(*model_proto_);
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_copy, GetGlobalContext(), subgraph_context_);
|
||||
try {
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_copy,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
subgraph_context_.has_dynamic_input_shape = false;
|
||||
|
||||
} else if (ModelHasSymbolicInputDims(subgraph)) {
|
||||
|
|
@ -97,15 +103,19 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
if (GetGlobalContext().device_type.find("CPU") != std::string::npos) {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has symbolic input dims and "
|
||||
<< "device_type is CPU.";
|
||||
#if (defined OV_API_20)
|
||||
if (GetGlobalContext().enable_dynamic_shapes) {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
|
||||
<< "Creating backend Dynamic Shapes";
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, GetGlobalContext(), subgraph_context_);
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
|
||||
<< "Backend created for graph " << subgraph_context_.subgraph_name;
|
||||
}
|
||||
#endif
|
||||
if (GetGlobalContext().enable_dynamic_shapes) {
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Starting backend initialization. "
|
||||
<< "Creating backend Dynamic Shapes";
|
||||
try {
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
|
||||
<< "Backend created for graph " << subgraph_context_.subgraph_name;
|
||||
}
|
||||
}
|
||||
} else if (ModelHasSymbolicInputDims(subgraph) &&
|
||||
GetGlobalContext().device_type.find("GPU") != std::string::npos) {
|
||||
|
|
@ -115,7 +125,13 @@ BackendManager::BackendManager(const onnxruntime::Node& fused_node,
|
|||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model has concreate input dims. Initializing backend for graph " << subgraph_context_.subgraph_name;
|
||||
|
||||
subgraph_context_.has_dynamic_input_shape = false;
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, GetGlobalContext(), subgraph_context_);
|
||||
try {
|
||||
concrete_backend_ = BackendFactory::MakeBackend(*model_proto_,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -172,8 +188,8 @@ bool BackendManager::ModelHasSymbolicInputDims(const onnxruntime::GraphViewer& s
|
|||
}
|
||||
|
||||
std::unique_ptr<ONNX_NAMESPACE::ModelProto>
|
||||
BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node,
|
||||
const onnxruntime::GraphViewer& subgraph,
|
||||
BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node,
|
||||
const onnxruntime::GraphViewer& subgraph,
|
||||
const logging::Logger& logger) const {
|
||||
auto model = subgraph.CreateModel(logger);
|
||||
|
||||
|
|
@ -195,7 +211,7 @@ BackendManager::GetModelProtoFromFusedNode(const onnxruntime::Node& fused_node,
|
|||
}
|
||||
|
||||
std::vector<std::vector<int64_t>> GetInputTensorShapes(Ort::KernelContext& context) {
|
||||
|
||||
|
||||
const auto input_count = context.GetInputCount();
|
||||
std::vector<std::vector<int64_t>> input_shapes;
|
||||
input_shapes.reserve(input_count);
|
||||
|
|
@ -273,10 +289,8 @@ void BackendManager::Compute(OrtKernelContext* context) {
|
|||
bool use_dynamic_backend = true;
|
||||
if (GetGlobalContext().enable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape &&
|
||||
GetGlobalContext().device_type.find("CPU") != std::string::npos) {
|
||||
#if (defined OV_API_20)
|
||||
concrete_backend_->Infer(context);
|
||||
use_dynamic_backend = false;
|
||||
#endif
|
||||
concrete_backend_->Infer(context);
|
||||
use_dynamic_backend = false;
|
||||
}
|
||||
else if (use_dynamic_backend && subgraph_context_.has_dynamic_input_shape) {
|
||||
std::vector<std::vector<int64_t>> tensor_shapes = GetInputTensorShapes(ctx);
|
||||
|
|
@ -297,8 +311,13 @@ void BackendManager::Compute(OrtKernelContext* context) {
|
|||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP] "
|
||||
<< "Backend created for graph " << subgraph_context_.subgraph_name;
|
||||
auto modelproto_with_concrete_shapes = ReWriteInputShapeInfo(*model_proto_, tensor_shapes);
|
||||
dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes,
|
||||
GetGlobalContext(), subgraph_context_);
|
||||
try {
|
||||
dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes,
|
||||
GetGlobalContext(),
|
||||
subgraph_context_);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
backend_map_.insert({key, dynamic_backend});
|
||||
} else {
|
||||
dynamic_backend = search->second;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,16 @@
|
|||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "backend_utils.h"
|
||||
|
||||
#if defined (OV_API_20)
|
||||
using Exception = ov::Exception;
|
||||
#elif defined (OPENVINO_2021_4)
|
||||
using Exception = InferenceEngine::Exception;
|
||||
using WaitMode = InferenceEngine::InferRequest::WaitMode;
|
||||
#else
|
||||
using Exception = InferenceEngine::details::InferenceEngineException;
|
||||
using WaitMode = InferenceEngine::IInferRequest::WaitMode;
|
||||
#endif
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace openvino_ep {
|
||||
namespace backend_utils {
|
||||
|
|
@ -25,11 +35,6 @@ bool IsDebugEnabled() {
|
|||
}
|
||||
return false;
|
||||
}
|
||||
void DumpOnnxModelProto(const ONNX_NAMESPACE::ModelProto& model_proto, std::string file_name) {
|
||||
std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary);
|
||||
model_proto.SerializeToOstream(outfile);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
bool IsCILogEnabled() {
|
||||
|
|
@ -71,145 +76,74 @@ bool IsDirExists(const std::string& pathname) {
|
|||
return false;
|
||||
}
|
||||
|
||||
void CreateDirectory(const std::string& ov_compiled_blobs_dir) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "'ov_compiled_blobs' directory doesn't exist at the executable path, so creating one";
|
||||
#if defined(_WIN32)
|
||||
if (_mkdir(ov_compiled_blobs_dir.c_str()) == 0) { // Creating a directory
|
||||
LOGS_DEFAULT(INFO) << log_tag << "created a directory named 'ov_compiled_blobs' at the executable path";
|
||||
} else {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Error creating a directory named 'ov_compiled_blobs' at the executable path";
|
||||
throw std::runtime_error("Could not create the directory");
|
||||
}
|
||||
#else
|
||||
if (mkdir(ov_compiled_blobs_dir.c_str(), 0777) == 0) { // Creating a directory
|
||||
LOGS_DEFAULT(INFO) << log_tag << "created a directory named 'ov_compiled_blobs' at the executable path";
|
||||
} else {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Error creating a directory named 'ov_compiled_blobs' at the executable path";
|
||||
throw std::runtime_error("Could not create the directory");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
struct static_cast_int64 {
|
||||
template <typename T1> // T1 models type statically convertible to T
|
||||
int64_t operator()(const T1& x) const { return static_cast<int64_t>(x); }
|
||||
};
|
||||
|
||||
std::shared_ptr<InferenceEngine::CNNNetwork>
|
||||
CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map) {
|
||||
std::shared_ptr<OVNetwork>
|
||||
CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context,
|
||||
const SubGraphContext& subgraph_context,
|
||||
std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map) {
|
||||
if(IsCILogEnabled()) {
|
||||
std::cout << "CreateNgraphFunc" << std::endl;
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
if (IsDebugEnabled()) {
|
||||
DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx");
|
||||
}
|
||||
#endif
|
||||
|
||||
std::shared_ptr<ngraph::Function> ng_function;
|
||||
#if defined (OPENVINO_2021_4)
|
||||
const std::string model = model_proto.SerializeAsString();
|
||||
const std::string model = model_proto.SerializeAsString();
|
||||
try {
|
||||
auto cnn_network = global_context.ie_core.ReadModel(model);
|
||||
ng_function = cnn_network->getFunction();
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(model_proto);
|
||||
#endif
|
||||
if ((subgraph_context.precision == InferenceEngine::Precision::FP16) &&
|
||||
(global_context.device_type.find("MYRIAD") == std::string::npos)) {
|
||||
//FP16 transformations
|
||||
ov::pass::ConvertFP32ToFP16 pass_obj;
|
||||
pass_obj.run_on_model(cnn_network);
|
||||
cnn_network->validate_nodes_and_infer_types();
|
||||
|
||||
if (global_context.device_type.find("GPU") != std::string::npos &&
|
||||
subgraph_context.precision == InferenceEngine::Precision::FP16) {
|
||||
//FP16 transformations
|
||||
ngraph::pass::ConvertFP32ToFP16().run_on_function(ng_function);
|
||||
ng_function->validate_nodes_and_infer_types();
|
||||
}
|
||||
|
||||
if (!global_context.is_wholly_supported_graph) {
|
||||
std::map<std::string, std::string> result_to_output;
|
||||
for (auto& result : ng_function->get_results()) {
|
||||
result_to_output[result->get_friendly_name()] = result->input_value(0).get_node_shared_ptr()->get_friendly_name();
|
||||
}
|
||||
ngraph::pass::ConstantFolding().run_on_function(ng_function);
|
||||
auto& results = const_cast<::ngraph::ResultVector&>(ng_function->get_results());
|
||||
size_t index = results.size() - 1;
|
||||
#if defined (OV_API_20)
|
||||
for (auto it = results.rbegin(); it != results.rend(); ++it) {
|
||||
if (auto const_node = std::dynamic_pointer_cast<ngraph::op::Constant>((*it)->input_value(0).get_node_shared_ptr())) {
|
||||
const_outputs_map[(*it)->get_friendly_name()] = const_node;
|
||||
results.erase(results.begin() + index);
|
||||
auto proc = ov::preprocess::PrePostProcessor(cnn_network);
|
||||
for (size_t i=0; i < cnn_network->inputs().size(); i++) {
|
||||
if(cnn_network->inputs()[i].get_element_type() == ov::element::f16) {
|
||||
proc.input(i).tensor().set_element_type(ov::element::f32);
|
||||
proc.input(i).preprocess().convert_element_type(ov::element::f16);
|
||||
}
|
||||
}
|
||||
--index;
|
||||
|
||||
for (size_t i=0; i < cnn_network->outputs().size(); i++) {
|
||||
if(cnn_network->outputs()[i].get_element_type() == ov::element::f16) {
|
||||
proc.output(i).postprocess().convert_element_type(ov::element::f32);
|
||||
}
|
||||
}
|
||||
cnn_network = proc.build();
|
||||
}
|
||||
#else
|
||||
|
||||
//Check for Constant Folding
|
||||
if (!global_context.is_wholly_supported_graph) {
|
||||
ov::pass::ConstantFolding pass_const_obj;
|
||||
pass_const_obj.run_on_model(cnn_network);
|
||||
auto& results = const_cast<ov::ResultVector&>(cnn_network.get()->get_results());
|
||||
size_t index = results.size() - 1;
|
||||
|
||||
for (auto it = results.rbegin(); it != results.rend(); ++it) {
|
||||
if (auto const_node = std::dynamic_pointer_cast<ngraph::op::Constant>((*it)->input_value(0).get_node_shared_ptr())) {
|
||||
const_outputs_map[result_to_output.at((*it)->get_friendly_name())] = const_node;
|
||||
const_outputs_map[(*it)->get_friendly_name()] = const_node;
|
||||
results.erase(results.begin() + index);
|
||||
}
|
||||
--index;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
return std::make_shared<InferenceEngine::CNNNetwork>(ng_function);
|
||||
};
|
||||
|
||||
#if defined (OV_API_20)
|
||||
std::shared_ptr<OVNetwork>
|
||||
CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map) {
|
||||
if(IsCILogEnabled()) {
|
||||
std::cout << "CreateNgraphFunc" << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
#ifndef NDEBUG
|
||||
if (IsDebugEnabled()) {
|
||||
DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx");
|
||||
}
|
||||
#if defined OPENVINO_2022_3
|
||||
if (IsDebugEnabled()) {
|
||||
std::string name = cnn_network->get_friendly_name();
|
||||
ov::pass::Serialize serializer(name + ".xml", name + ".bin");
|
||||
serializer.run_on_model(cnn_network);
|
||||
ngraph::plot_graph(cnn_network, name+"_executable" + ".dot");
|
||||
}
|
||||
#endif
|
||||
|
||||
const std::string model = model_proto.SerializeAsString();
|
||||
auto cnn_network = global_context.ie_core.ReadModel(model);
|
||||
|
||||
if ((subgraph_context.precision == InferenceEngine::Precision::FP16) &&
|
||||
(global_context.device_type.find("MYRIAD") == std::string::npos)) {
|
||||
//FP16 transformations
|
||||
ov::pass::ConvertFP32ToFP16 pass_obj;
|
||||
pass_obj.run_on_model(cnn_network);
|
||||
cnn_network->validate_nodes_and_infer_types();
|
||||
|
||||
auto proc = ov::preprocess::PrePostProcessor(cnn_network);
|
||||
for (size_t i=0; i < cnn_network->inputs().size(); i++) {
|
||||
if(cnn_network->inputs()[i].get_element_type() == ov::element::f16) {
|
||||
proc.input(i).tensor().set_element_type(ov::element::f32);
|
||||
proc.input(i).preprocess().convert_element_type(ov::element::f16);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i=0; i < cnn_network->outputs().size(); i++) {
|
||||
if(cnn_network->outputs()[i].get_element_type() == ov::element::f16) {
|
||||
proc.output(i).postprocess().convert_element_type(ov::element::f32);
|
||||
}
|
||||
}
|
||||
cnn_network = proc.build();
|
||||
#endif
|
||||
return cnn_network;
|
||||
}catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
|
||||
//Check for Constant Folding
|
||||
if (!global_context.is_wholly_supported_graph) {
|
||||
ov::pass::ConstantFolding pass_const_obj;
|
||||
pass_const_obj.run_on_model(cnn_network);
|
||||
auto& results = const_cast<ov::ResultVector&>(cnn_network.get()->get_results());
|
||||
size_t index = results.size() - 1;
|
||||
|
||||
for (auto it = results.rbegin(); it != results.rend(); ++it) {
|
||||
if (auto const_node = std::dynamic_pointer_cast<ngraph::op::Constant>((*it)->input_value(0).get_node_shared_ptr())) {
|
||||
const_outputs_map[(*it)->get_friendly_name()] = const_node;
|
||||
results.erase(results.begin() + index);
|
||||
}
|
||||
--index;
|
||||
}
|
||||
}
|
||||
return cnn_network;
|
||||
}
|
||||
#endif
|
||||
|
||||
InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type, std::string device) {
|
||||
ONNX_NAMESPACE::DataType type_string = ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(onnx_type);
|
||||
|
|
@ -236,40 +170,7 @@ InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::
|
|||
} else if (*type_string == "int64" || *type_string == "tensor(int64)") {
|
||||
return InferenceEngine::Precision::I32;
|
||||
} else {
|
||||
ORT_THROW(log_tag + "Unsupported Data type");
|
||||
}
|
||||
}
|
||||
|
||||
void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto,
|
||||
std::shared_ptr<InferenceEngine::CNNNetwork> network,
|
||||
std::unordered_map<std::string, int> output_names,
|
||||
std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map,
|
||||
std::string device) {
|
||||
// Configure input & output
|
||||
// Prepare input blobs
|
||||
|
||||
auto inputInfo = network->getInputsInfo();
|
||||
int input_idx = 0;
|
||||
for (auto iter = inputInfo.begin(); iter != inputInfo.end(); ++iter, ++input_idx) {
|
||||
// Get the onnx index for the corresponding input (ignoring initializers)
|
||||
auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().input(input_idx).type(), device);
|
||||
iter->second->setPrecision(precision);
|
||||
}
|
||||
|
||||
// Prepare output blobs
|
||||
auto outputInfo = network->getOutputsInfo();
|
||||
for (auto iter = outputInfo.begin(); iter != outputInfo.end(); ++iter) {
|
||||
auto output_name = iter->first;
|
||||
auto it = const_outputs_map.find(output_name);
|
||||
//Output is constant and don't need to set precision
|
||||
if (it != const_outputs_map.end())
|
||||
break;
|
||||
auto itr = output_names.find(output_name);
|
||||
if (itr == output_names.end()) {
|
||||
ORT_THROW(log_tag + "Output Names Mismatch: " + output_name + " doesn't exist");
|
||||
}
|
||||
auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().output(itr->second).type(), device);
|
||||
iter->second->setPrecision(precision);
|
||||
throw std::string(log_tag + "Unsupported Data type");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -281,11 +182,7 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size,
|
|||
|
||||
auto graph_output_blob = infer_request->GetTensor(output_name);
|
||||
|
||||
#if defined (OV_API_20)
|
||||
auto graph_output_dims = graph_output_blob->get_shape();
|
||||
#else
|
||||
auto graph_output_dims = graph_output_blob->getTensorDesc().getDims();
|
||||
#endif
|
||||
|
||||
if (batch_size > 1) {
|
||||
// Add the batch size as dim 0.
|
||||
|
|
@ -298,7 +195,7 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size,
|
|||
}
|
||||
auto it = output_names.find(output_name);
|
||||
if (it == output_names.end()) {
|
||||
ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX");
|
||||
throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX");
|
||||
}
|
||||
int index = it->second;
|
||||
return context.GetOutput(index, output_shape.get(), num_dims);
|
||||
|
|
@ -310,16 +207,14 @@ GetOutputTensor(Ort::KernelContext& context,
|
|||
std::unordered_map<std::string, int> output_names,
|
||||
std::shared_ptr<ngraph::Node> node) {
|
||||
|
||||
#if (defined OV_API_20)
|
||||
// Find position of '/' in the output_name
|
||||
int pos = output_name.find("/");
|
||||
// Copy the substring from start to pos
|
||||
output_name = output_name.substr(0 , pos);
|
||||
#endif
|
||||
// Find position of '/' in the output_name
|
||||
int pos = output_name.find("/");
|
||||
// Copy the substring from start to pos
|
||||
output_name = output_name.substr(0 , pos);
|
||||
|
||||
auto it = output_names.find(output_name);
|
||||
if (it == output_names.end()) {
|
||||
ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX");
|
||||
throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX");
|
||||
}
|
||||
int index = it->second;
|
||||
auto shape = node->get_shape();
|
||||
|
|
@ -378,7 +273,7 @@ void FillOutputsWithConstantData(std::shared_ptr<ngraph::Node> node, Ort::Unowne
|
|||
break;
|
||||
}
|
||||
default:
|
||||
ORT_THROW(log_tag + "Unsupported output data type");
|
||||
throw std::string(log_tag + "Unsupported output data type");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -390,81 +285,6 @@ void FillOutputHelper(Ort::UnownedValue& out_tensor, std::shared_ptr<ngraph::Nod
|
|||
std::copy(res.begin(), res.end(), tensor_data);
|
||||
}
|
||||
|
||||
void FillInputBlob(InferenceEngine::Blob::Ptr& inputBlob, size_t batch_slice_idx,
|
||||
std::string input_name, Ort::KernelContext& context,
|
||||
InferenceEngine::Precision precision, const SubGraphContext& subgraph_context) {
|
||||
auto minput = InferenceEngine::as<InferenceEngine::MemoryBlob>(inputBlob);
|
||||
auto minputHolder = minput->wmap();
|
||||
|
||||
auto input_data = minputHolder.as<InferenceEngine::PrecisionTrait<InferenceEngine::Precision::FP32>::value_type*>();
|
||||
size_t input_data_size = inputBlob->byteSize();
|
||||
|
||||
auto tensor = context.GetInput(subgraph_context.input_names.at(input_name));
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
ORT_THROW(log_tag + "IO Buffering is not enabled, Please enable Input on CPU");
|
||||
}
|
||||
auto tensor_shape = tensor.GetTensorTypeAndShapeInfo();
|
||||
auto elem_type = tensor_shape.GetElementType();
|
||||
|
||||
if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) &&
|
||||
(precision == InferenceEngine::Precision::I32)) {
|
||||
const int64_t* tensor_data_64 = tensor.GetTensorData<int64_t>();
|
||||
auto data_len = (input_data_size * 2) / sizeof(int64_t);
|
||||
const int64_t* batch_memory_offset = tensor_data_64 + data_len * batch_slice_idx;
|
||||
|
||||
std::copy(batch_memory_offset, batch_memory_offset + data_len, (uint32_t*)input_data);
|
||||
} else {
|
||||
// Copy input data into OpenVINO's input buffer
|
||||
const char* tensor_data = tensor.GetTensorData<char>();
|
||||
const char* batch_memory_offset = tensor_data + input_data_size * batch_slice_idx;
|
||||
std::memcpy(input_data, batch_memory_offset, input_data_size);
|
||||
}
|
||||
}
|
||||
|
||||
void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, Ort::UnownedValue& output_tensor,
|
||||
InferenceEngine::Precision precision, size_t batch_slice_idx) {
|
||||
auto moutput = InferenceEngine::as<InferenceEngine::MemoryBlob>(outputBlob);
|
||||
|
||||
auto moutputHolder = moutput->rmap();
|
||||
|
||||
const auto output_data = moutputHolder.as<const InferenceEngine::PrecisionTrait<InferenceEngine::Precision::FP32>::value_type*>();
|
||||
|
||||
size_t output_data_size = outputBlob->byteSize();
|
||||
auto tensor_shape = output_tensor.GetTensorTypeAndShapeInfo();
|
||||
const auto elem_type = tensor_shape.GetElementType();
|
||||
|
||||
if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) &&
|
||||
(precision == InferenceEngine::Precision::I32)) {
|
||||
int64_t* tensor_data = output_tensor.GetTensorMutableData<int64_t>();
|
||||
auto data_len = output_data_size / sizeof(int32_t);
|
||||
int64_t* batch_memory_offset = tensor_data + data_len * batch_slice_idx;
|
||||
|
||||
std::transform((int32_t*)output_data, ((int32_t*)output_data) + data_len, batch_memory_offset, static_cast_int64());
|
||||
|
||||
} else {
|
||||
char* tensor_data = output_tensor.GetTensorMutableData<char>();
|
||||
char* batch_memory_offset = tensor_data + output_data_size * batch_slice_idx;
|
||||
|
||||
std::memcpy(batch_memory_offset, output_data, output_data_size);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string, InferenceEngine::InferenceEngineProfileInfo>>
|
||||
perfCountersSorted(std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap) {
|
||||
using perfItem = std::pair<std::string, InferenceEngine::InferenceEngineProfileInfo>;
|
||||
std::vector<perfItem> sorted;
|
||||
for (auto& kvp : perfMap) sorted.push_back(kvp);
|
||||
|
||||
std::stable_sort(sorted.begin(), sorted.end(),
|
||||
[](const perfItem& l, const perfItem& r) {
|
||||
return l.second.execution_index < r.second.execution_index;
|
||||
});
|
||||
|
||||
return sorted;
|
||||
}
|
||||
|
||||
#if defined (OV_API_20)
|
||||
void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx,
|
||||
std::string input_name, Ort::KernelContext& context,
|
||||
const SubGraphContext& subgraph_context) {
|
||||
|
|
@ -474,7 +294,7 @@ void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx,
|
|||
auto tensor = context.GetInput(subgraph_context.input_names.at(input_name));
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
ORT_THROW(log_tag + "IO Buffering is not enabled, Please enable Input on CPU");
|
||||
throw std::string(log_tag + "IO Buffering is not enabled, Please enable Input on CPU");
|
||||
}
|
||||
// Copy input data into OpenVINO's input buffer
|
||||
const char* tensor_data = tensor.GetTensorData<char>();
|
||||
|
|
@ -527,65 +347,15 @@ void printPerformanceCounts(const std::vector<OVProfilingInfo>& performanceMap,
|
|||
totalTime += it.real_time.count();
|
||||
}
|
||||
}
|
||||
stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl;
|
||||
std::cout << std::endl;
|
||||
std::cout << "Full device name: " << deviceName << std::endl;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
#endif
|
||||
|
||||
void printPerformanceCounts(const std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>& performanceMap,
|
||||
std::ostream& stream, std::string deviceName) {
|
||||
long long totalTime = 0;
|
||||
// Print performance counts
|
||||
stream << std::endl
|
||||
<< "performance counts:" << std::endl
|
||||
<< std::endl;
|
||||
|
||||
auto performanceMapSorted = perfCountersSorted(performanceMap);
|
||||
|
||||
for (const auto& it : performanceMapSorted) {
|
||||
std::string toPrint(it.first);
|
||||
const int maxLayerName = 30;
|
||||
|
||||
if (it.first.length() >= maxLayerName) {
|
||||
toPrint = it.first.substr(0, maxLayerName - 4);
|
||||
toPrint += "...";
|
||||
}
|
||||
stream << std::setw(maxLayerName) << std::left << toPrint;
|
||||
switch (it.second.status) {
|
||||
case InferenceEngine::InferenceEngineProfileInfo::EXECUTED:
|
||||
stream << std::setw(15) << std::left << "EXECUTED";
|
||||
break;
|
||||
case InferenceEngine::InferenceEngineProfileInfo::NOT_RUN:
|
||||
stream << std::setw(15) << std::left << "NOT_RUN";
|
||||
break;
|
||||
case InferenceEngine::InferenceEngineProfileInfo::OPTIMIZED_OUT:
|
||||
stream << std::setw(15) << std::left << "OPTIMIZED_OUT";
|
||||
break;
|
||||
}
|
||||
stream << std::setw(30) << std::left << "layerType: " + std::string(it.second.layer_type) + " ";
|
||||
stream << std::setw(20) << std::left << "realTime: " + std::to_string(it.second.realTime_uSec);
|
||||
stream << std::setw(20) << std::left << "cpu: " + std::to_string(it.second.cpu_uSec);
|
||||
stream << " execType: " << it.second.exec_type << std::endl;
|
||||
if (it.second.realTime_uSec > 0) {
|
||||
totalTime += it.second.realTime_uSec;
|
||||
}
|
||||
}
|
||||
stream << std::setw(20) << std::left << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl;
|
||||
stream << std::setw(20) << "Total time: " + std::to_string(totalTime) << " microseconds" << std::endl;
|
||||
std::cout << std::endl;
|
||||
std::cout << "Full device name: " << deviceName << std::endl;
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName) {
|
||||
#if defined (OV_API_20)
|
||||
auto performanceMap = request->GetNewObj().get_profiling_info();
|
||||
printPerformanceCounts(performanceMap, stream, deviceName);
|
||||
#else
|
||||
auto performanceMap = request->GetObj().GetPerformanceCounts();
|
||||
printPerformanceCounts(performanceMap, stream, deviceName);
|
||||
#endif
|
||||
auto performanceMap = request->GetNewObj().get_profiling_info();
|
||||
printPerformanceCounts(performanceMap, stream, deviceName);
|
||||
}
|
||||
|
||||
} // namespace backend_utils
|
||||
|
|
|
|||
|
|
@ -27,25 +27,16 @@ const std::string log_tag = "[OpenVINO-EP] ";
|
|||
bool IsDebugEnabled();
|
||||
#endif
|
||||
|
||||
//Internal diagnostic function.
|
||||
// Internal diagnostic function.
|
||||
bool IsCILogEnabled();
|
||||
|
||||
bool UseCompiledNetwork();
|
||||
|
||||
std::string GetCurrentWorkingDir();
|
||||
// std::string GetCurrentWorkingDir();
|
||||
|
||||
bool IsDirExists(const std::string& pathname);
|
||||
// bool IsDirExists(const std::string& pathname);
|
||||
|
||||
void CreateDirectory(const std::string& ov_compiled_blobs_dir);
|
||||
|
||||
void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto,
|
||||
std::shared_ptr<InferenceEngine::CNNNetwork> network,
|
||||
std::unordered_map<std::string, int> output_names,
|
||||
std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map,
|
||||
std::string device);
|
||||
|
||||
std::shared_ptr<InferenceEngine::CNNNetwork>
|
||||
CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map);
|
||||
// void CreateDirectory(const std::string& ov_compiled_blobs_dir);
|
||||
|
||||
int GetFirstAvailableDevice(GlobalContext& global_context);
|
||||
|
||||
|
|
@ -69,7 +60,6 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size,
|
|||
std::string output_name,
|
||||
std::unordered_map<std::string, int> output_names);
|
||||
|
||||
#if defined (OV_API_20)
|
||||
void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx,
|
||||
std::string input_name, Ort::KernelContext& context,
|
||||
const SubGraphContext& subgraph_context);
|
||||
|
|
@ -78,24 +68,11 @@ void FillOutputBlob(OVTensorPtr outputBlob, Ort::UnownedValue& output_tensor,
|
|||
size_t batch_slice_idx);
|
||||
|
||||
std::shared_ptr<OVNetwork>
|
||||
CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map);
|
||||
CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context,
|
||||
std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map);
|
||||
|
||||
void printPerformanceCounts(const std::vector<OVProfilingInfo>& performanceMap,
|
||||
std::ostream& stream, std::string deviceName);
|
||||
#endif
|
||||
|
||||
void printPerformanceCounts(const std::map<std::string, InferenceEngine::InferenceEngineProfileInfo>& performanceMap,
|
||||
std::ostream& stream, std::string deviceName);
|
||||
|
||||
std::vector<std::pair<std::string, InferenceEngine::InferenceEngineProfileInfo>>
|
||||
perfCountersSorted(std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap);
|
||||
|
||||
void FillInputBlob(InferenceEngine::Blob::Ptr& inputBlob, size_t batch_slice_idx,
|
||||
std::string input_name, Ort::KernelContext& context,
|
||||
InferenceEngine::Precision precision, const SubGraphContext& subgraph_context);
|
||||
|
||||
void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, Ort::UnownedValue& output_tensor,
|
||||
InferenceEngine::Precision precision, size_t batch_slice_idx);
|
||||
|
||||
void printPerformanceCounts(OVInferRequestPtr request, std::ostream& stream, std::string deviceName);
|
||||
|
||||
|
|
|
|||
|
|
@ -21,11 +21,16 @@ BackendFactory::MakeBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
} else if (type == "CPU" || type.find("GPU") != std::string::npos || type == "MYRIAD" ||
|
||||
type.find("HETERO") != std::string::npos || type.find("MULTI") != std::string::npos ||
|
||||
type.find("AUTO") != std::string::npos) {
|
||||
return std::make_shared<BasicBackend>(model_proto, global_context, subgraph_context);
|
||||
std::shared_ptr<IBackend> concrete_backend_;
|
||||
try {
|
||||
concrete_backend_=std::make_shared<BasicBackend>(model_proto, global_context, subgraph_context);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
return concrete_backend_;
|
||||
} else {
|
||||
ORT_THROW("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type);
|
||||
throw std::string("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace openvino_ep
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -24,56 +24,61 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
const SubGraphContext& subgraph_context)
|
||||
: global_context_(global_context), subgraph_context_(subgraph_context) {
|
||||
std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type;
|
||||
bool vpu_status = false;
|
||||
std::string model_blob_name;
|
||||
std::string ov_compiled_blobs_dir = "";
|
||||
|
||||
if (hw_target == "MYRIAD")
|
||||
vpu_status = true;
|
||||
if (!ImportBlob(hw_target, vpu_status)) {
|
||||
#if defined (OPENVINO_2021_4)
|
||||
ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_);
|
||||
SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_,
|
||||
global_context_.device_type);
|
||||
#else
|
||||
try{
|
||||
#ifndef NDEBUG
|
||||
if (IsDebugEnabled()) {
|
||||
std::string file_name = subgraph_context.subgraph_name + "_static.onnx";
|
||||
std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary);
|
||||
model_proto.SerializeToOstream(outfile);
|
||||
// DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx");
|
||||
}
|
||||
#endif
|
||||
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
|
||||
#endif
|
||||
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
if (ValidateSubgraph(const_outputs_map_))
|
||||
return;
|
||||
|
||||
// OV Config
|
||||
OVConfig config;
|
||||
PopulateConfigValue(config);
|
||||
ov::AnyMap device_config;
|
||||
PopulateConfigValue(config, device_config);
|
||||
|
||||
//Enable caching
|
||||
EnableCaching();
|
||||
|
||||
//Setting OpenCL queue throttling for GPU
|
||||
#if defined (OV_API_20)
|
||||
EnableGPUThrottling(config);
|
||||
#endif
|
||||
EnableGPUThrottling(device_config);
|
||||
|
||||
#if defined(IO_BUFFER_ENABLED)
|
||||
if ((global_context.device_type.find("GPU") != std::string::npos) &&
|
||||
(global_context_.context != nullptr) &&
|
||||
(openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled";
|
||||
cl_context ctx = static_cast<cl_context>(global_context_.context);
|
||||
#ifdef OV_API_20
|
||||
remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_.ie_core.Get(), ctx);
|
||||
#else
|
||||
remote_context_ = InferenceEngine::gpu::make_shared_context(global_context_.ie_core.Get(), hw_target, ctx);
|
||||
#endif
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name);
|
||||
} else {
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name);
|
||||
}
|
||||
try {
|
||||
if ((global_context.device_type.find("GPU") != std::string::npos) &&
|
||||
(global_context_.context != nullptr) &&
|
||||
(openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled";
|
||||
cl_context ctx = static_cast<cl_context>(global_context_.context);
|
||||
#ifdef OV_API_20
|
||||
remote_context_ = new ov::intel_gpu::ocl::ClContext(global_context_.ie_core.Get(), ctx);
|
||||
#else
|
||||
remote_context_ = InferenceEngine::gpu::make_shared_context(global_context_.ie_core.Get(), hw_target, ctx);
|
||||
#endif
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name);
|
||||
} else {
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name);
|
||||
}
|
||||
}catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
#else
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name);
|
||||
try{
|
||||
exe_network_ = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name);
|
||||
} catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
#endif
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
|
||||
}
|
||||
|
||||
|
||||
//The infer_requests_ pool will be intialized with a default value of 8 infer_request's
|
||||
//The nireq value can also be configured to any num_of_threads during runtime
|
||||
|
|
@ -97,78 +102,46 @@ bool BasicBackend::ValidateSubgraph(std::map<std::string, std::shared_ptr<ngraph
|
|||
return false;
|
||||
}
|
||||
|
||||
void BasicBackend::PopulateCompiledDirectory(std::string hw_target, std::string& ov_compiled_blobs_dir,
|
||||
std::string& model_blob_name, bool& vpu_status) {
|
||||
std::ifstream blob_path;
|
||||
if(hw_target == "MYRIAD" && global_context_.use_compiled_network == true) {
|
||||
if(!openvino_ep::backend_utils::UseCompiledNetwork()) {
|
||||
std::size_t model_index = global_context_.onnx_model_path_name.find_last_of("/\\");
|
||||
std::string model_name= global_context_.onnx_model_path_name.substr(model_index+1);
|
||||
std::size_t model_extension_index = model_name.find_last_of(".");
|
||||
if(openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph) {
|
||||
model_blob_name = global_context_.onnx_model_name + "_" + "op_v_" + std::to_string(global_context_.onnx_opset_version) + "_" + model_name.substr(0,model_extension_index) + "_" + hw_target + "_" + subgraph_context_.subgraph_name + "_ov_" + "fully" + ".blob";
|
||||
}
|
||||
else {
|
||||
model_blob_name = global_context_.onnx_model_name + "_" + "op_v_" + std::to_string(global_context_.onnx_opset_version) + "_" + model_name.substr(0,model_extension_index) + "_" + hw_target + "_" + subgraph_context_.subgraph_name + "_ov_" + "partially" + ".blob";
|
||||
}
|
||||
if(global_context_.blob_dump_path == "" || global_context_.blob_dump_path == "\"" || global_context_.blob_dump_path.empty()) {
|
||||
ov_compiled_blobs_dir = openvino_ep::backend_utils::GetCurrentWorkingDir() + "/ov_compiled_blobs/";
|
||||
} else {
|
||||
ov_compiled_blobs_dir = global_context_.blob_dump_path + "/ov_compiled_blobs";
|
||||
}
|
||||
if(openvino_ep::backend_utils::IsDirExists(ov_compiled_blobs_dir)) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "'ov_compiled_blobs' directory already exists at the executable path";
|
||||
}
|
||||
else {
|
||||
CreateDirectory(ov_compiled_blobs_dir);
|
||||
}
|
||||
blob_path.open(ov_compiled_blobs_dir + "/" + model_blob_name);
|
||||
if (!blob_path.is_open()) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Device specific Compiled blob doesn't exist for this model";
|
||||
} else {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Device specific Compiled blob already exists for this model";
|
||||
vpu_status = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool BasicBackend::ImportBlob(std::string hw_target, bool vpu_status) {
|
||||
const std::string compiled_blob_path = onnxruntime::GetEnvironmentVar("OV_BLOB_PATH");
|
||||
if (vpu_status == true && openvino_ep::backend_utils::UseCompiledNetwork() && !compiled_blob_path.empty() &&
|
||||
openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Importing the pre-compiled blob from the path set by the user";
|
||||
exe_network_ = global_context_.ie_core.ImportModel(compiled_blob_path, hw_target, subgraph_context_.subgraph_name);
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Succesfully Created an executable network from a previously exported network";
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void BasicBackend::PopulateConfigValue(OVConfig& config) {
|
||||
void BasicBackend::PopulateConfigValue(OVConfig& config, ov::AnyMap& device_config) {
|
||||
// Set inference precision if device_type != AUTO
|
||||
// if (global_context_.device_type.find("GPU_FP16")!= std::string::npos){
|
||||
// device_config.emplace(ov::hint::inference_precision(global_context_.precision_str));
|
||||
// }
|
||||
device_config = {};
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
config["PERF_COUNT"] = CONFIG_VALUE(YES);
|
||||
device_config.emplace(ov::enable_profiling(true));
|
||||
}
|
||||
#endif
|
||||
if (global_context_.device_type.find("MYRIAD") != std::string::npos) {
|
||||
if (global_context_.device_type.find("MYRIAD") != std::string::npos) {
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
config["PERF_COUNT"] = CONFIG_VALUE(YES);
|
||||
}
|
||||
#endif
|
||||
if (subgraph_context_.set_vpu_config) {
|
||||
config["MYRIAD_DETECT_NETWORK_BATCH"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
if (global_context_.enable_vpu_fast_compile) {
|
||||
config["MYRIAD_HW_INJECT_STAGES"] = CONFIG_VALUE(NO);
|
||||
config["MYRIAD_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
//to check preprocessing inside model
|
||||
#if defined (OPENVINO_2021_4) || (OPENVINO_2022_1) || (OPENVINO_2022_2)
|
||||
config["MYRIAD_CHECK_PREPROCESSING_INSIDE_MODEL"] = CONFIG_VALUE(NO);
|
||||
#endif
|
||||
if (global_context_.enable_vpu_fast_compile) {
|
||||
config["MYRIAD_HW_INJECT_STAGES"] = CONFIG_VALUE(NO);
|
||||
config["MYRIAD_COPY_OPTIMIZATION"] = CONFIG_VALUE(NO);
|
||||
}
|
||||
//to check preprocessing inside model
|
||||
#if defined (OPENVINO_2022_1) || (OPENVINO_2022_2) || (OPENVINO_2022_3)
|
||||
config["MYRIAD_CHECK_PREPROCESSING_INSIDE_MODEL"] = CONFIG_VALUE(NO);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
void BasicBackend::EnableCaching() {
|
||||
if (global_context_.use_compiled_network == true) {
|
||||
if (global_context_.use_compiled_network == true && global_context_.is_wholly_supported_graph) {
|
||||
#if defined (OPENVINO_2022_3)
|
||||
#if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__)
|
||||
_putenv_s("OV_GPU_CACHE_MODEL", "1");
|
||||
#else
|
||||
setenv("OV_GPU_CACHE_MODEL", "1", 1);
|
||||
#endif
|
||||
#endif
|
||||
std::string cache_dir_path;
|
||||
if (global_context_.blob_dump_path.empty()) {
|
||||
cache_dir_path = "ov_compiled_blobs";
|
||||
|
|
@ -180,197 +153,171 @@ void BasicBackend::EnableCaching() {
|
|||
}
|
||||
}
|
||||
|
||||
#if defined (OV_API_20)
|
||||
void BasicBackend::EnableGPUThrottling(OVConfig& config) {
|
||||
void BasicBackend::EnableGPUThrottling(ov::AnyMap& device_config) {
|
||||
if (global_context_.enable_opencl_throttling == true && global_context_.device_type.find("GPU") != std::string::npos) {
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Enabled OpenCL queue throttling for GPU device";
|
||||
config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1";
|
||||
device_config[GPU_CONFIG_KEY(PLUGIN_THROTTLE)] = "1";
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on
|
||||
// an Infer Request indexed by infer_req_idx
|
||||
void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) {
|
||||
#if defined (OV_API_20)
|
||||
auto graph_input_info = exe_network_.Get().inputs();
|
||||
int input_idx = 0;
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
auto input_names = input_info_iter->get_names();
|
||||
std::string onnx_input_name;
|
||||
std::string input_name;
|
||||
// use names retrieved from original ONNX model to assign the right onnx input name for the graph
|
||||
for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) {
|
||||
if (it->second == input_idx) {
|
||||
onnx_input_name = it->first;
|
||||
break;
|
||||
try{
|
||||
auto graph_input_info = exe_network_.Get().inputs();
|
||||
int input_idx = 0;
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
auto input_names = input_info_iter->get_names();
|
||||
std::string onnx_input_name;
|
||||
std::string input_name;
|
||||
// use names retrieved from original ONNX model to assign the right onnx input name for the graph
|
||||
for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) {
|
||||
if (it->second == input_idx) {
|
||||
onnx_input_name = it->first;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
if (input_names.find(onnx_input_name) != input_names.end()) {
|
||||
input_name = onnx_input_name;
|
||||
} else {
|
||||
ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names");
|
||||
}
|
||||
size_t batch_slice_idx = 0;
|
||||
if (subgraph_context_.has_dynamic_input_shape &&
|
||||
global_context_.enable_dynamic_shapes == true &&
|
||||
global_context_.device_type.find("CPU") != std::string::npos) {
|
||||
auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name));
|
||||
auto tensor_info = tensor.GetTensorTypeAndShapeInfo();
|
||||
auto tensor_shape = tensor_info.GetShape();
|
||||
auto tensor_size = tensor_shape.size();
|
||||
auto tensor_iter = 0;
|
||||
ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0);
|
||||
for (auto i = tensor_shape.begin(); i != tensor_shape.end(); ++i) {
|
||||
input_tensor_shape[tensor_iter] = *i;
|
||||
tensor_iter+=1;
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
if (input_names.find(onnx_input_name) != input_names.end()) {
|
||||
input_name = onnx_input_name;
|
||||
} else {
|
||||
throw(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names");
|
||||
}
|
||||
auto input = ie_cnn_network_->get_parameters().at(0);
|
||||
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape);
|
||||
FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
infer_request->SetTensor(input_name, tensor_ptr);
|
||||
} else {
|
||||
OVTensorPtr graph_input_blob;
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
size_t batch_slice_idx = 0;
|
||||
if (subgraph_context_.has_dynamic_input_shape &&
|
||||
global_context_.enable_dynamic_shapes == true &&
|
||||
global_context_.device_type.find("CPU") != std::string::npos) {
|
||||
auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name));
|
||||
auto tensor_info = tensor.GetTensorTypeAndShapeInfo();
|
||||
auto tensor_shape = tensor_info.GetShape();
|
||||
auto tensor_size = tensor_shape.size();
|
||||
auto tensor_iter = 0;
|
||||
ov::Shape input_tensor_shape = ov::Shape(tensor_size, 0);
|
||||
for (auto i = tensor_shape.begin(); i != tensor_shape.end(); ++i) {
|
||||
input_tensor_shape[tensor_iter] = *i;
|
||||
tensor_iter+=1;
|
||||
}
|
||||
auto input = ie_cnn_network_->get_parameters().at(0);
|
||||
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape);
|
||||
FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
try{
|
||||
infer_request->SetTensor(input_name, tensor_ptr);
|
||||
} catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
} else {
|
||||
OVTensorPtr graph_input_blob;
|
||||
try{
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
} catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
}
|
||||
input_idx++;
|
||||
}
|
||||
// Start Async inference
|
||||
infer_request->StartAsync();
|
||||
}catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
input_idx++;
|
||||
}
|
||||
#else
|
||||
auto graph_input_info = exe_network_.Get().GetInputsInfo();
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
// Get OpenVINO's input buffer
|
||||
OVTensorPtr graph_input_blob;
|
||||
std::string input_name = input_info_iter->first;
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
auto precision = input_info_iter->second->getPrecision();
|
||||
size_t batch_slice = 0;
|
||||
FillInputBlob(graph_input_blob, batch_slice, input_name, context, precision, subgraph_context_);
|
||||
}
|
||||
#endif
|
||||
// Start Async inference
|
||||
infer_request->StartAsync();
|
||||
}
|
||||
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
//Wait for Remote Aynchronous inference completion
|
||||
void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) {
|
||||
#if defined (OV_API_20)
|
||||
auto graph_input_info = exe_network_.Get().inputs();
|
||||
int input_idx = 0;
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
auto input_names = input_info_iter->get_names();
|
||||
std::string onnx_input_name;
|
||||
std::string input_name;
|
||||
// use names retrieved from original ONNX model to assign the right onnx input name for the graph
|
||||
for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) {
|
||||
if (it->second == input_idx) {
|
||||
onnx_input_name = it->first;
|
||||
break;
|
||||
try{
|
||||
auto graph_input_info = exe_network_.Get().inputs();
|
||||
int input_idx = 0;
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
auto input_names = input_info_iter->get_names();
|
||||
std::string onnx_input_name;
|
||||
std::string input_name;
|
||||
// use names retrieved from original ONNX model to assign the right onnx input name for the graph
|
||||
for (auto it = subgraph_context_.input_names.begin(); it != subgraph_context_.input_names.end(); ++it) {
|
||||
if (it->second == input_idx) {
|
||||
onnx_input_name = it->first;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
if (input_names.find(onnx_input_name) != input_names.end()) {
|
||||
input_name = onnx_input_name;
|
||||
} else {
|
||||
ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names");
|
||||
}
|
||||
input_idx++;
|
||||
#else
|
||||
auto graph_input_info = exe_network_.Get().GetInputsInfo();
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
std::string input_name = input_info_iter->first;
|
||||
#endif
|
||||
// Kernel Context Input Buffer
|
||||
const auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name));
|
||||
// If the ORTValue wraps a device pointer
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
//Get the shared buffer pointer
|
||||
const void *tensor_data = tensor.GetTensorRawData();
|
||||
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
|
||||
//Create an Input Remote Blob
|
||||
#if defined(OV_API_20)
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
if (input_names.find(onnx_input_name) != input_names.end()) {
|
||||
input_name = onnx_input_name;
|
||||
} else {
|
||||
throw(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names");
|
||||
}
|
||||
input_idx++;
|
||||
// Kernel Context Input Buffer
|
||||
const auto tensor = context.GetInput(subgraph_context_.input_names.at(input_name));
|
||||
// If the ORTValue wraps a device pointer
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
//Get the shared buffer pointer
|
||||
const void *tensor_data = tensor.GetTensorRawData();
|
||||
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
|
||||
//Create an Input Remote Blob
|
||||
auto input = ie_cnn_network_->get_parameters().at(0);
|
||||
auto remote_blob = remote_context_->create_tensor(input->get_element_type(), input->get_shape(), *shared_buffer_const);
|
||||
ov::Tensor tensor = static_cast<ov::Tensor>(remote_blob);
|
||||
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor);
|
||||
#else
|
||||
cl::Buffer* shared_buffer = const_cast<cl::Buffer *>(shared_buffer_const);
|
||||
OVTensorPtr tensor_ptr = InferenceEngine::gpu::make_shared_blob(input_info_iter->second->getTensorDesc(), remote_context_, *shared_buffer);
|
||||
#endif
|
||||
infer_request->SetTensor(input_name, tensor_ptr);
|
||||
} else {
|
||||
OVTensorPtr graph_input_blob;
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
size_t batch_slice_idx = 0;
|
||||
#if defined (OV_API_20)
|
||||
infer_request->SetTensor(input_name, tensor_ptr);
|
||||
} else {
|
||||
OVTensorPtr graph_input_blob;
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
size_t batch_slice_idx = 0;
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
#else
|
||||
auto precision = input_info_iter->second->getPrecision();
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, precision, subgraph_context_);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
//Set the output blob as remote blob
|
||||
#if defined (OV_API_20)
|
||||
auto graph_output_info = exe_network_.Get().outputs();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
auto output_names = output_info_iter->get_names();
|
||||
std::string onnx_output_name;
|
||||
std::string output_name;
|
||||
bool output_name_found = false;
|
||||
// using the output name retrieved from ONNX original to match with the output names returned by OV tensors
|
||||
for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) {
|
||||
onnx_output_name = it->first;
|
||||
if (output_names.find(onnx_output_name) != output_names.end()) {
|
||||
//Assigning the output_name
|
||||
output_name = it->first;
|
||||
output_name_found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(!output_name_found) {
|
||||
ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names");
|
||||
}
|
||||
#else
|
||||
auto graph_output_info = exe_network_.Get().GetOutputsInfo();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
// Get Ort Output Tensor
|
||||
auto output_name = output_info_iter->first;
|
||||
#endif
|
||||
|
||||
size_t batch_size = 1;
|
||||
auto tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
// Check if ORT Value wraps a device pointer
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
const void *tensor_data = tensor.GetTensorRawData();
|
||||
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
|
||||
// Create a shared Blob, set the Infer Request Output Blob
|
||||
#if defined(OV_API_20)
|
||||
//Set the output blob as remote blob
|
||||
auto graph_output_info = exe_network_.Get().outputs();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
auto output_names = output_info_iter->get_names();
|
||||
std::string onnx_output_name;
|
||||
std::string output_name;
|
||||
bool output_name_found = false;
|
||||
// using the output name retrieved from ONNX original to match with the output names returned by OV tensors
|
||||
for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) {
|
||||
onnx_output_name = it->first;
|
||||
if (output_names.find(onnx_output_name) != output_names.end()) {
|
||||
//Assigning the output_name
|
||||
output_name = it->first;
|
||||
output_name_found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(!output_name_found) {
|
||||
throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names");
|
||||
}
|
||||
|
||||
size_t batch_size = 1;
|
||||
auto tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto mem_info = tensor.GetTensorMemoryInfo();
|
||||
// Check if ORT Value wraps a device pointer
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
const void *tensor_data = tensor.GetTensorRawData();
|
||||
const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
|
||||
// Create a shared Blob, set the Infer Request Output Blob
|
||||
auto output = ie_cnn_network_->get_results().at(0);
|
||||
auto remote_tensor = remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const);
|
||||
ov::Tensor tensor = static_cast<ov::Tensor>(remote_tensor);
|
||||
OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor);
|
||||
#else
|
||||
cl::Buffer* shared_buffer = const_cast<cl::Buffer *>(shared_buffer_const);
|
||||
OVTensorPtr tensor_ptr = InferenceEngine::gpu::make_shared_blob(output_info_iter->second->getTensorDesc(), remote_context_, *shared_buffer);
|
||||
#endif
|
||||
infer_request->SetTensor(output_name, tensor_ptr);
|
||||
try {
|
||||
infer_request->SetTensor(output_name, tensor_ptr);
|
||||
} catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start Async inference
|
||||
infer_request->StartAsync();
|
||||
// Start Async inference
|
||||
infer_request->StartAsync();
|
||||
}catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -378,76 +325,62 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
|
|||
// and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx
|
||||
void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRequestPtr infer_request) {
|
||||
// Wait for Async inference completion
|
||||
|
||||
infer_request->WaitRequest();
|
||||
#if defined (OV_API_20)
|
||||
auto graph_output_info = exe_network_.Get().outputs();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
OVTensorPtr graph_output_blob;
|
||||
auto output_names = output_info_iter->get_names();
|
||||
std::string onnx_output_name;
|
||||
std::string output_name;
|
||||
bool output_name_found = false;
|
||||
// using the output name retrieved from ONNX original to match with the output names returned by OV tensors
|
||||
for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) {
|
||||
onnx_output_name = it->first;
|
||||
if (output_names.find(onnx_output_name) != output_names.end()) {
|
||||
// Assigning the output_name
|
||||
output_name = it->first;
|
||||
output_name_found = true;
|
||||
break;
|
||||
try{
|
||||
infer_request->WaitRequest();
|
||||
auto graph_output_info = exe_network_.Get().outputs();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
OVTensorPtr graph_output_blob;
|
||||
auto output_names = output_info_iter->get_names();
|
||||
std::string onnx_output_name;
|
||||
std::string output_name;
|
||||
bool output_name_found = false;
|
||||
// using the output name retrieved from ONNX original to match with the output names returned by OV tensors
|
||||
for (auto it = subgraph_context_.output_names.begin(); it != subgraph_context_.output_names.end(); ++it) {
|
||||
onnx_output_name = it->first;
|
||||
if (output_names.find(onnx_output_name) != output_names.end()) {
|
||||
// Assigning the output_name
|
||||
output_name = it->first;
|
||||
output_name_found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!output_name_found) {
|
||||
ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX. "
|
||||
"[ONNX Output: ] " + onnx_output_name + " doesn't exist in the "
|
||||
"list of OpenVINO output tensor names");
|
||||
}
|
||||
graph_output_blob = infer_request->GetTensor(output_name);
|
||||
size_t batch_size = 1;
|
||||
auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto mem_info = output_tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
return;
|
||||
} else {
|
||||
size_t batch_slice = 0;
|
||||
FillOutputBlob(graph_output_blob, output_tensor, batch_slice);
|
||||
}
|
||||
}
|
||||
#else
|
||||
auto graph_output_info = exe_network_.Get().GetOutputsInfo();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
// Get OpenVINO's output blob
|
||||
OVTensorPtr graph_output_blob;
|
||||
auto output_name = output_info_iter->first;
|
||||
graph_output_blob = infer_request->GetTensor(output_name);
|
||||
size_t batch_size = 1;
|
||||
auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto mem_info = output_tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
return;
|
||||
} else {
|
||||
auto precision = output_info_iter->second->getPrecision();
|
||||
size_t batch_slice = 0;
|
||||
FillOutputBlob(graph_output_blob, output_tensor, precision, batch_slice);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!const_outputs_map_.empty()) {
|
||||
for (auto item : const_outputs_map_) {
|
||||
auto out_name = item.first;
|
||||
auto node = item.second;
|
||||
auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node);
|
||||
if (!output_name_found) {
|
||||
throw(log_tag + "Output names mismatch between OpenVINO and ONNX. "
|
||||
"[ONNX Output: ] " + onnx_output_name + " doesn't exist in the "
|
||||
"list of OpenVINO output tensor names");
|
||||
}
|
||||
try{
|
||||
graph_output_blob = infer_request->GetTensor(output_name);
|
||||
} catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
size_t batch_size = 1;
|
||||
auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto mem_info = output_tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
ORT_THROW(log_tag + "IO Buffering is not supported for constant subgraphs");
|
||||
return;
|
||||
} else {
|
||||
FillOutputsWithConstantData(node, output_tensor);
|
||||
size_t batch_slice = 0;
|
||||
FillOutputBlob(graph_output_blob, output_tensor, batch_slice);
|
||||
}
|
||||
}
|
||||
|
||||
if (!const_outputs_map_.empty()) {
|
||||
for (auto item : const_outputs_map_) {
|
||||
auto out_name = item.first;
|
||||
auto node = item.second;
|
||||
auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node);
|
||||
auto mem_info = output_tensor.GetTensorMemoryInfo();
|
||||
if (mem_info.GetAllocatorName() == OpenVINO_GPU) {
|
||||
throw(log_tag + "IO Buffering is not supported for constant subgraphs");
|
||||
} else {
|
||||
FillOutputsWithConstantData(node, output_tensor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch (const char* msg) {
|
||||
throw(msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -463,8 +396,12 @@ void BasicBackend::Infer(OrtKernelContext* ctx) {
|
|||
for (auto item : const_outputs_map_) {
|
||||
auto out_name = item.first;
|
||||
auto node = item.second;
|
||||
auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node);
|
||||
FillOutputsWithConstantData(node, output_tensor);
|
||||
try {
|
||||
auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node);
|
||||
FillOutputsWithConstantData(node, output_tensor);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
}
|
||||
// Get Output tensors
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Inference successful";
|
||||
|
|
@ -482,15 +419,30 @@ void BasicBackend::Infer(OrtKernelContext* ctx) {
|
|||
if ((global_context_.device_type.find("GPU") != std::string::npos) &&
|
||||
(global_context_.context != nullptr) &&
|
||||
(openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph)) {
|
||||
StartRemoteAsyncInference(context, infer_request);
|
||||
try {
|
||||
StartRemoteAsyncInference(context, infer_request);
|
||||
} catch (std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
} else {
|
||||
StartAsyncInference(context, infer_request);
|
||||
try{
|
||||
StartAsyncInference(context, infer_request);
|
||||
} catch(std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
}
|
||||
#else
|
||||
StartAsyncInference(context, infer_request);
|
||||
try{
|
||||
StartAsyncInference(context, infer_request);
|
||||
} catch(std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
#endif
|
||||
|
||||
CompleteAsyncInference(context, infer_request);
|
||||
try{
|
||||
CompleteAsyncInference(context, infer_request);
|
||||
}catch(std::string const & msg) {
|
||||
throw msg;
|
||||
}
|
||||
|
||||
// Get Output tensors
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Inference successful";
|
||||
|
|
|
|||
|
|
@ -32,11 +32,9 @@ class BasicBackend : public IBackend {
|
|||
bool ImportBlob(std::string hw_target, bool vpu_status);
|
||||
void PopulateCompiledDirectory(std::string, std::string&, std::string&, bool&);
|
||||
bool ValidateSubgraph(std::map<std::string, std::shared_ptr<ngraph::Node>>& const_outputs_map);
|
||||
void PopulateConfigValue(OVConfig& config);
|
||||
void PopulateConfigValue(OVConfig& config, ov::AnyMap& device_config);
|
||||
void EnableCaching();
|
||||
#if defined(OV_API_20)
|
||||
void EnableGPUThrottling(OVConfig& config);
|
||||
#endif
|
||||
void EnableGPUThrottling(ov::AnyMap& device_config);
|
||||
void StartAsyncInference(Ort::KernelContext& context, std::shared_ptr<OVInferRequest> infer_request);
|
||||
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
|
|
@ -78,11 +76,7 @@ class InferRequestsQueue {
|
|||
void printstatus() {
|
||||
std::cout << "printing elements of the vector (infer_requests_): " << std::endl;
|
||||
for (auto i = infer_requests_.begin(); i != infer_requests_.end(); ++i) {
|
||||
#if defined (OV_API_20)
|
||||
i->get()->QueryStatus();
|
||||
#else
|
||||
std::cout << *i << "\n";
|
||||
#endif
|
||||
}
|
||||
std::cout << '\n';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,19 +35,16 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
// of Infer Requests only if the VAD-M accelerator is being used.
|
||||
// sets number of maximum parallel inferences
|
||||
num_inf_reqs_ = 8;
|
||||
|
||||
#if defined(OV_API_20)
|
||||
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
|
||||
#else
|
||||
ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_);
|
||||
SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type);
|
||||
#ifndef NDEBUG
|
||||
if (IsDebugEnabled()) {
|
||||
std::string file_name = subgraph_context.subgraph_name + "_static.onnx";
|
||||
std::fstream outfile(file_name, std::ios::out | std::ios::trunc | std::ios::binary);
|
||||
model_proto.SerializeToOstream(outfile);
|
||||
// DumpOnnxModelProto(model_proto, subgraph_context.subgraph_name + "_static.onnx");
|
||||
}
|
||||
#endif
|
||||
OVConfig config;
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
config["PERF_COUNT"] = CONFIG_VALUE(YES);
|
||||
}
|
||||
#endif
|
||||
ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
|
||||
|
||||
|
||||
if (const_outputs_map_.size() == subgraph_context_.output_names.size())
|
||||
subgraph_context_.is_constant = true;
|
||||
|
|
@ -56,6 +53,14 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
if (subgraph_context_.is_constant)
|
||||
return;
|
||||
std::string& hw_target = (global_context_.device_id != "") ? global_context_.device_id : global_context_.device_type;
|
||||
OVConfig config;
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
config["PERF_COUNT"] = CONFIG_VALUE(YES);
|
||||
}
|
||||
#endif
|
||||
ov::AnyMap device_config;
|
||||
|
||||
// Loading model to the plugin
|
||||
//If graph is fully supported and batching is enabled, load the network onto all VPU's and infer
|
||||
std::vector<OVExeNetwork> exe_networks;
|
||||
|
|
@ -63,7 +68,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
for (int j = 0; j < 8; j++) {
|
||||
OVExeNetwork exe_network;
|
||||
config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[j];
|
||||
exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name);
|
||||
exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name);
|
||||
exe_networks.push_back(exe_network);
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
|
||||
|
|
@ -79,9 +84,9 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
else {
|
||||
i = GetFirstAvailableDevice(global_context);
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Device Tag is: " << i;
|
||||
config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i];
|
||||
OVExeNetwork exe_network;
|
||||
exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, subgraph_context_.subgraph_name);
|
||||
config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i];
|
||||
exe_network = global_context_.ie_core.LoadNetwork(ie_cnn_network_, hw_target, config, device_config, subgraph_context_.subgraph_name);
|
||||
LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin";
|
||||
OVInferRequestPtr infRequest;
|
||||
infRequest = std::make_shared<OVInferRequest>(exe_network.CreateInferRequest());
|
||||
|
|
@ -95,8 +100,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
|
|||
void VADMBackend::StartAsyncInference(Ort::KernelContext& context,
|
||||
size_t batch_slice_idx, size_t infer_req_idx) {
|
||||
auto infer_request = infer_requests_[infer_req_idx];
|
||||
|
||||
#if defined (OV_API_20)
|
||||
|
||||
auto graph_input_info = ie_cnn_network_->inputs();
|
||||
int input_idx = 0;
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
|
|
@ -111,32 +115,20 @@ void VADMBackend::StartAsyncInference(Ort::KernelContext& context,
|
|||
break;
|
||||
}
|
||||
}
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
// using the input name retrieved from ONNX original to match with the input names returned by OV tensors
|
||||
if (input_names.find(onnx_input_name) != input_names.end()) {
|
||||
input_name = onnx_input_name;
|
||||
} else {
|
||||
ORT_THROW(log_tag + "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + " doesn't exist in the list of OpenVINO input tensor names");
|
||||
}
|
||||
OVTensorPtr graph_input_blob;
|
||||
OVTensorPtr graph_input_blob;
|
||||
graph_input_blob = infer_request->GetTensor(input_name);
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_);
|
||||
input_idx++;
|
||||
}
|
||||
#else
|
||||
auto graph_input_info = ie_cnn_network_->getInputsInfo();
|
||||
for (auto input_info_iter = graph_input_info.begin();
|
||||
input_info_iter != graph_input_info.end(); ++input_info_iter) {
|
||||
// Get OpenVINO's input buffer
|
||||
std::string input_name = input_info_iter->first;
|
||||
auto precision = input_info_iter->second->getPrecision();
|
||||
auto graph_input_blob = infer_request->GetTensor(input_name);
|
||||
FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, precision, subgraph_context_);
|
||||
}
|
||||
#endif
|
||||
|
||||
// Start Async inference
|
||||
infer_request->StartAsync();
|
||||
|
||||
}
|
||||
|
||||
// Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx
|
||||
|
|
@ -148,8 +140,7 @@ void VADMBackend::CompleteAsyncInference(Ort::KernelContext& context,
|
|||
|
||||
// Wait for Async inference completion
|
||||
infer_request->WaitRequest();
|
||||
|
||||
#if defined (OV_API_20)
|
||||
|
||||
auto graph_output_info = ie_cnn_network_->outputs();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
|
|
@ -175,19 +166,7 @@ void VADMBackend::CompleteAsyncInference(Ort::KernelContext& context,
|
|||
auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
FillOutputBlob(graph_output_blob, output_tensor, batch_slice_idx);
|
||||
}
|
||||
#else
|
||||
auto graph_output_info = ie_cnn_network_->getOutputsInfo();
|
||||
for (auto output_info_iter = graph_output_info.begin();
|
||||
output_info_iter != graph_output_info.end(); ++output_info_iter) {
|
||||
// Get OpenVINO's output blob
|
||||
OVTensorPtr graph_output_blob;
|
||||
auto output_name = output_info_iter->first;
|
||||
graph_output_blob = infer_request->GetTensor(output_name);
|
||||
auto output_tensor = GetOutputTensor(context, batch_size, infer_request, output_name, subgraph_context_.output_names);
|
||||
auto precision = output_info_iter->second->getPrecision();
|
||||
FillOutputBlob(graph_output_blob, output_tensor, precision, batch_slice_idx);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!const_outputs_map_.empty()) {
|
||||
for (auto item : const_outputs_map_) {
|
||||
auto out_name = item.first;
|
||||
|
|
@ -220,7 +199,7 @@ void VADMBackend::Infer(OrtKernelContext* context) {
|
|||
LOGS_DEFAULT(INFO) << log_tag << "Running graph " << subgraph_context_.subgraph_name;
|
||||
LOGS_DEFAULT(INFO) << log_tag << "In Infer";
|
||||
Ort::KernelContext ctx(context);
|
||||
|
||||
|
||||
std::lock_guard<std::mutex> lock(compute_lock_);
|
||||
|
||||
size_t batch_size = 1;
|
||||
|
|
@ -228,11 +207,7 @@ void VADMBackend::Infer(OrtKernelContext* context) {
|
|||
if (subgraph_context_.enable_batching) {
|
||||
// Calculate the batch_size from the input tensor shape.
|
||||
auto tensor = ctx.GetInput(subgraph_context_.input_indexes[0]);
|
||||
#if defined (OV_API_20)
|
||||
batch_size = DeduceBatchSize(tensor, ie_cnn_network_->get_result()->get_shape());
|
||||
#else
|
||||
batch_size = DeduceBatchSize(tensor, ie_cnn_network_->getInputsInfo().begin()->second->getTensorDesc().getDims());
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t full_parallel_runs = batch_size / num_inf_reqs_;
|
||||
|
|
|
|||
|
|
@ -122,11 +122,7 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
|
|||
#endif
|
||||
openvino_ep::BackendManager::GetGlobalContext().onnx_opset_version = graph_viewer.DomainToVersionMap().at(kOnnxDomain);
|
||||
|
||||
#if defined (OPENVINO_2021_4)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2021_4");
|
||||
result = obj.Execute();
|
||||
#elif defined (OPENVINO_2022_1)
|
||||
#if defined (OPENVINO_2022_1)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_1");
|
||||
result = obj.Execute();
|
||||
|
|
@ -134,6 +130,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
|
|||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_2");
|
||||
result = obj.Execute();
|
||||
#elif defined (OPENVINO_2022_3)
|
||||
openvino_ep::GetCapability obj(graph_viewer,
|
||||
openvino_ep::BackendManager::GetGlobalContext().device_type, "V_2022_3");
|
||||
result = obj.Execute();
|
||||
#endif
|
||||
|
||||
return result;
|
||||
|
|
@ -148,11 +148,7 @@ common::Status OpenVINOExecutionProvider::Compile(
|
|||
|
||||
NodeComputeInfo compute_info;
|
||||
|
||||
#if defined(OV_API_20)
|
||||
openvino_ep::BackendManager::GetGlobalContext().use_api_2 = true;
|
||||
#else
|
||||
openvino_ep::BackendManager::GetGlobalContext().use_api_2 = false;
|
||||
#endif
|
||||
|
||||
std::shared_ptr<openvino_ep::BackendManager> backend_manager = std::make_shared<openvino_ep::BackendManager>(fused_node, graph_body_viewer, *GetLogger());
|
||||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ struct OpenVINOExecutionProviderInfo {
|
|||
precision_ = "FP32";
|
||||
#elif defined OPENVINO_CONFIG_CPU_FP16
|
||||
device_type_ = "CPU";
|
||||
precision_ = "FP16";
|
||||
precision_ = "FP16";
|
||||
#elif defined OPENVINO_CONFIG_GPU_FP32
|
||||
device_type_ = "GPU";
|
||||
precision_ = "FP32";
|
||||
|
|
@ -112,7 +112,7 @@ struct OpenVINOExecutionProviderInfo {
|
|||
precision_ = "FP32";
|
||||
} else if (dev_type == "CPU_FP16") {
|
||||
device_type_ = "CPU";
|
||||
precision_ = "FP16";
|
||||
precision_ = "FP16";
|
||||
} else if (dev_type == "GPU_FP32") {
|
||||
device_type_ = "GPU";
|
||||
precision_ = "FP32";
|
||||
|
|
@ -140,14 +140,18 @@ struct OpenVINOExecutionProviderInfo {
|
|||
} else if (dev_type == "VAD-F_FP32") {
|
||||
device_type_ = "HETERO:FPGA,CPU";
|
||||
precision_ = "FP32";
|
||||
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0 || dev_type.find("AUTO") == 0) {
|
||||
} else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0) {
|
||||
std::vector<std::string> devices = parseDevices(dev_type);
|
||||
precision_ = "FP16";
|
||||
if (devices[0] == "CPU") {
|
||||
precision_ = "FP32";
|
||||
}
|
||||
device_type_ = dev_type;
|
||||
} else {
|
||||
} else if ( dev_type.find("AUTO") == 0){
|
||||
std::vector<std::string> devices = parseDevices(dev_type);
|
||||
precision_ = "FP32";
|
||||
device_type_ = dev_type;
|
||||
}else {
|
||||
ORT_THROW("Invalid device string: " + dev_type);
|
||||
}
|
||||
LOGS_DEFAULT(INFO) << "[OpenVINO-EP]"
|
||||
|
|
|
|||
|
|
@ -9,9 +9,6 @@
|
|||
|
||||
#if defined (OV_API_20)
|
||||
using Exception = ov::Exception;
|
||||
#elif defined (OPENVINO_2021_4)
|
||||
using Exception = InferenceEngine::Exception;
|
||||
using WaitMode = InferenceEngine::InferRequest::WaitMode;
|
||||
#else
|
||||
using Exception = InferenceEngine::details::InferenceEngineException;
|
||||
using WaitMode = InferenceEngine::IInferRequest::WaitMode;
|
||||
|
|
@ -23,176 +20,110 @@ namespace onnxruntime {
|
|||
const std::string log_tag = "[OpenVINO-EP] ";
|
||||
std::shared_ptr<OVNetwork> OVCore::ReadModel(const std::string& model) const {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
OVTensor weights;
|
||||
return oe.read_model(model, weights);
|
||||
#else
|
||||
OVTensorPtr blob = {nullptr};
|
||||
auto network = oe.ReadNetwork(model, blob);
|
||||
auto obj = std::make_shared<OVNetwork>(network);
|
||||
return obj;
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what()));
|
||||
throw std::string(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what()));
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + "[OpenVINO-EP] Unknown exception while Reading network");
|
||||
throw std::string(log_tag + "[OpenVINO-EP] Unknown exception while Reading network");
|
||||
}
|
||||
}
|
||||
|
||||
OVExeNetwork OVCore::LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network, std::string& hw_target, OVConfig config, std::string name) {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
auto obj = oe.compile_model(ie_cnn_network, hw_target, config);
|
||||
OVExeNetwork exe(obj);
|
||||
return exe;
|
||||
#else
|
||||
auto obj = oe.LoadNetwork(*ie_cnn_network, hw_target, config);
|
||||
OVExeNetwork exe(obj);
|
||||
return exe;
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Exception while Loading Network for graph " + name);
|
||||
}
|
||||
}
|
||||
|
||||
OVExeNetwork OVCore::ImportModel(const std::string& compiled_blob, std::string hw_target, std::string name) {
|
||||
OVExeNetwork OVCore::LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network, std::string& hw_target, OVConfig& config, ov::AnyMap& device_config, std::string name) {
|
||||
ov::CompiledModel obj;
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
std::ifstream blob_stream_obj(compiled_blob);
|
||||
auto obj = oe.import_model(blob_stream_obj, hw_target, {});
|
||||
return OVExeNetwork(obj);
|
||||
#else
|
||||
auto obj = oe.ImportNetwork(compiled_blob, hw_target, {});
|
||||
return OVExeNetwork(obj);
|
||||
#endif
|
||||
} catch (Exception &e) {
|
||||
ORT_THROW(log_tag + " Exception while Importing Network for graph: " + name + ": " + e.what());
|
||||
} catch(...) {
|
||||
ORT_THROW(log_tag + " Exception while Importing Network for graph: " + name);
|
||||
if (hw_target.find("MYRIAD")==std::string::npos || hw_target.find("VAD-M_FP16")==std::string::npos){
|
||||
obj = oe.compile_model(ie_cnn_network, hw_target, device_config);
|
||||
} else {
|
||||
obj = oe.compile_model(ie_cnn_network, hw_target, config);
|
||||
}
|
||||
OVExeNetwork exe(obj);
|
||||
return exe;
|
||||
} catch (const Exception& e) {
|
||||
throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what());
|
||||
} catch (...) {
|
||||
throw std::string(log_tag + " Exception while Loading Network for graph " + name);
|
||||
}
|
||||
}
|
||||
|
||||
void OVCore::SetCache(std::string cache_dir_path) {
|
||||
#if defined(OV_API_20)
|
||||
oe.set_property(ov::cache_dir(cache_dir_path));
|
||||
#else
|
||||
oe.SetConfig({{CONFIG_KEY(CACHE_DIR), cache_dir_path}});
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
OVExeNetwork OVCore::LoadNetwork(std::shared_ptr<OVNetwork>& model, OVRemoteContextPtr context, std::string& name) {
|
||||
try {
|
||||
#if defined(OV_API_20)
|
||||
auto obj = oe.compile_model(model, *context);
|
||||
return OVExeNetwork(obj);
|
||||
#else
|
||||
auto obj = oe.LoadNetwork(*model, context);
|
||||
return OVExeNetwork(obj);
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what());
|
||||
throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Exception while Loading Network for graph " + name);
|
||||
}
|
||||
throw std::string(log_tag + " Exception while Loading Network for graph " + name);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
std::vector<std::string> OVCore::GetAvailableDevices() {
|
||||
#if defined (OV_API_20)
|
||||
auto obj = oe.get_available_devices();
|
||||
return obj;
|
||||
#else
|
||||
auto obj = oe.GetAvailableDevices();
|
||||
return obj;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
OVInferRequest OVExeNetwork::CreateInferRequest() {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
auto infReq = obj.create_infer_request();
|
||||
OVInferRequest inf_obj(infReq);
|
||||
return inf_obj;
|
||||
#else
|
||||
auto infReq = obj.CreateInferRequest();
|
||||
OVInferRequest inf_obj(infReq);
|
||||
return inf_obj;
|
||||
#endif
|
||||
auto infReq = obj.create_infer_request();
|
||||
OVInferRequest inf_obj(infReq);
|
||||
return inf_obj;
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what());
|
||||
throw std::string(log_tag + "Exception while creating InferRequest object: " + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + "Exception while creating InferRequest object.");
|
||||
throw std::string(log_tag + "Exception while creating InferRequest object.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OVTensorPtr OVInferRequest::GetTensor(const std::string& input_name) {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
auto tobj = ovInfReq.get_tensor(input_name);
|
||||
OVTensorPtr blob = std::make_shared<OVTensor>(tobj);
|
||||
return blob;
|
||||
#else
|
||||
auto blob = infReq.GetBlob(input_name);
|
||||
return blob;
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what());
|
||||
throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name);
|
||||
throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name);
|
||||
}
|
||||
}
|
||||
|
||||
void OVInferRequest::SetTensor(const std::string& name, OVTensorPtr& blob) {
|
||||
try {
|
||||
#if defined(OV_API_20)
|
||||
ovInfReq.set_tensor(name, *(blob.get()));
|
||||
#else
|
||||
infReq.SetBlob(name, blob);
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name + e.what());
|
||||
throw std::string(log_tag + " Cannot set Remote Blob for output: " + name + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name);
|
||||
throw std::string(log_tag + " Cannot set Remote Blob for output: " + name);
|
||||
}
|
||||
}
|
||||
|
||||
void OVInferRequest::StartAsync() {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
ovInfReq.start_async();
|
||||
#else
|
||||
infReq.StartAsync();
|
||||
#endif
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Couldn't start Inference: " + e.what());
|
||||
throw std::string(log_tag + " Couldn't start Inference: " + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " In Error Couldn't start Inference");
|
||||
throw std::string(log_tag + " In Error Couldn't start Inference");
|
||||
}
|
||||
}
|
||||
|
||||
void OVInferRequest::WaitRequest() {
|
||||
try {
|
||||
#if defined (OV_API_20)
|
||||
ovInfReq.wait();
|
||||
#else
|
||||
infReq.Wait(WaitMode::RESULT_READY);
|
||||
#endif
|
||||
ovInfReq.wait();
|
||||
} catch (const Exception& e) {
|
||||
ORT_THROW(log_tag + " Wait Model Failed: " + e.what());
|
||||
throw std::string(log_tag + " Wait Model Failed: " + e.what());
|
||||
} catch (...) {
|
||||
ORT_THROW(log_tag + " Wait Mode Failed");
|
||||
}
|
||||
throw std::string(log_tag + " Wait Mode Failed");
|
||||
}
|
||||
}
|
||||
|
||||
void OVInferRequest::QueryStatus() {
|
||||
#if defined (OV_API_20)
|
||||
std::cout << "ovInfReq.query_state()" << " ";
|
||||
#else
|
||||
std::cout << &infReq << " ";
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
#include <vector>
|
||||
|
||||
#include <inference_engine.hpp>
|
||||
#if defined (OPENVINO_2022_1) || (OPENVINO_2022_2)
|
||||
#if defined (OPENVINO_2022_1) || (OPENVINO_2022_2) || (OPENVINO_2022_3)
|
||||
#define OV_API_20
|
||||
#include "openvino/openvino.hpp"
|
||||
#include "openvino/pass/convert_fp32_to_fp16.hpp"
|
||||
|
|
@ -15,10 +15,8 @@
|
|||
#ifdef IO_BUFFER_ENABLED
|
||||
#include <gpu/gpu_context_api_ocl.hpp>
|
||||
#include <gpu/gpu_config.hpp>
|
||||
#if defined (OV_API_20)
|
||||
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include <string>
|
||||
|
||||
|
|
@ -28,102 +26,56 @@ class OVCore;
|
|||
class OVInferRequest;
|
||||
class OVExeNetwork;
|
||||
|
||||
#if defined (OV_API_20)
|
||||
typedef InferenceEngine::Precision OVPrecision;
|
||||
typedef ov::Tensor OVTensor;
|
||||
typedef ov::ProfilingInfo OVProfilingInfo;
|
||||
typedef ov::AnyMap OVConfig;
|
||||
typedef ov::Model OVNetwork;
|
||||
#else
|
||||
typedef InferenceEngine::Precision OVPrecision;
|
||||
typedef InferenceEngine::Blob OVTensor;
|
||||
typedef InferenceEngine::InferenceEngineProfileInfo OVProfilingInfo;
|
||||
typedef std::map<std::string, std::string> OVConfig;
|
||||
typedef InferenceEngine::CNNNetwork OVNetwork;
|
||||
#endif
|
||||
typedef std::shared_ptr<OVInferRequest> OVInferRequestPtr;
|
||||
typedef std::shared_ptr<OVTensor> OVTensorPtr;
|
||||
typedef InferenceEngine::Precision OVPrecision;
|
||||
typedef ov::Tensor OVTensor;
|
||||
typedef ov::ProfilingInfo OVProfilingInfo;
|
||||
typedef ov::AnyMap OVConfig;
|
||||
typedef ov::Model OVNetwork;
|
||||
typedef std::shared_ptr<OVInferRequest> OVInferRequestPtr;
|
||||
typedef std::shared_ptr<OVTensor> OVTensorPtr;
|
||||
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
#ifdef OV_API_20
|
||||
typedef ov::intel_gpu::ocl::ClContext* OVRemoteContextPtr;
|
||||
typedef ov::RemoteContext OVRemoteContext;
|
||||
#else
|
||||
typedef InferenceEngine::RemoteContext::Ptr OVRemoteContextPtr;
|
||||
typedef InferenceEngine::RemoteContext OVRemoteContext;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
class OVCore {
|
||||
#if defined (OV_API_20)
|
||||
ov::Core oe;
|
||||
#else
|
||||
InferenceEngine::Core oe;
|
||||
#endif
|
||||
public:
|
||||
std::shared_ptr<OVNetwork> ReadModel(const std::string& model_stream) const;
|
||||
OVExeNetwork LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network, std::string& hw_target, OVConfig config, std::string name);
|
||||
OVExeNetwork ImportModel(const std::string& compiled_blob, std::string hw_target, std::string name);
|
||||
OVExeNetwork LoadNetwork(std::shared_ptr<OVNetwork>& ie_cnn_network, std::string& hw_target, OVConfig& config, ov::AnyMap& device_config, std::string name);
|
||||
void SetCache(std::string cache_dir_path);
|
||||
#ifdef IO_BUFFER_ENABLED
|
||||
OVExeNetwork LoadNetwork(std::shared_ptr<OVNetwork>& model, OVRemoteContextPtr context, std::string& name);
|
||||
#endif
|
||||
std::vector<std::string> GetAvailableDevices();
|
||||
#if defined (OV_API_20)
|
||||
ov::Core& Get() {
|
||||
return oe;
|
||||
}
|
||||
#else
|
||||
InferenceEngine::Core& Get() {
|
||||
return oe;
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
class OVExeNetwork {
|
||||
#if (defined OV_API_20)
|
||||
ov::CompiledModel obj;
|
||||
#else
|
||||
InferenceEngine::ExecutableNetwork obj;
|
||||
#endif
|
||||
public:
|
||||
#if defined (OV_API_20)
|
||||
OVExeNetwork(ov::CompiledModel md) { obj = md; }
|
||||
OVExeNetwork() { obj = ov::CompiledModel(); }
|
||||
ov::CompiledModel& Get() { return obj; }
|
||||
#else
|
||||
OVExeNetwork(InferenceEngine::ExecutableNetwork md) { obj = md; }
|
||||
OVExeNetwork() { obj = InferenceEngine::ExecutableNetwork(); }
|
||||
InferenceEngine::ExecutableNetwork& Get() { return obj ; }
|
||||
#endif
|
||||
OVInferRequest CreateInferRequest();
|
||||
};
|
||||
|
||||
class OVInferRequest {
|
||||
#if defined (OV_API_20)
|
||||
ov::InferRequest ovInfReq;
|
||||
#else
|
||||
InferenceEngine::InferRequest infReq;
|
||||
#endif
|
||||
public:
|
||||
OVTensorPtr GetTensor(const std::string& name);
|
||||
void SetTensor(const std::string& name, OVTensorPtr& blob);
|
||||
void StartAsync();
|
||||
void WaitRequest();
|
||||
void QueryStatus();
|
||||
#if defined (OV_API_20)
|
||||
explicit OVInferRequest(ov::InferRequest obj) { ovInfReq = obj; }
|
||||
OVInferRequest() { ovInfReq = ov::InferRequest(); }
|
||||
ov::InferRequest& GetNewObj() {
|
||||
return ovInfReq;
|
||||
}
|
||||
#else
|
||||
explicit OVInferRequest(InferenceEngine::InferRequest obj) { infReq = obj; }
|
||||
OVInferRequest() { infReq = InferenceEngine::InferRequest(); }
|
||||
InferenceEngine::InferRequest& GetObj() {
|
||||
return infReq;
|
||||
}
|
||||
#endif
|
||||
};
|
||||
}
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -26,14 +26,14 @@ namespace openvino_ep {
|
|||
GetCapability::GetCapability(const GraphViewer& graph_viewer_param, std::string device_type_param,
|
||||
const std::string version_param):
|
||||
graph_viewer_(graph_viewer_param), device_type_(device_type_param){
|
||||
if (version_param == "V_2021_4") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2021_4, device_type_);
|
||||
} else if (version_param == "V_2022_1") {
|
||||
if (version_param == "V_2022_1") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_1, device_type_);
|
||||
} else if (version_param == "V_2022_2") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_);
|
||||
} else if (version_param == "V_2022_3") {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_);
|
||||
} else {
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_2, device_type_);
|
||||
data_ops_ = new DataOps(graph_viewer_, V_2022_3, device_type_);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -128,7 +128,9 @@ std::vector<SupportedOp> supported_op_mode = {
|
|||
{"GlobalMaxPool", V_2022_1, {"CPU", "GPU"}},
|
||||
{"Greater", V_2020_4, {"All"}},
|
||||
{"GreaterOrEqual", V_2022_1, {"All"}},
|
||||
{"GridSample", V_2022_3, {"CPU"}},
|
||||
{"Identity", V_2020_4, {"All"}},
|
||||
{"If", V_2022_3, {"All"}},
|
||||
{"ImageScaler", V_2022_1, {"All"}},
|
||||
{"InstanceNormalization", V_2020_4, {"All"}},
|
||||
{"HardSigmoid", V_2020_4, {"CPU", "GPU"}},
|
||||
|
|
@ -161,6 +163,7 @@ std::vector<SupportedOp> supported_op_mode = {
|
|||
{"Pad", V_2020_4, {"All"}},
|
||||
{"Pow", V_2020_4, {"All"}},
|
||||
{"PRelu", V_2020_4, {"All"}},
|
||||
{"QLinearMatMul", V_2022_3, {"CPU"}},
|
||||
{"QuantizeLinear", V_2021_4, {"CPU", "GPU"}},
|
||||
{"Range", V_2021_2, {"MYRIAD"}},
|
||||
{"Range", V_2022_1, {"All"}},
|
||||
|
|
@ -317,18 +320,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
|
||||
//populate unsupportedmode_t
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
for (size_t i = 0; i < node->InputDefs().size(); i++) {
|
||||
if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Abs", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//Abs is not supproted with INT8 or INT32 as input data type on GPU
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
|
|
@ -343,7 +335,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Abs", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//tensor type does not support select last index
|
||||
auto& attributes = node->GetAttributes();
|
||||
|
|
@ -359,34 +351,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"ArgMin", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//auto pad null value is not supported
|
||||
const auto& attributes = node->GetAttributes();
|
||||
auto auto_attr = attributes.find("auto_pad");
|
||||
if (auto_attr->second().s() == "") {
|
||||
return true;
|
||||
}
|
||||
// default value of ceil_mode (0) is supported.
|
||||
auto ceil_attr = attributes.find("ceil_mode");
|
||||
if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0) return true;
|
||||
return (!dimension_unsupported(node));
|
||||
}};
|
||||
op_list_.insert({"AveragePool", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//Only float 16, float and double data types are supported
|
||||
const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos;
|
||||
const bool data_is_float16 = node->InputDefs()[0]->Type()->find("float16") != std::string::npos;
|
||||
const bool data_is_double = node->InputDefs()[0]->Type()->find("double") != std::string::npos;
|
||||
return !(data_is_float || data_is_float16 || data_is_double);
|
||||
}};
|
||||
op_list_.insert({"Clip", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
//int64 data type is not supported on GPU
|
||||
|
|
@ -405,46 +370,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Clip", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& ) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
const auto& attributes = node->GetAttributes();
|
||||
auto conv_filter = attributes.find("kernel_shape");
|
||||
auto& ints = conv_filter->second().ints();
|
||||
//If the kernel size is not 2D, the op is rejected in case of MYRIAD
|
||||
if(ints.size() !=2) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
//If the device is GPU
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
bool if_bias = false;
|
||||
const auto& attributes = node->GetAttributes();
|
||||
auto conv_filter = attributes.find("kernel_shape");
|
||||
if (conv_filter != attributes.end()) {
|
||||
auto& ints = conv_filter->second().ints();
|
||||
//check if the Input for the op has bias
|
||||
if(node->InputDefs().size() > 2) {
|
||||
if(node->InputDefs()[2]->Name() == "B")
|
||||
if_bias = true;
|
||||
}
|
||||
//If the kernel size is 1D and the input has bias and the output precision is FP32, the op is rejected.
|
||||
if(ints.size() == 1 && if_bias) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)
|
||||
return true;
|
||||
}
|
||||
//If the kernel size is 3D and the input doesnot have bias, the op is rejected in case of GPU
|
||||
if(ints.size() == 3 && !if_bias)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Conv", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet& ) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
const auto& attributes = node->GetAttributes();
|
||||
|
|
@ -477,74 +403,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Conv", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
auto& attributes = node->GetAttributes();
|
||||
if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Conv", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
if (GetInputCount(node, initializers) > 1)
|
||||
return true;
|
||||
}
|
||||
bool if_bias = false;
|
||||
const auto& attributes = node->GetAttributes();
|
||||
auto out_shape_attr = attributes.find("output_shape");
|
||||
|
||||
// If the device is GPU
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto conv_filter = attributes.find("kernel_shape");
|
||||
if (conv_filter != attributes.end()) {
|
||||
auto& kernel_size = conv_filter->second().ints();
|
||||
|
||||
//If 3D convolution, reject the op
|
||||
if(kernel_size.size() == 3)
|
||||
return true;
|
||||
//In 1D conv, if the pads are asymmetric, then the op is rejected
|
||||
if(kernel_size.size() == 1) {
|
||||
if (attributes.count("pads") > 0) {
|
||||
auto& pads_attr = attributes.at("pads");
|
||||
auto int_size = pads_attr.ints_size();
|
||||
if (int_size > 1 && (pads_attr.ints(0) != pads_attr.ints(1)))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
//check if the Input for the op has bias
|
||||
if(node->InputDefs().size() > 2) {
|
||||
if(node->InputDefs()[2]->Name() == "B")
|
||||
if_bias = true;
|
||||
}
|
||||
//If the kernel size is 2D, the input has no bias, the padding is 0 and the op has dilations, the op is rejected
|
||||
if(kernel_size.size() == 2 && !if_bias) {
|
||||
if (attributes.count("pads") > 0) {
|
||||
auto& pads_attr = attributes.at("pads");
|
||||
auto int_size = pads_attr.ints_size();
|
||||
// comparing if all the 4 values in the padding are equal to 1
|
||||
if (int_size == 4 && (pads_attr.ints(0) == pads_attr.ints(1) == pads_attr.ints(2) == pads_attr.ints(3))) {
|
||||
if(pads_attr.ints(0) == 1)
|
||||
return false;
|
||||
}
|
||||
// If the op has dilations as an attribute after the above checks, then the op is rejected.
|
||||
auto dilation_attr = attributes.find("dilations");
|
||||
if (dilation_attr != attributes.end())
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"ConvTranspose", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
if (GetInputCount(node, initializers) > 1)
|
||||
|
|
@ -576,69 +435,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"ConvTranspose", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
auto& attributes = node->GetAttributes();
|
||||
if (attributes.count("auto_pad") == 0 || attributes.at("auto_pad").s() == "") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"ConvTranspose", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
// all ConvInteger zero points need to be constants
|
||||
if (node->InputDefs().size() == 3) {
|
||||
return (initializers.find(node->InputDefs()[2]->Name()) == initializers.end());
|
||||
} else if (node->InputDefs().size() == 4) {
|
||||
return initializers.find(node->InputDefs()[2]->Name()) == initializers.end() ||
|
||||
initializers.find(node->InputDefs()[3]->Name()) == initializers.end();
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"ConvInteger", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
const auto& input = node->InputDefs()[0];
|
||||
auto graph_inputs = graph_viewer_.GetInputs();
|
||||
auto it = find(graph_inputs.begin(), graph_inputs.end(), input);
|
||||
if (it != graph_inputs.end()) {
|
||||
const auto& indices_arg = node->InputDefs()[1];
|
||||
if (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)
|
||||
return true;
|
||||
}
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
//If the output of Gather op is INT8 or UINT8, it is rejected for GPU.
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 ||
|
||||
output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Gather", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& indices_arg = node->InputDefs()[0];
|
||||
const auto& output_arg = node->OutputDefs()[0];
|
||||
if (indices_arg->TypeAsProto()->tensor_type().elem_type() != output_arg->TypeAsProto()->tensor_type().elem_type())
|
||||
return true;
|
||||
if ((indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) ||
|
||||
(indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}};
|
||||
op_list_.insert({"GatherElements", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
const auto& indices_arg = node->InputDefs()[0];
|
||||
|
|
@ -656,32 +453,23 @@ void DataOps::populate_op_mode_supported() {
|
|||
}};
|
||||
op_list_.insert({"GatherElements", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& input = node->InputDefs()[0];
|
||||
const auto& output = node->OutputDefs()[0];
|
||||
auto graph_inputs = this->graph_viewer_.GetInputs();
|
||||
auto graph_outputs = this->graph_viewer_.GetOutputs();
|
||||
auto input_it = find(graph_inputs.begin(), graph_inputs.end(), input);
|
||||
auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output);
|
||||
if (input_it != graph_inputs.end() && output_it != graph_outputs.end())
|
||||
return true;
|
||||
return false;
|
||||
if (device_id_.find("GPU") != std::string::npos && node->OpType()=="If"){
|
||||
// Only Equal op is supported as input for IF op in GPU
|
||||
for (auto nit = node->InputNodesBegin(); nit != node->InputNodesEnd(); ++nit){
|
||||
if (nit->OpType() == "Equal"){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}};
|
||||
op_list_.insert({"Identity", obj});
|
||||
op_list_.insert({"If", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
//Loop has to be initializer
|
||||
const auto& cond = node->InputDefs()[1];
|
||||
return (initializers.find(cond->Name()) == initializers.end());
|
||||
}};
|
||||
op_list_.insert({"Loop", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//If the Input size of LSTM is greater than 3, it is rejected.
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
|
|
@ -693,35 +481,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"LSTM", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//MaxPool "indices" output is not currently supported.
|
||||
//if (node->OutputDefs().size() > 1)
|
||||
// return true;
|
||||
const auto& attributes = node->GetAttributes();
|
||||
/* default value of ceil_mode (0) is supported.
|
||||
auto ceil_attr = attributes.find("ceil_mode");
|
||||
if (ceil_attr != attributes.end() && ceil_attr->second().i() != 0)
|
||||
return true;*/
|
||||
auto auto_attr = attributes.find("auto_pad");
|
||||
//auto pad null value is not supported
|
||||
if (auto_attr->second().s() == "")
|
||||
return true;
|
||||
// dilations attrs are not supported in nGraph
|
||||
if (attributes.find("dilations") != attributes.end())
|
||||
return true;
|
||||
// uint8 as output type for maxpool op is not supported on GPU
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)
|
||||
return true;
|
||||
}
|
||||
return (!this->dimension_unsupported(node));
|
||||
}};
|
||||
op_list_.insert({"MaxPool", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& attributes = node->GetAttributes();
|
||||
// dilations attrs are not supported yet for Maxpool
|
||||
|
|
@ -732,47 +492,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"MaxPool", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
if (device_id_.find("MYRIAD") == std::string::npos) {
|
||||
if (GetInputCount(node, initializers) == 1)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Max", obj});
|
||||
op_list_.insert({"Min", obj});
|
||||
op_list_.insert({"Mean", obj});
|
||||
op_list_.insert({"Sum", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//All matmuls except float have computation missmatch
|
||||
const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos;
|
||||
const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos;
|
||||
return (A_is_float && B_is_float) ? false : true;
|
||||
}};
|
||||
op_list_.insert({"MatMul", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
// all MatMulInteger zero points need to be constants
|
||||
if (node->InputDefs().size() == 3) {
|
||||
// not found in initializers -> not const
|
||||
return initializers.find(node->InputDefs()[2]->Name()) == initializers.end();
|
||||
} else if (node->InputDefs().size() == 4) {
|
||||
// not found in initializers -> not const
|
||||
return ((initializers.find(node->InputDefs()[2]->Name()) == initializers.end()) ||
|
||||
(initializers.find(node->InputDefs()[2]->Name()) == initializers.end()));
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"MatMulInteger", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
|
|
@ -788,32 +508,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Mod", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
//If the output of Neg op is INT8, it is rejected for GPU.
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Neg", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
auto graph_outputs = graph_viewer_.GetOutputs();
|
||||
const auto& output = node->OutputDefs()[0];
|
||||
auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output);
|
||||
if (output_it != graph_outputs.end())
|
||||
return true;
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"NonMaxSuppression", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
|
|
@ -832,7 +527,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Pow", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
// Max op with one input is not supporting for GPU_FP16
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
|
|
@ -848,7 +543,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Max", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
// Min op with one input is not supporting for GPU_FP16
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
|
|
@ -864,7 +559,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Min", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
// Sum op with one input is not supporting for GPU_FP16
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
|
|
@ -880,24 +575,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Sum", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
auto slope = node->InputDefs()[1];
|
||||
//PRelu slope has to be an initializer or needs to come from a constant node
|
||||
if (initializers.count(slope->Name()))
|
||||
return false;
|
||||
else {
|
||||
for (auto input_node = node->InputNodesBegin(); input_node != node->InputNodesEnd(); ++input_node) {
|
||||
if (GetInputCount(this->graph_viewer_.GetNode((*input_node).Index()), initializers) == 0)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}};
|
||||
op_list_.insert({"PRelu", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto slope = node->InputDefs()[1];
|
||||
|
|
@ -916,20 +594,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"PRelu", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
bool non_const_zero_point = false;
|
||||
// check if any of the zero points is NOT in the initializers list
|
||||
non_const_zero_point |= initializers.find(node->InputDefs()[2]->Name()) == initializers.end();
|
||||
non_const_zero_point |= initializers.find(node->InputDefs()[5]->Name()) == initializers.end();
|
||||
non_const_zero_point |= initializers.find(node->InputDefs()[7]->Name()) == initializers.end();
|
||||
// QLinearMatMul is not supported if any of the zero points is a dynamic input
|
||||
return non_const_zero_point;
|
||||
}};
|
||||
op_list_.insert({"QLinearMatMul", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& input_arg = node->InputDefs()[1];
|
||||
auto shape = input_arg->Shape();
|
||||
|
|
@ -945,7 +610,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Reshape", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
auto& attributes = node->GetAttributes();
|
||||
if (attributes.count("mode") ==1 && attributes.at("mode").s() == "linear") {
|
||||
|
|
@ -958,21 +623,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Resize", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
//If the output of ReduceMax op is INT8 or UINT8, it is rejected for GPU.
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 ||
|
||||
output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"ReduceMax", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos || device_id_.find("MYRIAD") != std::string::npos) {
|
||||
//INT32 dataype is not supported as input
|
||||
|
|
@ -986,35 +637,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"ReduceLogSumExp", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
const auto& attributes = node->GetAttributes();
|
||||
auto axis_attr = attributes.find("axis");
|
||||
//Negative axis is not supported
|
||||
if (axis_attr->second().i() < 0)
|
||||
return true;
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
const auto& input_arg = node->InputDefs()[2];
|
||||
auto updates_shape = input_arg->Shape();
|
||||
const auto& output_arg = node->OutputDefs()[0];
|
||||
auto out_shape = output_arg->Shape();
|
||||
//If updates attribute dim value greater than output_shape dim value, we reject
|
||||
if(node->InputDefs()[2]->Name() == "updates")
|
||||
{
|
||||
size_t updates_size = updates_shape->dim_size();
|
||||
if(updates_size == 2) {
|
||||
if(updates_shape->dim(1).dim_value() > out_shape->dim(1).dim_value())
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Scatter", obj});
|
||||
op_list_.insert({"ScatterElements", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("MYRIAD") != std::string::npos) {
|
||||
const auto& attributes = node->GetAttributes();
|
||||
|
|
@ -1043,7 +666,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"ScatterElements", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
|
|
@ -1058,7 +681,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Scatter", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//If the Input of Shrink op is UINT8, it is rejected (Due to output mismatch)
|
||||
for (size_t i = 0; i < node->InputDefs().size(); i++) {
|
||||
|
|
@ -1070,7 +693,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Shrink", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet& initializers) {
|
||||
//start, end, axes need to be a initializer
|
||||
bool cond_for_slice = false;
|
||||
|
|
@ -1095,17 +718,6 @@ void DataOps::populate_op_mode_supported() {
|
|||
}};
|
||||
op_list_.insert({"Slice", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//Shape can't have empty axes attribute
|
||||
const auto& attributes = node->GetAttributes();
|
||||
if (attributes.count("axes") == 0)
|
||||
return true;
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Squeeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
|
|
@ -1121,27 +733,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Squeeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
if (device_id_.find("GPU") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
//If the output of Transpose op is INT8, it is rejected for GPU.
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}};
|
||||
op_list_.insert({"Transpose", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
return (!this->dimension_unsupported(node));
|
||||
}};
|
||||
op_list_.insert({"Unsqueeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//If the operator is unsqueeze
|
||||
//If axes is an input, then we cannot produce a static graph. Conversion fails in convert_function_to_cnn_network.
|
||||
|
|
@ -1155,7 +747,7 @@ void DataOps::populate_op_mode_supported() {
|
|||
op_list_.insert({"Unsqueeze", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4, V_2022_1, V_2022_2},
|
||||
UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
//check for attributes
|
||||
auto& upsample_attr = node->GetAttributes();
|
||||
|
|
@ -1187,17 +779,13 @@ void DataOps::populate_op_mode_supported() {
|
|||
}};
|
||||
op_list_.insert({"Upsample", obj});
|
||||
}
|
||||
{
|
||||
UnsupportedOpMode obj = {{V_2021_4},
|
||||
[this](const Node* node, const InitializedTensorSet&) {
|
||||
return (!this->dimension_unsupported(node));
|
||||
}};
|
||||
op_list_.insert({"ReduceSum", obj});
|
||||
}
|
||||
}
|
||||
|
||||
bool DataOps::op_is_supported(std::string name, std::vector<SupportedOp>& op_list) {
|
||||
bool auto_support = false;
|
||||
bool multi_support = false;
|
||||
for (size_t i = 0; i < op_list.size(); i++) {
|
||||
|
||||
if (op_list[i].optype == name) {
|
||||
if (op_list[i].version <= version_id_) {
|
||||
auto it = op_list[i].device_type.begin();
|
||||
|
|
@ -1205,46 +793,63 @@ bool DataOps::op_is_supported(std::string name, std::vector<SupportedOp>& op_lis
|
|||
//status variable is set to True if it's Hetero/Multi/Auto device type
|
||||
bool status = false;
|
||||
|
||||
//if device supported is all then we support it
|
||||
if (*it == "All") {
|
||||
return true;
|
||||
}
|
||||
|
||||
//The operator to be marked true, it should be supported by either of the devices specified with HETERO
|
||||
if (device_id_.find("HETERO") == 0) {
|
||||
status = true;
|
||||
if (device_id_.find(*it) != std::string::npos) {
|
||||
if (device_id_.find(*it) != std::string::npos || (*it == "All")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
//The operator to be marked true, it should be supported by all the devices specified with MULTI/AUTO
|
||||
if (device_id_.find("MULTI") == 0 || device_id_.find("AUTO") == 0) {
|
||||
if (device_id_.find("MULTI") == 0) {
|
||||
status = true;
|
||||
if (device_id_.find(*it) == std::string::npos) {
|
||||
return false;
|
||||
if ((*it == "All") || device_id_.find(*it) != std::string::npos) {
|
||||
multi_support = true;
|
||||
}
|
||||
}
|
||||
|
||||
//The operator to be marked true, it should be supported by atleast CPU device specified with AUTO
|
||||
if (device_id_.find("AUTO") == 0) {
|
||||
if (std::string(*it).find("CPU")==std::string::npos){
|
||||
auto_support = false;
|
||||
}
|
||||
else if((*it == "All") || (device_id_.find(*it) != std::string::npos)){
|
||||
auto_support = true;
|
||||
}
|
||||
}
|
||||
//if device supported is all then we support it
|
||||
if (*it == "All") {
|
||||
return true;
|
||||
}
|
||||
//check for device supported
|
||||
if (status == false) {
|
||||
if (device_id_.find(*it) != std::string::npos) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
it++;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (device_id_.find("AUTO") == 0 && auto_support == true) {
|
||||
return true;
|
||||
}
|
||||
if (device_id_.find("MULTI") == 0 && multi_support == true) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) {
|
||||
const auto* type_proto = node_arg->TypeAsProto();
|
||||
if (!type_proto) {
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
std::cout << "Node is not a proto " << std::endl;
|
||||
}
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1297,9 +902,6 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) {
|
|||
return false;
|
||||
|
||||
} else if (device_id_ == "GPU") {
|
||||
auto prec_str = openvino_ep::BackendManager::GetGlobalContext().precision_str;
|
||||
if (prec_str == "FP32" && dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)
|
||||
return false;
|
||||
for (auto const& var : supported_types_gpu_) {
|
||||
if ((var.first <= version_id_) &&
|
||||
(var.second == dtype)) {
|
||||
|
|
@ -1417,6 +1019,11 @@ bool DataOps::node_is_supported(const std::map<std::string, std::set<std::string
|
|||
});
|
||||
|
||||
if (!are_types_supported) {
|
||||
#ifndef NDEBUG
|
||||
if (openvino_ep::backend_utils::IsDebugEnabled()) {
|
||||
std::cout << "DType is not supported" << std::endl;
|
||||
}
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1572,8 +1179,11 @@ bool DataOps::SpecialConditionForClusterSizeOne(std::unordered_set<std::string>&
|
|||
}
|
||||
} else if (node->OpType() == "MaxPool" && device_id_.find("MYRIAD") != std::string::npos) {
|
||||
auto output_data_type = node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
|
||||
if (output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT ||
|
||||
output_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) {
|
||||
if (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT ||
|
||||
output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) {
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ enum versionNum {
|
|||
V_2021_3,
|
||||
V_2021_4,
|
||||
V_2022_1,
|
||||
V_2022_2
|
||||
V_2022_2,
|
||||
V_2022_3
|
||||
};
|
||||
|
||||
using VersionNum = enum versionNum;
|
||||
|
|
|
|||
|
|
@ -115,8 +115,10 @@ GetPartitionedClusters(const std::vector<NodeIndex>& topological_order, const st
|
|||
if (!this_cluster.empty()) {
|
||||
ng_clusters.push_back(std::move(this_cluster));
|
||||
}
|
||||
// Point prev to node idx past this unsuported node.
|
||||
prev = ++it;
|
||||
if (it != topological_order.end()) {
|
||||
// Point prev to node idx past this unsuported node.
|
||||
prev = ++it;
|
||||
}
|
||||
}
|
||||
|
||||
//Tail
|
||||
|
|
|
|||
|
|
@ -140,6 +140,9 @@ def create_backend_test(test_name=None):
|
|||
if backend.supports_device("OPENVINO_CPU_FP32"):
|
||||
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP32")
|
||||
|
||||
if backend.supports_device("OPENVINO_CPU_FP16"):
|
||||
current_failing_tests += apply_filters(filters, "current_failing_tests_OPENVINO_CPU_FP16")
|
||||
|
||||
if backend.supports_device("MIGRAPHX"):
|
||||
current_failing_tests += apply_filters(filters, "current_failing_tests_MIGRAPHX")
|
||||
|
||||
|
|
|
|||
|
|
@ -261,6 +261,8 @@
|
|||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_operator_permute2",
|
||||
"^test_leakyrelu",
|
||||
"^test_leakyrelu_example",
|
||||
"^test_operator_repeat",
|
||||
"^test_operator_repeat_dim_overflow",
|
||||
"^test_add_bcast.*",
|
||||
|
|
@ -279,7 +281,10 @@
|
|||
"^test_unique_not_sorted_without_axis",
|
||||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_scatter_elements_with_negative_indices",
|
||||
"^test_sce.*",
|
||||
"^test_squeeze",
|
||||
"^test_squeeze_negative_axes",
|
||||
"^test_nllloss.*",
|
||||
"^test_gather_negative_indices.*",
|
||||
"^test_reduce_sum_do_not_keepdims*", // Does not support axes as input
|
||||
|
|
@ -303,6 +308,14 @@
|
|||
"^test_maxpool_with_argmax_2d_precomputed_strides", //Disabled as it throws segfault
|
||||
"^test_maxpool_with_argmax_2d_precomputed_pads" //Disabled as it throws segfault
|
||||
],
|
||||
"current_failing_tests_OPENVINO_CPU_FP16": [
|
||||
"^test_gridsample*",
|
||||
"^test_reduce_sum_do_not_keepdims_example",
|
||||
"^test_reduce_sum_do_not_keepdims_random",
|
||||
"^test_scatter_elements_with_negative_indices",
|
||||
"^test_squeeze",
|
||||
"^test_squeeze_negative_axes"
|
||||
],
|
||||
"current_failing_tests_OPENVINO_CPU_FP32": [
|
||||
"^test_operator_permute2",
|
||||
"^test_operator_repeat",
|
||||
|
|
@ -312,6 +325,7 @@
|
|||
"^test_max_float64.*",
|
||||
"^test_min_float64.*",
|
||||
"^test_gather_negative_indices.*",
|
||||
"^test_gridsample*",
|
||||
"^test_sce.*",
|
||||
"^test_nllloss.*",
|
||||
"^test_upsample_nearest.*",
|
||||
|
|
|
|||
38
setup.py
38
setup.py
|
|
@ -567,25 +567,25 @@ if enable_training:
|
|||
# To support the package consisting of both openvino and training modules part of it
|
||||
package_name = "onnxruntime-training"
|
||||
|
||||
disable_local_version = environ.get("ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION", "0")
|
||||
disable_local_version = (
|
||||
disable_local_version == "1"
|
||||
or disable_local_version.lower() == "true"
|
||||
or disable_local_version.lower() == "yes"
|
||||
)
|
||||
# local version should be disabled for internal feeds.
|
||||
if not disable_local_version:
|
||||
# we want put default training packages to pypi. pypi does not accept package with a local version.
|
||||
if not default_training_package_device or nightly_build:
|
||||
if cuda_version:
|
||||
# removing '.' to make Cuda version number in the same form as Pytorch.
|
||||
local_version = "+cu" + cuda_version.replace(".", "")
|
||||
elif rocm_version:
|
||||
# removing '.' to make Rocm version number in the same form as Pytorch.
|
||||
local_version = "+rocm" + rocm_version.replace(".", "")
|
||||
else:
|
||||
# cpu version for documentation
|
||||
local_version = "+cpu"
|
||||
disable_local_version = environ.get("ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION", "0")
|
||||
disable_local_version = (
|
||||
disable_local_version == "1"
|
||||
or disable_local_version.lower() == "true"
|
||||
or disable_local_version.lower() == "yes"
|
||||
)
|
||||
# local version should be disabled for internal feeds.
|
||||
if not disable_local_version:
|
||||
# we want put default training packages to pypi. pypi does not accept package with a local version.
|
||||
if not default_training_package_device or nightly_build:
|
||||
if cuda_version:
|
||||
# removing '.' to make Cuda version number in the same form as Pytorch.
|
||||
local_version = "+cu" + cuda_version.replace(".", "")
|
||||
elif rocm_version:
|
||||
# removing '.' to make Rocm version number in the same form as Pytorch.
|
||||
local_version = "+rocm" + rocm_version.replace(".", "")
|
||||
else:
|
||||
# cpu version for documentation
|
||||
local_version = "+cpu"
|
||||
|
||||
if package_name == "onnxruntime-tvm":
|
||||
packages += ["onnxruntime.providers.tvm"]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ jobs:
|
|||
parameters:
|
||||
AgentPool : 'Linux-CPU-2019'
|
||||
JobName: 'Linux_CI_Dev'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2022.2.0 -x "--use_openvino CPU_FP32 --build_wheel"'
|
||||
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2022.3.0 -x "--use_openvino CPU_FP32 --build_wheel"'
|
||||
DoNugetPack: 'false'
|
||||
ArtifactName: 'drop-linux'
|
||||
TimeoutInMinutes: 120
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
ARG UBUNTU_VERSION=20.04
|
||||
FROM ubuntu:${UBUNTU_VERSION}
|
||||
|
||||
ARG OPENVINO_VERSION=2022.2.0
|
||||
ARG OPENVINO_VERSION=2022.3.0
|
||||
ARG PYTHON_VERSION=3.8
|
||||
|
||||
ADD scripts /tmp/scripts
|
||||
|
|
@ -21,9 +21,9 @@ ENV DEBIAN_FRONTEND=noninteractive
|
|||
ENV PATH /usr/local/gradle/bin:$PATH
|
||||
|
||||
RUN cd /opt && mkdir -p intel && cd intel && \
|
||||
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \
|
||||
tar xzf l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz && \
|
||||
mv l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64 openvino_2022.2.0 && \
|
||||
wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \
|
||||
tar xzf l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \
|
||||
mv l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 && \
|
||||
cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y
|
||||
|
||||
WORKDIR /root
|
||||
|
|
|
|||
|
|
@ -4,13 +4,14 @@ ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbi
|
|||
ADD scripts /tmp/scripts
|
||||
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts
|
||||
|
||||
ARG PYTHON_VER_PATH="cp36-cp36m"
|
||||
ARG PYTHON_VERSION="3.6"
|
||||
ARG PYTHON_VER_PATH="cp38-cp38"
|
||||
ARG PYTHON_VERSION="3.8"
|
||||
ARG BUILD_UID=1001
|
||||
ARG BUILD_USER=onnxruntimedev
|
||||
ARG OV_DEVICE_PRECISION="CPU_FP32"
|
||||
ARG ENABLE_TRAINING=true
|
||||
ARG ORT_BRANCH="main"
|
||||
ARG ORT_BRANCH="rel-1.13.1"
|
||||
ARG OV_VERSION="2022.2.0"
|
||||
RUN adduser --uid $BUILD_UID $BUILD_USER
|
||||
WORKDIR /home/$BUILD_USER
|
||||
ENV PYTHON_EXE="/opt/python/$PYTHON_VER_PATH/bin/python$PYTHON_VERSION"
|
||||
|
|
@ -36,15 +37,15 @@ RUN ${PYTHON_EXE} -m pip install onnx numpy wheel
|
|||
USER $BUILD_USER
|
||||
RUN cd $WORKDIR && git clone https://github.com/openvinotoolkit/openvino.git && \
|
||||
cd openvino && \
|
||||
git checkout 2022.1.0 && \
|
||||
git checkout $OV_VERSION && \
|
||||
git submodule init && \
|
||||
git submodule update --recursive
|
||||
|
||||
RUN cd $WORKDIR && cd openvino && mkdir build && cd build && \
|
||||
cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DCMAKE_INSTALL_PREFIX=/home/onnxruntimedev/openvino_2022.1.0 && \
|
||||
cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=$PYTHON_EXE -DCMAKE_INSTALL_PREFIX=/home/onnxruntimedev/openvino_$OV_VERSION && \
|
||||
make -j8 && make install
|
||||
|
||||
ENV INTEL_OPENVINO_DIR /home/onnxruntimedev/openvino_2022.1.0
|
||||
ENV INTEL_OPENVINO_DIR /home/onnxruntimedev/openvino_$OV_VERSION
|
||||
ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64:$INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH
|
||||
ENV TBB_LIBS $INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib
|
||||
ENV InferenceEngine_DIR $INTEL_OPENVINO_DIR/runtime/cmake
|
||||
|
|
|
|||
|
|
@ -708,14 +708,16 @@ def generate_files(line_list, args):
|
|||
+ '\\native" />'
|
||||
)
|
||||
# usb-ma2x8x.mvcmd
|
||||
files_list.append(
|
||||
"<file src="
|
||||
+ '"'
|
||||
+ os.path.join(dll_list_path, "usb-ma2x8x.mvcmd")
|
||||
+ runtimes_target
|
||||
+ args.target_architecture
|
||||
+ '\\native" />'
|
||||
)
|
||||
# OpenVINO 2022.3 doesn't have usb-ma2x8x.mvcmd
|
||||
if "2022.3" not in openvino_path:
|
||||
files_list.append(
|
||||
"<file src="
|
||||
+ '"'
|
||||
+ os.path.join(dll_list_path, "usb-ma2x8x.mvcmd")
|
||||
+ runtimes_target
|
||||
+ args.target_architecture
|
||||
+ '\\native" />'
|
||||
)
|
||||
for tbb_element in os.listdir(tbb_list_path):
|
||||
if tbb_element.endswith("dll"):
|
||||
files_list.append(
|
||||
|
|
|
|||
Loading…
Reference in a new issue