diff --git a/BUILD.md b/BUILD.md
index e97935bcf3..ed7764aee6 100644
--- a/BUILD.md
+++ b/BUILD.md
@@ -146,7 +146,7 @@ ONNX Runtime supports OpenVINO Execution Provider to enable deep learning infere
The OpenVINO Execution Provider can be built using the following commands:
-- Install the OpenVINO 2018 R5.0.1 release along with its dependencies from ([https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)).
+- Currently supports and validated on two versions of OpenVINO: OpenVINO 2018 R5.0.1 and OpenVINO 2019 R1.1(Recommended). Install the OpenVINO release along with its dependencies from ([https://software.intel.com/en-us/openvino-toolkit](https://software.intel.com/en-us/openvino-toolkit)).
- Install the model optimizer prerequisites for ONNX by running
/deployment_tools/model_optimizer/install_prerequisites/install_prerequisites_onnx.sh
@@ -155,11 +155,11 @@ The OpenVINO Execution Provider can be built using the following commands:
source setupvars.sh
-- To configure Intel® Processor Graphics(GPU), please follow the installation steps from (https://docs.openvinotoolkit.org/2018_R5/_docs_install_guides_installing_openvino_linux.html#GPU-steps)
+- To configure Intel® Processor Graphics(GPU), please follow the installation steps from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-GPU-steps)
-- To configure Intel® MovidiusTM USB, please follow the getting started guide from (https://docs.openvinotoolkit.org/2018_R5/_docs_install_guides_installing_openvino_linux.html#Movidius-steps)
+- To configure Intel® MovidiusTM USB, please follow the getting started guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#additional-NCS-steps)
-- To configure Intel® Vision Accelerator Design based on 8 MovidiusTM MyriadX VPUs, please follow the configuration guide from (https://docs.openvinotoolkit.org/2018_R5/_docs_install_guides_installing_openvino_linux.html#Vision-Accelerator-Design-steps)
+- To configure Intel® Vision Accelerator Design based on 8 MovidiusTM MyriadX VPUs, please follow the configuration guide from (https://docs.openvinotoolkit.org/2019_R1.1/_docs_install_guides_installing_openvino_linux.html#install-VPU)
- Build ONNX Runtime using the below command.
diff --git a/cmake/external/openvino.cmake b/cmake/external/openvino.cmake
index 0b24b5683c..929011867a 100644
--- a/cmake/external/openvino.cmake
+++ b/cmake/external/openvino.cmake
@@ -4,7 +4,7 @@
include (ExternalProject)
set(OPENVINO_URL https://github.com/opencv/dldt.git)
-set(OPENVINO_TAG 2018_R5)
+set(OPENVINO_TAG 2019_R1.1)
set(OPENVINO_SHARED_LIB libinference_engine.so)
diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake
index ddf0642642..615491d3b0 100644
--- a/cmake/onnxruntime_providers.cmake
+++ b/cmake/onnxruntime_providers.cmake
@@ -96,7 +96,7 @@ if (onnxruntime_USE_CUDA)
)
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs})
source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_cuda_contrib_ops_cc_srcs} ${onnxruntime_cuda_contrib_ops_cu_srcs})
-
+
# disable contrib ops conditionally
if(onnxruntime_DISABLE_CONTRIB_OPS)
add_library(onnxruntime_providers_cuda ${onnxruntime_providers_cuda_cc_srcs} ${onnxruntime_providers_cuda_cu_srcs})
@@ -260,10 +260,20 @@ if (onnxruntime_USE_OPENVINO)
# Below variables point to directories within the OpenVINO installation directory
# whose value is set in INTEL_CVSDK_DIR variable by running the setupvars.sh script
- if (onnxruntime_USE_OPENVINO_BINARY)
- set(OPENVINO_INCLUDE_DIR $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/include)
- set(OPENVINO_LIB_DIR $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64/)
+if (onnxruntime_USE_OPENVINO_BINARY)
+ if ($ENV{INTEL_CVSDK_DIR} MATCHES "2019.1")
+ message($ENV{INTEL_CVSDK_DIR})
+ set(OPENVINO_INCLUDE_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/include)
+ set(OPENVINO_TBB_INCLUDE_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/include)
+ set(OPENVINO_LIB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64/)
+ set(OPENVINO_TBB_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib)
+ set(OPENVINO_MKL_TINY_DIR $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib)
endif()
+ if ($ENV{INTEL_CVSDK_DIR} MATCHES "2018.5")
+ set(OPENVINO_INCLUDE_DIR $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/include)
+ set(OPENVINO_LIB_DIR $ENV{INTEL_CVSDK_DIR}/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64/)
+ endif()
+endif()
find_package(PythonLibs REQUIRED)
source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs})
@@ -271,11 +281,17 @@ if (onnxruntime_USE_OPENVINO)
onnxruntime_add_include_to_target(onnxruntime_providers_openvino gsl onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
add_dependencies(onnxruntime_providers_openvino ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime")
- target_include_directories(onnxruntime_providers_openvino PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS})
+ target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR} ${OPENVINO_TBB_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS})
install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX)
- link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR})
- target_link_libraries(onnxruntime_providers_openvino -linference_engine ${PYTHON_LIBRARIES})
+ if ($ENV{INTEL_CVSDK_DIR} MATCHES "2019.1")
+ link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR} ${OPENVINO_TBB_DIR} ${OPENVINO_MKL_TINY_DIR})
+ target_link_libraries(onnxruntime_providers_openvino -linference_engine -ltbb ${PYTHON_LIBRARIES})
+ endif()
+ if ($ENV{INTEL_CVSDK_DIR} MATCHES "2018.5")
+ link_directories(onnxruntime_providers_openvino ${OPENVINO_LIB_DIR})
+ target_link_libraries(onnxruntime_providers_openvino -linference_engine ${PYTHON_LIBRARIES})
+ endif()
file(COPY ${onnxruntime_providers_openvino_py_srcs} DESTINATION ${onnxruntime_BINARY_DIR})
endif()
diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino
index 9edbcffe35..aba1f0c18a 100644
--- a/dockerfiles/Dockerfile.openvino
+++ b/dockerfiles/Dockerfile.openvino
@@ -6,7 +6,7 @@
FROM ubuntu:16.04
RUN apt update && \
- apt -y install python3.5 python3-pip zip x11-apps lsb-core wget cpio sudo libboost-python-dev libpng-dev zlib1g-dev git libnuma1 ocl-icd-libopencl1 clinfo libboost-filesystem1.58.0 libboost-thread1.58.0 protobuf-compiler libprotoc-dev && pip3 install numpy networkx opencv-python pytest && locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8
+ apt -y install python3.5 python3-pip zip x11-apps lsb-core wget cpio sudo libboost-python-dev libpng-dev zlib1g-dev git libnuma1 ocl-icd-libopencl1 clinfo libboost-filesystem1.58.0 libboost-thread1.58.0 protobuf-compiler libprotoc-dev libusb-1.0-0-dev&& pip3 install numpy networkx opencv-python pytest && locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8
ARG DEVICE=CPU_FP32
ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime
@@ -14,7 +14,7 @@ ARG ONNXRUNTIME_BRANCH=master
ENV pattern="COMPONENTS=DEFAULTS"
-ENV replacement="COMPONENTS=intel-inference_engine_sdk__noarch;intel-inference_engine_cpu__noarch;intel-inference_engine_gpu__noarch;intel-inference_engine_vpu__noarch;intel-inference_engine_gna__noarch;intel-inference_engine_hddl__noarch;intel-model_optimizer__noarch;intel-opencv_ubuntu_16_rel__noarch"
+ENV replacement="COMPONENTS=intel-openvino-ie-sdk-ubuntu-xenial__x86_64;intel-openvino-ie-rt-cpu-ubuntu-xenial__x86_64;intel-openvino-ie-rt-gpu-ubuntu-xenial__x86_64;intel-openvino-ie-rt-vpu-ubuntu-xenial__x86_64;intel-openvino-ie-rt-hddl-ubuntu-xenial__x86_64;intel-openvino-model-optimizer__x86_64;intel-openvino-opencv-lib-ubuntu-xenial__x86_64"
COPY l_openvino_*.tgz .
RUN tar -xzf l_openvino_toolkit*.tgz && \
rm -rf l_openvino_toolkit*.tgz && \
@@ -22,26 +22,26 @@ RUN tar -xzf l_openvino_toolkit*.tgz && \
sed -i "s/$pattern/$replacement/" silent.cfg && \
sed -i 's/decline/accept/g' silent.cfg && \
./install.sh -s silent.cfg && \
- /bin/bash -c "source /opt/intel/computer_vision_sdk/bin/setupvars.sh" && \
- ./install_cv_sdk_dependencies.sh && \
+ ./install_openvino_dependencies.sh && \
cd - && \
- rm -rf l_openvino_toolkit*
+ rm -rf l_openvino_toolkit* && \
+ cd /opt/intel/openvino/deployment_tools/model_optimizer/install_prerequisites && ./install_prerequisites_onnx.sh
ENV LD_LIBRARY_PATH=/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
-ENV INSTALLDIR=/opt/intel/computer_vision_sdk
-ENV INTEL_CVSDK_DIR=${INSTALLDIR}
-ENV LD_LIBRARY_PATH=${INSTALLDIR}/deployment_tools/model_optimizer/model_optimizer_caffe/bin:${LD_LIBRARY_PATH}
-ENV ModelOptimizer_ROOT_DIR=${INSTALLDIR}/deployment_tools/model_optimizer/model_optimizer_caffe
+ENV INTEL_OPENVINO_DIR=/opt/intel/openvino
+ENV INTEL_CVSDK_DIR=/opt/intel/openvino
+ENV LD_LIBRARY_PATH=${INSTALL_OPENVINO_DIR}/deployment_tools/model_optimizer/model_optimizer_caffe/bin:${LD_LIBRARY_PATH}
+ENV ModelOptimizer_ROOT_DIR=${INSTALL_OPENVINO_DIR}/deployment_tools/model_optimizer/model_optimizer_caffe
ENV InferenceEngine_DIR=${INTEL_CVSDK_DIR}/deployment_tools/inference_engine/share
-ENV IE_PLUGINS_PATH=${INTEL_CVSDK_DIR}/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64
-ENV LD_LIBRARY_PATH=/opt/intel/opencl:${INSTALLDIR}/deployment_tools/inference_engine/external/cldnn/lib:${INSTALLDIR}/inference_engine/external/gna/lib:${INSTALLDIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${INSTALLDIR}/deployment_tools/inference_engine/external/omp/lib:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH}
-ENV OpenCV_DIR=${INSTALLDIR}/opencv/share/OpenCV
-ENV LD_LIBRARY_PATH=${INSTALLDIR}/opencv/lib:${INSTALLDIR}/opencv/share/OpenCV/3rdparty/lib:${LD_LIBRARY_PATH}
+ENV IE_PLUGINS_PATH=${INTEL_CVSDK_DIR}/deployment_tools/inference_engine/lib/intel64
+ENV LD_LIBRARY_PATH=/opt/intel/opencl:${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/cldnn/lib:${INSTALL_OPENVINO_DIR}/inference_engine/external/gna/lib:${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/omp/lib:${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH}
+ENV OpenCV_DIR=${INSTALL_OPENVINO_DIR}/opencv/share/OpenCV
+ENV LD_LIBRARY_PATH=${INSTALL_OPENVINO_DIR}/opencv/lib:${INSTALL_OPENVINO_DIR}/opencv/share/OpenCV/3rdparty/lib:${LD_LIBRARY_PATH}
ENV PATH=${INTEL_CVSDK_DIR}/deployment_tools/model_optimizer:$PATH
ENV PYTHONPATH=${INTEL_CVSDK_DIR}/deployment_tools/model_optimizer:$PYTHONPATH
ENV PYTHONPATH=$INTEL_CVSDK_DIR/python/python3.5:${INTEL_CVSDK_DIR}/python/python3.5/ubuntu16:${PYTHONPATH}
-ENV HDDL_INSTALL_DIR=${INSTALLDIR}/deployment_tools/inference_engine/external/hddl
-ENV LD_LIBRARY_PATH=${INSTALLDIR}/deployment_tools/inference_engine/external/hddl/lib:$LD_LIBRARY_PATH
+ENV HDDL_INSTALL_DIR=${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl
+ENV LD_LIBRARY_PATH=${INSTALL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl/lib:$LD_LIBRARY_PATH
RUN wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-gmmlib_19.1.1_amd64.deb
RUN wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-core_1.0.2-1787_amd64.deb
@@ -59,8 +59,7 @@ ENV LANG en_US.UTF-8
RUN wget https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz && \
tar -xf cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake && rm -rf /cmake-3.13.2-Linux-x86_64.tar.gz
-RUN /bin/bash -c "source /opt/intel/computer_vision_sdk/bin/setupvars.sh" && \
- git clone --recursive -b $ONNXRUNTIME_BRANCH $ONNXRUNTIME_REPO /onnxruntime && \
+RUN git clone --recursive -b $ONNXRUNTIME_BRANCH $ONNXRUNTIME_REPO /onnxruntime && \
cd /onnxruntime/cmake/external/onnx && python3 setup.py install && \
cd /onnxruntime && ./build.sh --config RelWithDebInfo --update --build --parallel --use_openvino $DEVICE --build_wheel && pip3 install /onnxruntime/build/Linux/RelWithDebInfo/dist/*-linux_x86_64.whl && rm -rf /onnxruntime
diff --git a/docs/execution_providers/OpenVINO-ExecutionProvider.md b/docs/execution_providers/OpenVINO-ExecutionProvider.md
index 5cfa516af3..a3691f7db1 100644
--- a/docs/execution_providers/OpenVINO-ExecutionProvider.md
+++ b/docs/execution_providers/OpenVINO-ExecutionProvider.md
@@ -6,9 +6,9 @@ OpenVINO Execution Provider enables deep learning inference on Intel CPUs, Intel
Below table shows the ONNX layers supported using OpenVINO Execution Provider and the mapping between ONNX layers and OpenVINO layers. The below table also lists the Intel hardware support for each of the layers. CPU refers to Intel®
Atom, Core, and Xeon processors. GPU refers to the Intel Integrated Graphics. VPU refers to USB based Intel® MovidiusTM
-VPUs as well as Intel® Vision accelerator Design with Intel Movidius TM MyriadX VPU.
+VPUs as well as Intel® Vision accelerator Design with Intel Movidius TM MyriadX VPU.
-| **ONNX Layers** | **OpenVINO Layers** | **CPU** | **GPU** | **VPU** |
+| **ONNX Layers** | **OpenVINO Layers** | **CPU** | **GPU** | **VPU** |
| --- | --- | --- | --- | --- |
| Add | Eltwise (operation=sum) | Yes | Yes | Yes
| AveragePool | Pooling(pool\_method=avg) | Yes | Yes | Yes
@@ -33,7 +33,7 @@ VPUs as well as Intel® Vision accelerator Design with Intel Movidiu
| UnSqueeze | Reshape | Yes | Yes | Yes
| LeakyRelu | ReLU | Yes | Yes | Yes
-*MatMul is supported in GPU only when the following layer is an Add layer in the topology.
+*MatMul is supported in GPU only when the following layer is an Add layer in the topology.
# Topology Support
@@ -41,17 +41,17 @@ Below topologies are supported from ONNX open model zoo using OpenVINO Execution
## Image Classification Networks
-| **Topology** | **CPU** | **GPU** | **VPU** |
-| --- | --- | --- | --- |
+| **Topology** | **CPU** | **GPU** | **VPU** |
+| --- | --- | --- | --- |
| bvlc\_alexnet | Yes | Yes | Yes
| bvlc\_googlenet | Yes | Yes | Yes
-| bvlc\_reference\_caffenet | Yes | Yes | Yes
-| bvlc\_reference\_rcnn\_ilsvrc13 | Yes | Yes | Yes
+| bvlc\_reference\_caffenet | Yes | Yes | Yes
+| bvlc\_reference\_rcnn\_ilsvrc13 | Yes | Yes | Yes
| densenet121 | Yes | Yes | Yes
-| Inception\_v1 | Yes | Yes | No
+| Inception\_v1 | Yes | Yes | Yes**
| Inception\_v2 | Yes | Yes | Yes
| Shufflenet | Yes | Yes | Yes
-| Zfnet512 | Yes | Yes | Yes
+| Zfnet512 | Yes | Yes | Yes
| Squeeznet 1.1 | Yes | Yes | Yes
| Resnet18v1 | Yes | Yes | Yes
| Resnet34v1 | Yes | Yes | Yes
@@ -62,29 +62,32 @@ Below topologies are supported from ONNX open model zoo using OpenVINO Execution
| Resnet34v2 | Yes | Yes | Yes
| Resnet50v2 | Yes | Yes | Yes
| Resnet101v2 | Yes | Yes | Yes
-| Resnet152v2 | Yes | Yes | Yes
+| Resnet152v2 | Yes | Yes | Yes
| Mobilenetv2 | Yes | Yes | Yes
| vgg16 | Yes | Yes | Yes
| vgg19 | Yes | Yes | Yes
+
## Image Recognition Networks
-| **Topology** | **CPU** | **GPU** | **VPU** |
-| --- | --- | --- | --- |
-| MNIST | Yes | Yes | No
+| **Topology** | **CPU** | **GPU** | **VPU** |
+| --- | --- | --- | --- |
+| MNIST | Yes | Yes | Yes**
+
+**Inception_v1 and MNIST are supported in OpenVINO R1.1 and are not supported in OpenVINO R5.0.1.
## Object Detection Networks
-| **Topology** | **CPU** | **GPU** | **VPU** |
-| --- | --- | --- | --- |
+| **Topology** | **CPU** | **GPU** | **VPU** |
+| --- | --- | --- | --- |
|TinyYOLOv2 | Yes | Yes | Yes
-| ResNet101\_DUC\_HDC | Yes | Yes | No
+| ResNet101\_DUC\_HDC | Yes | No | No
# Application code changes for VAD-R performance scaling
-VAD-R has 8 VPUs and is suitable for applications that require multiple inferences to run in parallel. We use batching approach for performance scaling on VAD-R.
+VAD-R has 8 VPUs and is suitable for applications that require multiple inferences to run in parallel. We use batching approach for performance scaling on VAD-R.
-Below python code snippets provide sample classification code to batch input images, load a model and process the output results.
+Below python code snippets provide sample classification code to batch input images, load a model and process the output results.
~~~
import onnxruntime as rt
@@ -95,7 +98,7 @@ import sys
import cv2
import numpy
import time
-import glob
+import glob
~~~
### Load the input onnx model
@@ -111,19 +114,19 @@ for i in range(iters):
images = [cv2.imread(file) for file in glob.glob(str(sys.argv[2])+'/*.jpg')]
for img in images:
# resizing the image
- img = cv2.resize(img, (224,224))
- # convert image to numpy
- x = numpy.asarray(img).astype(numpy.float32)
- x = numpy.transpose(x, (2,0,1))
+ img = cv2.resize(img, (224,224))
+ # convert image to numpy
+ x = numpy.asarray(img).astype(numpy.float32)
+ x = numpy.transpose(x, (2,0,1))
# expand the dimension and batch the images
- x = numpy.expand_dims(x,axis=0)
- if y is None:
- y = x
- else:
- y = numpy.concatenate((y,x), axis=0)
+ x = numpy.expand_dims(x,axis=0)
+ if y is None:
+ y = x
+ else:
+ y = numpy.concatenate((y,x), axis=0)
~~~
-### Start Inference
+### Start Inference
~~~
res = sess.run([sess.get_outputs()[0].name], {sess.get_inputs()[0].name: y})
~~~
diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
index f18d9bffae..d434c9e4e4 100644
--- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
+++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
@@ -64,146 +64,131 @@ static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::
}
//Gets the input count of given node
-int GetInputCount(const Node* node, const InitializedTensorSet& initializer_set){
-
- int count = 0;
- for(const auto& input : node->InputDefs()){
- auto name = input->Name();
- auto it = initializer_set.find(name);
- if(it == initializer_set.end()){
- count++;
- }
+int GetInputCount(const Node* node, const InitializedTensorSet& initializer_set) {
+ int count = 0;
+ for (const auto& input : node->InputDefs()) {
+ auto name = input->Name();
+ auto it = initializer_set.find(name);
+ if (it == initializer_set.end()) {
+ count++;
}
- return count;
+ }
+ return count;
}
//Checks whether the dimensions of a given node are supported in OpenVINO
-bool IsDimensionSupported(const Node* node, std::string dev_id){
+bool IsDimensionSupported(const Node* node, std::string dev_id) {
+ auto node_inputs = node->InputDefs();
+ size_t input_dims = 0;
+ if (node_inputs[0]->Shape() != nullptr) {
+ input_dims = node_inputs[0]->Shape()->dim_size();
+ }
- auto node_inputs = node->InputDefs();
- size_t input_dims = 0;
- if(node_inputs[0]->Shape() != nullptr){
- input_dims = node_inputs[0]->Shape()->dim_size();
+ if (node->OpType().find("Pool") != std::string::npos) {
+ if (dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (input_dims != 3 && input_dims != 4)
+ return false;
+ } else if (input_dims != 4 && input_dims != 5) {
+ return false;
+ }
+ }
+
+ //Only support 4D and 5D Transposes
+ if (node->OpType() == "Transpose") {
+ if (input_dims == 2 || input_dims == 3 || input_dims > 5)
+ return false;
+ }
+
+ if (node->OpType() == "Unsqueeze") {
+ auto attributes = node->GetAttributes();
+ auto axes = attributes["axes"].ints();
+ if (input_dims + axes.size() > 5)
+ return false;
+ if (dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
+ return false;
+ }
+ }
+
+ if (node->OpType() == "Reshape") {
+ //Don't support Reshape without output dims
+ auto node_outputs = node->OutputDefs();
+ if (node_outputs[0]->Shape() != nullptr && node_outputs[0]->Shape()->dim_size() == 0)
+ return false;
+
+ if (dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
+ return false;
+ }
+ }
+
+ if (node->OpType() == "Softmax") {
+ //First dimension of Softmax input has to be 1
+ if (input_dims != 0) {
+ if (node_inputs[0]->Shape()->dim(0).dim_value() != 1)
+ return false;
}
- if(node->OpType().find("Pool") != std::string::npos){
-
- if(dev_id == "MYRIAD" || dev_id == "HDDL"){
- if(input_dims != 3 && input_dims != 4)
- return false;
- } else if(input_dims != 4 && input_dims != 5){
- return false;
- }
+ //3D input not supported on GPU, MYRIAD and HDDL
+ if (dev_id == "GPU" || dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (input_dims == 3)
+ return false;
}
-
- //Only support 4D and 5D Transposes
- if(node->OpType() == "Transpose"){
-
- if(input_dims == 2 || input_dims == 3 || input_dims > 5)
- return false;
+ }
+ //Only 2D MatMul is supported
+ if (node->OpType() == "MatMul") {
+ for (size_t i = 0; i < node_inputs.size(); i++) {
+ if (node_inputs[i]->Shape() != nullptr) {
+ if (node_inputs[i]->Shape()->dim_size() != 2)
+ return false;
+ }
}
+ }
- if(node->OpType() == "Unsqueeze"){
-
- auto attributes = node->GetAttributes();
- auto axes = attributes["axes"].ints();
- if(input_dims + axes.size() > 5)
- return false;
- if(dev_id == "MYRIAD" || dev_id == "HDDL"){
- if(node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
- return false;
- }
+ if (node->OpType() == "Flatten") {
+ if (dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
+ return false;
}
+ }
- if(node->OpType() == "Reshape"){
-
- //Don't support Reshape without output dims
- auto node_outputs = node->OutputDefs();
- if(node_outputs[0]->Shape() != nullptr && node_outputs[0]->Shape()->dim_size() == 0)
- return false;
-
- if(dev_id == "MYRIAD" || dev_id == "HDDL"){
-
- if(node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
- return false;
-
- }
- }
-
- if(node->OpType() == "Softmax"){
-
- //First dimension of Softmax input has to be 1
- if(input_dims != 0 ){
- if(node_inputs[0]->Shape()->dim(0).dim_value() != 1)
- return false;
- }
-
- //3D input not supported on MYRIAD and HDDL
- if(dev_id == "MYRIAD" || dev_id == "HDDL"){
- if(input_dims == 3)
- return false;
- }
- }
-
- //Only 2D MatMul is supported
- if(node->OpType() == "MatMul"){
- for(size_t i = 0; i < node_inputs.size(); i++){
-
- if(node_inputs[i]->Shape() != nullptr){
- if(node_inputs[i]->Shape()->dim_size() != 2)
- return false;
- }
- }
- }
-
- if(node->OpType() == "Flatten"){
-
- if(dev_id == "MYRIAD" || dev_id == "HDDL"){
- if(node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim(0).dim_value() != 1)
- return false;
- }
- }
-
- return true;
+ return true;
}
//Checks whether the node is supported by OpenVINO
-bool IsOpSupported(std::string name){
+bool IsOpSupported(std::string name) {
+ std::set supported_ops = {
+ "Add",
+ "BatchNormalization",
+ "Conv",
+ "GlobalAveragePool",
+ "Relu",
+ "Reshape",
+ "Flatten",
+ "Gemm",
+ "MaxPool",
+ "AveragePool",
+ "Concat",
+ "Dropout",
+ "LRN",
+ "Softmax",
+ "Mul",
+ "Sum",
+ "Transpose",
+ "Identity",
+ "MatMul",
+ "Unsqueeze",
+ "ImageScaler",
+ "LeakyRelu",
+ "GlobalMaxPool"};
- std::set supported_ops = {
- "Add",
- "BatchNormalization",
- "Conv",
- "GlobalAveragePool",
- "Relu",
- "Reshape",
- "Flatten",
- "Gemm",
- "MaxPool",
- "AveragePool",
- "Concat",
- "Dropout",
- "LRN",
- "Softmax",
- "Mul",
- "Sum",
- "Transpose",
- "Identity",
- "MatMul",
- "Unsqueeze",
- "ImageScaler",
- "LeakyRelu",
- "GlobalMaxPool"};
-
- auto iter = supported_ops.find(name);
- return iter != supported_ops.end();
+ auto iter = supported_ops.find(name);
+ return iter != supported_ops.end();
}
+//Checks if the entire graph is supported by OpenVINO EP and throws eception if any.
-//Checks if the entire graph is supported by OpenVINO EP and returns false if it is not.
-
-bool IsGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string dev_id){
-
+void CheckGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string dev_id) {
const auto& initializers = graph_viewer.GetAllInitializedTensors();
auto node_indexes = graph_viewer.GetNodesInTopologicalOrder();
@@ -216,123 +201,124 @@ bool IsGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string
int num_inputs = graph_viewer.GetInputs().size();
int num_outputs = graph_viewer.GetOutputs().size();
- //GPU Plugin does not support 1D and 5D input
- if(dev_id == "GPU"){
+ //GPU Plugin does not support 1D and 5D input
+ if (dev_id == "GPU") {
+ for (int i = 0; i < num_inputs; i++) {
+ input_dims = graph_proto->input(i).type().tensor_type().shape().dim_size();
- for(int i = 0; i < num_inputs; i++){
- input_dims = graph_proto->input(i).type().tensor_type().shape().dim_size();
-
- if(input_dims == 1 || input_dims == 5)
- return false;
- }
+ if (input_dims == 1 || input_dims == 5) {
+ throw "GPU plugin doesn't support 1D and 5D input";
+ }
}
+ }
- //GPU Plugin does not support 5D output
- if(dev_id == "GPU"){
+ //GPU Plugin does not support 5D output
+ if (dev_id == "GPU") {
+ for (int i = 0; i < num_outputs; i++) {
+ output_dims = graph_proto->output(i).type().tensor_type().shape().dim_size();
- for(int i = 0; i < num_outputs; i++){
- output_dims = graph_proto->output(i).type().tensor_type().shape().dim_size();
-
- if(output_dims == 5)
- return false;
- }
+ if (output_dims == 5) {
+ throw "GPU plugin doesn't support 5D output";
+ }
}
-
+ }
for (auto index : node_indexes) {
const auto node = graph_viewer.GetNode(index);
//Check if the Operation is Supported by OpenVINO
if (!IsOpSupported(node->OpType())) {
- return false;
+ {
+ throw "Operation is not supported by OpenVINO";
+ }
}
auto node_inputs = node->InputDefs();
//Zero dimension check
- for(size_t i = 0; i < node_inputs.size(); i++){
- if(node_inputs[i]->Shape() != nullptr){
-
- if(node_inputs[i]->Shape()->dim_size() == 0)
- return false;
+ for (size_t i = 0; i < node_inputs.size(); i++) {
+ if (node_inputs[i]->Shape() != nullptr) {
+ if (node_inputs[i]->Shape()->dim_size() == 0) {
+ throw "Node_input is zero dimension";
}
+ }
}
-
-
//BatchNormalization cannot take more than 1 input
- if(node->OpType() == "BatchNormalization"){
-
- if(GetInputCount(node,initializers) > 1)
- return false;
+ if (node->OpType() == "BatchNormalization") {
+ if (GetInputCount(node, initializers) > 1) {
+ throw "BatchNormalization: Cannot take more than 1 input";
+ }
}
-
//Conv cannot take more than 1 input
- if(node->OpType() == "Conv"){
-
- if(GetInputCount(node,initializers) > 1)
- return false;
+ if (node->OpType() == "Conv") {
+ if (GetInputCount(node, initializers) > 1) {
+ throw "Conv: Cannot take more than 1 input";
+ }
}
-
//Reshape should have shape as initializer
- if(node->OpType() == "Reshape"){
+ if (node->OpType() == "Reshape") {
+ int input_count = GetInputCount(node, initializers);
- int input_count = GetInputCount(node,initializers);
+ if (input_count > 1) {
+ throw "Reshape: Shape should be an initializer";
+ }
- if(input_count > 1)
- return false;
-
- //Myriad and HDDL plugins do not support Reshape with two initializers
- if(dev_id == "MYRIAD" || dev_id == "HDDL")
- if(input_count == 0)
- return false;
-
- if(!IsDimensionSupported(node,dev_id)){
- return false;
+ //Myriad and HDDL plugins do not support Reshape with two initializers
+ if (dev_id == "MYRIAD" || dev_id == "HDDL")
+ if (input_count == 0) {
+ throw "Myriad and HDDL plugins do not support Reshape with two initializers ";
}
+
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Reshape: Dimensions are not supported";
+ }
}
- if(node->OpType() == "Flatten"){
+ if (node->OpType() == "Flatten") {
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Flatten: Dimensions are not supported";
+ }
- if(!IsDimensionSupported(node,dev_id))
- return false;
-
- //Only default axis is supported for MYRIAD and HDDL plugins
- auto attributes = node->GetAttributes();
- auto axis = attributes["axis"].i();
- if (dev_id == "MYRIAD" || dev_id == "HDDL") {
- if (axis != 1)
- return false;
+ //Only default axis is supported for MYRIAD and HDDL plugins
+ auto attributes = node->GetAttributes();
+ auto axis = attributes["axis"].i();
+ if (dev_id == "MYRIAD" || dev_id == "HDDL") {
+ if (axis != 1) {
+ throw "Only default axis is supported for MYRIAD and HDDL plugins";
}
+ }
}
//MatMul is only supported if it is followed by Add
if (node->OpType() == "MatMul") {
for (size_t i = 0; i < node->InputDefs().size(); i++) {
if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) {
- return false;
+ throw "Input data type should be float";
}
}
auto iter = node->OutputNodesBegin();
if (iter == node->OutputNodesEnd()) {
- return false;
+ throw "MatMul should be followed by Add";
}
for (auto it = node->OutputNodesBegin(); it != node->OutputNodesEnd(); ++it) {
const auto out_node = graph_viewer.GetNode((*it).Index());
if (out_node->OpType() != "Add") {
- return false;
+ {
+ throw "Matmul should be followed by Add";
+ }
}
}
- if(!IsDimensionSupported(node,dev_id))
- return false;
-
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Dimension is not supported";
+ }
}
//Dropout , Identity and Concat can't have graph inputs
@@ -341,7 +327,9 @@ bool IsGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string
for (const auto& input : node->InputDefs()) {
auto it = find(graph_inputs.begin(), graph_inputs.end(), input);
if (it != graph_inputs.end()) {
- return false;
+ {
+ throw "Dropout, Identity and Concat can't have graph inputs";
+ }
}
}
}
@@ -349,42 +337,48 @@ bool IsGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string
//Attribute auto pad for MaxPool and Average Pool must not be empty or SAME_LOWER
//Only support 4D and 5D blobs for CPU,GPU
//Only support 3D and 4D blobs for MYRIAD and HDDL
- if (node->OpType() == "MaxPool" || node->OpType() == "AveragePool"){
+ if (node->OpType() == "MaxPool" || node->OpType() == "AveragePool") {
auto attributes = node->GetAttributes();
auto auto_pad = attributes["auto_pad"].s();
- if (auto_pad == "" || auto_pad == "SAME_LOWER")
- return false;
+ if (auto_pad == "" || auto_pad == "SAME_LOWER") {
+ throw "Attribute auto pad shouldn't be empty or SAME_LOWER for MaxPool and Average Pool";
+ }
auto strides_ints = attributes["strides"].ints();
- if(auto_pad == "SAME_UPPER" && strides_ints.size() == 0)
- return false;
+ if (auto_pad == "SAME_UPPER" && strides_ints.size() == 0) {
+ throw "Pooling: Generic Error";
+ }
//Dilations have to be 1
auto dilations_ints = attributes["dilations"].ints();
if (dilations_ints.size() != 0) {
- if (dilations_ints[0] > 1)
- return false;
+ if (dilations_ints[0] > 1) {
+ throw "Pooling: Generic error";
+ }
}
//Don't support ceil_mode = 1
auto ceil_mode = attributes["ceil_mode"].i();
- if (ceil_mode != 0)
- return false;
+ if (ceil_mode != 0) {
+ throw "Pooling: Ceil mode should be 1";
+ }
//Don't support multiple outputs for Pooling
- if (node->OutputDefs().size() > 1)
- return false;
+ if (node->OutputDefs().size() > 1) {
+ throw "Pooling: Multiple outputs not supported";
+ }
- if(!IsDimensionSupported(node,dev_id))
- return false;
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Pooling: Dimensions not supported";
+ }
}
//Only support 4D and 5D blobs for CPU,GPU
//Only support 3D and 4D blobs for MYRIAD and HDDL
- if(node->OpType() == "GlobalMaxPool" || node->OpType() == "GlobalAveragePool"){
-
- if(!IsDimensionSupported(node,dev_id))
- return false;
+ if (node->OpType() == "GlobalMaxPool" || node->OpType() == "GlobalAveragePool") {
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Pooling: Only support 4D and 5D blobs for CPU,GPU, Only support 3D and 4D blobs for MYRIAD and HDDL";
+ }
}
//Transpose with no attr is not supported
@@ -392,45 +386,50 @@ bool IsGraphSupported(const onnxruntime::GraphViewer& graph_viewer, std::string
auto attributes = node->GetAttributes();
auto perm = attributes["perm"].ints();
if (perm.size() == 0 || perm.size() > 5) {
- return false;
+ throw " Transpose: Tranpose with no attr is not supported. Perm size shouldn't be zero or greater than five";
}
//String data type is not supported
const auto* type_proto = node->InputDefs()[0]->TypeAsProto();
if (type_proto->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_STRING) {
- return false;
+ throw "Transpose: String data type is not supported ";
}
- if(!IsDimensionSupported(node,dev_id))
- return false;
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Transpose: Dimensions are not supported ";
+ }
}
-
if (node->OpType() == "Unsqueeze") {
-
- if(!IsDimensionSupported(node,dev_id))
- return false;
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Unsqueeze: Dimensions are not supported ";
+ }
const auto* type_proto = node->InputDefs()[0]->TypeAsProto();
- if (type_proto->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT)
- return false;
+ if (type_proto->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) {
+ throw "Unsqueeze: Datatype should be float";
+ }
}
//Only support 2D input and axis 1
if (node->OpType() == "Softmax") {
-
- if(!IsDimensionSupported(node,dev_id))
- return false;
+ if (!IsDimensionSupported(node, dev_id)) {
+ throw "Softmax: Dimensions are not supported ";
+ }
auto attributes = node->GetAttributes();
auto axis = attributes["axis"].i();
- if (axis != 1)
- return false;
+ if (axis != 1) {
+ throw "Softmax: Only default axis is supported";
+ }
}
+ //Don't support only one input
+ if (node->OpType() == "Sum") {
+ if (node->InputDefs().size() == 1) {
+ throw "Sum: Doesn't support only one input ";
+ }
+ }
}
-
- return true;
-
}
std::vector> OpenVINOExecutionProvider::GetCapability(
@@ -460,14 +459,17 @@ std::vector> OpenVINOExecutionProvider::GetCa
#endif
int counter = 0;
+
std::unique_ptr sub_graph = std::make_unique();
auto model_proto = GetModelProtoFromFusedNode(graph_viewer);
- std::set fused_inputs, fused_outputs;
+ std::set fused_inputs, fused_outputs;
- if (!IsGraphSupported(graph_viewer,device_id)) {
- LOGS_DEFAULT(WARNING) << openvino_ep::OpenVINOGraph::log_tag << "Rejecting as graph has unsupported operations.";
+ try {
+ CheckGraphSupported(graph_viewer, device_id);
+ } catch (const char* error_msg) {
+ LOGS_DEFAULT(WARNING) << openvino_ep::OpenVINOGraph::log_tag << "Rejecting as graph has unsupported operations." << error_msg;
return result;
}
@@ -479,7 +481,7 @@ std::vector> OpenVINOExecutionProvider::GetCa
// Try converting with OpenVINO's Model Optimizer
try {
openvino_ep::OpenVINOGraph::ConvertONNXModelToOpenVINOIR(model_proto_strbuf, xml_string, weights_string, precision_fp32);
- } catch (const char* msg) {
+ } catch (const char* msg) {
// Model Optimizer cannot convert this model.
LOGS_DEFAULT(WARNING) << openvino_ep::OpenVINOGraph::log_tag << "Rejecting as Model Optimizer cannot convert this model." << msg;
return result;
diff --git a/onnxruntime/core/providers/openvino/openvino_graph.cc b/onnxruntime/core/providers/openvino/openvino_graph.cc
index a8dfa066e1..53e5e6388b 100644
--- a/onnxruntime/core/providers/openvino/openvino_graph.cc
+++ b/onnxruntime/core/providers/openvino/openvino_graph.cc
@@ -20,14 +20,12 @@
#include "openvino_graph.h"
-namespace onnxruntime{
+namespace onnxruntime {
namespace openvino_ep {
-
const std::string OpenVINOGraph::log_tag = "[OpenVINO-EP] ";
OpenVINOGraph::OpenVINOGraph(const onnxruntime::Node* fused_node) {
-
device_id_ = "CPU";
precision_ = InferenceEngine::Precision::FP32;
std::string precision_str = "FP32";
@@ -99,7 +97,26 @@ OpenVINOGraph::OpenVINOGraph(const onnxruntime::Node* fused_node) {
openvino_network_ = BuildOpenVINONetworkWithMO();
// Create hardware specific OpenVINO network representation
- infer_requests_ = GetExecutableHandle(openvino_network_, device_id_);
+ GetExecutableHandle(openvino_network_);
+
+ std::vector plugin_path = GetEnvLdLibraryPath();
+ plugin_path.push_back("");
+ plugin_ = InferenceEngine::PluginDispatcher(
+ plugin_path)
+ .getPluginByDevice(device_id_);
+
+ //Loading model to the plugin
+ InferenceEngine::ExecutableNetwork exeNetwork = plugin_.LoadNetwork(*openvino_network_, {});
+
+ LOGS_DEFAULT(INFO) << log_tag << "Network loaded into accelerator plug-in succesfully";
+
+ //Create infer request
+ for (size_t i = 0; i < num_inf_reqs_; i++) {
+ auto infRequest = exeNetwork.CreateInferRequestPtr();
+
+ infer_requests_.push_back(infRequest);
+ }
+ LOGS_DEFAULT(INFO) << log_tag << "Infer requests created: " << num_inf_reqs_;
}
std::vector OpenVINOGraph::GetEnvLdLibraryPath() const {
@@ -116,8 +133,7 @@ std::vector OpenVINOGraph::GetEnvLdLibraryPath() const {
}
void OpenVINOGraph::ConvertONNXModelToOpenVINOIR(const std::string& onnx_model,
- std::string& openvino_xml, std::string& openvino_bin, bool precision_fp32) {
-
+ std::string& openvino_xml, std::string& openvino_bin, bool precision_fp32) {
Py_Initialize();
if (!Py_IsInitialized()) {
throw "Python environment initialization failure";
@@ -180,7 +196,6 @@ void OpenVINOGraph::ConvertONNXModelToOpenVINOIR(const std::string& onnx_model,
}
std::shared_ptr OpenVINOGraph::BuildOpenVINONetworkWithMO() {
-
const auto& attributes = fused_node_->GetAttributes();
std::string xml_string = attributes.at("xml_str").s();
std::string weights_string = attributes.at("weights_str").s();
@@ -200,37 +215,27 @@ std::shared_ptr OpenVINOGraph::BuildOpenVINONetwork
InferenceEngine::Precision OpenVINOGraph::ConvertPrecisionONNXToOpenVINO(
ONNX_NAMESPACE::DataType onnx_type) {
-
- if(*onnx_type == "float" || *onnx_type == "tensor(float)") {
+ if (*onnx_type == "float" || *onnx_type == "tensor(float)") {
return InferenceEngine::Precision::FP32;
- } else if ( *onnx_type == "float16" || *onnx_type == "tensor(float16)") {
+ } else if (*onnx_type == "float16" || *onnx_type == "tensor(float16)") {
return InferenceEngine::Precision::FP16;
- } else if ( *onnx_type == "int32" || *onnx_type == "tensor(int32)") {
+ } else if (*onnx_type == "int32" || *onnx_type == "tensor(int32)") {
return InferenceEngine::Precision::I32;
- } else if ( *onnx_type == "int16" || *onnx_type == "tensor(int16)") {
+ } else if (*onnx_type == "int16" || *onnx_type == "tensor(int16)") {
return InferenceEngine::Precision::I16;
- } else if ( *onnx_type == "int8" || *onnx_type == "tensor(int8)") {
+ } else if (*onnx_type == "int8" || *onnx_type == "tensor(int8)") {
return InferenceEngine::Precision::I8;
- } else if ( *onnx_type == "uint16" || *onnx_type == "tensor(uint16)") {
+ } else if (*onnx_type == "uint16" || *onnx_type == "tensor(uint16)") {
return InferenceEngine::Precision::U16;
- } else if ( *onnx_type == "uint8" || *onnx_type == "tensor(uint8)") {
+ } else if (*onnx_type == "uint8" || *onnx_type == "tensor(uint8)") {
return InferenceEngine::Precision::U8;
} else {
throw "Unsupported Data type";
}
}
-std::vector OpenVINOGraph::GetExecutableHandle(
- std::shared_ptr network, const std::string& device) {
-
-
- // Load Plugin for inference engine
- std::vector plugin_path = GetEnvLdLibraryPath();
- plugin_path.push_back("");
- InferenceEngine::InferencePlugin plugin = InferenceEngine::PluginDispatcher(
- plugin_path)
- .getPluginByDevice(device);
-
+void OpenVINOGraph::GetExecutableHandle(
+ std::shared_ptr network) {
LOGS_DEFAULT(INFO) << log_tag << "Loaded plugins";
// Configure input & output
@@ -241,7 +246,6 @@ std::vector OpenVINOGraph::GetExecutableHand
int input_idx = 0;
for (auto iter = inputInfo.begin(); iter != inputInfo.end(); ++iter, ++input_idx) {
-
// Get the onnx index for the corresponding input (ignoring initializers)
auto tracked_input_idx = input_indexes_[input_idx];
auto precision = ConvertPrecisionONNXToOpenVINO(onnx_input_defs[tracked_input_idx]->Type());
@@ -276,7 +280,6 @@ std::vector OpenVINOGraph::GetExecutableHand
int output_idx = 0;
for (auto iter = outputInfo.begin(); iter != outputInfo.end(); ++iter, ++output_idx) {
-
auto precision = ConvertPrecisionONNXToOpenVINO(onnx_output_defs[output_idx]->Type());
iter->second->setPrecision(precision);
@@ -302,21 +305,6 @@ std::vector OpenVINOGraph::GetExecutableHand
throw "Invalid Dims type for output data map for: " + iter->first;
}
}
-
- // Loading model to the plugin
- InferenceEngine::ExecutableNetwork exeNetwork = plugin.LoadNetwork(*network,
- {});
-
- LOGS_DEFAULT(INFO) << log_tag << "Network loaded into accelerator plug-in succesfully";
-
- // Create infer request
- std::vector infer_requests;
- for (size_t i = 0; i < num_inf_reqs_; i++) {
- infer_requests.push_back(exeNetwork.CreateInferRequestPtr());
- }
- LOGS_DEFAULT(INFO) << log_tag << "Infer requests created: " << num_inf_reqs_;
-
- return infer_requests;
}
size_t OpenVINOGraph::DeduceBatchSize(Ort::CustomOpApi ort, const OrtValue* input_tensor,
@@ -341,14 +329,12 @@ size_t OpenVINOGraph::DeduceBatchSize(Ort::CustomOpApi ort, const OrtValue* inpu
void OpenVINOGraph::StartAsyncInference(Ort::CustomOpApi ort, const OrtValue* input_tensors[],
size_t batch_slice_idx,
size_t infer_req_idx) {
-
auto infer_request = infer_requests_[infer_req_idx];
auto graph_input_info = openvino_network_->getInputsInfo();
size_t i = 0;
for (auto input_info_iter = graph_input_info.begin();
input_info_iter != graph_input_info.end(); ++input_info_iter, ++i) {
-
// Get OpenVINO's input buffer
auto graph_input_blob = infer_request->GetBlob(input_info_iter->first);
auto graph_input_buffer =
@@ -371,7 +357,6 @@ void OpenVINOGraph::StartAsyncInference(Ort::CustomOpApi ort, const OrtValue* in
void OpenVINOGraph::CompleteAsyncInference(Ort::CustomOpApi ort, OrtValue* output_tensors[],
size_t batch_slice_idx,
size_t infer_req_idx) {
-
auto infer_request = infer_requests_[infer_req_idx];
// Wait for Async inference completion
@@ -484,4 +469,4 @@ void OpenVINOGraph::Infer(Ort::CustomOpApi ort, OrtKernelContext* context) {
}
} // namespace openvino_ep
-} // namespace onnxruntime
+} // namespace onnxruntime
diff --git a/onnxruntime/core/providers/openvino/openvino_graph.h b/onnxruntime/core/providers/openvino/openvino_graph.h
index 2f53b62fed..58943b376e 100644
--- a/onnxruntime/core/providers/openvino/openvino_graph.h
+++ b/onnxruntime/core/providers/openvino/openvino_graph.h
@@ -34,9 +34,8 @@ class OpenVINOGraph {
InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(ONNX_NAMESPACE::DataType onnx_type);
- std::vector GetExecutableHandle(
- std::shared_ptr network,
- const std::string& device);
+ void GetExecutableHandle(
+ std::shared_ptr network);
size_t DeduceBatchSize(Ort::CustomOpApi ort, const OrtValue* input_tensor,
InferenceEngine::SizeVector graph_dims);
@@ -54,6 +53,7 @@ class OpenVINOGraph {
const onnxruntime::Node* fused_node_;
std::shared_ptr openvino_network_;
size_t num_inf_reqs_;
+ InferenceEngine::InferencePlugin plugin_;
std::vector infer_requests_;
std::string device_id_;
mutable std::mutex compute_lock_;
diff --git a/onnxruntime/core/providers/openvino/openvino_mo/openvino_emitter.py b/onnxruntime/core/providers/openvino/openvino_mo/openvino_emitter.py
index ea78c30117..26605d57a2 100644
--- a/onnxruntime/core/providers/openvino/openvino_mo/openvino_emitter.py
+++ b/onnxruntime/core/providers/openvino/openvino_mo/openvino_emitter.py
@@ -2,29 +2,40 @@
Copyright(C) 2019 Intel Corporation
Licensed under the MIT License
"""
+
+from mo.utils.version import get_version
+from mo.utils.utils import refer_to_faq_msg
+from mo.utils.unsupported_ops import UnsupportedOps
+from mo.graph.graph import *
+from mo.front.extractor import update_ie_fields
+from xml.etree.ElementTree import Element, SubElement, tostring
+import xml.dom.minidom
+import hashlib
import sys
import os
-
-import hashlib
-import xml.dom.minidom
-from xml.etree.ElementTree import Element, SubElement, tostring
-
-from mo.front.extractor import update_ie_fields
-from mo.graph.graph import *
-from mo.utils.unsupported_ops import UnsupportedOps
-from mo.utils.utils import refer_to_faq_msg
-from mo.utils.version import get_version
-
ov_root = os.environ['INTEL_CVSDK_DIR']
-mo_path = os.path.join(ov_root, "deployment_tools", "model_optimizer")
+if '2019' in ov_root:
+ version = 'R1'
+else:
+ version = 'R5'
+mo_path = ov_root + "/deployment_tools/model_optimizer"
sys.path.append(mo_path)
def create_const_nodes(graph: nx.MultiDiGraph, start_data_nodes_are_not_allowed: bool = True):
for node_name in list(graph.nodes()):
- node = NodeWrap(graph, node_name)
- if (node.has('kind') and node.kind == 'data' and ((len(node.out_edges()) == 1 and 'bin' not in node.out_edge(0)) or node.has_and_set('is_output')) and len(node.in_nodes()) == 0):
+ if 'R5' in version:
+ node = NodeWrap(graph, node_name)
+ else:
+ node = Node(graph, node_name)
+ if (
+ node.has('kind') and
+ node.kind == 'data' and (
+ (len(node.out_edges()) == 1 and 'bin' not in node.out_edge(0)) or
+ node.has_and_set('is_output')
+ ) and
+ len(node.in_nodes()) == 0):
if node.has_valid('value'):
const_node_name = node.id + '_const'
@@ -74,7 +85,10 @@ def serialize_constants_recursively(weights, graph: nx.MultiDiGraph, data_type,
elif (data_type == np.float16):
precision = 2
for node in nodes:
- node = NodeWrap(graph, node)
+ if 'R5' in version:
+ node = NodeWrap(graph, node)
+ else:
+ node = Node(graph, node)
if node.kind == 'data' and node.value is not None and any('bin' in d for u, v, d in graph.out_edges(node.node, data=True)):
blob = node.value
@@ -101,7 +115,10 @@ def serialize_constants_recursively(weights, graph: nx.MultiDiGraph, data_type,
graph, node.soft_get('name'), node.id, node.shape, node.offset, node.size))
for node in nodes:
- node = NodeWrap(graph, node)
+ if 'R5' in version:
+ node = NodeWrap(graph, node)
+ else:
+ node = Node(graph, node)
# Dump blobs recursively if sub-graphs are present in the node
if node.has_valid('sub_graphs'):
for sub_graph_attr_name in node.sub_graphs:
@@ -131,6 +148,8 @@ def xml_shape(shape: np.ndarray, element: xml.etree.ElementTree.Element):
dim = SubElement(element, 'dim')
if d <= 0:
d = 1
+ # raise Error('The value "{}" for shape is less or equal to 0. May be the input shape of the topology is '
+ # 'wrong.'.format(d))
if int(d) != d:
raise Error('The value "{}" for shape is not integer.'.format(d))
if not isinstance(d, np.int64):
@@ -140,10 +159,24 @@ def xml_shape(shape: np.ndarray, element: xml.etree.ElementTree.Element):
dim.text = str(d)
+def sorted_inputs(node):
+ if 'R5' in version:
+ return get_sorted_inputs(node)
+ else:
+ return node.get_sorted_inputs(node)
+
+
+def sorted_outputs(node):
+ if 'R5' in version:
+ return get_sorted_outputs(node)
+ else:
+ return node.get_sorted_outputs(node)
+
+
def xml_ports(node: Node, element: xml.etree.ElementTree.Element, edges: xml.etree.ElementTree.Element):
# input ports
inputs = None # will create input section only if at least one input is available
- for u, d in get_sorted_inputs(node):
+ for u, d in sorted_inputs(node):
if 'bin' not in d and ('xml_skip' not in d or not d['xml_skip']):
if inputs is None:
inputs = SubElement(element, 'input')
@@ -166,7 +199,7 @@ def xml_ports(node: Node, element: xml.etree.ElementTree.Element, edges: xml.etr
# output ports
outputs = None
- for v, d in get_sorted_outputs(node):
+ for v, d in sorted_outputs(node):
if 'xml_skip' not in d or not d['xml_skip']:
if outputs is None:
outputs = SubElement(element, 'output')
@@ -180,7 +213,7 @@ def xml_ports(node: Node, element: xml.etree.ElementTree.Element, edges: xml.etr
def xml_consts(graph: nx.MultiDiGraph, node: Node, element: xml.etree.ElementTree.Element):
blobs = None # sub-element that will be created on-demand
- for u, d in get_sorted_inputs(node):
+ for u, d in sorted_inputs(node):
if 'bin' in d:
if not blobs:
blobs = SubElement(element, 'blobs')
@@ -346,7 +379,10 @@ def serialize_network(graph, net_element, unsupported):
return
nodes = sorted(graph.nodes())
for node in nodes:
- node = NodeWrap(graph, node)
+ if 'R5' in version:
+ node = NodeWrap(graph, node)
+ else:
+ node = Node(graph, node)
if not node.has('IE'):
continue
if node.kind == 'op' and (not node.has('type') or node.type is None):
@@ -394,12 +430,15 @@ def generate_ie_ir(graph: nx.MultiDiGraph, file_name: str, input_names: tuple =
def port_renumber(graph: nx.MultiDiGraph):
for node in list(graph.nodes()):
- node = NodeWrap(graph, node)
+ if 'R5' in version:
+ node = NodeWrap(graph, node)
+ else:
+ node = Node(graph, node)
if node.kind == 'op':
base = 0
- for u, d in get_sorted_inputs(node):
+ for u, d in sorted_inputs(node):
d['in'] = base
base += 1
- for v, d in get_sorted_outputs(node):
+ for v, d in sorted_outputs(node):
d['out'] = base
base += 1
diff --git a/onnxruntime/core/providers/openvino/openvino_mo/openvino_mo.py b/onnxruntime/core/providers/openvino/openvino_mo/openvino_mo.py
index fa998265ec..b1bd8db440 100644
--- a/onnxruntime/core/providers/openvino/openvino_mo/openvino_mo.py
+++ b/onnxruntime/core/providers/openvino/openvino_mo/openvino_mo.py
@@ -12,35 +12,25 @@ from __future__ import print_function
from __future__ import unicode_literals
from mo.pipeline.common import determined_sort, get_fw_tensor_debug_info, get_sorted_outputs, collect_sub_graphs, relabel_nodes_inplace_safe
-from mo.middle.passes import tensor_names, convert_data_type
-from mo.graph.graph import Node, unique_id
from openvino_emitter import port_renumber, serialize_mean_image, create_const_nodes, serialize_network, add_meta_data, generate_ie_ir, serialize_constants, serialize_constants_recursively
import openvino_emitter
-import networkx as nx
from operator import itemgetter
-from mo.graph.graph import check_empty_graph
-from mo.utils.error import Error
from mo.utils import class_registration
from mo.middle.passes.shape import convert_reshape, reverse_input_channels, \
fuse_sequence_of_reshapes, merge_nodes_permutations, permute_data_nodes_attrs, permute_op_nodes_attrs
from mo.middle.passes.mean_scale_values import move_scaleshift_to_preprocess
-from mo.middle.passes.eliminate import get_nodes_with_attributes
-from mo.middle.passes.infer import scale_input, override_placeholder_shapes, convert_mul_add_to_power, \
- add_mean_scale_values, override_batch, exit_bound_edges, control_flow_infer # , partial_infer, update_fully_connected_shapes
from mo.middle.passes.fusing.mark_unfused_nodes import mark_unfused_nodes
from mo.middle.passes.fusing.fuse_linear_seq import fuse_mul_add_sequence
from mo.middle.passes.fusing.fuse_linear_ops import fuse_linear_ops
from mo.middle.passes.fusing.fuse_grouped_conv import grouped_convolutions_fusing
from mo.middle.passes.fusing.decomposition import convert_batch_norm, convert_scale_shift_to_mul_add
-from mo.middle.passes.eliminate import graph_clean_up, remove_op_nodes, remove_useless_split
-from mo.middle.passes.conv import convert_add_to_scaleshift, convert_gemm_to_fully_connected, \
- convert_muladd_to_scaleshift_or_power, fuse_pad, convert_dilated_convolution, convert_mul_to_scaleshift
+from mo.middle.passes.infer import partial_infer, update_fully_connected_shapes
+from mo.middle.passes import infer, tensor_names, convert_data_type
from mo.front.onnx.loader import load_onnx_model, protobuf2nx
from mo.front.onnx.extractor import common_onnx_fields, onnx_op_extractor, onnx_op_extractors
from mo.front.extractor import add_output_ops, add_input_ops, \
extract_node_attrs, create_tensor_nodes, remove_output_ops, user_data_repack
-from mo.front.common.register_custom_ops import update_extractors_with_extensions
-from mo.front.common.register_custom_ops import check_for_duplicates
+from mo.front.common.register_custom_ops import check_for_duplicates, update_extractors_with_extensions
from extensions.middle.NormalizeFullyConnected import NormalizeFullyConnected
from extensions.middle.EltwiseInputNormalization import EltwiseInputNormalize
from mo.utils.versions_checker import check_requirements
@@ -52,10 +42,10 @@ from mo.utils.error import Error, FrameworkError
from mo.utils.cli_parser import get_placeholder_shapes, get_tuple_values, get_model_name, \
get_common_cli_options, get_caffe_cli_options, get_tf_cli_options, get_mxnet_cli_options, get_kaldi_cli_options, \
get_onnx_cli_options, get_mean_scale_dictionary, parse_tuple_pairs, get_meta_info
-from mo.utils import import_extensions
from mo.utils.versions_checker import check_python_version
import onnx
import numpy as np
+import networkx as nx
from collections import OrderedDict
import traceback
import logging as log
@@ -64,12 +54,35 @@ import argparse
import sys
import os
ov_root = os.environ['INTEL_CVSDK_DIR']
-mo_path = os.path.join(ov_root, "deployment_tools", "model_optimizer")
-mo_extensions = os.path.join(mo_path, "extensions")
+if '2019.1' in ov_root:
+ version = 'R1'
+elif '2018.5' in ov_root:
+ version = 'R5'
+else:
+ version = 'unsupported'
+ print('You are using unsupported version of OpenVINO. Please refer to BUILD.md for supported versions of OpenVINO.')
+mo_path = ov_root + "/deployment_tools/model_optimizer"
+mo_extensions = mo_path + "/extensions"
sys.path.append(mo_path)
-# from mo.back.ie_ir_ver_2.emitter import port_renumber, serialize_mean_image, create_const_nodes #, generate_ie_ir, serialize_constants, serialize_constants_recursively
+if 'R5' in version:
+ from mo.utils import import_extensions
+ from mo.middle.passes.conv import convert_add_to_scaleshift, convert_gemm_to_fully_connected, \
+ convert_muladd_to_scaleshift_or_power, fuse_pad, convert_dilated_convolution, convert_mul_to_scaleshift
+ from mo.middle.passes.eliminate import graph_clean_up, remove_op_nodes, remove_useless_split, get_nodes_with_attributes
+ from mo.middle.passes.infer import scale_input, override_placeholder_shapes, convert_mul_add_to_power, \
+ add_mean_scale_values, override_batch, exit_bound_edges, control_flow_infer
+ from mo.graph.graph import check_empty_graph, Node, unique_id
+else:
+ from mo.utils import import_extensions, class_registration
+ from mo.middle.passes.conv import convert_add_or_mul_to_scaleshift, convert_muladd_to_scaleshift_or_power, fuse_pad
+ from mo.middle.passes.eliminate import remove_const_ops, mark_output_reachable_nodes, mark_undead_nodes, mark_const_producer_nodes, \
+ eliminate_dead_nodes, add_constant_operations, shape_inference, remove_op_nodes, get_nodes_with_attributes
+ from mo.middle.passes.infer import override_placeholder_shapes, convert_mul_add_to_power, override_batch, exit_bound_edges, control_flow_infer
+ from extensions.back.CreateConstNodes import CreateConstNodesReplacement
+ from mo.middle.pattern_match import for_graph_and_each_sub_graph_recursively
+ from mo.graph.graph import check_empty_graph, Node, Graph
def is_fully_defined_shape(shape: np.ndarray):
@@ -78,160 +91,11 @@ def is_fully_defined_shape(shape: np.ndarray):
return True
-def partial_infer(graph: nx.MultiDiGraph, start_node: str = None):
-
- cycle_nodes = get_nodes_with_attributes(graph, is_cyclic=True)
- cycle_nodes = [Node(graph, node).out_node().id for node in cycle_nodes]
- ebunch_cyclic = list(graph.out_edges(
- nbunch=cycle_nodes, data=True, keys=True))
- ebunch_reconnected = exit_bound_edges(
- graph, sources=cycle_nodes, end_node_attrs={'op': 'Exit'})
- graph.remove_edges_from(ebunch_cyclic)
- graph.add_edges_from(ebunch_reconnected)
-
- try:
- nodes = list(nx.topological_sort(graph))
- except:
- raise Error('Graph contains a cycle. Can not proceed. ' +
- refer_to_faq_msg(97))
-
- graph.remove_edges_from(ebunch_reconnected)
- graph.add_edges_from(ebunch_cyclic)
-
- # Mark all nodes as not inferred yet
- if start_node is not None:
- start_index = nodes.index(start_node)
- nx.set_node_attributes(G=graph.subgraph(
- nodes[start_index:]), name='is_partial_inferred', values=False)
- else:
- nx.set_node_attributes(
- G=graph, name='is_partial_inferred', values=False)
- debug_logger = log.getLogger().isEnabledFor(log.DEBUG)
-
- nx.set_node_attributes(G=graph, name='executable',
- values={n: True for n in get_nodes_with_attributes(graph, kind='data')})
-
- for n in nodes:
- # Data Flow Infer
- try:
- node = Node(graph, n)
- node_name = node.soft_get('name')
- if node.has('is_partial_inferred') and not node.is_partial_inferred:
- if node.has('infer') and node.infer is not None:
- log.debug('-' * 20)
- log.debug('Partial infer for {}'.format(
- node.soft_get('name')))
- log.debug('Op: {}'.format(node.soft_get('op')))
- node.infer(node)
- out_nodes = node.out_nodes()
-
- # propagate nchw_layout attributes to data nodes
- if node.has('nchw_layout'):
- for out_node in out_nodes.values():
- out_node['nchw_layout'] = node.nchw_layout
-
- # In debug print current node attributes, input shapes/values and output shape/values
- if debug_logger:
- log.debug('Inputs:')
- log_debug_dict(node.in_nodes(), 'input')
- log.debug('Outputs:')
- log_debug_dict(node.out_nodes(), 'output')
-
- for out_port, out_node in out_nodes.items():
- not_all_output_shapes = False
- if not out_node.has_valid('shape'):
- log.error('Shape is not defined for output {} of "{}".'.format(
- out_port, node_name))
- not_all_output_shapes = True
- elif not is_fully_defined_shape(out_node.shape):
- log.error(
- ('Shape {} is not fully defined for output {} of "{}". ' +
- 'Use --input_shape with positive integers to override model input shapes.').format(
- out_node.shape,
- out_port,
- node_name
- )
- )
- not_all_output_shapes = True
-
- if not_all_output_shapes:
- raise Error('Not all output shapes were inferred or fully defined for node "{}". ' +
- refer_to_faq_msg(40),
- node_name)
- elif node.kind != 'data':
- raise Error(
- 'There is no registered "infer" function for node "{}" with op = "{}". ' +
- 'Please implement this function in the extensions. ' +
- refer_to_faq_msg(37),
- node_name,
- node.soft_get('op')
- )
- node.is_partial_inferred = True
-
- except Exception as err:
- log.error('Cannot infer shapes or values for node "{}".'.format(
- node.soft_get('name')))
- log.error(str(err))
- log.error('')
- log.error('It can happen due to bug in custom shape infer function {}.'.format(
- node.soft_get('infer')))
- log.error('Or because the node inputs have incorrect values/shapes.')
- log.error(
- 'Or because input shapes are incorrect (embedded to the model or passed via --input_shape).')
- debug_messages = '\n'.join(
- ['Layer "' + node_name + '": ' + node_attrs['debug_message'] for node_name, node_attrs in
- graph.nodes(data=True) if 'debug_message' in node_attrs])
- if debug_messages != "":
- log.error('')
- log.error('Other possible failure reasons are listed below:')
- log.error(debug_messages)
- if not debug_logger:
- log.error(
- 'Run Model Optimizer with --log_level=DEBUG for more information.')
- else:
- log.debug('Node "{}" attributes: {}'.format(
- node.soft_get('name'), node.graph.node[node.id]))
- raise Error('Stopped shape/value propagation at "{}" node. '.format(node.soft_get('name')) +
- refer_to_faq_msg(38)) from err
- control_flow_infer(graph, n)
-
- not_fully_inferred = get_nodes_with_attributes(
- graph, is_not_fully_inferred=True)
- for n in not_fully_inferred:
- node = Node(graph, n)
- if node.has('infer') and node.infer is not None:
- node.infer(node)
-
- # delete_not_executable(graph)
- return graph
-
-
-def update_fully_connected_shapes(graph: nx.MultiDiGraph):
- nodes = nx.topological_sort(graph)
- while True:
- should_infer = False
- for n in nodes:
- node = Node(graph, n)
- if node.has('type') and node.type == 'FullyConnected' and node.in_node(0).shape.size == 3:
- log.debug("node.in_node(0).shape = {}".format(
- node.in_node(0).shape))
- log.debug("channel_dims = {}".format(node.channel_dims))
- assert (node.in_node(0).shape.size ==
- 3 and node.channel_dims > 0)
- node.in_node(0).shape = np.delete(node.in_node(0).shape, 1)
- if node.out_node().shape.size == 3:
- node.channel_dims = node.channel_dims - 1
- log.debug(
- "Initiated partial infer from update_fully_connected_shapes")
- graph = partial_infer(graph, node.in_node(0).id)
- should_infer = True
- break
- if not should_infer:
- break
+infer.is_fully_defined_shape = is_fully_defined_shape
def prepare_emit_ir(graph: nx.MultiDiGraph, data_type: str, output_dir: str, output_model_name: str,
- mean_data: [list, None]=None, input_names: list=[], meta_info: dict=dict()):
+ mean_data: [list, None] = None, input_names: list = [], meta_info: dict = dict()):
for sub_graph in [graph] + collect_sub_graphs(graph):
create_const_nodes(
@@ -257,7 +121,6 @@ def prepare_emit_ir(graph: nx.MultiDiGraph, data_type: str, output_dir: str, out
if mean_data:
mean_offset, mean_size = serialize_mean_image(
bin_file, mean_data=mean_data)
-
xml_string = generate_ie_ir(graph=graph,
file_name=os.path.join(
output_dir, '{}.xml'.format(output_model_name)),
@@ -266,16 +129,33 @@ def prepare_emit_ir(graph: nx.MultiDiGraph, data_type: str, output_dir: str, out
mean_size=mean_size,
meta_info=meta_info)
- # tensor_names.output_tensor_names_map(graph, os.path.join(output_dir, '{}.mapping'.format(output_model_name)))
return weights, xml_string
-# argv: argparse.Namespace
+
+if 'R1' in version:
+ def graph_clean_up(graph: Graph, undead_node_types: list = None):
+ if undead_node_types is None:
+ undead_node_types = []
+
+ if 'Shape' in undead_node_types:
+ undead_node_types.remove('Shape')
+
+ mark_output_reachable_nodes(graph)
+ mark_undead_nodes(graph, undead_node_types)
+ mark_const_producer_nodes(graph)
+ eliminate_dead_nodes(graph)
+ # Add Const op for constant data nodes
+ add_constant_operations(graph)
+ shape_inference(graph)
+
+ def graph_clean_up_onnx(graph: Graph):
+ graph_clean_up(graph, ['Shape'])
-def driver(onnx_modelproto_bytes, precision: str, output_model_name: str, outputs: list, output_dir: str,
- scale: float,
- user_shapes: [None, list, np.array]=None,
- mean_scale_values: [dict, list]=()):
+def driver_R5(onnx_modelproto_bytes, precision: str, output_model_name: str, outputs: list, output_dir: str,
+ scale: float,
+ user_shapes: [None, list, np.array] = None,
+ mean_scale_values: [dict, list] = ()):
try:
model_proto = onnx.load_from_string(bytes(onnx_modelproto_bytes))
@@ -320,7 +200,6 @@ def driver(onnx_modelproto_bytes, precision: str, output_model_name: str, output
output_op_nodes = add_output_ops(graph, packed_outputs)
input_op_nodes = add_input_ops(graph, packed_user_shapes, True)
- # this call of 'graph_clean_up' removes child nodes of outputs which is useful when custom output is specified
graph_clean_up(graph)
check_empty_graph(graph, 'add_output_ops and add_input_ops')
extract_node_attrs(graph, lambda node: onnx_op_extractor(
@@ -404,6 +283,101 @@ def driver(onnx_modelproto_bytes, precision: str, output_model_name: str, output
return weights, xml_string
+def driver_R1(onnx_modelproto_bytes, precision: str, output_model_name: str, outputs: list, output_dir: str,
+ scale: float,
+ user_shapes: [None, list, np.array] = None,
+ mean_scale_values: [dict, list] = ()):
+
+ try:
+ model_proto = onnx.load_from_string(bytes(onnx_modelproto_bytes))
+ except Exception as e:
+ print("[python] onnx exception: ", str(e))
+
+ model_graph = model_proto.graph # pylint: disable=no-member
+
+ update_extractors_with_extensions(onnx_op_extractors)
+
+ try:
+ graph = protobuf2nx(model_proto)
+ log.debug("Number of nodes in NX graph: {}".format(
+ graph.number_of_nodes()))
+ graph.__setattr__(
+ 'name', output_model_name if output_model_name else model_proto.graph.name) # pylint: disable=no-member
+ graph.graph['layout'] = 'NCHW'
+ graph.graph['cmd_params'] = argparse.Namespace(batch=None, data_type='float', disable_fusing=False, disable_gfusing=False, disable_resnet_optimization=False, enable_concat_optimization=False, extensions=mo_extensions, finegrain_fusing=None, framework='onnx', freeze_placeholder_with_value=None, generate_deprecated_IR_V2=False,
+ input=None, input_model=None, input_shape=None, keep_shape_ops=False, log_level='ERROR', mean_scale_values={}, mean_values=(), model_name=None, move_to_preprocess=False, output=None, output_dir='.', placeholder_shapes=None, reverse_input_channels=False, scale=None, scale_values=(), silent=False, version=False)
+ graph.graph['fw'] = 'onnx'
+ graph.graph['feature_dim'] = 1 if graph.graph['layout'] == 'NCHW' else 3
+ graph.graph['ir_version'] = 4
+ extract_node_attrs(graph, lambda node: (
+ True, common_onnx_fields(node)))
+ except Exception as e:
+ raise Error(
+ 'Cannot pre-process ONNX graph after reading from model file "{}". '
+ 'File is corrupt or has unsupported format. Details: {}. ' +
+ refer_to_faq_msg(44),
+ model_file_name,
+ str(e)
+ ) from e
+ graph.check_empty_graph(
+ 'protobuf2nx. It may happen due to problems with loaded model')
+ extract_node_attrs(graph, lambda node: onnx_op_extractor(
+ node, check_for_duplicates(onnx_op_extractors)))
+
+ # --------------------------------- LOAD END ------------------------------------------------------
+ class_registration.apply_replacements(
+ graph, class_registration.ClassType.FRONT_REPLACER)
+ partial_infer(graph)
+ graph.check_empty_graph('partial_infer')
+ class_registration.apply_replacements(
+ graph, class_registration.ClassType.MIDDLE_REPLACER)
+
+ fuse_pad(graph)
+ graph_clean_up_onnx(graph)
+
+ mark_unfused_nodes(graph, 'False')
+ convert_batch_norm(graph)
+ graph_clean_up_onnx(graph)
+
+ convert_muladd_to_scaleshift_or_power(graph)
+ graph_clean_up_onnx(graph)
+
+ convert_mul_add_to_power(graph)
+ graph_clean_up_onnx(graph)
+
+ convert_reshape(graph)
+ graph_clean_up_onnx(graph)
+ convert_add_or_mul_to_scaleshift(graph) # scale = 1
+ graph_clean_up_onnx(graph)
+
+ fuse_pad(graph)
+ graph_clean_up_onnx(graph)
+
+ fuse_sequence_of_reshapes(graph)
+ graph_clean_up_onnx(graph)
+
+ pattern = EltwiseInputNormalize()
+ pattern.find_and_replace_pattern(graph)
+
+ merge_nodes_permutations(graph)
+ permute_data_nodes_attrs(graph)
+ permute_op_nodes_attrs(graph)
+
+ class_registration.apply_replacements(
+ graph, class_registration.ClassType.BACK_REPLACER)
+
+ for_graph_and_each_sub_graph_recursively(graph, remove_const_ops)
+
+ CreateConstNodesReplacement().find_and_replace_pattern(graph)
+
+ for_graph_and_each_sub_graph_recursively(graph, remove_output_ops)
+
+ weights, xml_string = prepare_emit_ir(graph=graph, data_type=precision, output_dir=output_dir, output_model_name=output_model_name,
+ meta_info={'unset': []})
+
+ return weights, xml_string
+
+
def driver_entry(onnx_modelproto_bytes, precision: str):
start_time = datetime.datetime.now()
@@ -414,11 +388,19 @@ def driver_entry(onnx_modelproto_bytes, precision: str):
scale_values = {}
mean_scale = {}
- from mo.front.onnx.register_custom_ops import update_registration
- import_extensions.load_dirs('onnx', [mo_extensions], update_registration)
- weights, xml_string = driver(onnx_modelproto_bytes, precision, model_name, outputs, ".", None,
- user_shapes=placeholder_shapes,
- mean_scale_values=mean_scale)
+ if 'R5' in version:
+ from mo.front.onnx.register_custom_ops import update_registration
+ import_extensions.load_dirs(
+ 'onnx', [mo_extensions], update_registration)
+ weights, xml_string = driver_R5(onnx_modelproto_bytes, precision, model_name, outputs, ".", None,
+ user_shapes=placeholder_shapes,
+ mean_scale_values=mean_scale)
+ else:
+ from mo.front.onnx.register_custom_ops import get_front_classes
+ import_extensions.load_dirs('onnx', [mo_extensions], get_front_classes)
+ weights, xml_string = driver_R1(onnx_modelproto_bytes, precision, model_name, outputs, ".", None,
+ user_shapes=placeholder_shapes,
+ mean_scale_values=mean_scale)
return weights, xml_string
@@ -459,5 +441,11 @@ if __name__ == "__main__":
sys.exit(ret_code)
from mo.utils.cli_parser import get_onnx_cli_parser
+ if '2019' in ov_root:
+ print('2019 R1 version')
+ else:
+ print('2018 R5 version')
weights_string, final_string = convert_fp32()
+ print(weights_string)
+
sys.exit(0)
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
index 27cf31c9e5..b993c360ef 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
@@ -2,7 +2,7 @@ ARG OS_VERSION=16.04
FROM ubuntu:${OS_VERSION}
ARG PYTHON_VERSION=3.5
-ARG OPENVINO_VERSION=2018_R5
+ARG OPENVINO_VERSION=2019_R1.1
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
@@ -14,9 +14,10 @@ RUN /tmp/scripts/install_openvino.sh -o ${OPENVINO_VERSION} && \
WORKDIR /root
-ENV INTEL_CVSDK_DIR /data/dldt
+ENV INTEL_CVSDK_DIR /data/dldt/openvino_2019.1.144
+ENV INTEL_OPENVINO_DIR /data/dldt/openvino_2019.1.144
-ENV LD_LIBRARY_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64:$INTEL_CVSDK_DIR/deployment_tools/inference_engine/temp/omp/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH $INTEL_CVSDK_DIR/deployment_tools/inference_engine/lib/intel64:$INTEL_CVSDK_DIR/deployment_tools/inference_engine/temp/omp/lib:$INTEL_CVSDK_DIR/deployment_tools/inference_engine/external/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH
ENV PATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$PATH
ENV PYTHONPATH $INTEL_CVSDK_DIR/deployment_tools/model_optimizer:$INTEL_CVSDK_DIR/tools:$PYTHONPATH
diff --git a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh b/tools/ci_build/github/linux/docker/scripts/install_openvino.sh
index d205bcdf78..0c44d8d97c 100755
--- a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh
+++ b/tools/ci_build/github/linux/docker/scripts/install_openvino.sh
@@ -7,10 +7,11 @@ o) OPENVINO_VERSION=${OPTARG};;
esac
done
-OPENVINO_VERSION=${OPENVINO_VERSION:=2018_R5}
-git clone https://github.com/opencv/dldt.git /data/dldt
+OPENVINO_VERSION=${OPENVINO_VERSION:=2019_R1.1}
+git clone https://github.com/opencv/dldt.git /data/dldt/openvino_2019.1.144
-export INTEL_CVSDK_DIR=/data/dldt
+export INTEL_CVSDK_DIR=/data/dldt/openvino_2019.1.144
+apt-get update && apt-get -y install libusb-1.0-0-dev
cd ${INTEL_CVSDK_DIR}/inference-engine
git submodule init
@@ -30,7 +31,9 @@ mv model-optimizer model_optimizer && mv model_optimizer deployment_tools/
cd ${INTEL_CVSDK_DIR}/deployment_tools/model_optimizer/install_prerequisites && ./install_prerequisites_onnx.sh
cd ${INTEL_CVSDK_DIR}/deployment_tools/inference_engine
-mkdir -p lib/ubuntu_16.04/intel64
-mv bin/intel64/Release/lib/* lib/ubuntu_16.04/intel64
+mkdir -p lib/intel64
+mkdir -p external/tbb/lib
+mv bin/intel64/Release/lib/* lib/intel64
+mv temp/tbb/lib/* external/tbb/lib
-cd ~
\ No newline at end of file
+cd ~
diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh
index d3e50efd3e..e98c720ab0 100755
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@@ -21,7 +21,7 @@ x) BUILD_EXTR_PAR=${OPTARG};;
c) CUDA_VER=${OPTARG};;
# x86 or other, only for ubuntu16.04 os
a) BUILD_ARCH=${OPTARG};;
-# openvino version tag: 2018_R5, 2019_R1 (Default is 2018_R5)
+# openvino version tag: 2018_R5, 2019_R1.1 (Default is 2019_R1.1)
v) OPENVINO_VERSION=${OPTARG};;
esac
done