diff --git a/BUILD.md b/BUILD.md index 8f30421af9..87783f6675 100644 --- a/BUILD.md +++ b/BUILD.md @@ -338,20 +338,21 @@ See more information on the nGraph Execution Provider [here](./docs/execution_pr See more information on the OpenVINO Execution Provider [here](./docs/execution_providers/OpenVINO-ExecutionProvider.md). #### Prerequisites -1. Install the Intel® Distribution of OpenVINOTM Toolkit **Release 2020.4** for the appropriate OS and target hardware : +1. Install the Intel® Distribution of OpenVINOTM Toolkit **Release 2021.1** for the appropriate OS and target hardware : * [Linux - CPU, GPU, VPU, VAD-M](https://software.intel.com/en-us/openvino-toolkit/choose-download/free-download-linux) * [Linux - FPGA](https://software.intel.com/en-us/openvino-toolkit/choose-download/free-download-linux-fpga) * [Windows - CPU, GPU, VPU, VAD-M](https://software.intel.com/en-us/openvino-toolkit/choose-download/free-download-windows). - Follow [documentation](https://docs.openvinotoolkit.org/2020.4/index.html) for detailed instructions. + Follow [documentation](https://docs.openvinotoolkit.org/2021.1/index.html) for detailed instructions. - *2020.4 is the recommended OpenVINO version. [OpenVINO 2020.2](https://docs.openvinotoolkit.org/2020.2/index.html) is minimal OpenVINO version requirement.* + *2021.1 is the recommended OpenVINO version. [OpenVINO 2020.2](https://docs.openvinotoolkit.org/2020.2/index.html) is minimal OpenVINO version requirement.* + *The minimum ubuntu version to support 2021.1 is 18.04.* 2. Configure the target hardware with specific follow on instructions: - * To configure Intel® Processor Graphics(GPU) please follow these instructions: [Windows](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_windows.html#Install-GPU), [Linux](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_linux.html#additional-GPU-steps) - * To configure Intel® MovidiusTM USB, please follow this getting started guide: [Linux](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_linux.html#additional-NCS-steps) - * To configure Intel® Vision Accelerator Design based on 8 MovidiusTM MyriadX VPUs, please follow this configuration guide: [Windows](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_windows.html#hddl-myriad), [Linux](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_linux.html#install-VPU). Follow steps 3 and 4 to complete the configuration. - * To configure Intel® Vision Accelerator Design with an Intel® Arria® 10 FPGA, please follow this configuration guide: [Linux](https://docs.openvinotoolkit.org/2020.4/openvino_docs_install_guides_installing_openvino_linux_fpga.html) + * To configure Intel® Processor Graphics(GPU) please follow these instructions: [Windows](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_windows.html#Install-GPU), [Linux](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_linux.html#additional-GPU-steps) + * To configure Intel® MovidiusTM USB, please follow this getting started guide: [Linux](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_linux.html#additional-NCS-steps) + * To configure Intel® Vision Accelerator Design based on 8 MovidiusTM MyriadX VPUs, please follow this configuration guide: [Windows](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_windows.html#hddl-myriad), [Linux](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_linux.html#install-VPU). Follow steps 3 and 4 to complete the configuration. + * To configure Intel® Vision Accelerator Design with an Intel® Arria® 10 FPGA, please follow this configuration guide: [Linux](https://docs.openvinotoolkit.org/2021.1/openvino_docs_install_guides_installing_openvino_linux_fpga.html) 3. Initialize the OpenVINO environment by running the setupvars script as shown below: * For Linux run: diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6f25b2f4d2..91f6786a30 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -968,6 +968,9 @@ if(onnxruntime_USE_OPENVINO) elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2020.4") set(OPENVINO_VERSION "2020.4") add_definitions(-DOPENVINO_2020_4=1) + elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2021.1") + set(OPENVINO_VERSION "2021.1") + add_definitions(-DOPENVINO_2021_1=1) else() message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}") endif() diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index c449611009..4cdf1f57bd 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -561,6 +561,7 @@ if (onnxruntime_USE_OPENVINO) if ((OPENVINO_VERSION VERSION_GREATER_EQUAL "2020.3") OR (WIN32)) # Link to nGraph from OpenVINO installation list(APPEND OPENVINO_INCLUDE_DIR_LIST $ENV{INTEL_OPENVINO_DIR}/deployment_tools/ngraph/include) + list(APPEND OPENVINO_INCLUDE_DIR_LIST $ENV{INTEL_OPENVINO_DIR}/deployment_tools/ngraph/include/ngraph/frontend) list(APPEND OPENVINO_LIB_DIR_LIST $ENV{INTEL_OPENVINO_DIR}/deployment_tools/ngraph/lib) if (WIN32) list(APPEND OPENVINO_LIB_LIST ngraph.lib) diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino index 3a358238e1..13526da838 100644 --- a/dockerfiles/Dockerfile.openvino +++ b/dockerfiles/Dockerfile.openvino @@ -15,7 +15,7 @@ ARG MY_ROOT=/code ENV PATH /opt/miniconda/bin:/code/cmake-3.14.3-Linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH=/opt/miniconda/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH -ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2020.4.287 +ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.1.110 ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/share ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64 ENV LD_LIBRARY_PATH=/opt/intel/opencl:${INTEL_OPENVINO_DIR}/inference_engine/external/gna/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib:$INTEL_OPENVINO_DIR/deployment_tools/ngraph/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/omp/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH} @@ -24,6 +24,7 @@ ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/opencv/lib:${INTEL_OPENVINO_DIR}/openc ENV HDDL_INSTALL_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl/lib:$LD_LIBRARY_PATH ENV LANG en_US.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ apt -y install apt-transport-https ca-certificates python3 python3-pip zip x11-apps lsb-core wget cpio sudo libboost-python-dev libpng-dev zlib1g-dev git libnuma1 ocl-icd-libopencl1 clinfo libboost-filesystem1.65-dev libboost-thread1.65-dev protobuf-compiler libprotoc-dev autoconf automake libtool libjson-c-dev unattended-upgrades && \ @@ -31,12 +32,12 @@ RUN apt update && \ rm -rf /var/lib/apt/lists/* && \ # Install OpenVINO cd ${MY_ROOT} && \ - wget https://apt.repos.intel.com/openvino/2020/GPG-PUB-KEY-INTEL-OPENVINO-2020 && \ - apt-key add GPG-PUB-KEY-INTEL-OPENVINO-2020 && rm GPG-PUB-KEY-INTEL-OPENVINO-2020 && \ + wget https://apt.repos.intel.com/openvino/2021/GPG-PUB-KEY-INTEL-OPENVINO-2021 && \ + apt-key add GPG-PUB-KEY-INTEL-OPENVINO-2021 && rm GPG-PUB-KEY-INTEL-OPENVINO-2021 && \ cd /etc/apt/sources.list.d && \ - echo "deb https://apt.repos.intel.com/openvino/2020 all main">intel-openvino-2020.list && \ + echo "deb https://apt.repos.intel.com/openvino/2021 all main">intel-openvino-2021.list && \ apt update && \ - apt -y install intel-openvino-dev-ubuntu18-2020.4.287 && \ + apt -y install intel-openvino-dev-ubuntu18-2021.1.110 && \ cd ${INTEL_OPENVINO_DIR}/install_dependencies && ./install_openvino_dependencies.sh && \ cd ${INTEL_OPENVINO_DIR} && rm -rf documentation && cd deployment_tools/ && rm -rf model_optimizer open_model_zoo demo && cd inference_engine && rm -rf samples && \ # Install GPU runtime and drivers @@ -54,14 +55,9 @@ RUN apt update && \ dpkg -i /tmp/opencl/*.deb && \ ldconfig && \ rm -rf /tmp/opencl && \ -# Install CMake cd ${MY_ROOT} && \ locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 && \ - pip3 install cython numpy && \ - mkdir -p /opt/cmake/bin && \ - cd ${MY_ROOT} && \ - wget https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz && \ - tar -xf cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake && rm -rf ${MY_ROOT}/cmake-3.13.2-Linux-x86_64.tar.gz && \ + pip3 install cython && \ # Download and build ONNX Runtime cd ${MY_ROOT} && \ git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} && \ @@ -69,4 +65,4 @@ RUN apt update && \ cd onnxruntime/cmake/external/onnx && python3 setup.py install && \ cd ${MY_ROOT}/onnxruntime && ./build.sh --config Release --update --build --parallel --use_openvino ${DEVICE} --build_wheel && \ pip install build/Linux/Release/dist/*-linux_x86_64.whl && \ - cd ${MY_ROOT}/ && rm -rf cmake-3.14.3-Linux-x86_64 onnxruntime + cd ${MY_ROOT}/ && rm -rf onnxruntime diff --git a/dockerfiles/README.md b/dockerfiles/README.md index be793553ef..687828eade 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -120,7 +120,7 @@ Therefore, ONNX RT Execution Provider for **nGraph** will be deprecated starting Retrieve your docker image in one of the following ways. - - Choose Dockerfile.openvino as the dockerfile for building an OpenVINO 2020.4 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is master. + - Choose Dockerfile.openvino as the dockerfile for building an OpenVINO 2021.1 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is master. ``` docker build --rm -t onnxruntime --build-arg DEVICE=$DEVICE -f Dockerfile.openvino . ``` @@ -176,7 +176,7 @@ Therefore, ONNX RT Execution Provider for **nGraph** will be deprecated starting ### OpenVINO on VAD-M Accelerator Version -1. Download OpenVINO **Full package** for version **2020.3** for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html) +1. Download OpenVINO **Full package** for version **2021.1** for Linux on host machine from [this link](https://software.intel.com/en-us/openvino-toolkit/choose-download) and install it with the help of instructions from [this link](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux.html) 2. Install the drivers on the host machine according to the reference in [here](https://docs.openvinotoolkit.org/latest/_docs_install_guides_installing_openvino_linux_ivad_vpu.html) diff --git a/docs/execution_providers/OpenVINO-ExecutionProvider.md b/docs/execution_providers/OpenVINO-ExecutionProvider.md index 5b3d76ec89..e409cae97a 100644 --- a/docs/execution_providers/OpenVINO-ExecutionProvider.md +++ b/docs/execution_providers/OpenVINO-ExecutionProvider.md @@ -125,6 +125,7 @@ VPUs as well as Intel® Vision accelerator Design with Intel Movidiu | Equal | Yes | Yes | Yes | | Erf | Yes | Yes | Yes | | Exp | Yes | Yes | Yes | +| Expand | No | No | Yes | | Flatten | Yes | Yes | Yes | | Floor | Yes | Yes | Yes | | Gather | Yes | Yes | Yes | @@ -145,6 +146,8 @@ VPUs as well as Intel® Vision accelerator Design with Intel Movidiu | Min | Yes | Yes | Yes | | Mul | Yes | Yes | Yes | | Neg | Yes | Yes | Yes | +| NonMaxSuppression | No | No | Yes | +| NonZero | Yes | No | Yes | | Not | Yes | Yes | No | | OneHot | Yes | Yes | Yes | | Pad | Yes | Yes | Yes | @@ -160,7 +163,9 @@ VPUs as well as Intel® Vision accelerator Design with Intel Movidiu | ReduceSumSquare | Yes | No | Yes | | Relu | Yes | Yes | Yes | | Reshape | Yes | Yes | Yes | -| Resize | Yes | No | No | +| Resize | Yes | No | Yes | +| RoiAlign | No | No | Yes | +| Scatter | No | No | Yes | | Selu | Yes | Yes | No | | Shape | Yes | Yes | Yes | | Sigmoid | Yes | Yes | Yes | @@ -216,6 +221,7 @@ Below topologies from ONNX open model zoo are fully supported on OpenVINO Execut | zfnet512 | Yes | Yes | Yes | Yes* | | arcface | Yes | Yes | Yes | Yes* | + ## Image Recognition Networks | **MODEL NAME** | **CPU** | **GPU** | **VPU** | **FPGA** | | --- | --- | --- | --- | --- | @@ -226,6 +232,15 @@ Below topologies from ONNX open model zoo are fully supported on OpenVINO Execut | --- | --- | --- | --- | --- | | tiny_yolov2 | Yes | Yes | Yes | Yes* | +## Image Manipulation Networks +| **MODEL NAME** | **CPU** | **GPU** | **VPU** | **FPGA** | +| --- | --- | --- | --- | --- | +| mosaic | Yes | No | No | No* | +| candy | Yes | No | No | No* | +| rain_princess | Yes | No | No | No* | +| pointilism | Yes | No | No | No* | +| udnie | Yes | No | No | No* | + *FPGA only runs in HETERO mode wherein the layers that are not supported on FPGA fall back to OpenVINO CPU. ## CSharp API diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc index 96b7afcb55..ccb1154e90 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.cc +++ b/onnxruntime/core/providers/openvino/backend_utils.cc @@ -41,6 +41,12 @@ void DumpOnnxModelProto(const ONNX_NAMESPACE::ModelProto& model_proto, std::stri #endif +struct static_cast_int64 +{ + template // T1 models type statically convertible to T + int64_t operator()(const T1& x) const { return static_cast(x); } +}; + std::shared_ptr CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map>& const_outputs_map) { @@ -74,21 +80,23 @@ CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalCont ng_function->validate_nodes_and_infer_types(); } -#if defined(OPENVINO_2020_4) - std::map result_to_output; - for(auto& result : ng_function->get_results()){ - result_to_output[result->get_friendly_name()] = result->input_value(0).get_node_shared_ptr()->get_friendly_name(); - } - - ngraph::pass::ConstantFolding().run_on_function(ng_function); - auto& results = const_cast<::ngraph::ResultVector&>(ng_function->get_results()); - size_t index = results.size() - 1; - for (auto it = results.rbegin(); it != results.rend(); ++it){ - if(auto const_node = std::dynamic_pointer_cast((*it)->input_value(0).get_node_shared_ptr())){ - const_outputs_map[result_to_output.at((*it)->get_friendly_name())] = const_node; - results.erase(results.begin() + index); +#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) + if(!global_context.is_wholly_supported_graph){ + std::map result_to_output; + for(auto& result : ng_function->get_results()){ + result_to_output[result->get_friendly_name()] = result->input_value(0).get_node_shared_ptr()->get_friendly_name(); + } + + ngraph::pass::ConstantFolding().run_on_function(ng_function); + auto& results = const_cast<::ngraph::ResultVector&>(ng_function->get_results()); + size_t index = results.size() - 1; + for (auto it = results.rbegin(); it != results.rend(); ++it){ + if(auto const_node = std::dynamic_pointer_cast((*it)->input_value(0).get_node_shared_ptr())){ + const_outputs_map[result_to_output.at((*it)->get_friendly_name())] = const_node; + results.erase(results.begin() + index); + } + --index; } - --index; } #endif @@ -102,7 +110,7 @@ CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalCont } } -InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type) { +InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type, std::string device) { ONNX_NAMESPACE::DataType type_string = ONNX_NAMESPACE::Utils::DataTypeUtils::ToType(onnx_type); if (*type_string == "float" || *type_string == "tensor(float)") { return InferenceEngine::Precision::FP32; @@ -119,7 +127,11 @@ InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE:: } else if (*type_string == "uint8" || *type_string == "tensor(uint8)") { return InferenceEngine::Precision::U8; } else if (*type_string == "bool" || *type_string == "tensor(bool)") { - return InferenceEngine::Precision::U8; + if (device == "MYRIAD") { + return InferenceEngine::Precision::I32; + } else { + return InferenceEngine::Precision::U8; + } } else if (*type_string == "int64" || *type_string == "tensor(int64)") { return InferenceEngine::Precision::I32; } else { @@ -130,7 +142,8 @@ InferenceEngine::Precision ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE:: void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, std::shared_ptr network, std::unordered_map output_names, - std::map>& const_outputs_map) { + std::map>& const_outputs_map, + std::string device) { // Configure input & output // Prepare input blobs @@ -141,7 +154,7 @@ void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, int input_idx = 0; for (auto iter = inputInfo.begin(); iter != inputInfo.end(); ++iter, ++input_idx) { // Get the onnx index for the corresponding input (ignoring initializers) - auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().input(input_idx).type()); + auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().input(input_idx).type(), device); iter->second->setPrecision(precision); } @@ -149,75 +162,79 @@ void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, auto outputInfo = network->getOutputsInfo(); for (auto iter = outputInfo.begin(); iter != outputInfo.end(); ++iter) { auto output_name = iter->first; -#if defined(OPENVINO_2020_4) +#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) auto it = const_outputs_map.find(output_name); //Output is constant and don't need to set precision if(it != const_outputs_map.end()) break; #endif - auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().output(output_names.at(output_name)).type()); + auto itr = output_names.find(output_name); + if(itr == output_names.end()){ + ORT_THROW(log_tag + "Output Names Mismatch: " + output_name + " doesn't exist"); + } + auto precision = ConvertPrecisionONNXToOpenVINO(model_proto.graph().output(itr->second).type(), device); iter->second->setPrecision(precision); } } -std::vector -GetOutputTensors(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_size, +OrtValue* +GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_size, InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network, - std::unordered_map output_names, std::map> const_output_map) { - std::vector output_tensors; + std::string output_name, + std::unordered_map output_names) { - if(output_names.size() != const_output_map.size()){ - auto graph_output_info = ie_cnn_network->getOutputsInfo(); + OrtValue* output_tensor; - size_t i = 0; - for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter, ++i) { - auto graph_output_blob = infer_request->GetBlob(output_info_iter->first); - auto graph_output_dims = graph_output_blob->getTensorDesc().getDims(); - - if (batch_size > 1) { - // Add the batch size as dim 0. - graph_output_dims.insert(graph_output_dims.begin(), batch_size); - } - size_t num_dims = graph_output_dims.size(); - auto output_shape = new int64_t[num_dims]; - for (size_t j = 0; j < num_dims; j++) { - output_shape[j] = static_cast(graph_output_dims[j]); - } - auto it = output_names.find(output_info_iter->first); - if (it == output_names.end()) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); - } - int index = it->second; - - output_tensors.push_back(ort.KernelContext_GetOutput(context, index, output_shape, num_dims)); - delete output_shape; - } + auto graph_output_blob = infer_request->GetBlob(output_name); + auto graph_output_dims = graph_output_blob->getTensorDesc().getDims(); + if (batch_size > 1) { + // Add the batch size as dim 0. + graph_output_dims.insert(graph_output_dims.begin(), batch_size); } -#if defined(OPENVINO_2020_4) - for(auto item : const_output_map){ - auto it = output_names.find(item.first); - if(it == output_names.end()) { - ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); - } - int index = it->second; - auto node = item.second; - auto shape = node->get_shape(); - - size_t num_dims = shape.size(); - auto output_shape = new int64_t[num_dims]; - for(size_t j = 0; j < num_dims; j++){ - output_shape[j] = static_cast(shape[j]); - } - - output_tensors.push_back(ort.KernelContext_GetOutput(context, index, output_shape, num_dims)); - delete output_shape; + size_t num_dims = graph_output_dims.size(); + auto output_shape = new int64_t[num_dims]; + for (size_t j = 0; j < num_dims; j++) { + output_shape[j] = static_cast(graph_output_dims[j]); } -#endif - return output_tensors; + auto it = output_names.find(output_name); + if (it == output_names.end()) { + ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); + } + int index = it->second; + + output_tensor = ort.KernelContext_GetOutput(context, index, output_shape, num_dims); + delete[] output_shape; + + return output_tensor; } +#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) +OrtValue* +GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context, + std::string output_name, + std::unordered_map output_names, + std::shared_ptr node){ + + OrtValue* output_tensor; + auto it = output_names.find(output_name); + if (it == output_names.end()) { + ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); + } + int index = it->second; + auto shape = node->get_shape(); + + size_t num_dims = shape.size(); + auto output_shape = new int64_t[num_dims]; + for(size_t j = 0; j < num_dims; j++){ + output_shape[j] = static_cast(shape[j]); + } + output_tensor = ort.KernelContext_GetOutput(context, index, output_shape, num_dims); + delete[] output_shape; + + return output_tensor; +} +#endif + int GetFirstAvailableDevice(GlobalContext& global_context){ int i = 0; @@ -242,7 +259,7 @@ int GetFirstAvailableDevice(GlobalContext& global_context){ return i; } -#if defined(OPENVINO_2020_4) +#if (defined OPENVINO_2020_4) || (defined OPENVINO_2021_1) void FillOutputsWithConstantData(Ort::CustomOpApi& ort, std::shared_ptr node, OrtValue* out_tensor){ @@ -274,7 +291,7 @@ void FillOutputsWithConstantData(Ort::CustomOpApi& ort, std::shared_ptr void FillOutputHelper(Ort::CustomOpApi& ort, OrtValue* out_tensor, std::shared_ptr node){ @@ -285,6 +302,73 @@ void FillOutputHelper(Ort::CustomOpApi& ort, OrtValue* out_tensor, std::shared_p } #endif +void FillInputBlob(InferenceEngine::Blob::Ptr& inputBlob, size_t request_id, size_t batch_slice_idx, + std::string input_name, Ort::CustomOpApi& ort, OrtKernelContext* context, + InferenceEngine::Precision precision, const SubGraphContext& subgraph_context){ + + auto minput = InferenceEngine::as(inputBlob); + auto minputHolder = minput->wmap(); + + auto input_data = minputHolder.as::value_type*>(); + size_t input_data_size = inputBlob->byteSize(); + +#if (defined OPENVINO_2020_2) || (defined OPENVINO_2020_3) + const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context.input_indexes[request_id]); +#else + ORT_UNUSED_PARAMETER(request_id); + const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context.input_names.at(input_name)); +#endif + auto tensor_shape = ort.GetTensorTypeAndShape(tensor); + auto elem_type = ort.GetTensorElementType(tensor_shape); + + if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && + (precision == InferenceEngine::Precision::I32)) { + + const int64_t* tensor_data_64 = ort.GetTensorData(tensor); + auto data_len = (input_data_size * 2) / sizeof(int64_t); + const int64_t* batch_memory_offset = tensor_data_64 + data_len * batch_slice_idx; + + std::copy(batch_memory_offset, batch_memory_offset+data_len, (uint32_t*)input_data); + } else { + + // Copy input data into OpenVINO's input buffer + const char* tensor_data = ort.GetTensorData(tensor); + const char* batch_memory_offset = tensor_data + input_data_size * batch_slice_idx; + std::memcpy(input_data, batch_memory_offset, input_data_size); + } +} + +void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, OrtValue* output_tensor, + Ort::CustomOpApi& ort, InferenceEngine::Precision precision, size_t batch_slice_idx){ + + auto moutput = InferenceEngine::as(outputBlob); + + auto moutputHolder = moutput->rmap(); + + const auto output_data = moutputHolder. + as::value_type*>(); + + size_t output_data_size = outputBlob->byteSize(); + auto tensor_shape = ort.GetTensorTypeAndShape(output_tensor); + auto elem_type = ort.GetTensorElementType(tensor_shape); + + if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && + (precision == InferenceEngine::Precision::I32)) { + + int64_t* tensor_data = ort.GetTensorMutableData(output_tensor); + auto data_len = output_data_size/sizeof(int32_t); + int64_t* batch_memory_offset = tensor_data + data_len * batch_slice_idx; + + std::transform((int32_t*)output_data,((int32_t*)output_data) + data_len, batch_memory_offset, static_cast_int64()); + + } else { + char* tensor_data = ort.GetTensorMutableData(output_tensor); + char* batch_memory_offset = tensor_data + output_data_size * batch_slice_idx; + + std::memcpy(batch_memory_offset, output_data, output_data_size); + } +} + } // namespace backend_utils } // namespace openvino_ep } // namespace onnxruntime diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h index c7d87889fc..2efdafeeab 100644 --- a/onnxruntime/core/providers/openvino/backend_utils.h +++ b/onnxruntime/core/providers/openvino/backend_utils.h @@ -20,7 +20,8 @@ bool IsDebugEnabled(); void SetIODefs(const ONNX_NAMESPACE::ModelProto& model_proto, std::shared_ptr network, std::unordered_map output_names, - std::map>& const_outputs_map); + std::map>& const_outputs_map, + std::string device); std::shared_ptr CreateCNNNetwork(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, const SubGraphContext& subgraph_context, std::map node, OrtValue* out_tensor); template void FillOutputHelper(Ort::CustomOpApi& ort, OrtValue* out_tensor, std::shared_ptr node); + +OrtValue* +GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context, + std::string output_name, + std::unordered_map output_names, + std::shared_ptr node); #endif InferenceEngine::Precision -ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type); +ConvertPrecisionONNXToOpenVINO(const ONNX_NAMESPACE::TypeProto& onnx_type, std::string device); -std::vector GetOutputTensors(Ort::CustomOpApi& ort, - OrtKernelContext* context, size_t batch_size, - InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network, - std::unordered_map output_names, std::map> const_output_map); +OrtValue* +GetOutputTensor(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_size, + InferenceEngine::InferRequest::Ptr infer_request, + std::string output_name, + std::unordered_map output_names); + + +void FillInputBlob(InferenceEngine::Blob::Ptr& inputBlob, size_t request_id, size_t batch_slice_idx, + std::string input_name, Ort::CustomOpApi& ort, OrtKernelContext* context, + InferenceEngine::Precision precision, const SubGraphContext& subgraph_context); + +void FillOutputBlob(InferenceEngine::Blob::Ptr& outputBlob, OrtValue* output_tensor, + Ort::CustomOpApi& ort, InferenceEngine::Precision precision, size_t batch_slice_idx); } // namespace backend_utils } // namespace openvino_ep diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index 3e9a832cdb..d89536e688 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -24,23 +24,17 @@ namespace openvino_ep { using namespace backend_utils; - -struct static_cast_int64 -{ - template // T1 models type statically convertible to T - int64_t operator()(const T1& x) const { return static_cast(x); } -}; - BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, GlobalContext& global_context, const SubGraphContext& subgraph_context) : global_context_(global_context), subgraph_context_(subgraph_context) { ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_); - SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_); + SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type); + InferenceEngine::ExecutableNetwork exe_network; -#if defined(OPENVINO_2020_4) +#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) if(const_outputs_map_.size() == subgraph_context_.output_names.size()) subgraph_context_.is_constant = true; #endif @@ -83,20 +77,18 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on // an Infer Request indexed by infer_req_idx -void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, - OrtKernelContext* context, - InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network) { - auto graph_input_info = ie_cnn_network->getInputsInfo(); +void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) { - size_t i = 0; + auto graph_input_info = ie_cnn_network_->getInputsInfo(); + + size_t index = 0; for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter, ++i) { + input_info_iter != graph_input_info.end(); ++input_info_iter, ++index) { // Get OpenVINO's input buffer InferenceEngine::Blob::Ptr graph_input_blob; std::string input_name = input_info_iter->first; try { - graph_input_blob = infer_request->GetBlob(input_name); + graph_input_blob = infer_request_->GetBlob(input_name); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); @@ -104,38 +96,12 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name); } auto precision = input_info_iter->second->getPrecision(); - auto graph_input_buffer = graph_input_blob->buffer() - .as::value_type*>(); - size_t input_data_size = graph_input_blob->byteSize(); - - #if (defined OPENVINO_2020_2) || (defined OPENVINO_2020_3) - const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context_.input_indexes[i]); - #else - const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context_.input_names.at(input_name)); - #endif - - auto tensor_shape = ort.GetTensorTypeAndShape(tensor); - auto elem_type = ort.GetTensorElementType(tensor_shape); - - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - - const int64_t* tensor_data_64 = ort.GetTensorData(tensor); - auto data_len = (input_data_size * 2) / sizeof(int64_t) ; - - std::copy(tensor_data_64, tensor_data_64+data_len, (uint32_t*)graph_input_buffer); - } else { - - // Copy input data into OpenVINO's input buffer - const char* tensor_data = ort.GetTensorData(tensor); - std::memcpy(graph_input_buffer, tensor_data, input_data_size); - - } - + size_t batch_slice = 0; + FillInputBlob(graph_input_blob, index, batch_slice, input_name, ort, context, precision, subgraph_context_); } // Start Async inference try { - infer_request->StartAsync(); + infer_request_->StartAsync(); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Couldn't start Inference: " + e.what()); } catch (...) { @@ -145,64 +111,44 @@ void BasicBackend::StartAsyncInference(Ort::CustomOpApi& ort, // Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx // and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx -void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, - std::vector output_tensors, - InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network) { +void BasicBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context) { // Wait for Async inference completion try { - infer_request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); + infer_request_->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); } catch (InferenceEngine::details::InferenceEngineException e) { ORT_THROW(log_tag + " Exception with completing Inference: " + e.what()); } catch (...) { ORT_THROW(log_tag + " Exception with completing Inference"); } - auto graph_output_info = ie_cnn_network->getOutputsInfo(); + auto graph_output_info = ie_cnn_network_->getOutputsInfo(); - size_t i = 0; for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter, ++i) { + output_info_iter != graph_output_info.end(); ++output_info_iter) { // Get OpenVINO's output blob InferenceEngine::Blob::Ptr graph_output_blob; + auto output_name = output_info_iter->first; try { - graph_output_blob = infer_request->GetBlob(output_info_iter->first); + graph_output_blob = infer_request_->GetBlob(output_name); } catch (InferenceEngine::details::InferenceEngineException e) { - ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_info_iter->first + e.what()); + ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what()); } catch (...) { - ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_info_iter->first); + ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name); } - - auto graph_output_buffer = graph_output_blob->buffer() - .as::value_type*>(); - - size_t output_data_size = graph_output_blob->byteSize(); - - auto tensor_shape = ort.GetTensorTypeAndShape(output_tensors[i]); - auto elem_type = ort.GetTensorElementType(tensor_shape); + size_t batch_size = 1; + auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request_, output_name, subgraph_context_.output_names); auto precision = output_info_iter->second->getPrecision(); - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - - int64_t* tensor_data = ort.GetTensorMutableData(output_tensors[i]); - - auto data_len = output_data_size/sizeof(int32_t); - std::transform((int32_t*)graph_output_buffer,((int32_t*)graph_output_buffer) + data_len, tensor_data, static_cast_int64()); - - } else { - char* tensor_data = ort.GetTensorMutableData(output_tensors[i]); - std::memcpy(tensor_data, graph_output_buffer, output_data_size); - - } + size_t batch_slice = 0; + FillOutputBlob(graph_output_blob, output_tensor, ort, precision, batch_slice); } -#if defined(OPENVINO_2020_4) +#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) if(!const_outputs_map_.empty()){ - size_t j = i; for(auto item : const_outputs_map_){ + auto out_name = item.first; auto node = item.second; - FillOutputsWithConstantData(ort,node,output_tensors[j]); - j++; + auto output_tensor = GetOutputTensor(ort, context, out_name, subgraph_context_.output_names, node); + FillOutputsWithConstantData(ort,node,output_tensor); } } #endif @@ -216,21 +162,19 @@ void BasicBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) { LOGS_DEFAULT(INFO) << log_tag << "In Infer"; std::lock_guard lock(compute_lock_); - size_t batch_size = 1; - auto output_tensors = GetOutputTensors(ort, context, batch_size, infer_request_, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_); if(subgraph_context_.is_constant){ -#if defined(OPENVINO_2020_4) - size_t i = 0; +#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) for(auto item : const_outputs_map_){ + auto out_name = item.first; auto node = item.second; - FillOutputsWithConstantData(ort,node, output_tensors[i]); - i++; + auto output_tensor = GetOutputTensor(ort, context, out_name, subgraph_context_.output_names, node); + FillOutputsWithConstantData(ort,node, output_tensor); } #endif } else{ - StartAsyncInference(ort, context, infer_request_, ie_cnn_network_); - CompleteAsyncInference(ort, output_tensors, infer_request_, ie_cnn_network_); + StartAsyncInference(ort, context); + CompleteAsyncInference(ort, context); } // Get Output tensors LOGS_DEFAULT(INFO) << log_tag << "Inference successful"; diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h index 924ad39f0f..8782f51271 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.h +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h @@ -22,13 +22,9 @@ class BasicBackend : public IBackend { void Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) override; private: - void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, - InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network); + void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context); - void CompleteAsyncInference(Ort::CustomOpApi& ort, std::vector output_tensors, - InferenceEngine::InferRequest::Ptr infer_request, - std::shared_ptr ie_cnn_network); + void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context); GlobalContext& global_context_; SubGraphContext subgraph_context_; diff --git a/onnxruntime/core/providers/openvino/backends/vadm_backend.cc b/onnxruntime/core/providers/openvino/backends/vadm_backend.cc index d04827600c..7c820e13a0 100644 --- a/onnxruntime/core/providers/openvino/backends/vadm_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/vadm_backend.cc @@ -1,6 +1,5 @@ // Copyright(C) 2019 Intel Corporation // Licensed under the MIT License - #include #include #include @@ -16,19 +15,17 @@ #include "../contexts.h" #include "../backend_utils.h" #include "vadm_backend.h" +#if defined(OPENVINO_2021_1) +#include +#else #include +#endif namespace onnxruntime { namespace openvino_ep { using namespace backend_utils; -struct static_cast_int64 -{ - template // T1 models type statically convertible to T - int64_t operator()(const T1& x) const { return static_cast(x); } -}; - VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, GlobalContext& global_context, const SubGraphContext& subgraph_context) @@ -49,7 +46,7 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, ie_cnn_network_ = CreateCNNNetwork(model_proto, global_context_, subgraph_context_, const_outputs_map_); - SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_); + SetIODefs(model_proto, ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_, global_context_.device_type); std::map config; #if defined(OPENVINO_2020_4) @@ -66,7 +63,11 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, if(global_context_.is_wholly_supported_graph && subgraph_context_.enable_batching){ for(int j = 0; j < 8; j++){ InferenceEngine::ExecutableNetwork exe_network; + #if defined(OPENVINO_2021_1) + config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[j]; + #else config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[j]; + #endif try { exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, "HDDL", config); } catch (InferenceEngine::details::InferenceEngineException e) { @@ -95,7 +96,11 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, else { i = GetFirstAvailableDevice(global_context); LOGS_DEFAULT(INFO) << log_tag << "Device Tag is: " << i; + #if defined(OPENVINO_2021_1) + config[InferenceEngine::HDDL_DEVICE_TAG] = global_context_.deviceTags[i]; + #else config[VPU_HDDL_CONFIG_KEY(DEVICE_TAG)] = global_context_.deviceTags[i]; + #endif InferenceEngine::ExecutableNetwork exe_network; try { exe_network = global_context_.ie_core.LoadNetwork(*ie_cnn_network_, "HDDL", config); @@ -121,15 +126,13 @@ VADMBackend::VADMBackend(const ONNX_NAMESPACE::ModelProto& model_proto, // Starts an asynchronous inference request for data in slice indexed by batch_slice_idx on // an Infer Request indexed by infer_req_idx void VADMBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, - size_t batch_slice_idx, size_t infer_req_idx, - std::vector& infer_requests, - std::shared_ptr ie_cnn_network) { - auto infer_request = infer_requests[infer_req_idx]; - auto graph_input_info = ie_cnn_network->getInputsInfo(); + size_t batch_slice_idx, size_t infer_req_idx) { + auto infer_request = infer_requests_[infer_req_idx]; + auto graph_input_info = ie_cnn_network_->getInputsInfo(); - size_t i = 0; + size_t index = 0; for (auto input_info_iter = graph_input_info.begin(); - input_info_iter != graph_input_info.end(); ++input_info_iter, ++i) { + input_info_iter != graph_input_info.end(); ++input_info_iter, ++index) { // Get OpenVINO's input buffer InferenceEngine::Blob::Ptr graph_input_blob; std::string input_name = input_info_iter->first; @@ -141,35 +144,7 @@ void VADMBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* c ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name); } auto precision = input_info_iter->second->getPrecision(); - auto graph_input_buffer = - graph_input_blob->buffer().as::value_type*>(); - - #if (defined OPENVINO_2020_2) || (defined OPENVINO_2020_3) - const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context_.input_indexes[i]); - #else - const OrtValue* tensor = ort.KernelContext_GetInput(context, subgraph_context_.input_names.at(input_name)); - #endif - - size_t input_data_size = graph_input_blob->byteSize(); - auto tensor_shape = ort.GetTensorTypeAndShape(tensor); - auto elem_type = ort.GetTensorElementType(tensor_shape); - - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - - const int64_t* tensor_data_64 = ort.GetTensorData(tensor); - auto data_len = (input_data_size * 2) / sizeof(int64_t); - const int64_t* batch_memory_offset = tensor_data_64 + data_len * batch_slice_idx; - - std::copy(batch_memory_offset, batch_memory_offset+data_len, (uint32_t*)graph_input_buffer); - } else { - - // Copy input data into OpenVINO's input buffer - const char* tensor_data = ort.GetTensorData(tensor); - const char* batch_memory_offset = tensor_data + input_data_size * batch_slice_idx; - - std::memcpy(graph_input_buffer, batch_memory_offset, input_data_size); - } + FillInputBlob(graph_input_blob, index, batch_slice_idx, input_name, ort, context, precision, subgraph_context_); } // Start Async inference @@ -184,11 +159,11 @@ void VADMBackend::StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* c // Wait for asynchronous inference completion on an Infer Request object indexed by infer_req_idx // and copy the results into a slice location within the batched output buffer indexed by batch_slice_idx -void VADMBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, std::vector output_tensors, - size_t batch_slice_idx, - size_t infer_req_idx, std::vector& infer_requests, - std::shared_ptr ie_cnn_network) { - auto infer_request = infer_requests[infer_req_idx]; +void VADMBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, + size_t batch_slice_idx, size_t infer_req_idx, + size_t batch_size) { + + auto infer_request = infer_requests_[infer_req_idx]; // Wait for Async inference completion try { @@ -198,53 +173,34 @@ void VADMBackend::CompleteAsyncInference(Ort::CustomOpApi& ort, std::vectorgetOutputsInfo(); + auto graph_output_info = ie_cnn_network_->getOutputsInfo(); - size_t i = 0; for (auto output_info_iter = graph_output_info.begin(); - output_info_iter != graph_output_info.end(); ++output_info_iter, ++i) { + output_info_iter != graph_output_info.end(); ++output_info_iter) { // Get OpenVINO's output blob InferenceEngine::Blob::Ptr graph_output_blob; + auto output_name = output_info_iter->first; try { - graph_output_blob = infer_request->GetBlob(output_info_iter->first); + graph_output_blob = infer_request->GetBlob(output_name); } catch (InferenceEngine::details::InferenceEngineException e) { - ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_info_iter->first + e.what()); + ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name + e.what()); } catch (...) { - ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_info_iter->first); + ORT_THROW(log_tag + " Cannot access IE Blob for output: " + output_name); } - auto graph_output_buffer = - graph_output_blob->buffer().as::value_type*>(); - size_t output_data_size = graph_output_blob->byteSize(); - auto tensor_shape = ort.GetTensorTypeAndShape(output_tensors[i]); - auto elem_type = ort.GetTensorElementType(tensor_shape); + auto output_tensor = GetOutputTensor(ort, context, batch_size, infer_request, output_name, subgraph_context_.output_names); auto precision = output_info_iter->second->getPrecision(); - if ((elem_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) && - (precision == InferenceEngine::Precision::I32)) { - - int64_t* tensor_data = ort.GetTensorMutableData(output_tensors[i]); - auto data_len = output_data_size/sizeof(int32_t); - int64_t* batch_memory_offset = tensor_data + data_len * batch_slice_idx; - - std::transform((int32_t*)graph_output_buffer,((int32_t*)graph_output_buffer) + data_len, batch_memory_offset, static_cast_int64()); - - } else { - char* tensor_data = ort.GetTensorMutableData(output_tensors[i]); - char* batch_memory_offset = tensor_data + output_data_size * batch_slice_idx; - - // Copy output results back to ONNX-RT's output buffers - std::memcpy(batch_memory_offset, graph_output_buffer, output_data_size); - } + FillOutputBlob(graph_output_blob, output_tensor, ort, precision, batch_slice_idx); } #if defined(OPENVINO_2020_4) if(!const_outputs_map_.empty()){ - size_t j = i; for(auto item : const_outputs_map_){ + auto out_name = item.first; auto node = item.second; - FillOutputsWithConstantData(ort,node,output_tensors[j]); - j++; + auto output_tensor = GetOutputTensor(ort, context, out_name, subgraph_context_.output_names, node); + FillOutputsWithConstantData(ort,node,output_tensor); } } #endif @@ -292,17 +248,13 @@ void VADMBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) { size_t full_parallel_runs = batch_size / num_inf_reqs_; size_t remainder_parallel_runs = batch_size % num_inf_reqs_; - // All infer_requests process identical tensor slices from the batch. - // So using info from first infer_request to allocate all output tensors. - auto output_tensors = GetOutputTensors(ort, context, batch_size, infer_requests_[0], ie_cnn_network_, subgraph_context_.output_names, const_outputs_map_); - if(subgraph_context_.is_constant){ -#if defined(OPENVINO_2020_4) - size_t i = 0; +#if defined(OPENVINO_2020_4) || defined(OPENVINO_2021_1) for(auto item : const_outputs_map_){ + auto out_name = item.first; auto node = item.second; - FillOutputsWithConstantData(ort,node, output_tensors[i]); - i++; + auto output_tensor = GetOutputTensor(ort, context, out_name, subgraph_context_.output_names, node); + FillOutputsWithConstantData(ort,node, output_tensor); } #endif } @@ -314,22 +266,22 @@ void VADMBackend::Infer(Ort::CustomOpApi& ort, OrtKernelContext* context) { for (size_t set = 0; set < full_parallel_runs; set++) { for (size_t inf_req_idx = 0; inf_req_idx < num_inf_reqs_; inf_req_idx++) { size_t batch_slice_idx = set * num_inf_reqs_ + inf_req_idx; - StartAsyncInference(ort, context, batch_slice_idx, inf_req_idx, infer_requests_, ie_cnn_network_); + StartAsyncInference(ort, context, batch_slice_idx, inf_req_idx); } for (size_t inf_req_idx = 0; inf_req_idx < num_inf_reqs_; inf_req_idx++) { size_t batch_slice_idx = set * num_inf_reqs_ + inf_req_idx; - CompleteAsyncInference(ort, output_tensors, batch_slice_idx, inf_req_idx, infer_requests_, ie_cnn_network_); + CompleteAsyncInference(ort, context, batch_slice_idx, inf_req_idx, batch_size); } } // Run parallel inferences for remaining batch slices for (size_t inf_req_idx = 0; inf_req_idx < remainder_parallel_runs; inf_req_idx++) { size_t batch_slice_idx = full_parallel_runs * num_inf_reqs_ + inf_req_idx; - StartAsyncInference(ort, context, batch_slice_idx, inf_req_idx, infer_requests_, ie_cnn_network_); + StartAsyncInference(ort, context, batch_slice_idx, inf_req_idx); } for (size_t inf_req_idx = 0; inf_req_idx < remainder_parallel_runs; inf_req_idx++) { size_t batch_slice_idx = full_parallel_runs * num_inf_reqs_ + inf_req_idx; - CompleteAsyncInference(ort, output_tensors, batch_slice_idx, inf_req_idx, infer_requests_, ie_cnn_network_); + CompleteAsyncInference(ort, context, batch_slice_idx, inf_req_idx, batch_size); } } LOGS_DEFAULT(INFO) << log_tag << "Inference successful"; diff --git a/onnxruntime/core/providers/openvino/backends/vadm_backend.h b/onnxruntime/core/providers/openvino/backends/vadm_backend.h index 93bb45a534..67cd676003 100644 --- a/onnxruntime/core/providers/openvino/backends/vadm_backend.h +++ b/onnxruntime/core/providers/openvino/backends/vadm_backend.h @@ -1,6 +1,5 @@ // Copyright(C) 2019 Intel Corporation // Licensed under the MIT License - #pragma once #include @@ -24,14 +23,11 @@ class VADMBackend : public IBackend { private: void StartAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, - size_t batch_slice_idx, size_t infer_req_idx, - std::vector& infer_requests, - std::shared_ptr ie_cnn_network); + size_t batch_slice_idx, size_t infer_req_idx); - void CompleteAsyncInference(Ort::CustomOpApi& ort, std::vector output_tensors, + void CompleteAsyncInference(Ort::CustomOpApi& ort, OrtKernelContext* context, size_t batch_slice_idx, size_t infer_req_idx, - std::vector& infer_requests, - std::shared_ptr ie_cnn_network); + size_t batch_size); GlobalContext& global_context_; SubGraphContext subgraph_context_; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 1bc7c6438f..4bcf42f7ad 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -64,6 +64,9 @@ OpenVINOExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_v #elif defined OPENVINO_2020_4 result = openvino_ep::GetCapability_2020_4(graph_viewer, openvino_ep::BackendManager::GetGlobalContext().device_type); +#elif defined OPENVINO_2021_1 + result = openvino_ep::GetCapability_2021_1(graph_viewer, + openvino_ep::BackendManager::GetGlobalContext().device_type); #endif return result; diff --git a/onnxruntime/core/providers/openvino/ov_versions/capabilities.h b/onnxruntime/core/providers/openvino/ov_versions/capabilities.h index 3f9c17ab80..113c88593a 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/capabilities.h +++ b/onnxruntime/core/providers/openvino/ov_versions/capabilities.h @@ -13,6 +13,10 @@ GetCapability_2020_2(const onnxruntime::GraphViewer& graph_viewer, const std::st std::vector> GetCapability_2020_4(const onnxruntime::GraphViewer& graph_viewer, const std::string device_type); +#elif defined OPENVINO_2021_1 +std::vector> +GetCapability_2021_1(const onnxruntime::GraphViewer& graph_viewer, const std::string device_id); + #endif } //namespace openvino_ep diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability_2021_1.cc b/onnxruntime/core/providers/openvino/ov_versions/capability_2021_1.cc new file mode 100644 index 0000000000..a54c52344d --- /dev/null +++ b/onnxruntime/core/providers/openvino/ov_versions/capability_2021_1.cc @@ -0,0 +1,913 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#if defined OPENVINO_2021_1 + +#include "core/framework/compute_capability.h" +#include "core/framework/tensorprotoutils.h" +#include "core/graph/graph_viewer.h" +#include "core/graph/model.h" +#include "core/graph/graph_utils.h" +#include "../backend_utils.h" +#include "../backend_manager.h" +#include "capabilities.h" +#include "utils.h" + +#if defined(_MSC_VER) +#pragma warning(disable : 4244 4245 5208) +#elif __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include +#include +#if defined(_MSC_VER) +#pragma warning(default : 4244 4245) +#elif __GNUC__ +#pragma GCC diagnostic pop +#endif + +namespace onnxruntime { +namespace openvino_ep { + +bool IsDimensionSupported(const Node* node) { + auto node_inputs = node->InputDefs(); + size_t input_dims = 0; + if (node_inputs[0]->Shape() == nullptr) { + return true; + } else { + input_dims = node_inputs[0]->Shape()->dim_size(); + if (node->OpType().find("Pool") != std::string::npos) { + if (input_dims != 4 && input_dims != 5) + return false; + } + + if (node->OpType() == "Unsqueeze") { + auto attributes = node->GetAttributes(); + auto axes = attributes["axes"].ints(); + if (input_dims + axes.size() > 5) + return false; + } + + } + return true; +} + +//Ops which are not supported by OpenVINO EP +bool IsOpSupported(std::string name, std::string device) { + std::set common_supported_ops = { + "Add", + "And", + "AveragePool", + "BatchNormalization", + "Cast", + "Clip", + "Concat", + "Constant", + "ConstantOfShape", + "Conv", + "ConvTranspose", + "DepthToSpace", + "Div", + "Dropout", + "Elu", + "Equal", + "Erf", + "Exp", + "Flatten", + "Floor", + "Gather", + "Gemm", + "GlobalAveragePool", + "Greater", + "Identity", + "InstanceNormalization", + "LeakyRelu", + "Less", + "Log", + "LRN", + "LSTM", + "MatMul", + "Max", + "MaxPool", + "Mean", + "Min", + "Mul", + "Neg", + "OneHot", + "Pad", + "Pow", + "PRelu", + "Reciprocal", + "ReduceMax", + "ReduceMean", + "ReduceMin", + "ReduceSum", + "Relu", + "Reshape", + "Shape", + "Sigmoid", + "Slice", + "Softmax", + "SpaceToDepth", + "Split", + "Sqrt", + "Squeeze", + "Sub", + "Sum", + "Tanh", + "TopK", + "Transpose", + "Unsqueeze", + }; + + std::set supported_ops_cpu = { + "Abs", + "Acos", + "Acosh", + "ArgMax", + "ArgMin", + "Asin", + "Asinh", + "Atan", + "Atanh", + "Cos", + "Cosh", + "GlobalLpPool", + "HardSigmoid", + "Not", + "ReduceLogSum", + "ReduceProd", + "ReduceSumSquare", + "Resize", + "Selu", + "Sign", + "Sinh", + "Softsign", + "Tan", + "NonZero", + "Upsample" + }; + + + std::set supported_ops_gpu = { + "Abs", + "Asin", + "Asinh", + "Atan", + "Ceil", + "GlobalLpPool", + "HardSigmoid", + "Not", + "Selu", + "Tan", + }; + std::set supported_ops_vpu = { + "Expand", + "NonMaxSuppression", + "NonZero", + "ReduceLogSum", + "ReduceSumSquare", + "Resize", + "RoiAlign", + "Scatter", + "SinFloat", + }; + + std::set supported_ops = {}; + + if (device == "CPU") { + std::merge(common_supported_ops.begin(), common_supported_ops.end(), + supported_ops_cpu.begin(), supported_ops_cpu.end(), + std::inserter(supported_ops,supported_ops.begin())); + } else if (device == "GPU") { + std::merge(common_supported_ops.begin(), common_supported_ops.end(), + supported_ops_gpu.begin(), supported_ops_gpu.end(), + std::inserter(supported_ops, supported_ops.begin())); + } else if (device == "MYRIAD" || device == "HDDL") { + std::merge(common_supported_ops.begin(), common_supported_ops.end(), + supported_ops_vpu.begin(), supported_ops_vpu.end(), + std::inserter(supported_ops, supported_ops.begin())); + } + + return supported_ops.find(name) != supported_ops.end(); +} + +// Returns true only if op is in a mode that is not currently supported +static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer& graph_viewer, const std::string& device_id) { + const auto& optype = node->OpType(); + + const auto& initializers = graph_viewer.GetAllInitializedTensors(); + + if (optype == "MaxPool") { + //MaxPool "indices" output is not currently supported. + if (node->OutputDefs().size() > 1) { + return true; + } + + const auto& attributes = node->GetAttributes(); + + const auto ceil_attr = attributes.find("ceil_mode"); + // default value of ceil_mode (0) is supported. + if (ceil_attr != attributes.end() && ceil_attr->second.i() != 0) { + return true; + } + + //auto pad null value is not supported + const auto auto_attr = attributes.find("auto_pad"); + if (auto_attr->second.s() == "") { + return true; + } + // dilations attrs are not supported in nGraph + if (attributes.find("dilations") != attributes.end()) { + return true; + } + if (!IsDimensionSupported(node)) + return true; + } else if (optype == "Abs") { + for (size_t i = 0; i < node->InputDefs().size(); i++) { + if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) + return true; + } + } else if (optype == "Max" || optype == "Min" || optype == "Mean" || optype == "Sum") { + if (GetInputCount(node, initializers) == 1) + return true; + if (optype == "Max" || optype == "Min") { + for (size_t i = 0; i < node->InputDefs().size(); i++) { + auto dtype = node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type(); + if (dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8 || + dtype == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16) + return true; + } + } + } else if (optype == "Clip") { + //Only float 16, float and double data types are supported + const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool data_is_float16 = node->InputDefs()[0]->Type()->find("float16") != std::string::npos; + const bool data_is_double = node->InputDefs()[0]->Type()->find("double") != std::string::npos; + return !(data_is_float || data_is_float16 || data_is_double); + } else if (optype == "Conv" || optype == "ConvTranspose") { + if (GetInputCount(node, initializers) > 1) + return true; + auto attributes = node->GetAttributes(); + if (attributes["auto_pad"].s() == "") { + return true; + } + } else if (optype == "ReduceMin") { + //Only FP32, INT32 and U8 data types are supported + const bool data_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool data_is_int32 = node->InputDefs()[0]->Type()->find("int32") != std::string::npos; + const bool data_is_u8 = node->InputDefs()[0]->Type()->find("uint8") != std::string::npos; + return !(data_is_float || data_is_int32 || data_is_u8); + } else if (optype == "MatMul") { + //All matmuls except float have computation missmatch + const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; + return (A_is_float && B_is_float) ? false : true; + + } else if (optype == "Pow") { + //Only supported if the data type of both inputs is same + auto x_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + auto y_data_type = node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + return x_data_type != y_data_type; + } else if (optype == "PRelu") { + auto slope = node->InputDefs()[1]; + + //PRelu slope has to be an initializer or needs to come from a constant node + if (initializers.count(slope->Name())) + return false; + else { + for (auto input_node = node->InputNodesBegin(); input_node != node->InputNodesEnd(); ++input_node) { + if (GetInputCount(graph_viewer.GetNode((*input_node).Index()), initializers) == 0) { + return false; + } + } + } + return true; + } else if (optype == "Identity") { + const auto& input = node->InputDefs()[0]; + const auto& output = node->OutputDefs()[0]; + auto graph_inputs = graph_viewer.GetInputs(); + auto graph_outputs = graph_viewer.GetOutputs(); + auto input_it = find(graph_inputs.begin(), graph_inputs.end(), input); + auto output_it = find(graph_outputs.begin(), graph_outputs.end(), output); + if(input_it != graph_inputs.end() && output_it != graph_outputs.end()) + return true; + } else if (optype == "Resize") { + //Resize opset 11 is not supported + if(node->InputDefs().size() > 2) + return true; + } else if (optype == "Unsqueeze") { + if (!IsDimensionSupported(node)) + return true; + } else if (optype == "Mod") { + //Only fmod=1 is supported + auto attributes = node->GetAttributes(); + auto fmod = attributes["fmod"].i(); + if (fmod != 1) + return true; + //Only FP32 data type is allowed + for (const auto& input : node->InputDefs()) { + if (input->Type()->find("float") == std::string::npos) + return true; + } + } else if (optype == "Squeeze") { + //Shape can't have empty axes attribute + const auto& attributes = node->GetAttributes(); + if (attributes.count("axes") == 0) + return true; + } else if (optype == "Slice") { + //start, end, axes need to be a initializer + const auto &data_arg = node->InputDefs()[0]; + auto graph_inputs = graph_viewer.GetInputs(); + bool cond_for_slice = false; + + auto it = find(graph_inputs.begin(), graph_inputs.end(), data_arg); + if(it != graph_inputs.end()){ + if(node->InputDefs().size() > 1){ + const auto &start_arg = node->InputDefs()[1]; + const auto &end_arg = node->InputDefs()[2]; + cond_for_slice |= initializers.find(start_arg->Name()) == initializers.end(); + cond_for_slice |= initializers.find(end_arg->Name()) == initializers.end(); + } + if (node->InputDefs().size() > 3) { + const auto &axes_arg = node->InputDefs()[3]; + cond_for_slice |= initializers.find(axes_arg->Name()) == initializers.end(); + } + } + + return cond_for_slice; + } else if (optype == "AveragePool") { + // ceil_mode attribute is not supported in nGraph + const auto& attributes = node->GetAttributes(); + //auto pad null value is not supported + const auto auto_attr = attributes.find("auto_pad"); + if (auto_attr->second.s() == "") { + return true; + } + const auto ceil_attr = attributes.find("ceil_mode"); + // default value of ceil_mode (0) is supported. + if (ceil_attr != attributes.end() && ceil_attr->second.i() != 0) { + return true; + } + if (!IsDimensionSupported(node)) + return true; + } else if (optype == "QLinearMatMul") { + const auto& a_zero_point = node->InputDefs()[2]; + const auto& b_zero_point = node->InputDefs()[5]; + const auto& y_zero_point = node->InputDefs()[7]; + + bool non_const_zero_point = false; + + // check if any of the zero points is NOT in the initializers list + non_const_zero_point |= initializers.find(a_zero_point->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(b_zero_point->Name()) == initializers.end(); + non_const_zero_point |= initializers.find(y_zero_point->Name()) == initializers.end(); + + // QLinearMatMul is not supported if any of the zero points is a dynamic input + return non_const_zero_point; + } else if (optype == "MatMulInteger") { + // all MatMulInteger zero points need to be constants + const auto inputs = node->InputDefs(); + if (inputs.size() == 3) { + const auto& a_zero_point = node->InputDefs()[2]; + + // not found in initializers -> not const + return initializers.find(a_zero_point->Name()) == initializers.end(); + } else if (inputs.size() == 4) { + const auto& a_zero_point = node->InputDefs()[2]; + const auto& b_zero_point = node->InputDefs()[3]; + + // not found in initializers -> not const + return initializers.find(a_zero_point->Name()) == initializers.end() || + initializers.find(b_zero_point->Name()) == initializers.end(); + } // else -> azp & bzp are 0 by default according to ONNX spec + } else if (optype == "ConvInteger") { + // all ConvInteger zero points need to be constants + const auto inputs = node->InputDefs(); + if (inputs.size() == 3) { + const auto& x_zero_point = node->InputDefs()[2]; + + // not found in initializers -> not const + return initializers.find(x_zero_point->Name()) == initializers.end(); + } else if (inputs.size() == 4) { + const auto& x_zero_point = node->InputDefs()[2]; + const auto& w_zero_point = node->InputDefs()[3]; + + // not found in initializers -> not const + return initializers.find(x_zero_point->Name()) == initializers.end() || + initializers.find(w_zero_point->Name()) == initializers.end(); + } // else -> xzp & wzp are 0 by default according to ONNX spec + } else if (optype == "ArgMax" || optype == "ArgMin") { + //tensor type does not support select last index + auto attributes = node->GetAttributes(); + auto last_index_arg = attributes["select_last_index"].i(); + if (last_index_arg != 0) + return true; + // tensor type supports float as input for argmax and argmin + auto dtype = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + if (dtype != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) { + return true; + } +} else if ((optype == "Equal") || (optype == "And")) { + + using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; + auto supportedOps = std::set>{ + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_FLOAT }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_INT8 }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8 }, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8 }, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8 }, + {onnx_dtype::TensorProto_DataType_INT8, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT8 }, + {onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32 }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT }, + {onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_UINT8 }}; + + if (optype == "Equal") { + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT32, onnx_dtype::TensorProto_DataType_INT32 }), + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_INT64, onnx_dtype::TensorProto_DataType_INT64 }); + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_BOOL, onnx_dtype::TensorProto_DataType_INT64, onnx_dtype::TensorProto_DataType_INT64 }), + supportedOps.insert(std::vector{onnx_dtype::TensorProto_DataType_UINT8, onnx_dtype::TensorProto_DataType_FLOAT, onnx_dtype::TensorProto_DataType_FLOAT }); + } + + onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + + const std::vector typePair{output_data_type, input_0_data_type, input_1_data_type}; + const auto match = supportedOps.find(typePair); + if (match == supportedOps.end()) { + return true; + } else + return false; + } else if(optype == "Gather") { + + if(device_id == "GPU"){ + const auto& input = node->InputDefs()[0]; + auto graph_inputs = graph_viewer.GetInputs(); + auto it = find(graph_inputs.begin(), graph_inputs.end(), input); + if(it != graph_inputs.end()){ + const auto &indices_arg = node->InputDefs()[1]; + if (indices_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64) + return true; + } + } + } else if(optype == "Upsample") { + + //check for attributes + auto upsample_attr = node->GetAttributes(); + auto upsample_arg = upsample_attr["scales"]; + auto float_size = upsample_arg.floats_size(); + if (float_size > 2 && (upsample_arg.floats(0) != 1.f || upsample_arg.floats(1) != 1.f)) + return true; + + //check for input dimensions + const auto &x_arg = node->InputDefs()[0]; + + auto shape = x_arg->Shape(); + if (shape != nullptr) { + //input tensor rank cannot be of one dimension + if (shape->dim_size() == 1) { + return true; + } + } + // x_arg supports only float, int8 and float16 type + if ((x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || + (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) || + (x_arg->TypeAsProto()->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { + return false; + } else { + return true; + } + } + //Op doesn't fall into known any of unsupported modes. + return false; +} + +static bool IsTypeSupported(const NodeArg* node_arg, bool is_initializer, const std::string& device_id) { + const auto* type_proto = node_arg->TypeAsProto(); + if (!type_proto) { + return false; + } + + if (is_initializer) { + switch (type_proto->tensor_type().elem_type()) { + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64: + return true; + default: + +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Initializer Data Type is not supported" << std::endl; + } +#endif + return false; + } + } else { + std::set supported_types_cpu = { + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64, + }; + + std::set supported_types_gpu = { + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32, + ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64, + }; + auto dtype = type_proto->tensor_type().elem_type(); + + if (device_id == "CPU" || device_id == "MYRIAD" || device_id == "HDDL") { + if (supported_types_cpu.find(dtype) != supported_types_cpu.end()) + return true; + else { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "I/O data type is not supported" << std::endl; + } +#endif + return false; + } + } else if (device_id == "GPU") { + if (supported_types_gpu.find(dtype) != supported_types_gpu.end()) + return true; + else { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "I/O data type is not supported" << std::endl; + } +#endif + return false; + } + } + return true; + } +} + +static bool IsNodeSupported(const std::map>& op_map, + const onnxruntime::GraphViewer& graph_viewer, + const NodeIndex node_idx, std::string& device_id) { + const auto& node = graph_viewer.GetNode(node_idx); + const auto& optype = node->OpType(); + +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Node " << optype << std::endl; + } +#endif + + const auto& domain = node->Domain(); + + /* + 0. Check if node is in the unsupported list + 1. Check input and output data types are supported. + 2. Check if there is unsupported dimension in input and output shapes + 3. Check Op is supported + 3a. Check if Op is of known unsupported modes (edge cases). If yes return false right away. + 3b. If above is not true, check if the op is available in nGraph. + */ + + //Check 0 + if (!IsOpSupported(optype, device_id)) { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Node is not in the supported ops list" << std::endl; + } +#endif + return false; + } + + //Check 1 + bool are_types_supported = true; + + node->ForEachDef([&are_types_supported, &graph_viewer, &device_id](const onnxruntime::NodeArg& node_arg, bool is_input) { + bool is_initializer = false; + if (is_input) { + if (graph_viewer.IsConstantInitializer(node_arg.Name(), true)) + is_initializer = true; + } + are_types_supported &= IsTypeSupported(&node_arg, is_initializer, device_id); + }); + + if (!are_types_supported) { + return false; + } + + //Check 2 + + bool has_unsupported_dimension = false; + node->ForEachDef([&has_unsupported_dimension, &graph_viewer, &device_id, &optype](const onnxruntime::NodeArg& node_arg, bool is_input) { + if (is_input) { + if (graph_viewer.IsConstantInitializer(node_arg.Name(), true)) + return; + } + auto shape = node_arg.Shape(); + if (shape != nullptr) { + //Can't have no dimensions + if (shape->dim_size() == 0) { + if(optype == "Unsqueeze" || optype == "Squeeze" || optype == "Cast" || + optype == "Gather" || optype == "Mul" || optype == "Sub" || + optype == "Min" || optype == "Div" || optype == "Floor") + return; + has_unsupported_dimension = true; + return; + } else { + //Zero dimension check + for (const auto& dim : shape->dim()) { + if (utils::HasDimValue(dim) && dim.dim_value() == 0) { + has_unsupported_dimension = true; + return; + } + } + } + } + }); + if (has_unsupported_dimension) { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Dimension check failed" << std::endl; + } +#endif + + return false; + } + + //Check 3a + if (domain == kOnnxDomain && IsUnsupportedOpMode(node, graph_viewer, device_id)) { +#ifndef NDEBUG + if (openvino_ep::backend_utils::IsDebugEnabled()) { + std::cout << "Failed in unsupported op mode" << std::endl; + } +#endif + + return false; + } + + //Check 3b + const auto opset = op_map.find(domain); + if (opset == op_map.end() || opset->second.find(optype) == opset->second.end()) { + return false; + } else { + return true; + } +} + +static std::vector +GetUnsupportedNodeIndices(const GraphViewer& graph_viewer, std::string device, /*out*/ std::unordered_set& ng_required_initializers) { + const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer)); + + std::vector unsupported_nodes_idx; + + for (const auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) { + if (IsNodeSupported(ng_supported_ops, graph_viewer, node_idx, device)) { + // Collect inputs that are initializers + graph_viewer.GetNode(node_idx)->ForEachDef([&ng_required_initializers, &graph_viewer](const onnxruntime::NodeArg& node_arg, bool is_input) { + if(is_input && graph_viewer.GetAllInitializedTensors().count(node_arg.Name())) { + ng_required_initializers.insert(node_arg.Name()); + } }, true); + } else { + unsupported_nodes_idx.push_back(node_idx); + } + } + + return unsupported_nodes_idx; +} + + +std::vector> +GetCapability_2021_1(const onnxruntime::GraphViewer& graph_viewer, std::string device_id) { + + std::vector> result; + if (graph_viewer.IsSubgraph()) { + return result; + } + + // Need access to model_path_ + for (const auto& tensor : graph_viewer.GetAllInitializedTensors()) { + if (tensor.second->has_data_location() && tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(WARNING) << "[OpenVINO-EP] Initializers with external data location are not currently supported"; + return result; + } + } + + // This is a list of initializers that nGraph considers as constants. Example weights, reshape shape etc. + std::unordered_set ng_required_initializers; + + const auto unsupported_nodes = GetUnsupportedNodeIndices(graph_viewer, device_id, ng_required_initializers); + #ifndef NDEBUG + if(openvino_ep::backend_utils::IsDebugEnabled()){ + std::cout << "No of unsupported nodes " << unsupported_nodes.size() << std::endl; + for(size_t i = 0; i < unsupported_nodes.size(); i++){ + const auto& node = graph_viewer.GetNode(unsupported_nodes[i]); + std::cout << "Unsupported node op " << node->OpType() << std::endl; + } + } + #endif + + //If all ops are supported, no partitioning is required. Short-circuit and avoid splitting. + if (unsupported_nodes.empty()) { + std::vector inputs; + std::vector outputs; + //Fill inputs with names + std::for_each(graph_viewer.GetInputs().begin(), graph_viewer.GetInputs().end(), + [&inputs](const NodeArg* node_arg) { inputs.push_back(node_arg->Name()); }); + + /* In scenarios, when there are no inputs or all inputs being initializers, + ConstantFolding optimization in onnxruntime pre-computes the value.*/ + if (inputs.empty()) { + return result; + } + + const auto& nodes = graph_viewer.GetNodesInTopologicalOrder(); + //Nodes that work well in models but not as a single node + if (nodes.size() == 1) { + const auto& node = graph_viewer.GetNode(nodes[0]); + if(IsOpSupportedOnlyInModel(node->OpType())) + return result; + //If reshape is not an intermediate node, shape needs to be an initializer + if(node->OpType() == "Reshape"){ + const auto& shape_arg = node->InputDefs()[1]; + if(ng_required_initializers.find(shape_arg->Name()) == ng_required_initializers.end()) + return result; + } else if (node->OpType() == "Expand") { + const auto& output = node->OutputDefs()[0]; + if (output->TypeAsProto()->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) + return result; + } else if (node->OpType() == "RoiAlign") { + using onnx_dtype = ONNX_NAMESPACE::TensorProto_DataType; + + onnx_dtype input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype input_2_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[2]->TypeAsProto()->tensor_type().elem_type(); + onnx_dtype output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + + if ((input_0_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || + (input_1_data_type != onnx_dtype::TensorProto_DataType_FLOAT16) || + (input_2_data_type != onnx_dtype::TensorProto_DataType_FLOAT) || + (output_data_type != onnx_dtype::TensorProto_DataType_FLOAT16)) + return result; + } else if ((node->OpType() == "Greater") || (node->OpType() == "Less")) { + + if (device_id == "MYRIAD") { + + auto input_0_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + auto input_1_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->InputDefs()[1]->TypeAsProto()->tensor_type().elem_type(); + auto output_data_type = (ONNX_NAMESPACE::TensorProto_DataType)node->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + + if (!((output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT) || + (output_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16))) { + return result; + } + + if ((input_0_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16) || + (input_1_data_type != ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)) { + return result; + } + } + } + } + + //Initializers need to be part of meta_def->inputs + std::for_each(ng_required_initializers.begin(), ng_required_initializers.end(), + [&inputs](const std::string& initializer) { inputs.push_back(initializer); }); + + //Fill outputs with names + std::for_each(graph_viewer.GetOutputs().begin(), graph_viewer.GetOutputs().end(), + [&outputs](const NodeArg* node_arg) { outputs.push_back(node_arg->Name()); }); + + // Create and add this graph to result. + AppendClusterToSubGraph(graph_viewer.GetNodesInTopologicalOrder(), inputs, outputs, result); + + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Model is fully supported by OpenVINO"; + openvino_ep::BackendManager::GetGlobalContext().is_wholly_supported_graph = true; + + } else { // unsupported_nodes_idx.empty() + + std::vector modified_unsupported_nodes; + for (const auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) { + if(find(unsupported_nodes.begin(), unsupported_nodes.end(), node_idx) != unsupported_nodes.end()){ + modified_unsupported_nodes.push_back(node_idx); + } + else{ + const auto& node = graph_viewer.GetNode(node_idx); + const auto& optype = node->OpType(); + if(optype == "TopK" || optype == "NonZero"){ + modified_unsupported_nodes.push_back(node_idx); + } + if(optype == "Gather"){ + if(device_id == "MYRIAD"){ + auto input_data_type = node->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + if(input_data_type == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8){ + modified_unsupported_nodes.push_back(node_idx); + } + } + } + } + } + auto ng_clusters = GetPartitionedClusters(graph_viewer.GetNodesInTopologicalOrder(), modified_unsupported_nodes); + + auto connected_clusters = GetConnectedClusters(graph_viewer, ng_clusters); + + //Myriad plugin can only load 10 subgraphs + if (device_id == "MYRIAD" && connected_clusters.size() > 10) { + std::sort(connected_clusters.begin(), connected_clusters.end(), + [](const std::vector& v1, const std::vector& v2) -> bool { + return v1.size() > v2.size(); + }); + } + int no_of_clusters = 0; + + for (auto this_cluster : connected_clusters) { + if (device_id == "MYRIAD" && no_of_clusters == 10) { + break; + } + std::vector cluster_graph_inputs, cluster_inputs, const_inputs, cluster_outputs; + //If subgraph has less then three, graph is considered trivial + if (this_cluster.size() < 3) { + continue; + } + GetInputsOutputsOfCluster(graph_viewer, this_cluster, ng_required_initializers, cluster_graph_inputs, cluster_inputs, const_inputs, cluster_outputs); + + bool omit_subgraph = false; + std::map slice_map; + //Omitting zero dim subgraphs + for (auto index : this_cluster) { + const auto& node = graph_viewer.GetNode(index); + const auto& optype = node->OpType(); + if (optype == "Mul" || optype == "Transpose" || optype == "Unsqueeze" || + optype == "Cast" || optype == "Concat" || optype == "Gather" || + optype == "Div" || optype == "Sub" || optype == "Identity") { + + if(optype == "Identity" && device_id != "CPU") + continue; + + if((optype == "Div" || optype == "Sub") && (device_id != "MYRIAD" && device_id != "GPU")) + continue; + for (const auto& input : node->InputDefs()) { + auto input_name = input->Name(); + auto it = find(cluster_graph_inputs.begin(), cluster_graph_inputs.end(), input_name); + if (it != cluster_graph_inputs.end()) { + omit_subgraph = true; + break; + } + } + } + + if(optype == "Conv" || optype == "Identity"){ + auto output_name = node->OutputDefs()[0]->Name(); + auto it = find(cluster_outputs.begin(), cluster_outputs.end(), output_name); + if(it != cluster_outputs.end() && node->GetOutputEdgesCount() != 0){ + omit_subgraph = true; + break; + } + } + + if(optype == "Slice"){ + auto input = node->InputDefs()[0]; + auto input_name = input->Name(); + const bool is_data_int32 = input->Type()->find("int32") != std::string::npos; + auto it = find(cluster_graph_inputs.begin(), cluster_graph_inputs.end(), input_name); + if(it != cluster_graph_inputs.end()){ + if(device_id == "MYRIAD" && is_data_int32){ + omit_subgraph = true; + break; + } + if(slice_map.count(input_name) == 0){ + slice_map[input_name] = 1; + } + else{ + omit_subgraph = true; + break; + } + } + } + } + if (omit_subgraph) + continue; + + /* In scenarios, when there are no inputs or all inputs being initializers, + ConstantFolding optimization in onnxruntime pre-computes the value.*/ + if (!cluster_inputs.empty()){ + AppendClusterToSubGraph(this_cluster, cluster_inputs, cluster_outputs, result); + no_of_clusters++; + } + } + LOGS_DEFAULT(INFO) << "[OpenVINO-EP] Supported subgraphs on OpenVINO: " << no_of_clusters; + } + + return result; +} + +} // namespace onnxruntime +} // namespace openvino_ep + +#endif //defined OPENVINO_2021_1 diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.cc b/onnxruntime/core/providers/openvino/ov_versions/utils.cc index 8e157ee081..d3752fbdf1 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/utils.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/utils.cc @@ -43,16 +43,18 @@ bool IsOpSupportedOnlyInModel(std::string name){ "ConstantOfShape", "Dropout", "EyeLike", + "Exp", "Identity", + "NonMaxSuppression", + "NonZero", + "Not", "OneHot", + "Pad", "ReduceMin", + "Resize", "Shape", "Split", - "TopK", - "Resize", - "Exp", - "Pad", - "Not" + "TopK" }; return ops_supported_only_in_model.find(name) != ops_supported_only_in_model.end(); } diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index 967def5ba1..52e59cac66 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -1869,7 +1869,8 @@ TEST(InferenceSessionTests, TestLenientShapeInferencing) { old_opset.AddInput("data", input_shape, input_data); old_opset.AddOutput("output", invalid_output_shape, output_data); // TensorRT doesn't handle Unsqueeze - old_opset.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); + // OpenVINO: Disabled temporarily + old_opset.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } #ifdef USE_CUDA diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index d6d7ec93ba..043865e080 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -1519,7 +1519,11 @@ TEST(MathOpTest, Equal_int64) { test.AddInput("A", dims, {1, 0, -1, -1}); test.AddInput("B", dims, {1, 1, 2, -1}); test.AddOutput("C", dims, {true, false, false, true}); - test.Run(); + #if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); + #else + test.Run(); + #endif } TEST(MathOpTest, Equal_float) { diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index c64aba38d2..46c07ddbfe 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -664,6 +664,8 @@ TEST_P(ModelTest, Run) { ORT_TSTR("split_zero_size_splits"), ORT_TSTR("convtranspose_3d")}; static const ORTCHAR_T* openvino_disabled_tests[] = {ORT_TSTR("tf_mobilenet_v1_1.0_224"), + ORT_TSTR("yolov3"), + ORT_TSTR("LSTM_Seq_lens_unpacked"), ORT_TSTR("tinyyolov3"), ORT_TSTR("faster_rcnn"), ORT_TSTR("mask_rcnn"), diff --git a/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc b/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc index e5d611adff..5cc5afca49 100644 --- a/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc @@ -134,7 +134,11 @@ static void scatter_negative_axis(const char* op_name, int op_version) { test.AddInput("indices", {1, 2}, {1, 3}); test.AddInput("updates", {1, 2}, {1.1f, 2.1f}); test.AddOutput("y", {1, 5}, {1.0f, 1.1f, 3.0f, 2.1f, 5.0f}); - test.Run(); + #if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) //TBD temporarily disabling for openvino + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); + #else + test.Run(); + #endif } TEST(Scatter, NegativeAxis) { @@ -200,7 +204,11 @@ static void scatter_valid_negative_index(const char* op_name, int op_version) { test.AddInput("indices", {1, 1, 1}, {-1}); test.AddInput("updates", {1, 1, 1}, {5.0f}); test.AddOutput("y", {4, 2, 1}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 5.0f, 0.0f}); - test.Run(); + #if defined(OPENVINO_CONFIG_MYRIAD) || defined(OPENVINO_CONFIG_VAD_M) //TBD temporarily disabling for openvino + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); + #else + test.Run(); + #endif } TEST(Scatter, ValidNegativeIndex) { diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index e6ec0eb8dc..331c243281 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -116,6 +116,8 @@ "^test_unique_not_sorted_without_axis", "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. "^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op. + "^test_sce.*", + "^test_nllloss.*", "^test_gather_negative_indices.*" ], "current_failing_tests_OPENVINO_CPU_FP32": [ @@ -125,7 +127,10 @@ "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. "^test_softmax_cross_entropy.*", // Does not support 5-D or above tensors for SUB op. "^test_mvn.*", - "^test_gather_negative_indices.*" + "^test_gather_negative_indices.*", + "^test_sce.*", + "^test_nllloss.*", + "^test_upsample_nearest.*" ], // ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported "tests_with_pre_opset7_dependencies": [ diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 7f43195405..51d9877ac1 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU' JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d openvino -v 2020.4 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d openvino -v 2021.1 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino index d8660c4745..c67e1f8959 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino @@ -2,33 +2,44 @@ ARG OS_VERSION=18.04 FROM ubuntu:${OS_VERSION} ARG PYTHON_VERSION=3.5 -ARG OPENVINO_VERSION=2020.4 +ARG OPENVINO_VERSION=2021.1 ADD scripts /tmp/scripts RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION -d EdgeDevice && \ /tmp/scripts/install_deps.sh -p $PYTHON_VERSION -d EdgeDevice RUN apt update && apt install -y libnuma1 ocl-icd-libopencl1 && \ - rm -rf /var/lib/apt/lists/* - -RUN /tmp/scripts/install_openvino.sh -o ${OPENVINO_VERSION} && \ - rm -rf /tmp/scripts + rm -rf /var/lib/apt/lists/* /tmp/scripts WORKDIR /root -ENV INTEL_OPENVINO_DIR /data/openvino/openvino_${OPENVINO_VERSION}.287 +ENV INTEL_OPENVINO_DIR /opt/intel/openvino_${OPENVINO_VERSION}.110 ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/intel64:$INTEL_OPENVINO_DIR/deployment_tools/ngraph/lib:$INTEL_OPENVINO_DIR/deployment_tools/inference_engine/external/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH ENV PYTHONPATH $INTEL_OPENVINO_DIR/tools:$PYTHONPATH ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/deployment_tools/inference_engine/lib/intel64 +ENV DEBIAN_FRONTEND=noninteractive -RUN wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-gmmlib_19.1.1_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-core_1.0.2-1787_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-igc-opencl_1.0.2-1787_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-opencl_19.15.12831_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/19.15.12831/intel-ocloc_19.15.12831_amd64.deb && \ +RUN wget https://apt.repos.intel.com/openvino/2021/GPG-PUB-KEY-INTEL-OPENVINO-2021 && \ + apt-key add GPG-PUB-KEY-INTEL-OPENVINO-2021 && rm GPG-PUB-KEY-INTEL-OPENVINO-2021 && \ + cd /etc/apt/sources.list.d && \ + echo "deb https://apt.repos.intel.com/openvino/2021 all main">intel-openvino-2021.list && \ + apt update && \ + apt install -y intel-openvino-dev-ubuntu18-2021.1.110 && \ + cd ${INTEL_OPENVINO_DIR}/install_dependencies && ./install_openvino_dependencies.sh + +RUN wget https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-gmmlib_19.3.2_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-core_1.0.2597_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-igc-opencl_1.0.2597_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-opencl_19.41.14441_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/19.41.14441/intel-ocloc_19.41.14441_amd64.deb && \ sudo dpkg -i *.deb && rm -rf *.deb +RUN mkdir -p /opt/cmake/bin && \ + wget https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz && \ + tar -xf cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake && rm -rf /cmake-3.13.2-Linux-x86_64.tar.gz && \ + ln -sf /opt/cmake/bin/* /usr/bin + ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev WORKDIR /home/$BUILD_USER diff --git a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh b/tools/ci_build/github/linux/docker/scripts/install_openvino.sh deleted file mode 100755 index 838ac696b6..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -set -e -while getopts o: parameter_Option -do case "${parameter_Option}" -in -o) OPENVINO_VERSION=${OPTARG};; -esac -done - -OPENVINO_VERSION=${OPENVINO_VERSION:=2020.4} -export INTEL_OPENVINO_DIR=/data/openvino/openvino_${OPENVINO_VERSION}.287 -export INTEL_OPENVINO_SRC_DIR=/data/openvino/openvino_src -git clone https://github.com/openvinotoolkit/openvino.git ${INTEL_OPENVINO_SRC_DIR} - -apt-get update && apt-get -y install libusb-1.0-0-dev - -cd $INTEL_OPENVINO_SRC_DIR -git checkout tags/$OPENVINO_VERSION -b $OPENVINO_VERSION -git submodule init -git submodule update --recursive - - -host_cpu=$(uname -m) -sudo -E apt update -sudo -E apt-get install -y \ - build-essential \ - curl \ - wget \ - libssl-dev \ - ca-certificates \ - git \ - libboost-regex-dev \ - gcc-multilib g++-multilib \ - libgtk2.0-dev \ - pkg-config \ - unzip \ - automake \ - libtool \ - autoconf \ - libcairo2-dev \ - libpango1.0-dev \ - libglib2.0-dev \ - libgtk2.0-dev \ - libswscale-dev \ - libavcodec-dev \ - libavformat-dev \ - libgstreamer1.0-0 \ - gstreamer1.0-plugins-base \ - libusb-1.0-0-dev \ - libopenblas-dev - -if apt-cache search --names-only '^libpng12-dev'| grep -q libpng12; then - sudo -E apt-get install -y libpng12-dev -else - sudo -E apt-get install -y libpng-dev -fi - -mkdir -p build -cd build - -mkdir -p $INTEL_OPENVINO_DIR - -cmake -DCMAKE_INSTALL_PREFIX=${INTEL_OPENVINO_DIR} -DNGRAPH_COMPONENT_PREFIX=deployment_tools/ngraph/ -DCMAKE_BUILD_TYPE=Release .. -make --jobs=$(nproc --all) -make install - -cd ~