diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt index b550e99ab5..5bfb805ce2 100644 --- a/ThirdPartyNotices.txt +++ b/ThirdPartyNotices.txt @@ -2810,4 +2810,212 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. + +_____ + +NervanaSystems/ngraph + +Copyright 2016-2019 Intel Corporation + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cgmanifest.json b/cgmanifest.json index 0d0877b6c7..ff4c1cbaf9 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -289,6 +289,15 @@ "repositoryUrl":"https://github.com/onnx/onnx-tensorrt.git" } } + }, + { + "component":{ + "type":"git", + "git":{ + "commitHash":"950fbf183274ab7d2092f99bab6c809ae87c7054", + "repositoryUrl":"https://github.com/NervanaSystems/ngraph.git" + } + } } ], "Version":1 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 8fd69d457c..96c89fefe7 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -51,6 +51,7 @@ option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON) option(onnxruntime_USE_MLAS "Use optimized blas library for GEMM and 2D Convolution" ON) option(onnxruntime_USE_MKLDNN "Build with MKL-DNN support" OFF) option(onnxruntime_USE_MKLML "Build MKL-DNN with MKL-ML binary dependency" OFF) +option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF) option(onnxruntime_USE_OPENBLAS "Use openblas" OFF) option(onnxruntime_DEV_MODE "Enable developer warnings and treat most of them as error." OFF) option(onnxruntime_USE_JEMALLOC "Use jecmalloc" OFF) @@ -475,6 +476,22 @@ if (onnxruntime_USE_MKLDNN) link_directories(${MKLDNN_LIB_DIR}) endif() +if (onnxruntime_USE_NGRAPH) + if (Win32) + message(FATAL_ERROR "nGraph is not currently supported on Windows.") + endif() + #if (onnxruntime_USE_OPENMP) + # message(FATAL_ERROR "Please set onnxruntime_USE_OPENMP=OFF for nGraph execution provider.") + #endif() + if (NOT onnxruntime_USE_FULL_PROTOBUF) + message(FATAL_ERROR "Please set onnxruntime_USE_FULL_PROTOBUF=ON for nGraph execution provider.") + endif() + add_definitions(-DUSE_NGRAPH=1) + include(ngraph) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES ngraph) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES ngraph) +endif() + if (onnxruntime_USE_OPENBLAS) add_definitions(-DUSE_OPENBLAS=1) if (WIN32) diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake new file mode 100644 index 0000000000..50ab13d5cc --- /dev/null +++ b/cmake/external/ngraph.cmake @@ -0,0 +1,57 @@ +# Copyright(C) 2019 Intel Corporation +# Licensed under the MIT License + +include (ExternalProject) + +set(ngraph_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/ngraph) +set(ngraph_INSTALL_DIR ${ngraph_ROOT_DIR}) +set(ngraph_INCLUDE_DIRS ${ngraph_INSTALL_DIR}/include) +set(ngraph_LIBRARIES ${ngraph_INSTALL_DIR}/lib) +set(ngraph_SRC ${CMAKE_CURRENT_BINARY_DIR}/ngraph/src/project_ngraph) +set(prebuilt_ONNX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/external/onnx") +set(prebuilt_ONNX_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/onnx") +set(ngraph_URL "https://github.com/NervanaSystems/ngraph.git") +set(ngraph_TAG "v0.18.1") + +# Libraries for python package. +set(NGRAPH_SHARED_LIB libngraph.so) +set(NGRAPH_CODEGEN_SHARED_LIB libcodegen.so) +set(NGRAPH_CPU_BACKEND_SHARED_LIB libcpu_backend.so) +set(NGRAPH_IOMP5MD_SHARED_LIB libiomp5.so) +set(NGRAPH_MKLDNN_SHARED_LIB libmkldnn.so) +set(NGRAPH_MKLML_SHARED_LIB libmklml_intel.so) +if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") + set(NGRAPH_TBB_SHARED_LIB libtbb_debug.so) + set(NGRAPH_TBB_SHARED_LIB_2 libtbb_debug.so.2) +else() + set(NGRAPH_TBB_SHARED_LIB libtbb.so) + set(NGRAPH_TBB_SHARED_LIB_2 libtbb.so.2) +endif() + +ExternalProject_Add(project_ngraph + PREFIX ngraph + GIT_REPOSITORY ${ngraph_URL} + GIT_TAG ${ngraph_TAG} + # Here we use onnx and protobuf built by onnxruntime to avoid linking with incompatible libraries. This might change in future. + PATCH_COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_onnx.cmake ${ngraph_SRC}/cmake/external_onnx.cmake + # TODO: Use cmake.file+copy as above. + COMMAND patch -p1 < ${PROJECT_SOURCE_DIR}/patches/ngraph/ngraph_protobuf.patch + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DNGRAPH_USE_PREBUILT_LLVM=TRUE + -DNGRAPH_USE_SYSTEM_PROTOBUF=FALSE + -DNGRAPH_ONNX_IMPORT_ENABLE=TRUE + -DNGRAPH_INTERPRETER_ENABLE=FALSE + -DNGRAPH_ONNXIFI_ENABLE=FALSE + -DNGRAPH_UNIT_TEST_ENABLE=FALSE + -DNGRAPH_TOOLS_ENABLE=FALSE + -DCMAKE_INSTALL_PREFIX=${ngraph_INSTALL_DIR} + -Dprebuilt_ONNX_BINARY_DIR=${prebuilt_ONNX_BINARY_DIR} + -Dprebuilt_ONNX_SOURCE_DIR=${prebuilt_ONNX_SOURCE_DIR} + DEPENDS onnx + ) + +add_library(ngraph SHARED IMPORTED) +set_property(TARGET ngraph PROPERTY IMPORTED_LOCATION ${ngraph_LIBRARIES}/${NGRAPH_SHARED_LIB}) +add_dependencies(ngraph project_ngraph) +include_directories(${ngraph_INCLUDE_DIRS}) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 36001381cd..952732b99d 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -59,6 +59,7 @@ target_link_libraries(onnxruntime PRIVATE ${onnxruntime_libs} ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} + ${PROVIDERS_NGRAPH} ${PROVIDERS_TENSORRT} onnxruntime_optimizer onnxruntime_providers diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index dd57f3d8cb..b8f3e1100b 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -22,6 +22,10 @@ if(onnxruntime_USE_MKLDNN) set(PROVIDERS_MKLDNN onnxruntime_providers_mkldnn) list(APPEND ONNXRUNTIME_PROVIDER_NAMES mkldnn) endif() +if(onnxruntime_USE_NGRAPH) + set(PROVIDERS_NGRAPH onnxruntime_providers_ngraph) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES ngraph) +endif() if(onnxruntime_USE_CUDA) set(PROVIDERS_CUDA onnxruntime_providers_cuda) list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda) @@ -176,6 +180,26 @@ if (onnxruntime_USE_TENSORRT) endif() endif() +if (onnxruntime_USE_NGRAPH) + include_directories("${CMAKE_CURRENT_BINARY_DIR}/onnx") + file(GLOB_RECURSE onnxruntime_providers_ngraph_cc_srcs + "${ONNXRUNTIME_ROOT}/core/providers/ngraph/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/ngraph/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_ngraph_cc_srcs}) + add_library(onnxruntime_providers_ngraph ${onnxruntime_providers_ngraph_cc_srcs}) + onnxruntime_add_include_to_target(onnxruntime_providers_ngraph onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_ngraph ngraph onnx ${onnxruntime_EXTERNAL_DEPENDENCIES}) + set_target_properties(onnxruntime_providers_ngraph PROPERTIES FOLDER "ONNXRuntime") + target_include_directories(onnxruntime_providers_ngraph PRIVATE ${ONNXRUNTIME_ROOT} ${ngraph_INCLUDE_DIRS}) + set_target_properties(onnxruntime_providers_ngraph PROPERTIES LINKER_LANGUAGE CXX) + + target_compile_options(onnxruntime_providers_ngraph PRIVATE "SHELL:-Wformat" "SHELL:-Wformat-security" "SHELL:-fstack-protector-strong" "SHELL:-D_FORTIFY_SOURCE=2") + target_link_options(onnxruntime_providers_ngraph PRIVATE "LINKER:-z, noexecstack " "LINKER:-z relro" "LINKER:-z now" "LINKER:-pie") + +endif() + if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) include(onnxruntime_providers_internal.cmake) endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index e4138ed10a..ace16b1b87 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -69,6 +69,7 @@ set(onnxruntime_pybind11_state_libs ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} ${PROVIDERS_TENSORRT} + ${PROVIDERS_NGRAPH} onnxruntime_optimizer onnxruntime_providers onnxruntime_util @@ -188,6 +189,23 @@ if (onnxruntime_USE_MKLDNN) $/onnxruntime/capi/ ) endif() + +if (onnxruntime_USE_NGRAPH) + add_custom_command( + TARGET onnxruntime_pybind11_state POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${ngraph_LIBRARIES}/${NGRAPH_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_CODEGEN_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_CPU_BACKEND_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_IOMP5MD_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_MKLDNN_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_MKLML_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_TBB_SHARED_LIB} + ${ngraph_LIBRARIES}/${NGRAPH_TBB_SHARED_LIB_2} + $/onnxruntime/capi/ + ) +endif() + if (onnxruntime_USE_TVM) add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index d74d4e216f..d0e8318b4c 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -124,6 +124,13 @@ file(GLOB_RECURSE onnxruntime_test_providers_cpu_src ) list(APPEND onnxruntime_test_providers_src ${onnxruntime_test_providers_cpu_src}) +if (onnxruntime_USE_NGRAPH) + file(GLOB_RECURSE onnxruntime_test_providers_ngraph_src + "${TEST_SRC_DIR}/providers/ngraph/*" + ) + list(APPEND onnxruntime_test_providers_src ${onnxruntime_test_providers_ngraph_src}) +endif() + # tests from lowest level library up. # the order of libraries should be maintained, with higher libraries being added first in the list @@ -172,6 +179,10 @@ if(onnxruntime_USE_MKLDNN) list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_mkldnn) endif() +if(onnxruntime_USE_NGRAPH) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_ngraph) +endif() + file(GLOB_RECURSE onnxruntime_test_tvm_src "${ONNXRUNTIME_ROOT}/test/tvm/*.h" "${ONNXRUNTIME_ROOT}/test/tvm/*.cc" @@ -187,6 +198,7 @@ set(ONNXRUNTIME_TEST_LIBS ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} ${PROVIDERS_TENSORRT} + ${PROVIDERS_NGRAPH} onnxruntime_optimizer onnxruntime_providers onnxruntime_util diff --git a/cmake/patches/ngraph/ngraph_onnx.cmake b/cmake/patches/ngraph/ngraph_onnx.cmake new file mode 100644 index 0000000000..b27cc16019 --- /dev/null +++ b/cmake/patches/ngraph/ngraph_onnx.cmake @@ -0,0 +1,28 @@ +set(BINARY_DIR "${prebuilt_ONNX_BINARY_DIR}") +set(ONNX_INCLUDE_DIR ${BINARY_DIR}) +set(ONNX_SOURCE_INCLUDE_DIR "${prebuilt_ONNX_SOURCE_DIR}/onnx") +include_directories("${ONNX_SOURCE_INCLUDE_DIR}") +set(ONNX_PROTO_INCLUDE_DIR ${ONNX_INCLUDE_DIR}) +set(ONNX_LIBRARY ${BINARY_DIR}/libonnx.a) +set(ONNX_PROTO_LIBRARY ${BINARY_DIR}/libonnx_proto.a) +set(ONNX_LIBRARIES ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}) + +if (NOT TARGET onnx::libonnx) + add_library(onnx::libonnx UNKNOWN IMPORTED) + set_target_properties(onnx::libonnx PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ONNX_INCLUDE_DIR} + IMPORTED_LOCATION ${ONNX_LIBRARY} + INCLUDE_DIRECTORIES "${ONNX_SOURCE_INCLUDE_DIR}" + INTERFACE_COMPILE_DEFINITIONS ONNX_ML=1) +endif() + +if (NOT TARGET onnx::libonnx_proto) + add_library(onnx::libonnx_proto UNKNOWN IMPORTED) + set_target_properties(onnx::libonnx_proto PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${ONNX_PROTO_INCLUDE_DIR} + IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY} + INCLUDE_DIRECTORIES "${ONNX_SOURCE_INCLUDE_DIR}" + INTERFACE_COMPILE_DEFINITIONS ONNX_ML=1) +endif() +add_library(ext_onnx UNKNOWN IMPORTED) +add_dependencies(ext_onnx onnx::libonnx) diff --git a/cmake/patches/ngraph/ngraph_protobuf.patch b/cmake/patches/ngraph/ngraph_protobuf.patch new file mode 100644 index 0000000000..0736ef5aea --- /dev/null +++ b/cmake/patches/ngraph/ngraph_protobuf.patch @@ -0,0 +1,13 @@ +diff --git a/cmake/external_protobuf.cmake b/cmake/external_protobuf.cmake +index 47977b3..1a66e1c 100644 +--- a/cmake/external_protobuf.cmake ++++ b/cmake/external_protobuf.cmake +@@ -23,7 +23,7 @@ include(ExternalProject) + + # This version of PROTOBUF is required by Microsoft ONNX Runtime. + set(NGRAPH_PROTOBUF_GIT_REPO_URL "https://github.com/protocolbuffers/protobuf") +-set(NGRAPH_PROTOBUF_GIT_TAG "v3.5.2") ++set(NGRAPH_PROTOBUF_GIT_TAG "v3.6.1") + + ExternalProject_Add( + ext_protobuf diff --git a/dockerfiles/Dockerfile.ngraph b/dockerfiles/Dockerfile.ngraph new file mode 100644 index 0000000000..c21002f0d7 --- /dev/null +++ b/dockerfiles/Dockerfile.ngraph @@ -0,0 +1,20 @@ +#------------------------------------------------------------------------- +# Copyright(C) 2019 Intel Corporation. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +FROM ubuntu:16.04 + + +RUN apt-get update && \ + apt-get install -y sudo \ + build-essential curl \ + libcurl4-openssl-dev \ + libssl-dev wget \ + python 3.6 python3-pip \ + python3-dev git +RUN pip3 install --upgrade pip +RUN pip3 install numpy onnx + +RUN pip3 install onnxruntime-ngraph +WORKDIR /code diff --git a/include/onnxruntime/core/framework/func_api.h b/include/onnxruntime/core/framework/func_api.h index c6a90180b1..d105139ffe 100644 --- a/include/onnxruntime/core/framework/func_api.h +++ b/include/onnxruntime/core/framework/func_api.h @@ -6,7 +6,14 @@ enum DType { TFloat32 = 0, TInt32 = 1, TDouble = 2, - TInt64 = 3 + TInt64 = 3, + TBool = 4, + TUint8 = 5, + TInt8 = 6, + TUint16 = 7, + TInt16 = 8, + TUint32 = 9, + TUint64 = 10 //TODO: more types }; diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index 328ce4a742..8abee9c33b 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -18,9 +18,11 @@ constexpr const char* kOnnxDomain = ""; constexpr const char* kOnnxDomainAlias = "ai.onnx"; constexpr const char* kMLDomain = "ai.onnx.ml"; constexpr const char* kMSDomain = "com.microsoft"; +constexpr const char* kNGraphDomain = "com.intel.ai"; constexpr const char* kCpuExecutionProvider = "CPUExecutionProvider"; constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider"; constexpr const char* kMklDnnExecutionProvider = "MKLDNNExecutionProvider"; +constexpr const char* kNGraphExecutionProvider = "NGRAPHExecutionProvider"; constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider"; constexpr const char* kBrainSliceExecutionProvider = "BrainSliceExecutionProvider"; constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider"; diff --git a/include/onnxruntime/core/graph/graph_viewer.h b/include/onnxruntime/core/graph/graph_viewer.h index 652c572af2..7e2a0364ed 100644 --- a/include/onnxruntime/core/graph/graph_viewer.h +++ b/include/onnxruntime/core/graph/graph_viewer.h @@ -102,6 +102,9 @@ class GraphViewer { return graph_->DomainToVersionMap(); } + /** Check if this is a Subgraph */ + bool IsSubgraph() const; + private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer); diff --git a/include/onnxruntime/core/providers/ngraph/ngraph_provider_factory.h b/include/onnxruntime/core/providers/ngraph/ngraph_provider_factory.h new file mode 100644 index 0000000000..0970362a2b --- /dev/null +++ b/include/onnxruntime/core/providers/ngraph/ngraph_provider_factory.h @@ -0,0 +1,17 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include "core/session/onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_NGraph, _In_ OrtSessionOptions* options, _In_ const char* ng_backend_type); + +#ifdef __cplusplus +} +#endif diff --git a/onnxruntime/core/framework/func_kernel.cc b/onnxruntime/core/framework/func_kernel.cc index 932568d2b1..70c55e45ad 100644 --- a/onnxruntime/core/framework/func_kernel.cc +++ b/onnxruntime/core/framework/func_kernel.cc @@ -22,6 +22,20 @@ DType ORT_type_to_c_type(MLDataType type) { return DType::TDouble; else if (type == DataTypeImpl::GetType()) return DType::TInt32; + else if (type == DataTypeImpl::GetType()) + return DType::TBool; + else if (type == DataTypeImpl::GetType()) + return DType::TUint8; + else if (type == DataTypeImpl::GetType()) + return DType::TInt8; + else if (type == DataTypeImpl::GetType()) + return DType::TUint16; + else if (type == DataTypeImpl::GetType()) + return DType::TInt16; + else if (type == DataTypeImpl::GetType()) + return DType::TUint32; + else if (type == DataTypeImpl::GetType()) + return DType::TUint64; else if (type == DataTypeImpl::GetType()) return DType::TInt64; else diff --git a/onnxruntime/core/graph/graph_viewer.cc b/onnxruntime/core/graph/graph_viewer.cc index 4beaf5243b..262a2591dd 100644 --- a/onnxruntime/core/graph/graph_viewer.cc +++ b/onnxruntime/core/graph/graph_viewer.cc @@ -104,4 +104,9 @@ const InitializedTensorSet& GraphViewer::GetAllInitializedTensors() const noexce const NodeArg* GraphViewer::GetNodeArg(const std::string& name) const { return graph_->GetNodeArg(name); } + +bool GraphViewer::IsSubgraph() const { + return graph_->IsSubgraph(); +} + } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc index d1dd4f637d..3175ee6ec8 100644 --- a/onnxruntime/core/optimizer/transformer_memcpy.cc +++ b/onnxruntime/core/optimizer/transformer_memcpy.cc @@ -58,6 +58,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr for (auto& provider : provider_types_) { if (provider != onnxruntime::kCpuExecutionProvider && provider != onnxruntime::kMklDnnExecutionProvider && + provider != onnxruntime::kNGraphExecutionProvider && provider != onnxruntime::kNupharExecutionProvider && provider != onnxruntime::kTensorrtExecutionProvider) { TransformerMemcpyImpl copy_impl(graph, provider); diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc new file mode 100644 index 0000000000..e7c681d510 --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.cc @@ -0,0 +1,203 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#pragma GCC diagnostic pop + +#include "ngraph_custom_op.h" +#include "core/common/logging/logging.h" + +namespace onnxruntime { +namespace ngraph_ep { + +static DType GetDataType(const ngraph::element::Type& ng_type) { + switch (ng_type.get_type_enum()) { + case ngraph::element::Type_t::f32: + return DType::TFloat32; + case ngraph::element::Type_t::f64: + return DType::TDouble; + case ngraph::element::Type_t::boolean: + return DType::TBool; + case ngraph::element::Type_t::u8: + return DType::TUint8; + case ngraph::element::Type_t::i8: + return DType::TInt8; + case ngraph::element::Type_t::u16: + return DType::TUint16; + case ngraph::element::Type_t::i16: + return DType::TInt16; + case ngraph::element::Type_t::u32: + return DType::TUint32; + case ngraph::element::Type_t::i32: + return DType::TInt32; + case ngraph::element::Type_t::u64: + return DType::TUint64; + case ngraph::element::Type_t::i64: + return DType::TInt64; + default: + throw "Unsupported DataType"; + } +} + +NGRAPHCustomOp::NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, + const std::shared_ptr& ng_backend) + : ng_backend_{ng_backend}, + model_proto_{model_proto} { + allocate_func_ = context->allocate_func; + release_func_ = context->release_func; + allocator_ = context->allocator_handle; + name_ = context->node_name; + + if (std::getenv("ONNXRUNTIME_NGRAPH_DUMP_OPS") != nullptr) { + std::fstream dump(name_ + ".onnx", std::ios::out | std::ios::trunc | std::ios::binary); + model_proto_.SerializeToOstream(&dump); + } +} + +NGRAPHCustomOp::~NGRAPHCustomOp() { + for (const auto& compiled_exe : ng_exe_map_) { + ng_backend_->remove_compiled_function(compiled_exe.second); + } +} + +//This method gets called in critical path of execution: Optimize +void NGRAPHCustomOp::Initialize(const ONNXRunTimeTensor* input_tensors, const size_t& num_inputs) const { + LOGS_DEFAULT(INFO) << "nGraph compiling customOp: " << name_; + + //Key for ng_exe_map + std::string uniq_input_shape; + + //Optimizing for general case of 4D tensors + uniq_input_shape.reserve(4 * sizeof(int64_t) * num_inputs + num_inputs); + + for (size_t i = 0; i < num_inputs; i++) { + const auto& ndim = input_tensors[i].ndim; + uniq_input_shape.append(reinterpret_cast(&ndim), sizeof(ndim)); + uniq_input_shape.append(reinterpret_cast(input_tensors[i].shape), ndim * sizeof(int64_t)); + } + + auto it = ng_exe_map_.insert({uniq_input_shape, nullptr}); //TODO: Limit the size of map with configurable size. + + //ng_exe with current shape already exists + if (!it.second) { + ng_curr_exe_ = it.first->second; + return; + } else { + auto graph_proto = model_proto_.mutable_graph(); + // Clear previous shapes if any and set new input shapes + for (size_t i = 0; i < num_inputs; i++) { + auto g_in_shape = graph_proto->mutable_input(i)->mutable_type()->mutable_tensor_type()->mutable_shape(); + g_in_shape->clear_dim(); + + for (size_t dim = 0; dim < input_tensors[i].ndim; dim++) { + g_in_shape->add_dim()->set_dim_value(input_tensors[i].shape[dim]); + } + } + + std::istringstream model_stream{model_proto_.SerializeAsString()}; + std::shared_ptr ng_function; + try { + ng_function = ngraph::onnx_import::import_onnx_model(model_stream); + } catch (const std::exception& exp) { + LOGS_DEFAULT(FATAL) << "[" << name_ << "] " + << "Exception while converting onnx to nGraph: " << std::string(exp.what()); + throw; + } catch (...) { + LOGS_DEFAULT(FATAL) << "[" << name_ << "] " + << "Unknown exception while converting onnx to nGraph"; + throw; + } + + for (auto& result : ng_function->get_results()) { + result->set_needs_default_layout(true); + } + + // Finally compile nGraph with backend. + try { + ng_curr_exe_ = ng_backend_->compile(ng_function); + } catch (const std::exception& exp) { + LOGS_DEFAULT(FATAL) << "Exception while compiling nGraph Op: " << name_ << std::string(exp.what()); + } catch (...) { + LOGS_DEFAULT(FATAL) << "Unknown exception while compiling nGraph Op: " << name_; + } + it.first->second = ng_curr_exe_; + } +} // namespace ngraph_ep + +//This method gets called in critical path of execution: Optimize +Status NGRAPHCustomOp::Compute(const ONNXRunTimeTensor* input_tensors, const size_t num_inputs, ONNXRunTimeTensor* const output_tensors, const size_t num_outputs) const { + ORT_UNUSED_PARAMETER(num_outputs); + + //TODO: Minimize locked region + std::lock_guard lock(compute_lock_); + + // Initialize nGraph function if it is not already initialized. + Initialize(input_tensors, num_inputs); + + ORT_ENFORCE(ng_curr_exe_ != nullptr); + + std::vector> ng_inputs; + std::vector> ng_outputs; + + // Write ONNXR input data to nGraph input tensors. + try { + auto& in_tensor = input_tensors; + for (const auto& ng_param : ng_curr_exe_->get_parameters()) { + ng_inputs.emplace_back(ng_backend_->create_tensor(ng_param->get_output_element_type(0), ng_param->get_output_shape(0), (in_tensor++)->data)); + } + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while copying input data to nGraph: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while copying input data to nGraph"); + } + + // Initialize output tensors + try { + //TODO: Optimize + auto onxr_output = output_tensors; + for (auto& ng_result : ng_curr_exe_->get_results()) { + const auto& dtype = ng_result->get_element_type(); + const auto& shape = ng_result->get_shape(); + + onxr_output->dtype = GetDataType(dtype); + onxr_output->ndim = shape.size(); + onxr_output->shape = new int64_t[onxr_output->ndim]; + + size_t num_elements = 1; + for (size_t dim = 0; dim < shape.size(); dim++) { + num_elements *= shape[dim]; + onxr_output->shape[dim] = shape[dim]; + } + + onxr_output->data = (*(allocate_func_))(allocator_, 64, num_elements * sizeof(onxr_output->dtype)); + + ng_outputs.emplace_back(ng_backend_->create_tensor(dtype, shape, onxr_output->data)); + ++onxr_output; + } + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while creating nGraph output Tensor: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while creating nGraph output Tensor"); + } + + // Run the graph through nGraph. + try { + if (!ng_curr_exe_->call(ng_outputs, ng_inputs)) + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Error while executing nGraph computation"); + } catch (const std::exception& exp) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Exception while executing nGraph computation: " + std::string(exp.what())); + } catch (...) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unknown exception while executing nGraph computation"); + } + + return Status::OK(); +} + +} // namespace ngraph_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/ngraph/ngraph_custom_op.h b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h new file mode 100644 index 0000000000..630a070cdc --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_custom_op.h @@ -0,0 +1,54 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#pragma GCC diagnostic pop +#include "core/framework/func_api.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { +namespace ngraph_ep { + +class NGRAPHCustomOp { + public: + NGRAPHCustomOp(const ComputeContext* context, const ONNX_NAMESPACE::ModelProto& model_proto, const std::shared_ptr& ng_backend); + + Status Compute(const ONNXRunTimeTensor* input_tensors, const size_t num_inputs, ONNXRunTimeTensor* const output_tensors, const size_t num_outputs) const; + + ~NGRAPHCustomOp(); + + private: + + void Initialize(const ONNXRunTimeTensor* input_tensors, const size_t& num_inputs) const; + + std::shared_ptr ng_backend_; + + mutable std::shared_ptr ng_curr_exe_ = nullptr; + + AllocateFunc allocate_func_ = nullptr; + + DestroyFunc release_func_ = nullptr; + + AllocatorHandle allocator_ = nullptr; + + std::string name_; + + /* + nGraph::Executable objects are specific to input shapes. + Here we keep of a cache of nGraph::Executable objects with key as input shapes. TODO: Configure size of this cache. + Logically, key = [i0.rank,[i0.dims],i1.rank,[i1.dims] ... iN.rank,[iN.dims]] raw bytes enclosed inside a string. + Example: input0.shape(1,2,3) input1.shape(4,5) + key = [3,1,2,3,2,4,5] +*/ + mutable std::unordered_map> ng_exe_map_; + + mutable std::mutex compute_lock_; + + mutable ONNX_NAMESPACE::ModelProto model_proto_; +}; +} // namespace ngraph_ep +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc new file mode 100644 index 0000000000..634048db9b --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.cc @@ -0,0 +1,550 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/framework/compute_capability.h" +#include "core/framework/allocatormgr.h" +#include "core/framework/kernel_registry.h" +#include "core/graph/graph_viewer.h" +#include "core/graph/model.h" +#include "ngraph_execution_provider.h" +#include "ngraph_custom_op.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include +#include +#pragma GCC diagnostic pop + +#define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, MIN(destsz, srcsz)) + +namespace onnxruntime { + +constexpr const char* NGRAPH = "nGraph"; + +NGRAPHExecutionProvider::NGRAPHExecutionProvider(const NGRAPHExecutionProviderInfo& info) + : IExecutionProvider{onnxruntime::kNGraphExecutionProvider} { + DeviceAllocatorRegistrationInfo default_allocator_info({OrtMemTypeDefault, + [](int) { return std::make_unique(std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeDefault)); }, + std::numeric_limits::max()}); + + InsertAllocator(CreateAllocator(default_allocator_info)); + + DeviceAllocatorRegistrationInfo cpu_allocator_info({OrtMemTypeCPUOutput, + [](int) { return std::make_unique(std::make_unique(NGRAPH, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeCPUOutput)); }, + std::numeric_limits::max()}); + + InsertAllocator(CreateAllocator(cpu_allocator_info)); + + ORT_ENFORCE(info.ng_backend_type == "CPU", "nGraph Execution Provider for onnxruntime currently is only supported for CPU backend."); + + try { + ng_backend_ = ngraph::runtime::Backend::create(info.ng_backend_type); + } catch (const std::exception& exp) { + LOGS_DEFAULT(FATAL) << "Exception while creating nGraph " << info.ng_backend_type << " Backend: " << std::string(exp.what()); + } catch (...) { + LOGS_DEFAULT(FATAL) << "Unknown exception while while creating nGraph " << info.ng_backend_type << " Backend"; + throw; + } +} + +std::shared_ptr NGRAPHExecutionProvider::GetKernelRegistry() const { + return std::make_shared(); +} + +/** + * Checks if a tensor represented by srcLocation can be copied into the dstLocation tensor + * @param src_location result of Location().name call on the source tensor + * @param dst_location result of Location().name call on the destination tensor + * @return true if src and dest locations combination allows copying + */ +bool TensorCopyPossible(const std::string& src_location, const std::string& dst_location) { + // contains allowed combinations of source and destination locations for tensors copying purposes + // the first element of a pair denotes a source, the second - destination + static const std::map allowed_copy_directions = { + {NGRAPH, CPU}, {NGRAPH, NGRAPH}, {CPU, NGRAPH}}; + + // copying of tensors is allowed only if the params match any of the allowed combinations + return std::any_of(allowed_copy_directions.begin(), + allowed_copy_directions.end(), [&](const auto& copy_direction) { + return src_location == copy_direction.first && dst_location == copy_direction.second; + }); +} + +Status NGRAPHExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst) const { + const size_t src_bytes = src.DataType()->Size() * src.Shape().Size(); + const size_t dst_bytes = dst.DataType()->Size() * dst.Shape().Size(); + if (src_bytes != dst_bytes) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "nGraph: Source and Destination data sizes are not equal - cannot copy tensors"); + } + + if (!TensorCopyPossible(src.Location().name, dst.Location().name)) { + ORT_NOT_IMPLEMENTED("Copying tensors between '", src.Location().name, "' and '", dst.Location().name, + "' is not implemented in NGRAPHExecutionProvider"); + } + + MEMCPY_S(dst.MutableDataRaw(), src.DataRaw(), dst_bytes, src_bytes); + + return Status::OK(); +} + +// Returns true only if op is in a mode that is not currently supported +static bool IsUnsupportedOpMode(const Node* node, const onnxruntime::GraphViewer& graph_viewer) { + const auto& optype = node->OpType(); + const auto& initializers = graph_viewer.GetAllInitializedTensors(); + + if (optype == "Reshape") { + //nGraph Reshape op currently requires shape info available in advance. + const auto& shape_arg = node->InputDefs()[1]; + return initializers.find(shape_arg->Name()) == initializers.end(); + } else if (optype == "MaxPool") { + //MaxPool "indices" output is not currently supported. + if (node->OutputDefs().size() > 1) { + return true; + } + + // ceil_mode and dilations attrs are not supported in nGraph + const auto& attributes = node->GetAttributes(); + if (attributes.find("ceil_mode") != attributes.end()) { + return true; + } else if (attributes.find("dilations") != attributes.end()) { + return true; + } + } else if (optype == "OneHot") { + //nGraph OneHot op currently requires depth info available in advance. + const auto& depth_arg = node->InputDefs()[1]; + return initializers.find(depth_arg->Name()) == initializers.end(); + } else if (optype == "TopK") { + //TopK opset 10 is currently not supported. + //K as input is currently not suppported. + return node->InputDefs().size() > 1; + } else if (optype == "LSTM") { + const auto& attributes = node->GetAttributes(); + const auto direction_attr = attributes.find("direction"); + if (direction_attr != attributes.end()) { + return direction_attr->second.s() == "reverse"; + } + } else if (optype == "MatMul") { + //All matmuls except float have computation missmatch + const bool A_is_float = node->InputDefs()[0]->Type()->find("float") != std::string::npos; + const bool B_is_float = node->InputDefs()[1]->Type()->find("float") != std::string::npos; + return (A_is_float && B_is_float) ? false : true; + } else if (optype == "Pad") { + //3D pad with negative padding have computation missmatch + const auto& attributes = node->GetAttributes(); + const auto pad_attr = attributes.find("pads"); + if (pad_attr != attributes.end() && (pad_attr->second.ints().size() > 4 || pad_attr->second.ints().size() == 3)) { + for (const auto& val : pad_attr->second.ints()) { + if (val < 0) + return true; + } + } + } else if (optype == "Cast") { + //support of casting to bool in nGraph is in progress + const auto& attributes = node->GetAttributes(); + const auto to_attr = attributes.find("to"); + return to_attr->second.i() == ONNX_NAMESPACE::TensorProto::BOOL; + } else if (optype == "Slice") { + //Slice in opset 10 is currently not supported. + //unsupported inputs: starts, ends, axes, steps + if (node->InputDefs().size() > 1) { + return true; + } + //nGraph does not properly handle the situation where any value of the "starts" attribute + //is higher than a corresponding value in the "ends" + const auto& attributes = node->GetAttributes(); + if (attributes.count("starts") == 0 || attributes.count("ends") == 0) { + return true; + } + + const auto& starts = attributes.find("starts")->second.ints(); + const auto& ends = attributes.find("ends")->second.ints(); + for (int i = 0; i < starts.size(); ++i) { + if (starts.Get(i) > ends.Get(i)) { + return true; + } + } + } else if (optype == "AveragePool") { + // ceil_mode attribute is not supported in nGraph + const auto& attributes = node->GetAttributes(); + if (attributes.find("ceil_mode") != attributes.end()) { + return true; + } + } + + //Op doesn't fall into known any of unsupported modes. + return false; +} + +static bool IsTypeSupported(const NodeArg* node_arg) { + const auto* type_proto = node_arg->TypeAsProto(); + if (!type_proto) { + return false; + } + + switch (type_proto->tensor_type().elem_type()) { + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_BOOL: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_DOUBLE: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT16: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT32: + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT64: + return true; + default: + return false; + } +} + +static bool IsNodeSupported(const std::map>& op_map, + const onnxruntime::GraphViewer& graph_viewer, + const NodeIndex node_idx) { + const auto& node = graph_viewer.GetNode(node_idx); + const auto& optype = node->OpType(); + const auto& domain = node->Domain(); + + /* + 1. Check input and output data types are supported. + 2. Check Op is supported + 2a. Check if Op is of known unsupported modes (edge cases). If yes return false right away. + 2b. If above is not true, check if the op is available in nGraph. + */ + + //Check 1 + bool are_types_supported = true; + + node->ForEachDef([&are_types_supported](const onnxruntime::NodeArg& node_arg, bool /*is_input*/) { + are_types_supported &= IsTypeSupported(&node_arg); + }); + + if (!are_types_supported) { + return false; + } + + //Check 2a + if (domain == kOnnxDomain && IsUnsupportedOpMode(node, graph_viewer)) { + return false; + } + + //Check 2b + const auto opset = op_map.find(domain); + if (opset == op_map.end() || opset->second.find(optype) == opset->second.end()) { + return false; + } else { + return true; + } +} + +static void AppendClusterToSubGraph(const std::vector& nodes, + const onnxruntime::GraphViewer& graph_viewer, + const std::vector& inputs, + const std::vector& outputs, + const std::unordered_set& ng_required_initializers, + std::vector>& result) { + static size_t op_counter = 0; + + // Create ng_required_initializers attribute of NGraphCustomOp + ONNX_NAMESPACE::AttributeProto initializers; + initializers.set_name("initializers"); + initializers.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS); + for (const auto& init : ng_required_initializers) { + auto tensor = initializers.add_tensors(); + *tensor = *(graph_viewer.GetAllInitializedTensors().at(init)); + } + + auto meta_def = std::make_unique(); + meta_def->attributes["initializers"] = initializers; + meta_def->name = "NGRAPHCustomOp_" + std::to_string(++op_counter); + meta_def->domain = kNGraphDomain; + meta_def->since_version = 1; + meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; + meta_def->inputs = inputs; + meta_def->outputs = outputs; + + std::unique_ptr sub_graph = std::make_unique(); + sub_graph->nodes = nodes; + sub_graph->SetMetaDef(meta_def); + result.push_back(std::make_unique(std::move(sub_graph))); +} + +static int GetOnnxOpSet(const GraphViewer& graph_viewer) { + const auto& dm_to_ver = graph_viewer.DomainToVersionMap(); + return dm_to_ver.at(kOnnxDomain); +} + +static std::map> GetNgSupportedOps(const int onnx_opset) { + std::map> ng_supported_ops; + ng_supported_ops.emplace(kOnnxDomain, ngraph::onnx_import::get_supported_operators(onnx_opset, kOnnxDomain)); + + const std::set ng_disabled_ops = {"DequantizeLinear", "QLinearConv", "QuantizeLinear"}; //Place-holder for ops not supported. + + for (const auto& disabled_op : ng_disabled_ops) { + ng_supported_ops.at(kOnnxDomain).erase(disabled_op); + } + + return ng_supported_ops; +} + +static std::vector GetUnsupportedNodeIndices(const GraphViewer& graph_viewer, /*out*/ std::unordered_set& ng_required_initializers) { + const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer)); + + std::vector unsupported_nodes_idx; + + for (const auto& node_idx : graph_viewer.GetNodesInTopologicalOrder()) { + if (IsNodeSupported(ng_supported_ops, graph_viewer, node_idx)) { + // Collect inputs that are initializers + graph_viewer.GetNode(node_idx)->ForEachDef([&ng_required_initializers, &graph_viewer](const onnxruntime::NodeArg& node_arg, bool is_input) { + if(is_input && graph_viewer.GetAllInitializedTensors().count(node_arg.Name())) { + ng_required_initializers.insert(node_arg.Name()); + } }, true); + } else { + unsupported_nodes_idx.push_back(node_idx); + } + } + + return unsupported_nodes_idx; +} + +/* Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts. + supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by nGraph +*/ +static std::vector> GetPartitionedClusters(const std::vector& topological_order, const std::vector& unsupported_nodes) { + std::vector> ng_clusters; + + auto prev = topological_order.begin(); + + for (const auto& unsup_node : unsupported_nodes) { + auto it = std::find(prev, topological_order.end(), unsup_node); + // Create a cluster vector[supported_node_idx, unsupported_node_idx) and append it to return list. + std::vector this_cluster{prev, it}; + if (!this_cluster.empty()) { + ng_clusters.push_back(std::move(this_cluster)); + } + // Point prev to node idx past this unsuported node. + prev = ++it; + } + + //Tail + std::vector this_cluster{prev, topological_order.end()}; + if (!this_cluster.empty()) { + ng_clusters.push_back(std::move(this_cluster)); + } + + return ng_clusters; +} + +static void GetInputsOutputsOfCluster(const GraphViewer& graph_viewer, + const std::vector& cluster, + const std::unordered_set& ng_required_initializers, + /*out*/ std::vector& cluster_inputs, + /*out*/ std::vector& cluster_outputs) { + std::unordered_set input_args; + std::unordered_set output_args; + std::unordered_set external_output_args; + + for (const auto& node_idx : cluster) { + const auto& node = graph_viewer.GetNode(node_idx); + + // Collect all inputs and outputs + node->ForEachDef( + [&input_args, &output_args](const NodeArg& node_arg, bool is_input) { + is_input ? input_args.insert(node_arg.Name()) : output_args.insert(node_arg.Name()); + }, + true); + + // Check if output of this node is used by nodes outside this_cluster. If yes add this to cluster outputs + for (auto it = node->OutputNodesBegin(); it != node->OutputNodesEnd(); ++it) { + const auto& ext_node = graph_viewer.GetNode((*it).Index()); + + if (std::find(cluster.begin(), cluster.end(), ext_node->Index()) == cluster.end()) { + // Node is external to this_cluster. Search through its inputs to find the output that is generated by this_cluster. + std::set ext_node_inputs; + ext_node->ForEachDef( + [&ext_node_inputs](const onnxruntime::NodeArg& arg, bool is_input) { + if (is_input) { + ext_node_inputs.insert(arg.Name()); + } + }, + true); + + for (const auto& out_def : node->OutputDefs()) { + if (ext_node_inputs.find(out_def->Name()) != ext_node_inputs.end()) { + external_output_args.insert(out_def->Name()); + } + } + } + } + } + + std::vector cluster_initializers; + + //Extract initializers used by this_cluster. + std::unordered_set original_graph_inputs; + for (const auto& node_arg : graph_viewer.GetInputsIncludingInitializers()) { + original_graph_inputs.insert(node_arg->Name()); + } + + const auto& initializers = graph_viewer.GetAllInitializedTensors(); + for (const auto& in_arg : input_args) { + if ((initializers.count(in_arg) && !original_graph_inputs.count(in_arg)) || + ng_required_initializers.count(in_arg)) { + cluster_initializers.push_back(in_arg); + } else if (!output_args.count(in_arg)) { + cluster_inputs.push_back(in_arg); + } + } + + std::copy(external_output_args.begin(), external_output_args.end(), std::back_inserter(cluster_outputs)); + for (const auto& node_arg : graph_viewer.GetOutputs()) { + const auto& name = node_arg->Name(); + if (output_args.count(name) && !external_output_args.count(name)) { + cluster_outputs.push_back(name); + } + } +} + +std::vector> +NGRAPHExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries) const { + ORT_UNUSED_PARAMETER(kernel_registries); + + std::vector> result; + + //TODO:(nivas) Handle If and Loop operators + if (graph_viewer.IsSubgraph()) { + return result; + } + + // Need access to model_path_ + for (const auto& tensor : graph_viewer.GetAllInitializedTensors()) { + if (tensor.second->has_data_location() && tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(WARNING) << "nGraph EP: Initializers with external data location are not currently supported"; + return result; + } + } + + /* This is a list of initializers that nGraph considers as constants. Example weights, reshape shape etc. + TODO: Support overridable initializers */ + std::unordered_set ng_required_initializers; + + const auto unsupported_nodes = GetUnsupportedNodeIndices(graph_viewer, ng_required_initializers); + + //If all ops are supported, no partitioning is required. Short-circuit and avoid splitting. + if (unsupported_nodes.empty()) { + std::vector inputs; + std::vector outputs; + + //Fill inputs with names + std::for_each(graph_viewer.GetInputs().begin(), graph_viewer.GetInputs().end(), + [&inputs](const NodeArg* node_arg) { inputs.push_back(node_arg->Name()); }); + + /* In scenarios, when there are no inputs or all inputs being initializers, + ConstantFolding optimization in onnxruntime pre-computes the value.*/ + if (inputs.empty()) { + return result; + } + + //Fill outputs with names + std::for_each(graph_viewer.GetOutputs().begin(), graph_viewer.GetOutputs().end(), + [&outputs](const NodeArg* node_arg) { outputs.push_back(node_arg->Name()); }); + + // Remove initializers from inputs if they are in ng_required_initializers + inputs.erase(std::remove_if(inputs.begin(), inputs.end(), [&ng_required_initializers](const std::string& name) -> bool { + return ng_required_initializers.count(name); + }), + inputs.end()); + + // Create and add this graph to result. + AppendClusterToSubGraph(graph_viewer.GetNodesInTopologicalOrder(), graph_viewer, inputs, outputs, ng_required_initializers, result); + + } else { // unsupported_nodes_idx.empty() + const auto ng_clusters = GetPartitionedClusters(graph_viewer.GetNodesInTopologicalOrder(), unsupported_nodes); + + for (const auto& this_cluster : ng_clusters) { + std::vector cluster_inputs, cluster_outputs; + GetInputsOutputsOfCluster(graph_viewer, this_cluster, ng_required_initializers, cluster_inputs, cluster_outputs); + + if (!cluster_inputs.empty()) { + AppendClusterToSubGraph(this_cluster, graph_viewer, cluster_inputs, cluster_outputs, ng_required_initializers, result); + } + } + } + + return result; +} + +static ONNX_NAMESPACE::ModelProto GetModelProtoFromFusedNode(const onnxruntime::Node* fused_node) { + const auto& attributes = fused_node->GetAttributes(); + const auto& initializers = attributes.at("initializers").tensors(); + + ONNX_NAMESPACE::ModelProto model_proto; + auto graph_proto = model_proto.mutable_graph(); + const auto& fused_graph = fused_node->GetFunctionBody()->Body(); + + for (const auto& node : fused_graph.Nodes()) { + node.ToProto(*(graph_proto->add_node())); + } + + for (const auto& input : fused_node->InputDefs()) { + auto valueInfoProto = graph_proto->add_input(); + *valueInfoProto = input->ToProto(); + } + + for (const auto& output : fused_node->OutputDefs()) { + auto valueInfoProto = graph_proto->add_output(); + *valueInfoProto = output->ToProto(); + } + + for (const auto& initializer : initializers) { + graph_proto->add_initializer()->CopyFrom(initializer); + } + + auto opset = model_proto.add_opset_import(); + opset->set_domain(kOnnxDomain); + opset->set_version(fused_graph.DomainToVersionMap().at(kOnnxDomain)); + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + return model_proto; +} + +Status NGRAPHExecutionProvider::Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) { + for (const auto& fused_node : fused_nodes) { + auto model_proto = GetModelProtoFromFusedNode(fused_node); + + NodeComputeInfo compute_info; + + // Local copy of backend since, class members cannot be captured. + auto ngraph_backend = ng_backend_; + compute_info.create_state_func = [model_proto, ngraph_backend](ComputeContext* context, FunctionState* state) { + auto* p = new onnxruntime::ngraph_ep::NGRAPHCustomOp(context, model_proto, ngraph_backend); + *state = p; + return 0; + }; + + compute_info.release_state_func = [](FunctionState state) { + if (state) + delete reinterpret_cast(state); + }; + + compute_info.compute_func = [](FunctionState state, ONNXRunTimeTensor* input_tensors, size_t num_inputs, ONNXRunTimeTensor* output_tensors, size_t num_outputs) { + onnxruntime::ngraph_ep::NGRAPHCustomOp* ng_custom_op = reinterpret_cast(state); + + const Status compute_status = ng_custom_op->Compute(input_tensors, num_inputs, output_tensors, num_outputs); + + return compute_status == Status::OK() ? 0 : 1; + }; + + node_compute_funcs.push_back(compute_info); + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h new file mode 100644 index 0000000000..906fa48823 --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_execution_provider.h @@ -0,0 +1,44 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once + +#include "core/framework/execution_provider.h" + +namespace ngraph { +namespace runtime { +class Backend; +} +} // namespace ngraph + +namespace onnxruntime { + +// Information needed to construct nGraph execution providers. +struct NGRAPHExecutionProviderInfo { + const std::string ng_backend_type; +}; + +// Logical device representation. +class NGRAPHExecutionProvider : public IExecutionProvider { + public: + explicit NGRAPHExecutionProvider(const NGRAPHExecutionProviderInfo& info); + ~NGRAPHExecutionProvider() = default; + + Status CopyTensor(const Tensor& src, Tensor& dst) const override; + + std::vector> + GetCapability(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries) const override; + + Status Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) override; + + const void* GetExecutionHandle() const noexcept override { return nullptr; } + + std::shared_ptr GetKernelRegistry() const override; + + private: + std::shared_ptr ng_backend_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/ngraph/ngraph_fwd.h b/onnxruntime/core/providers/ngraph/ngraph_fwd.h new file mode 100644 index 0000000000..1f2ab540d0 --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_fwd.h @@ -0,0 +1,12 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once + +namespace onnxruntime { +namespace ngraph_ep { +template +KernelCreateInfo BuildKernel(); +} +} // namespace onnxruntime + diff --git a/onnxruntime/core/providers/ngraph/ngraph_provider_factory.cc b/onnxruntime/core/providers/ngraph/ngraph_provider_factory.cc new file mode 100644 index 0000000000..987ce9651b --- /dev/null +++ b/onnxruntime/core/providers/ngraph/ngraph_provider_factory.cc @@ -0,0 +1,34 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include "core/providers/ngraph/ngraph_provider_factory.h" +#include +#include "ngraph_execution_provider.h" +#include "core/session/abi_session_options_impl.h" + +using namespace onnxruntime; + +namespace onnxruntime { +struct NGraphProviderFactory : IExecutionProviderFactory { + NGraphProviderFactory(std::string&& ng_backend_type) : ng_backend_type_(std::move(ng_backend_type)) {} + ~NGraphProviderFactory() = default; + + std::unique_ptr CreateProvider() override { + NGRAPHExecutionProviderInfo info{ng_backend_type_}; + return std::make_unique(info); + } + + private: + const std::string ng_backend_type_; +}; + +std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type) { + return std::make_shared(std::string{ng_backend_type}); +} + +} // namespace onnxruntime + +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_NGraph, _In_ OrtSessionOptions* options, _In_ const char* ng_backend_type) { + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_NGraph(ng_backend_type)); + return nullptr; +} diff --git a/onnxruntime/core/providers/ngraph/symbols.txt b/onnxruntime/core/providers/ngraph/symbols.txt new file mode 100644 index 0000000000..9445e69766 --- /dev/null +++ b/onnxruntime/core/providers/ngraph/symbols.txt @@ -0,0 +1 @@ +OrtSessionOptionsAppendExecutionProvider_NGraph \ No newline at end of file diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 8570901f52..4347295653 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -34,13 +34,20 @@ #define BACKEND_MKLML "" #endif +#if USE_NGRAPH +#define BACKEND_NGRAPH "-NGRAPH" +#include "core/providers/ngraph/ngraph_execution_provider.h" +#else +#define BACKEND_NGRAPH "" +#endif + #if USE_OPENBLAS #define BACKEND_OPENBLAS "-OPENBLAS" #else #define BACKEND_OPENBLAS "" #endif -#define BACKEND_DEVICE BACKEND_PROC BACKEND_MKLDNN BACKEND_MKLML BACKEND_OPENBLAS +#define BACKEND_DEVICE BACKEND_PROC BACKEND_MKLDNN BACKEND_MKLML BACKEND_NGRAPH BACKEND_OPENBLAS #include "core/session/onnxruntime_cxx_api.h" #include "core/providers/providers.h" #include "core/providers/cpu/cpu_execution_provider.h" @@ -55,6 +62,9 @@ #ifdef USE_MKLDNN #include "core/providers/mkldnn/mkldnn_provider_factory.h" #endif +#ifdef USE_NGRAPH +#include "core/providers/ngraph/ngraph_provider_factory.h" +#endif #ifdef USE_NUPHAR #include "core/providers/nuphar/nuphar_provider_factory.h" #endif @@ -67,6 +77,7 @@ std::shared_ptr CreateExecutionProviderFactory_CPU(in std::shared_ptr CreateExecutionProviderFactory_CUDA(int device_id); std::shared_ptr CreateExecutionProviderFactory_Tensorrt(); std::shared_ptr CreateExecutionProviderFactory_Mkldnn(int use_arena); +std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type); std::shared_ptr CreateExecutionProviderFactory_Nuphar(int device_id, const char*); std::shared_ptr CreateExecutionProviderFactory_BrainSlice(uint32_t ip, int, int, bool, const char*, const char*, const char*); } // namespace onnxruntime @@ -218,6 +229,11 @@ void InitializeSession(InferenceSession* sess) { RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Mkldnn(enable_cpu_mem_arena ? 1 : 0)); } #endif +#if USE_NGRAPH + { + RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_NGraph("CPU")); + } +#endif #if 0 //USE_NUPHAR { diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index dbf62cd127..1cd7607cb5 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -32,7 +32,7 @@ void usage() { "\t-r [repeat]: Specifies the number of times to repeat\n" "\t-v: verbose\n" "\t-n [test_case_name]: Specifies a single test case to run.\n" - "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn' or 'tensorrt'. Default: 'cpu'.\n" + "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn', 'tensorrt' or 'ngraph'. Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-h: help\n"); } @@ -80,6 +80,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { int p_models = GetNumCpuCores(); bool enable_cuda = false; bool enable_mkl = false; + bool enable_ngraph = false; bool enable_nuphar = false; bool enable_tensorrt = false; OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING; @@ -129,6 +130,8 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { enable_cuda = true; } else if (!CompareCString(optarg, ORT_TSTR("mkldnn"))) { enable_mkl = true; + } else if (!CompareCString(optarg, ORT_TSTR("ngraph"))) { + enable_ngraph = true; } else if (!CompareCString(optarg, ORT_TSTR("nuphar"))) { enable_nuphar = true; } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) { @@ -223,6 +226,14 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { return -1; #endif } + if (enable_ngraph) { //TODO: Re-order the priority? +#ifdef USE_NGRAPH + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_NGraph(sf, "CPU")); +#else + fprintf(stderr, "nGraph is not supported in this build"); + return -1; +#endif + } std::unordered_set cuda_flaky_tests = { "fp16_inception_v1", "fp16_shufflenet", "fp16_tiny_yolov2"}; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index c235fba6b1..adbc2087ec 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -28,9 +28,8 @@ namespace perftest { "Options:\n" "\t-m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'.\n" "\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. " - "Default:'duration'.\n" "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n" - "\t-e [cpu|cuda|mkldnn|tensorrt]: Specifies the provider 'cpu','cuda','mkldnn' or 'tensorrt'. Default:'cpu'.\n" + "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt' or 'ngraph'. Default:'cpu'.\n" "\t-b [tf|ort]: backend to use. Default:ort\n" "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" "\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n" @@ -68,6 +67,8 @@ namespace perftest { test_config.machine_config.provider_type_name = onnxruntime::kCudaExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("mkldnn"))) { test_config.machine_config.provider_type_name = onnxruntime::kMklDnnExecutionProvider; + } else if (!CompareCString(optarg, ORT_TSTR("ngraph"))) { + test_config.machine_config.provider_type_name = onnxruntime::kNGraphExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("brainslice"))) { test_config.machine_config.provider_type_name = onnxruntime::kBrainSliceExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) { diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 0a829bcfca..f20edd2914 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -36,6 +36,12 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(OrtEnv* env, const PerformanceTes ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Mkldnn(sf, enable_cpu_mem_arena ? 1 : 0)); #else ORT_THROW("MKL-DNN is not supported in this build\n"); +#endif + } else if (provider_name == onnxruntime::kNGraphExecutionProvider) { +#ifdef USE_NGRAPH + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_NGraph(sf, "CPU")); +#else + ORT_THROW("nGraph is not supported in this build"); #endif } else if (provider_name == onnxruntime::kCudaExecutionProvider) { #ifdef USE_CUDA diff --git a/onnxruntime/test/providers/ngraph/ngraph_execution_provider_test.cc b/onnxruntime/test/providers/ngraph/ngraph_execution_provider_test.cc new file mode 100644 index 0000000000..83bf930aaf --- /dev/null +++ b/onnxruntime/test/providers/ngraph/ngraph_execution_provider_test.cc @@ -0,0 +1,358 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include +#include "core/providers/ngraph/ngraph_execution_provider.h" +#include "test/providers/provider_test_utils.h" +#include "default_providers.h" +#include "gtest/gtest.h" +#include "core/session/inference_session.h" +#include "test/framework/test_utils.h" +#include "test/test_environment.h" +#include "core/util/math.h" + +using namespace ONNX_NAMESPACE; + +namespace onnxruntime { +namespace test { + +//This is a dummy op that just increments the tensor values by one. +class UnSupportedOp final : public OpKernel { + public: + UnSupportedOp(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* p_context) const { + const auto* X = p_context->Input(0); + + ORT_ENFORCE(X); + auto X_Data = X->Data(); + + auto& shape = X->Shape().GetDims(); + auto* Y = p_context->Output(0, shape); + auto* Y_Data = Y->MutableData(); + + size_t size = 1; + for (size_t i = 0; i < shape.size(); i++) { + size *= shape[i]; + } + + for (size_t i = 0; i < size; i++) { + Y_Data[i] = X_Data[i] + 1; + } + + return Status::OK(); + } +}; + +KernelDefBuilder UnSupportedOpDef() { + KernelDefBuilder def; + def.SetName("UnSupportedOp") + .SetDomain(onnxruntime::kOnnxDomain) + .SinceVersion(7) + .Provider(onnxruntime::kCpuExecutionProvider) + .TypeConstraint("T", DataTypeImpl::GetTensorType()); + return def; +} + +ONNX_NAMESPACE::OpSchema GetUnSupportedOpSchema() { + ONNX_NAMESPACE::OpSchema schema("UnSupportedOp", "unknown", 0); + schema.Input(0, "A", "Data.", "T"); + schema.Output(0, "C", "Result, has same dimensions and type as A", "T"); + schema.TypeConstraint("T", {"tensor(float)"}, "Constrain input and output types to float tensors."); + schema.SinceVersion(7); + return schema; +} + +void add_feeds(NameMLValMap& feeds, std::string name, std::vector dims, std::vector value) { + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), dims, value, &ml_value); + feeds.insert(std::make_pair(name, ml_value)); +} + +//TODO:(nivas) Refractor to use existing code +void RunTest(const std::string& model_path, const NameMLValMap& feeds, const std::vector& output_names, const std::vector>& expected_shapes, const std::vector>& expected_values) { + SessionOptions so; + InferenceSession session_object(so, &DefaultLoggingManager()); + + EXPECT_TRUE(session_object.RegisterExecutionProvider(DefaultNGraphExecutionProvider()).IsOK()); + + std::shared_ptr registry = std::make_shared(); + EXPECT_TRUE(session_object.RegisterCustomRegistry(registry).IsOK()); + auto unsupported_schema = GetUnSupportedOpSchema(); + std::vector schemas = {unsupported_schema}; + EXPECT_TRUE(registry->RegisterOpSet(schemas, onnxruntime::kOnnxDomain, 7, 8).IsOK()); + + auto def = UnSupportedOpDef(); + KernelCreateFn kernel_create_fn = [](const OpKernelInfo& info) -> OpKernel* { return new UnSupportedOp(info); }; + EXPECT_TRUE(registry->RegisterCustomKernel(def, kernel_create_fn).IsOK()); + + auto status = session_object.Load(model_path); + + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Load failed with status: " << status.ErrorMessage(); + return; + } + + status = session_object.Initialize(); + //TODO : Count number of nodes to ensure fusion and also verify that, fused node is assigned to nGraph_EP. + // Currently there is no api to get above info from here. + + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + RunOptions run_options{}; + run_options.run_tag = "nGraph EP test tag"; + run_options.run_log_verbosity_level = 1; + + std::vector fetches; + status = session_object.Run(run_options, feeds, output_names, &fetches); + + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Run failed with status: " << status.ErrorMessage(); + return; + } + + for (size_t idx = 0; idx < expected_values.size(); ++idx) { + auto& got_tensor = fetches[idx].Get(); + auto* got = got_tensor.Data(); + auto& expected = expected_values[idx]; + TensorShape expected_shape(expected_shapes[idx]); + ASSERT_EQ(got_tensor.Shape(), expected_shape); + for (size_t i = 0; i < expected.size(); i++) { + ASSERT_EQ(got[i], expected[i]); + } + } +} + +/* +Basic test: To ensure fusion. TODO: Count number of nodes after model initializaton. + (A) (A) + \ / + Add + (B) | + \ | + Mul + | + | + (Z) + +*/ +TEST(NGraphExecutionProviderTest, Basic_Test) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "B", {4}, {2.0f, 2.0f, 2.0f, 2.0f}); + + std::vector> expected_values = { + {4.0f, 8.0f, 12.0f, 16.0f}}; + + std::vector> expected_shapes = { + {4}}; + + RunTest("testdata/ngraph/Basic_Test.onnx", feeds, {"Z"}, expected_shapes, expected_values); +} + +/* + (A) (A) + \ / + Add + (B) | + \ | + Mul + | + | + UnSupportedOp + | + | + (Z) +Simple test-case for a graph with UnSupportedOp +*/ +TEST(NGraphExecutionProviderTest, Graph_with_UnSupportedOp) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "B", {4}, {2.0f, 2.0f, 2.0f, 2.0f}); + + std::vector> expected_values = { + {5.0f, 9.0f, 13.0f, 17.0f}}; + + std::vector> expected_shapes = { + {4}}; + + RunTest("testdata/ngraph/Graph_with_UnSupportedOp.onnx", feeds, {"Z"}, expected_shapes, expected_values); +} + +/* + (A) (A) + \ / + Add + (B) | + \ | + Mul + | + | + UnSupportedOp + | (C) + | / + Add + | + | + (Z) +Same as above, here are we have two sub-graphs that are run by nGraph execution provider +*/ +TEST(NGraphExecutionProviderTest, Two_Subgraphs) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "B", {4}, {2.0f, 2.0f, 2.0f, 2.0f}); + add_feeds(feeds, "C", {4}, {1.0f, 1.0f, 1.0f, 1.0f}); + + std::vector> expected_values = { + {6.0f, 10.0f, 14.0f, 18.0f}}; + + std::vector> expected_shapes = { + {4}}; + + RunTest("testdata/ngraph/Two_Subgraphs.onnx", feeds, {"Z"}, expected_shapes, expected_values); +} + +/* + (A) (A) + \ / + Add + (B) | + \ | + Mul ----------------\ + | | + | | + UnSupportedOp | + | (C) | + | / | + Add | + | | + | | + (Z) (Y) + +Output of the sub-graph is also graph output +*/ +TEST(NGraphExecutionProviderTest, ClusterOut_isAlso_GraphOut) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "B", {4}, {2.0f, 2.0f, 2.0f, 2.0f}); + add_feeds(feeds, "C", {4}, {1.0f, 1.0f, 1.0f, 1.0f}); + + std::vector> expected_values = { + {2.0f, 4.0f, 6.0f, 8.0f}, + {6.0f, 10.0f, 14.0f, 18.0f}}; + + std::vector> expected_shapes = { + {4}, + {4}}; + + RunTest("testdata/ngraph/ClusterOut_isAlso_GraphOut.onnx", feeds, {"Y", "Z"}, expected_shapes, expected_values); +} + +/* + (A) (A) + \ / + Add + (B) | \ + \ | \ + Mul \ + | \ + | \ + UnSupportedOp\ + (C) | \ + \ | \ + Add \ + | \ + | \ + [Z] [Y] + +Sub-graph in-out is also graph output +*/ +TEST(NGraphExecutionProviderTest, InOut_isAlso_GraphOut) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "B", {4}, {2.0f, 2.0f, 2.0f, 2.0f}); + add_feeds(feeds, "C", {4}, {1.0f, 1.0f, 1.0f, 1.0f}); + + std::vector> expected_values = { + {4.0f, 8.0f, 12.0f, 16.0f}, + {6.0f, 10.0f, 14.0f, 18.0f}}; + + std::vector> expected_shapes = { + {4}, + {4}}; + + RunTest("testdata/ngraph/InOut_isAlso_GraphOut.onnx", feeds, {"Y", "Z"}, expected_shapes, expected_values); +} + + +/* + (A) (A) + \ / + Add + | + | + Dropout + | \ Mask (Unused output) + | + UnSupportedOp + (C) | + \ | + Add + | + | + [Z] +Test-case to esnure ep working in case of unused or optional input that is NOT graph output. +*/ +TEST(NGraphExecutionProviderTest, Op_with_Optional_or_Unused_Outputs) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + add_feeds(feeds, "C", {4}, {1.0f, 1.0f, 1.0f, 1.0f}); + + std::vector> expected_values = { + {4.0f, 6.0f, 8.0f, 10.0f}}; + + std::vector> expected_shapes = { + {4}}; + + RunTest("testdata/ngraph/Op_with_Optional_or_Unused_Outputs.onnx", feeds, {"Z"}, expected_shapes, expected_values); +} + +/* + (A) (A) + \ / + Add(1) + _______|_______ + | | + | Add(2) + | | + UnSupportedOp(4) | + | Add(3) + | | + |_____________| + | + | + Add(5) + | + | + [Z] + +This test is to ensure, we do not have cyclic dependent sub-graphs. +Example: Sub-Graph-1{1,2,3,5} is invalid because the output of this cluster is input to UnSupportedOp whose output is again input to the same cluster. + +*/ +TEST(NGraphExecutionProviderTest, Independent_SubGraphs) { + NameMLValMap feeds; + add_feeds(feeds, "A", {4}, {1.0f, 2.0f, 3.0f, 4.0f}); + + std::vector> expected_values = { + {7.0f, 11.0f, 15.0f, 19.0f}}; + + std::vector> expected_shapes = { + {4}}; + + RunTest("testdata/ngraph/Independent_SubGraphs.onnx", feeds, {"Z"}, expected_shapes, expected_values); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 3103d65436..7ba0fd972b 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -315,7 +315,10 @@ void OpTester::ExecuteModel(Model& model, } } else { if (expect_result == ExpectResult::kExpectFailure) { - EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(expected_failure_string)); + // Disable expected_failure_string checks for MKL-DNN and nGraph EP's + if (provider_type != kMklDnnExecutionProvider && provider_type != kNGraphExecutionProvider) { + EXPECT_THAT(status.ErrorMessage(), testing::HasSubstr(expected_failure_string)); + } } else { LOGS_DEFAULT(ERROR) << "Run failed with status: " << status.ErrorMessage(); EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); @@ -411,6 +414,7 @@ void OpTester::Run(ExpectResult expect_result, kCpuExecutionProvider, kCudaExecutionProvider, kMklDnnExecutionProvider, + kNGraphExecutionProvider, kNupharExecutionProvider, kBrainSliceExecutionProvider, kTensorrtExecutionProvider, @@ -448,6 +452,8 @@ void OpTester::Run(ExpectResult expect_result, execution_provider = DefaultCudaExecutionProvider(); else if (provider_type == onnxruntime::kMklDnnExecutionProvider) execution_provider = DefaultMkldnnExecutionProvider(); + else if (provider_type == onnxruntime::kNGraphExecutionProvider) + execution_provider = DefaultNGraphExecutionProvider(); else if (provider_type == onnxruntime::kNupharExecutionProvider) execution_provider = DefaultNupharExecutionProvider(); else if (provider_type == onnxruntime::kBrainSliceExecutionProvider) @@ -470,10 +476,13 @@ void OpTester::Run(ExpectResult expect_result, if (provider_type == onnxruntime::kTensorrtExecutionProvider) continue; auto reg = execution_provider->GetKernelRegistry(); - const KernelCreateInfo* kci = reg->TryFindKernel(node, execution_provider->Type()); - if (!kci) { - valid = false; - break; + // nGraph EP doesn't have kernels registered + if (!reg->IsEmpty()) { + const KernelCreateInfo* kci = reg->TryFindKernel(node, execution_provider->Type()); + if (!kci) { + valid = false; + break; + } } } diff --git a/onnxruntime/test/testdata/ngraph/Basic_Test.onnx b/onnxruntime/test/testdata/ngraph/Basic_Test.onnx new file mode 100644 index 0000000000..c6fb88b25e --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/Basic_Test.onnx @@ -0,0 +1,19 @@ +nGraph EP test model:‰ + +A +A node1_outnode_1"Add + + node1_out +BZnode_2"MulnGraph EP test graphZ +A + + +Z +B + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/ClusterOut_isAlso_GraphOut.onnx b/onnxruntime/test/testdata/ngraph/ClusterOut_isAlso_GraphOut.onnx new file mode 100644 index 0000000000..a370bcda9c --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/ClusterOut_isAlso_GraphOut.onnx @@ -0,0 +1,32 @@ +nGraph EP test model:ò + +A +AYnode_1"Add + +Y +B node2_outnode_2"Mul +- + node2_out node3_outnode_3" UnSupportedOp + + node3_out +CZnode_4"AddnGraph EP test graphZ +A + + +Z +B + + +Z +C + + +b +Y + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/Graph_with_UnSupportedOp.onnx b/onnxruntime/test/testdata/ngraph/Graph_with_UnSupportedOp.onnx new file mode 100644 index 0000000000..42cfb71e26 --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/Graph_with_UnSupportedOp.onnx @@ -0,0 +1,21 @@ +nGraph EP test model:¸ + +A +A node1_outnode_1"Add +& + node1_out +B node2_outnode_2"Mul +% + node2_outZnode_3" UnSupportedOpnGraph EP test graphZ +A + + +Z +B + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/InOut_isAlso_GraphOut.onnx b/onnxruntime/test/testdata/ngraph/InOut_isAlso_GraphOut.onnx new file mode 100644 index 0000000000..2956bfe71f --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/InOut_isAlso_GraphOut.onnx @@ -0,0 +1,32 @@ +nGraph EP test model:ò + +A +A node1_outnode_1"Add + + node1_out +BYnode_2"Mul +% +Y node3_outnode_3" UnSupportedOp + + node3_out +CZnode_4"AddnGraph EP test graphZ +A + + +Z +B + + +Z +C + + +b +Y + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/Independent_SubGraphs.onnx b/onnxruntime/test/testdata/ngraph/Independent_SubGraphs.onnx new file mode 100644 index 0000000000..f8a036f7d7 Binary files /dev/null and b/onnxruntime/test/testdata/ngraph/Independent_SubGraphs.onnx differ diff --git a/onnxruntime/test/testdata/ngraph/Op_with_Optional_or_Unused_Outputs.onnx b/onnxruntime/test/testdata/ngraph/Op_with_Optional_or_Unused_Outputs.onnx new file mode 100644 index 0000000000..71634b2289 --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/Op_with_Optional_or_Unused_Outputs.onnx @@ -0,0 +1,23 @@ +nGraph EP test model:Ô + +A +A node1_outnode_1"Add +" + node1_outYXnode_2"Dropout +% +Y node3_outnode_3" UnSupportedOp + + node3_out +CZnode_4"AddnGraph EP test graphZ +A + + +Z +C + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/Two_Subgraphs.onnx b/onnxruntime/test/testdata/ngraph/Two_Subgraphs.onnx new file mode 100644 index 0000000000..ac97982072 --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/Two_Subgraphs.onnx @@ -0,0 +1,28 @@ +nGraph EP test model:ñ + +A +A node1_outnode_1"Add +& + node1_out +B node2_outnode_2"Mul +- + node2_out node3_outnode_3" UnSupportedOp + + node3_out +CZnode_4"AddnGraph EP test graphZ +A + + +Z +B + + +Z +C + + +b +Z + + +B \ No newline at end of file diff --git a/onnxruntime/test/testdata/ngraph/model_gen.py b/onnxruntime/test/testdata/ngraph/model_gen.py new file mode 100644 index 0000000000..be21632bf3 --- /dev/null +++ b/onnxruntime/test/testdata/ngraph/model_gen.py @@ -0,0 +1,107 @@ +import numpy as np +import onnx +from onnx import helper, numpy_helper + +def save_graph(graph_def, name) : + model_def = helper.make_model(graph_def, producer_name="nGraph EP test model") + model_def.opset_import[0].version = 7 + model_def.ir_version = 3 + onnx.save_model(model_def, name) + +A = helper.make_tensor_value_info('A', onnx.TensorProto.FLOAT, [4]) +B = helper.make_tensor_value_info('B', onnx.TensorProto.FLOAT, [4]) +C = helper.make_tensor_value_info('C', onnx.TensorProto.FLOAT, [4]) +Y = helper.make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [4]) +Z = helper.make_tensor_value_info('Z', onnx.TensorProto.FLOAT, [4]) + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['node1_out'], "node_1"), +helper.make_node('Mul',['node1_out','B'],['Z'], "node_2")], +"nGraph EP test graph", +[A,B], +[Z] +) + +save_graph(graph_def, "Basic_Test.onnx") + + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['node1_out'], "node_1"), +helper.make_node('Mul',['node1_out','B'],['node2_out'], "node_2"), +helper.make_node('UnSupportedOp',['node2_out'],['Z'], "node_3")], +"nGraph EP test graph", +[A,B], +[Z] +) + +save_graph(graph_def, "Graph_with_UnSupportedOp.onnx") + + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['node1_out'], "node_1"), +helper.make_node('Mul',['node1_out','B'],['node2_out'], "node_2"), +helper.make_node('UnSupportedOp',['node2_out'],['node3_out'], "node_3"), +helper.make_node('Add',['node3_out','C'],['Z'], "node_4")], +"nGraph EP test graph", +[A,B,C], +[Z] +) + +save_graph(graph_def, "Two_Subgraphs.onnx") + + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['Y'], "node_1"), +helper.make_node('Mul',['Y','B'],['node2_out'], "node_2"), +helper.make_node('UnSupportedOp',['node2_out'],['node3_out'], "node_3"), +helper.make_node('Add',['node3_out','C'],['Z'], "node_4")], +"nGraph EP test graph", +[A,B,C], +[Y,Z] +) + +save_graph(graph_def, "ClusterOut_isAlso_GraphOut.onnx") + + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['node1_out'], "node_1"), +helper.make_node('Mul',['node1_out','B'],['Y'], "node_2"), +helper.make_node('UnSupportedOp',['Y'],['node3_out'], "node_3"), +helper.make_node('Add',['node3_out','C'],['Z'], "node_4")], +"nGraph EP test graph", +[A,B,C], +[Y,Z] +) + +save_graph(graph_def, "InOut_isAlso_GraphOut.onnx") + + +graph_def = helper.make_graph( +[helper.make_node('Add',['A','A'],['node1_out'], "node_1"), +helper.make_node('Dropout',['node1_out'],['Y','X'], "node_2"), +helper.make_node('UnSupportedOp',['Y'],['node3_out'], "node_3"), +helper.make_node('Add',['node3_out','C'],['Z'], "node_4")], +"nGraph EP test graph", +[A,C], +[Z] +) + +save_graph(graph_def, "Op_with_Optional_or_Unused_Outputs.onnx") + +one_in = helper.make_tensor_value_info('one', onnx.TensorProto.FLOAT, [1]) +one_data = np.array([1],dtype=np.float32) +one_tensor = numpy_helper.from_array(one_data, "one") + +graph_def = helper.make_graph( +[helper.make_node('Add',['A', "A"],['node1_out'], "node_1"), +helper.make_node('UnSupportedOp',['node1_out'],['branch_a_out'], "node_2"), +helper.make_node('Add',["node1_out","one"],["branch_b_out1"], "node_3"), +helper.make_node('Add',["branch_b_out1","one"],["branch_b_out"],"node_4"), +helper.make_node('Add',["branch_a_out","branch_b_out"],["Z"], "node_5")], +"nGraph EP test graph", +[A, one_in], +[Z], +[one_tensor] +) + +save_graph(graph_def, "Independent_SubGraphs.onnx") diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index bdb7457853..86fb9377f1 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -11,6 +11,7 @@ namespace onnxruntime { std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); std::shared_ptr CreateExecutionProviderFactory_CUDA(int device_id); std::shared_ptr CreateExecutionProviderFactory_Mkldnn(int use_arena); +std::shared_ptr CreateExecutionProviderFactory_NGraph(const char* ng_backend_type); std::shared_ptr CreateExecutionProviderFactory_Nuphar(int device_id, const char*); std::shared_ptr CreateExecutionProviderFactory_BrainSlice(uint32_t ip, int, int, bool, const char*, const char*, const char*); std::shared_ptr CreateExecutionProviderFactory_Tensorrt(); @@ -46,6 +47,14 @@ std::unique_ptr DefaultMkldnnExecutionProvider(bool enable_a #endif } +std::unique_ptr DefaultNGraphExecutionProvider() { +#ifdef USE_NGRAPH + return CreateExecutionProviderFactory_NGraph("CPU")->CreateProvider(); +#else + return nullptr; +#endif +} + std::unique_ptr DefaultNupharExecutionProvider() { #ifdef USE_NUPHAR return CreateExecutionProviderFactory_Nuphar(0, "")->CreateProvider(); diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 4687b8f6b4..25f348eb6b 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -10,6 +10,7 @@ namespace test { std::unique_ptr DefaultCpuExecutionProvider(bool enable_arena = true); std::unique_ptr DefaultCudaExecutionProvider(); std::unique_ptr DefaultMkldnnExecutionProvider(bool enable_arena = true); +std::unique_ptr DefaultNGraphExecutionProvider(); std::unique_ptr DefaultNupharExecutionProvider(); std::unique_ptr DefaultBrainSliceExecutionProvider(); std::unique_ptr DefaultTensorrtExecutionProvider(); diff --git a/onnxruntime/test/util/include/providers.h b/onnxruntime/test/util/include/providers.h index e6eab2aa54..592c13cf44 100644 --- a/onnxruntime/test/util/include/providers.h +++ b/onnxruntime/test/util/include/providers.h @@ -10,6 +10,9 @@ #ifdef USE_MKLDNN #include "core/providers/mkldnn/mkldnn_provider_factory.h" #endif +#ifdef USE_NGRAPH +#include "core/providers/ngraph/ngraph_provider_factory.h" +#endif #ifdef USE_NUPHAR #include "core/providers/nuphar/nuphar_provider_factory.h" #endif diff --git a/setup.py b/setup.py index 1dea3f2945..92384379a6 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,9 @@ elif '--use_cuda' in sys.argv: package_name = 'ort-gpu-nightly' nightly_build = True sys.argv.remove('--nightly_build') +elif '--use_ngraph' in sys.argv: + package_name = 'onnxruntime-ngraph' + sys.argv.remove('--use_ngraph') if '--nightly_build' in sys.argv: package_name = 'ort-nightly' @@ -43,6 +46,8 @@ except ImportError: # Additional binaries if platform.system() == 'Linux': libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.so.0', 'libmklml_intel.so', 'libiomp5.so'] + # nGraph Libs + libs.extend(['libngraph.so', 'libcodegen.so', 'libcpu_backend.so', 'libmkldnn.so', 'libtbb_debug.so', 'libtbb_debug.so.2', 'libtbb.so', 'libtbb.so.2']) elif platform.system() == "Darwin": libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.0.dylib'] # TODO add libmklml and libiomp5 later. else: diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 78d652fa36..c098e963f7 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -116,6 +116,7 @@ Use the individual flags to only run the specified stages. parser.add_argument("--use_openblas", action='store_true', help="Build with OpenBLAS.") parser.add_argument("--use_mkldnn", action='store_true', help="Build with MKLDNN.") parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") + parser.add_argument("--use_ngraph", action='store_true', help="Build with nGraph.") parser.add_argument("--use_nsync", action='store_true', help="Build with NSYNC.") parser.add_argument("--use_preinstalled_eigen", action='store_true', help="Use pre-installed eigen.") parser.add_argument("--eigen_path", help="Path to pre-installed eigen.") @@ -311,6 +312,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_OPENBLAS=" + ("ON" if args.use_openblas else "OFF"), "-Donnxruntime_USE_MKLDNN=" + ("ON" if args.use_mkldnn else "OFF"), "-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"), + "-Donnxruntime_USE_NGRAPH=" + ("ON" if args.use_ngraph else "OFF"), "-Donnxruntime_USE_OPENMP=" + ("ON" if args.use_openmp else "OFF"), "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"), "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"), @@ -323,8 +325,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script) "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), "-Donnxruntime_BUILD_x86=" + ("ON" if args.x86 else "OFF"), - # TensorRT provider currently only supports full_protobuf option. - "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_tensorrt else "OFF"), + # nGraph and TensorRT providers currently only supports full_protobuf option. + "-Donnxruntime_USE_FULL_PROTOBUF=" + ("ON" if args.use_full_protobuf or args.use_ngraph or args.use_tensorrt else "OFF"), "-Donnxruntime_DISABLE_CONTRIB_OPS=" + ("ON" if args.disable_contrib_ops else "OFF"), "-Donnxruntime_MSVC_STATIC_RUNTIME=" + ("ON" if args.enable_msvc_static_runtime else "OFF"), ] @@ -490,7 +492,7 @@ def setup_tensorrt_vars(args): return tensorrt_home -def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_python_tests, enable_tvm = False, enable_tensorrt = False): +def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_python_tests, enable_tvm = False, enable_tensorrt = False, enable_ngraph = False): for config in configs: log.info("Running tests for %s configuration", config) cwd = get_config_build_dir(build_dir, config) @@ -505,7 +507,8 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enab if enable_python_tests: # Disable python tests for TensorRT because many tests are not supported yet - if enable_tensorrt: + if enable_tensorrt or enable_ngraph: + print("NOT RUNNING") return if is_windows(): cwd = os.path.join(cwd, config) @@ -566,7 +569,7 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para else: run_subprocess([exe] + cmd, cwd=cwd) -def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt, nightly_build = False): +def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_ngraph, use_tensorrt, nightly_build = False): for config in configs: cwd = get_config_build_dir(build_dir, config) @@ -577,6 +580,8 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt, n run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt', '--nightly_build'], cwd=cwd) elif use_cuda: run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda', '--nightly_build'], cwd=cwd) + elif use_ngraph: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_ngraph', '--nightly-build'], cwd=cwd) else: run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--nightly_build'], cwd=cwd) else: @@ -584,6 +589,8 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt, n run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt'], cwd=cwd) elif use_cuda: run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd) + elif use_ngraph: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_ngraph'], cwd=cwd) else: run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd) if is_ubuntu_1604(): @@ -739,7 +746,7 @@ def main(): if args.test : run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, args.enable_pybind if not args.skip_onnx_tests else False, - args.use_tvm, args.use_tensorrt) + args.use_tvm, args.use_tensorrt, args.use_ngraph) # run the onnx model tests if requested explicitly. if args.enable_onnx_tests and not args.skip_onnx_tests: # directory from ONNX submodule with ONNX test data @@ -754,6 +761,8 @@ def main(): run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False, 2) elif args.x86 or platform.system() == 'Darwin': run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, False, 1) + elif args.use_ngraph: + run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'ngraph', True, 1) # TODO: parallel executor test fails on MacOS else: run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True, 0) @@ -764,8 +773,8 @@ def main(): if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') - build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_tensorrt, nightly_build) - + build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_ngraph, args.use_tensorrt, nightly_build) + if args.gen_doc: generate_documentation(source_dir, build_dir, configs) diff --git a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml new file mode 100644 index 0000000000..d7ac1c5c9e --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml @@ -0,0 +1,30 @@ +jobs: +- job: Linux_nGraph_CI_Dev + pool: Linux-CPU + steps: + - template: templates/set-test-data-variables-step.yml + + - task: CmdLine@2 + displayName: 'Download azcopy' + inputs: + script: | + curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' + tar -zxvf azcopy.tar.gz --strip 1 + workingDirectory: $(Build.BinariesDirectory) + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' + arguments: --test_data_url $(TestDataUrl) + pythonInterpreter: '/usr/bin/python3' + workingDirectory: $(Build.BinariesDirectory) + + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d ngraph -r $(Build.BinariesDirectory) -x "--use_ngraph"' + displayName: 'Command Line Script' + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + + - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 11fbb8faad..00bda4df84 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -14,6 +14,7 @@ apt-get update && apt-get install -y software-properties-common add-apt-repository ppa:deadsnakes/ppa apt-get update && apt-get install -y --no-install-recommends \ autotools-dev \ + automake \ build-essential \ git apt-transport-https \ ca-certificates \ @@ -34,6 +35,8 @@ apt-get update && apt-get install -y --no-install-recommends \ libssl1.0.0 \ libkrb5-3 \ libicu55 \ + libtinfo-dev \ + libtool \ aria2 \ bzip2 \ unzip \ diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index a57cfb58e7..58f786a35c 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -39,6 +39,12 @@ else --use_openmp \ --cuda_home /usr/local/cuda \ --cudnn_home /usr/local/cuda --build_shared_lib $BUILD_EXTR_PAR + elif [ $BUILD_DEVICE = "ngraph" ]; then + python3 $SCRIPT_DIR/../../build.py --build_dir /build \ + --config Debug Release --build_shared_lib \ + --skip_submodule_sync --enable_onnx_tests \ + --build_wheel \ + --parallel --use_openmp $BUILD_EXTR_PAR else python3 $SCRIPT_DIR/../../build.py --build_dir /build \ --config Debug Release --build_shared_lib \ diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index d600f85963..8c0948d667 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -63,7 +63,7 @@ if [ -z "$NIGHTLY_BUILD" ]; then set NIGHTLY_BUILD=0 fi -if [ $BUILD_DEVICE = "cpu" ]; then +if [ $BUILD_DEVICE = "cpu" ] || [ $BUILD_DEVICE = "ngraph" ]; then docker rm -f "onnxruntime-$BUILD_DEVICE" || true docker run -h $HOSTNAME \ --name "onnxruntime-$BUILD_DEVICE" \ @@ -89,7 +89,7 @@ else /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" -o $BUILD_OS & fi -wait -n +wait $! EXIT_CODE=$?