From e8b0ae892380972ed695e1bd455f3534a9523f03 Mon Sep 17 00:00:00 2001 From: stevenlix Date: Thu, 14 Mar 2019 12:00:39 -0700 Subject: [PATCH] Trt execution provider (#382) * updated cmake files for trt * added trt execution provider * added trt basic test * removed trt_path action attribute * Add files via upload * Update build.py * Update trt_allocator.h * fixed issues found by reviewers * changed cast operator * added comment for custom kernel implementation * changed auto to auto& * changed to function compile APIs for TRT execution provider * changed to function compile APIs for TRT execution provider * added new DType DInt64 * adapted to the changes of onnxruntime_c_api * removed trt kernel (use function compile instead) * updated onnx-tensorrt submodule * set default memory type to TRT fused kernel * resolve merge conflict * fixed the issue that USE_CUDA conflicts with USE_TRT * construct graph by adding nodes in topological order * made changes for Windows * change buffers type * bypass HasImplementationOf check for TRT XP because TRT kernel is not registered * added domain to version info in rebuilt model proto * added trt to test option list * added DomainToVersionMap() to GraphViewer * removed Copy() * fixed broken code * format the code to clang format * used local reference to the frequently used values * fixed a couple of issues according to reviewers feedback * fixed a couple of issues according to reviewers feedback * added python binding for TRT and enable use_cuda when use_trt is on * fixed a redefinition issue * changed shared_ptr to unique_ptr on trt engines, and made a few changes required by reviewers * enabled trtexecution provider for unit tests * renamed trt to tensorrt * added tesorrt to python binding * update submodule onnx and onnx-tensorrt * made a couple of minor changes based on reviewer's feedback * added CUDA_CHECK * removed test code * fixed broken code after merge * updated onnx-tensorrt submodule * added post processing to align trt inputs/outputs with graph inputs/outputs * updated onnx submodule * added CUDA fallback for TensorRT and fixed TensorRT cmake issue * added ci pipeline for tensorrt and removed some redundent code from trt xp * fixed syntax issue * updated onnx-tensorrt submodule * fix trt build problem by: (#602) 1. Add additional /wd for debug build 2. Add io.h for additional targets 3. Bring back mb version of getopt * Update install_ubuntu.sh * Update linux-gpu-tensorrt-ci-pipeline.yml * Update linux-gpu-tensorrt-ci-pipeline.yml * Update run_build.sh * Update run_build.sh * Update run_build.sh * Update run_build.sh * fixed the issue that GetKernelRegistry returns nullptr * merged master to this branch * moved some data types to private * fixed tensorrt CI pipeline issue * customized test data for TensorRT pipeline * added onnx-tensorrt in json file and fixed an issue in ci script * added comments --- cgmanifest.json | 9 + cmake/CMakeLists.txt | 8 +- cmake/external/onnx | 2 +- cmake/external/onnx-tensorrt | 2 +- cmake/onnxruntime.cmake | 1 + cmake/onnxruntime_providers.cmake | 73 ++- cmake/onnxruntime_python.cmake | 1 + cmake/onnxruntime_unittests.cmake | 10 +- include/onnxruntime/core/framework/func_api.h | 3 +- include/onnxruntime/core/graph/constants.h | 2 +- include/onnxruntime/core/graph/graph_viewer.h | 5 + .../tensorrt/tensorrt_provider_factory.h | 15 + onnxruntime/core/framework/func_kernel.cc | 2 + .../core/framework/graph_partitioner.cc | 4 + onnxruntime/core/framework/utils.cc | 2 +- .../core/optimizer/transformer_memcpy.cc | 6 +- .../core/providers/tensorrt/symbols.txt | 1 + .../providers/tensorrt/tensorrt_allocator.h | 33 ++ .../tensorrt/tensorrt_execution_provider.cc | 447 ++++++++++++++++++ .../tensorrt/tensorrt_execution_provider.h | 100 ++++ .../tensorrt/tensorrt_provider_factory.cc | 33 ++ .../python/onnxruntime_pybind_state.cc | 10 + onnxruntime/test/framework/test_utils.cc | 6 +- onnxruntime/test/framework/test_utils.h | 8 +- onnxruntime/test/onnx/main.cc | 27 +- .../test/perftest/command_args_parser.cc | 6 +- .../test/perftest/performance_runner.cc | 7 +- .../test/providers/provider_test_utils.cc | 3 + .../test/providers/trt/trt_basic_test.cc | 103 ++++ onnxruntime/test/util/default_providers.cc | 18 +- .../test/util/include/default_providers.h | 2 +- onnxruntime/test/util/include/providers.h | 4 +- onnxruntime/test/win_getopt/mb/getopt.cc | 117 +++++ .../test/win_getopt/mb/include/getopt.h | 42 ++ .../test/win_getopt/mb/include/unistd.h | 5 + tools/ci_build/build.py | 49 +- .../linux-gpu-tensorrt-ci-pipeline.yml | 16 + .../win-gpu-tensorrt-ci-pipeline.yml | 144 ++++++ .../linux/docker/Dockerfile.ubuntu_tensorrt | 46 ++ .../linux/docker/scripts/install_ubuntu.sh | 1 + tools/ci_build/github/linux/run_build.sh | 10 + .../ci_build/github/linux/run_dockerbuild.sh | 17 +- 42 files changed, 1339 insertions(+), 61 deletions(-) create mode 100644 include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h create mode 100755 onnxruntime/core/providers/tensorrt/symbols.txt create mode 100755 onnxruntime/core/providers/tensorrt/tensorrt_allocator.h create mode 100755 onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc create mode 100755 onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h create mode 100755 onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc create mode 100644 onnxruntime/test/providers/trt/trt_basic_test.cc create mode 100644 onnxruntime/test/win_getopt/mb/getopt.cc create mode 100644 onnxruntime/test/win_getopt/mb/include/getopt.h create mode 100644 onnxruntime/test/win_getopt/mb/include/unistd.h create mode 100644 tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml create mode 100644 tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt diff --git a/cgmanifest.json b/cgmanifest.json index 9d5fd7c7d0..b336195444 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -280,6 +280,15 @@ "DownloadUrl":"https://svnweb.freebsd.org/base/release/12.0.0/lib/libc/stdlib/getopt.c?revision=341707&view=co" } } + }, + { + "component":{ + "type":"git", + "git":{ + "commitHash":"3ad2bd19d6f0a2475805e9aa5ae8734a0d60fbaa", + "repositoryUrl":"https://github.com/stevenlix/onnx-tensorrt.git" + } + } } ], "Version":1 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 49440bebc6..63ccf6a0fe 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -64,7 +64,7 @@ option(onnxruntime_BUILD_SHARED_LIB "Build a shared library" OFF) option(onnxruntime_ENABLE_MICROSOFT_INTERNAL "Use this option to enable/disable microsoft internal only code" OFF) option(onnxruntime_USE_NUPHAR "Build with Nupha" OFF) option(onnxruntime_USE_BRAINSLICE "Build with BrainSlice" OFF) -option(onnxruntime_USE_TRT "Build with TensorRT support" OFF) +option(onnxruntime_USE_TENSORRT "Build with TensorRT support" OFF) option(onnxruntime_CROSS_COMPILING "Cross compiling onnx runtime" OFF) set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) @@ -493,6 +493,12 @@ if (onnxruntime_USE_CUDA) endif() endif() +if (onnxruntime_USE_TENSORRT) + if (WIN32) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DELAYLOAD:nvinfer.dll /DELAYLOAD:nvinfer_plugin.dll") + endif() +endif() + if (onnxruntime_USE_TVM) if (WIN32 AND MSVC) # wd4100: identifier' : unreferenced formal parameter diff --git a/cmake/external/onnx b/cmake/external/onnx index 1ec81bc6d4..1cca8733c6 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit 1ec81bc6d49ccae23cd7801515feaadd13082903 +Subproject commit 1cca8733c692b57d3ebe3da044ef24f3af00006e diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt index 493487a720..3ad2bd19d6 160000 --- a/cmake/external/onnx-tensorrt +++ b/cmake/external/onnx-tensorrt @@ -1 +1 @@ -Subproject commit 493487a7203d8a059a9b2288807cb700857fe5ca +Subproject commit 3ad2bd19d6f0a2475805e9aa5ae8734a0d60fbaa diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 6ad7f009c2..957d1997a3 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -58,6 +58,7 @@ target_link_libraries(onnxruntime PRIVATE ${onnxruntime_libs} ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} + ${PROVIDERS_TENSORRT} onnxruntime_optimizer onnxruntime_providers onnxruntime_util diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index ab47cb06a7..8a1432a8d0 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -26,7 +26,10 @@ if(onnxruntime_USE_CUDA) set(PROVIDERS_CUDA onnxruntime_providers_cuda) list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda) endif() - +if(onnxruntime_USE_TENSORRT) + set(PROVIDERS_TENSORRT onnxruntime_providers_tensorrt) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES tensorrt) +endif() source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs}) # add using ONNXRUNTIME_ROOT so they show up under the 'contrib_ops' folder in Visual Studio source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs}) @@ -87,7 +90,7 @@ if (onnxruntime_USE_MKLDNN) ) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_mkldnn_cc_srcs}) - add_library(onnxruntime_providers_mkldnn ${onnxruntime_providers_mkldnn_cc_srcs}) + add_library(onnxruntime_providers_mkldnn ${onnxruntime_providers_mkldnn_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn gsl onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime") @@ -96,6 +99,72 @@ if (onnxruntime_USE_MKLDNN) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX) endif() +if (onnxruntime_USE_TENSORRT) + add_definitions(-DUSE_TENSORRT=1) + add_definitions("-DONNX_ML=1") + add_definitions("-DONNX_NAMESPACE=onnx") + include_directories(${PROJECT_SOURCE_DIR}/external/protobuf) + set(CUDA_INCLUDE_DIRS ${onnxruntime_CUDA_HOME}/include) + set(TENSORRT_ROOT ${onnxruntime_TENSORRT_HOME}) + include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx) + if (WIN32) + set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + set(OLD_CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996 /wd4244 /wd4267 /wd4099 /wd4551 /wd4505 /wd4515 /wd4706 /wd4456") + if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4701 /wd4805") + endif() + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -include algorithm") + set(PROTOBUF_LIBRARY libprotobuf) + set(DISABLED_WARNINGS_FOR_TRT /wd4267 /wd4244 /wd4996) + endif() + add_subdirectory(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt) + if (WIN32) + set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) + set(CMAKE_CUDA_FLAGS ${OLD_CMAKE_CUDA_FLAGS}) + unset(PROTOBUF_LIBRARY) + unset(OLD_CMAKE_CXX_FLAGS) + unset(OLD_CMAKE_CUDA_FLAGS) + set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199") + set_target_properties(nvonnxparser_runtime PROPERTIES LINK_FLAGS "/ignore:4199") + set_target_properties(trt_onnxify PROPERTIES LINK_FLAGS "/ignore:4199") + target_compile_definitions(trt_onnxify PRIVATE ONNXIFI_BUILD_LIBRARY=1) + target_sources(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc) + target_sources(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc) + target_include_directories(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include) + target_include_directories(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include) + target_compile_options(nvonnxparser_static PRIVATE /FIio.h) + target_compile_options(nvonnxparser PRIVATE /FIio.h) + target_compile_options(trt_onnxify PRIVATE /FIio.h) + target_compile_options(onnx2trt PRIVATE /FIio.h) + target_compile_options(getSupportedAPITest PRIVATE /FIio.h) + endif() + include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt) + include_directories(${TENSORRT_INCLUDE_DIR}) + set(trt_link_libs cudnn ${CMAKE_DL_LIBS} ${TENSORRT_LIBRARY}) + set(onnxparser_link_libs nvonnxparser_static nvonnxparser_plugin) + + file(GLOB_RECURSE onnxruntime_providers_tensorrt_cc_srcs + "${ONNXRUNTIME_ROOT}/core/providers/tensorrt/*.h" + "${ONNXRUNTIME_ROOT}/core/providers/tensorrt/*.cc" + ) + + source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_tensorrt_cc_srcs}) + add_library(onnxruntime_providers_tensorrt ${onnxruntime_providers_tensorrt_cc_srcs}) + target_link_libraries(onnxruntime_providers_tensorrt ${onnxparser_link_libs} ${trt_link_libs}) + onnxruntime_add_include_to_target(onnxruntime_providers_tensorrt onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf) + add_dependencies(onnxruntime_providers_tensorrt eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) + target_include_directories(onnxruntime_providers_tensorrt PRIVATE ${ONNXRUNTIME_ROOT} ${onnxruntime_CUDNN_HOME}/include ${eigen_INCLUDE_DIRS} PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/tensorrt DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) + set_target_properties(onnxruntime_providers_tensorrt PROPERTIES LINKER_LANGUAGE CXX) + set_target_properties(onnxruntime_providers_tensorrt PROPERTIES FOLDER "ONNXRuntime") + target_compile_definitions(onnxruntime_providers_tensorrt PRIVATE ONNXIFI_BUILD_LIBRARY=1) + target_compile_options(onnxruntime_providers_tensorrt PRIVATE ${DISABLED_WARNINGS_FOR_TRT}) + if (WIN32) + target_compile_options(onnxruntime_providers_tensorrt INTERFACE /wd4996) + endif() +endif() + if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) include(onnxruntime_providers_internal.cmake) endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 9b0eaca746..06de47993a 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -58,6 +58,7 @@ set(onnxruntime_pybind11_state_libs ${onnxruntime_libs} ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} + ${PROVIDERS_TENSORRT} onnxruntime_optimizer onnxruntime_providers onnxruntime_util diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 3f09ff4476..6515e03a6e 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -91,7 +91,7 @@ file(GLOB onnxruntime_test_ir_src file(GLOB onnxruntime_test_optimizer_src "${TEST_SRC_DIR}/optimizer/*.cc" - "${TEST_SRC_DIR}/optimizer/*.h" + "${TEST_SRC_DIR}/optimizer/*.h" ) set(onnxruntime_test_framework_src_patterns @@ -189,6 +189,7 @@ set(ONNXRUNTIME_TEST_LIBS ${onnxruntime_libs} ${PROVIDERS_CUDA} ${PROVIDERS_MKLDNN} + ${PROVIDERS_TENSORRT} onnxruntime_optimizer onnxruntime_providers onnxruntime_util @@ -205,6 +206,13 @@ set(onnxruntime_test_providers_libs ${ONNXRUNTIME_TEST_LIBS} ) +if(onnxruntime_USE_TENSORRT) + list(APPEND onnxruntime_test_framework_src_patterns ${TEST_SRC_DIR}/providers/tensorrt/*) + list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_tensorrt) + list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_tensorrt) + list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_tensorrt) +endif() + if( NOT WIN32 AND (HAS_FILESYSTEM_H OR HAS_EXPERIMENTAL_FILESYSTEM_H)) list(APPEND onnxruntime_test_providers_libs stdc++fs) endif() diff --git a/include/onnxruntime/core/framework/func_api.h b/include/onnxruntime/core/framework/func_api.h index 606645343c..c6a90180b1 100644 --- a/include/onnxruntime/core/framework/func_api.h +++ b/include/onnxruntime/core/framework/func_api.h @@ -5,7 +5,8 @@ namespace onnxruntime { enum DType { TFloat32 = 0, TInt32 = 1, - TDouble = 2 + TDouble = 2, + TInt64 = 3 //TODO: more types }; diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index 25b6f36c0f..328ce4a742 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -23,6 +23,6 @@ constexpr const char* kCudaExecutionProvider = "CUDAExecutionProvider"; constexpr const char* kMklDnnExecutionProvider = "MKLDNNExecutionProvider"; constexpr const char* kNupharExecutionProvider = "NupharExecutionProvider"; constexpr const char* kBrainSliceExecutionProvider = "BrainSliceExecutionProvider"; -constexpr const char* kTRTExecutionProvider = "TRTExecutionProvider"; +constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider"; } // namespace onnxruntime diff --git a/include/onnxruntime/core/graph/graph_viewer.h b/include/onnxruntime/core/graph/graph_viewer.h index 346cc53950..652c572af2 100644 --- a/include/onnxruntime/core/graph/graph_viewer.h +++ b/include/onnxruntime/core/graph/graph_viewer.h @@ -97,6 +97,11 @@ class GraphViewer { */ const NodeArg* GetNodeArg(const std::string& name) const; + /** Gets the map of operator domains to their opset versions. */ + const std::unordered_map& DomainToVersionMap() const noexcept { + return graph_->DomainToVersionMap(); + } + private: ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphViewer); diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h new file mode 100644 index 0000000000..bee1ae1b09 --- /dev/null +++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options); + +#ifdef __cplusplus +} +#endif + diff --git a/onnxruntime/core/framework/func_kernel.cc b/onnxruntime/core/framework/func_kernel.cc index 79878f8a82..932568d2b1 100644 --- a/onnxruntime/core/framework/func_kernel.cc +++ b/onnxruntime/core/framework/func_kernel.cc @@ -22,6 +22,8 @@ DType ORT_type_to_c_type(MLDataType type) { return DType::TDouble; else if (type == DataTypeImpl::GetType()) return DType::TInt32; + else if (type == DataTypeImpl::GetType()) + return DType::TInt64; else ORT_NOT_IMPLEMENTED("Unsupport MLType to c type."); } diff --git a/onnxruntime/core/framework/graph_partitioner.cc b/onnxruntime/core/framework/graph_partitioner.cc index 0a40a451b8..367bd4c334 100644 --- a/onnxruntime/core/framework/graph_partitioner.cc +++ b/onnxruntime/core/framework/graph_partitioner.cc @@ -180,6 +180,10 @@ Status GraphPartitioner::Partition(Graph& graph, bool export_dll, FuncManager& f //prepare the func kernel KernelDefBuilder builder; BuildFusedKernelDef(builder, *node); + if (node->GetExecutionProviderType() == onnxruntime::kTensorrtExecutionProvider) { + builder.SetDefaultInputsMemoryType(OrtMemTypeCPUInput); + builder.SetDefaultOutputMemoryType(OrtMemTypeCPUOutput); + } ORT_RETURN_IF_ERROR(fused_kernel_registry->Register( builder, [](const OpKernelInfo& info) { return new FunctionKernel(info); })); } diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index 1da0704ffa..2e613fe5f2 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -129,7 +129,7 @@ common::Status CopyOneInputAcrossDevices(const SessionState& session_state, } //no copy for TRT - if (required_provider_type == onnxruntime::kTRTExecutionProvider) { + if (required_provider_type == onnxruntime::kTensorrtExecutionProvider) { new_mlvalue = orig_mlvalue; break; } diff --git a/onnxruntime/core/optimizer/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc index 3c66849a4d..53417d6a81 100644 --- a/onnxruntime/core/optimizer/transformer_memcpy.cc +++ b/onnxruntime/core/optimizer/transformer_memcpy.cc @@ -59,7 +59,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr if (provider != onnxruntime::kCpuExecutionProvider && provider != onnxruntime::kMklDnnExecutionProvider && provider != onnxruntime::kNupharExecutionProvider && - provider != onnxruntime::kTRTExecutionProvider) { + provider != onnxruntime::kTensorrtExecutionProvider) { TransformerMemcpyImpl copy_impl(graph, provider); modified = copy_impl.ModifyGraph(registry_manager_); } @@ -84,7 +84,7 @@ common::Status MemcpyTransformer::ApplyImpl(Graph& graph, bool& modified, int gr Overview: The transformer transforms the input graph as follows: -(1) For every initializer W that is referenced by both provider and non-provider nodes, +(1) For every initializer W that is referenced by both provider and non-provider nodes, we create a duplicate initializer W2 and change all provider nodes to reference this duplicate copy. @@ -182,7 +182,7 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg } } else { // TODO: copy between devices? i.e. multiple GPUs - if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider && + if (node.GetExecutionProviderType() != onnxruntime::kCpuExecutionProvider && node.GetExecutionProviderType() != onnxruntime::kTensorrtExecutionProvider && !node.GetExecutionProviderType().empty()) { ORT_THROW("Execution type '", node.GetExecutionProviderType(), "' doesn't support memcpy "); } diff --git a/onnxruntime/core/providers/tensorrt/symbols.txt b/onnxruntime/core/providers/tensorrt/symbols.txt new file mode 100755 index 0000000000..8c11231f1f --- /dev/null +++ b/onnxruntime/core/providers/tensorrt/symbols.txt @@ -0,0 +1 @@ +OrtCreateTensorrtExecutionProviderFactory diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_allocator.h b/onnxruntime/core/providers/tensorrt/tensorrt_allocator.h new file mode 100755 index 0000000000..f6d70bdde0 --- /dev/null +++ b/onnxruntime/core/providers/tensorrt/tensorrt_allocator.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/framework/allocator.h" + +namespace onnxruntime { +constexpr const char* TRT = "Trt"; + +class TensorrtPinnedAllocator : public CPUAllocator { + public: + virtual const OrtAllocatorInfo& Info() const override { + static OrtAllocatorInfo tensorrt_cpu_allocator_info(TRT, + OrtAllocatorType::OrtDeviceAllocator, 0, + OrtMemType::OrtMemTypeCPU); + return tensorrt_cpu_allocator_info; + } +}; + +/*! \brief The default allocator doesn't allocate anything. It's used here to let allocation + planner get allocator information. +*/ +class TensorrtAllocator : public CPUAllocator { + public: + virtual const OrtAllocatorInfo& Info() const override { + static OrtAllocatorInfo tensorrt_default_allocator_info(TRT, + OrtAllocatorType::OrtDeviceAllocator, 0, + OrtMemType::OrtMemTypeDefault); + return tensorrt_default_allocator_info; + } +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc new file mode 100755 index 0000000000..d6a421a54e --- /dev/null +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -0,0 +1,447 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "tensorrt_execution_provider.h" +#include "tensorrt_allocator.h" +#include "core/framework/execution_provider.h" +#include "core/framework/op_kernel.h" +#include "core/framework/kernel_registry.h" +#include "core/framework/compute_capability.h" +#include "core/providers/cpu/cpu_execution_provider.h" +#include "core/platform/env.h" +#include "onnx/shape_inference/implementation.h" +#include "cuda_runtime_api.h" +#include "gsl/pointers" +#include "core/graph/model.h" +#include "cuda_runtime_api.h" + +using namespace std; +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { + +#define CHECK_CUDA(call) \ + do { \ + cudaError_t status = call; \ + if(status != cudaSuccess) { \ + return -1; \ + } \ + } while(0) + +TensorrtExecutionProvider::TensorrtExecutionProvider() + : IExecutionProvider{onnxruntime::kTensorrtExecutionProvider} { + DeviceAllocatorRegistrationInfo trt_device_info({OrtMemTypeCPU, [](int) { + return std::make_unique(); + }, + std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(trt_device_info)); + DeviceAllocatorRegistrationInfo default_device_info({OrtMemTypeDefault, [](int) { + return std::make_unique(); + }, + std::numeric_limits::max()}); + InsertAllocator(CreateAllocator(default_device_info)); +} + +TensorrtExecutionProvider::~TensorrtExecutionProvider() {} + +std::vector> +TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& /*kernel_registries*/) const { + // Construct modelproto from graph + onnxruntime::Model model(graph.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap()); + onnxruntime::Graph& graph_build = model.MainGraph(); + const std::vector& node_index = graph.GetNodesInTopologicalOrder(); + for (const auto& index : node_index) { + const Node* node = graph.GetNode(index); + graph_build.AddNode(*node); + } + ORT_ENFORCE(graph_build.Resolve().IsOK()); + ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + // Serialize modelproto to string + string string_buf; + model_proto.SerializeToString(&string_buf); + + // Get supported node list + SubGraphCollection_t supported_nodes_vector; + TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING); + auto trt_builder = unique_pointer(nvinfer1::createInferBuilder(trt_logger)); + auto trt_network = unique_pointer(trt_builder->createNetwork()); + auto trt_parser = unique_pointer(nvonnxparser::createParser(*trt_network, trt_logger)); + trt_parser->supportsModel(string_buf.data(), string_buf.size(), supported_nodes_vector); + model_proto.release_graph(); + + // Find inputs, initializers and outputs for each supported subgraph + std::vector> result; + std::set supported_nodes_set; + int counter = 0; + for (const auto& group : supported_nodes_vector) { + if (!group.empty()) { + std::set node_set(group.begin(), group.end()); //slx + //std::set node_set; + //for (const auto& index : group) { + // node_set.insert(node_index[index]); + //} + std::unique_ptr sub_graph = std::make_unique(); + // Find inputs and outputs of the subgraph + std::map fused_inputs, fused_outputs, fused_outputs_to_add; + std::set erased; + int input_order = 0; + int output_order = 0; + + for (const auto& index : group) { + supported_nodes_set.insert(index); + sub_graph->nodes.push_back(index); + const auto& node = graph.GetNode(index); + //supported_nodes_set.insert(node_index[index]); + //sub_graph->nodes.push_back(node_index[index]); + //const auto& node = graph.GetNode(node_index[index]); + + for (const auto& input : node->InputDefs()) { + const auto& it = fused_outputs.find(input); + + if (it != fused_outputs.end()) { + fused_outputs.erase(it); + erased.insert(input); + } + //only when input is neither in output list nor erased list, add the input to input list + else if (erased.find(input) == erased.end()) { + fused_inputs[input] = input_order++; + } + } + + // For output searching, there is a special case: + // If node's OutputEdges are more than its outputs, meaning certain output is used more than once, + // if the output is connected to nodes that don't belong to the subgraph, the output need to be added + // to the output list + if (node->GetOutputEdgesCount() > node->OutputDefs().size()) { + for (auto it = node->OutputEdgesBegin(), end = node->OutputEdgesEnd(); it != end; ++it) { + const auto& node_index = it->GetNode().Index(); + const auto& output = (it->GetNode()).InputDefs()[it->GetDstArgIndex()]; + + if (node_set.find(node_index) != node_set.end()) { + const auto& iter = fused_inputs.find(output); + + if (iter != fused_inputs.end()) { + fused_inputs.erase(iter); + erased.insert(output); + } else if (erased.find(output) == erased.end()) { + fused_outputs[output] = output_order++; + } + } else { + fused_outputs_to_add[output] = output_order++; + } + } + } else { + for (const auto& output : node->OutputDefs()) { + const auto& it = fused_inputs.find(output); + + if (it != fused_inputs.end()) { + fused_inputs.erase(it); + erased.insert(output); + } + // only when output is neither in input list nor erased list, add the output to output list + else if (erased.find(output) == erased.end()) { + fused_outputs[output] = output_order++; + } + } + } + } + + fused_outputs.insert(fused_outputs_to_add.begin(), fused_outputs_to_add.end()); + + // Sort inputs and outputs by the order they were added + std::multimap inputs, outputs; + + for (auto it = fused_inputs.begin(), end = fused_inputs.end(); it != end; ++it) { + inputs.insert(std::pair(it->second, it->first)); + } + + for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) { + outputs.insert(std::pair(it->second, it->first)); + } + + // Assign inputs and outputs to subgraph's meta_def + auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); + meta_def->name = "TRTKernel_" + std::to_string(counter++); + meta_def->domain = kMSDomain; + + for (const auto& input : inputs) { + meta_def->inputs.push_back(input.second->Name()); + } + + for (const auto& output : outputs) { + meta_def->outputs.push_back(output.second->Name()); + } + + meta_def->since_version = 1; + sub_graph->SetMetaDef(meta_def); + + result.push_back(std::make_unique(std::move(sub_graph))); + } + } + + return result; +} + +std::shared_ptr TensorrtExecutionProvider::GetKernelRegistry() const { + static std::shared_ptr kernel_registry = std::make_shared(); + return kernel_registry; +} + +common::Status TensorrtExecutionProvider::CopyTensor(const Tensor& src, Tensor& dst) const { + ORT_UNUSED_PARAMETER(src); + ORT_UNUSED_PARAMETER(dst); + return Status::OK(); +} + +common::Status TensorrtExecutionProvider::Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) { + for (const auto* fused_node : fused_nodes) { + std::vector input_indexes; + std::vector input_dim_sizes; + std::vector output_indexes; + std::vector output_dim_sizes; + std::vector> output_shapes; + + // Build map from input name to its index in input definitions + std::unordered_map input_map; + const auto& input_defs = fused_node->InputDefs(); + for (int i = 0, end = input_defs.size(); i < end; ++i) { + input_map[input_defs[i]->Name()] = i; + } + + // Build map from output name to its index in output definitions + std::unordered_map output_map; + const auto& output_defs = fused_node->OutputDefs(); + for (int i = 0, end = output_defs.size(); i < end; ++i) { + output_map[output_defs[i]->Name()] = i; + } + + // Reconstruct graph from fused node's function body + const auto* func_body = fused_node->GetFunctionBody(); + if (!func_body) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Function body is empty"); + } + const Graph& graph_body = func_body->Body(); + onnxruntime::Model model(graph_body.Name(), true, ModelMetaData(), IOnnxRuntimeOpSchemaRegistryList(), graph_body.DomainToVersionMap()); + onnxruntime::Graph& graph = model.MainGraph(); + + for (const auto& graph_body_node : graph_body.Nodes()) { + graph.AddNode(graph_body_node); + } + + ORT_ENFORCE(graph.Resolve().IsOK()); + + // Add initializer to graph + const auto& init_tensors = graph_body.GetAllInitializedTensors(); + for (const auto& tensor : init_tensors) { + graph.AddInitializedTensor(*(tensor.second)); + } + + // Add fused node's outputs to graph's outputs if the outputs are not included yet + // for the case that node's output is connected to more than one EdgeEnd nodes and some of them don't belong to the graph + ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + const auto& graph_output = model_proto.graph().output(); + std::set graph_outputs_set; + for (int i = 0, end = graph_output.size(); i < end; ++i) { + graph_outputs_set.insert(graph_output[i].name()); + } + + const auto& graph_value_info = model_proto.graph().value_info(); + std::vector output_to_add; + std::vector location; + int num_defs = output_defs.size(); + for (int i = num_defs - 1; i >= 0; --i) { + const std::string& output_name = output_defs[i]->Name(); + if (graph_outputs_set.find(output_name) == graph_outputs_set.end()) { + for (int j = 0, end = graph_value_info.size(); j < end; ++j) { + if (output_name == graph_value_info[j].name()) { + output_to_add.push_back(j); + location.push_back(num_defs - 1 - i); + } + } + } + } + + // Add outputs and move them to the right places + auto* mutable_output = model_proto.mutable_graph()->mutable_output(); + for (int i = 0, end = output_to_add.size(); i < end; ++i) { + *(mutable_output->Add()) = graph_value_info[output_to_add[i]]; + int start_index = (*mutable_output).size() - 1; + int end_index = start_index - location[i]; + for (int j = start_index; j > end_index; --j) { + mutable_output->SwapElements(j, j - 1); + } + } + + // Set version + model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + // Create TensorRT engine + string string_buf; + model_proto.SerializeToString(&string_buf); + + TensorrtLogger trt_logger(nvinfer1::ILogger::Severity::kWARNING); + auto trt_builder = unique_pointer(nvinfer1::createInferBuilder(trt_logger)); + auto trt_network = unique_pointer(trt_builder->createNetwork()); + auto trt_parser = unique_pointer(nvonnxparser::createParser(*trt_network, trt_logger)); + trt_parser->parse(string_buf.data(), string_buf.size()); + trt_builder->setMaxBatchSize(kMaxBatchSize); + trt_builder->setMaxWorkspaceSize(kMaxWorkSpaceSize); + auto trt_engine = unique_pointer(trt_builder->buildCudaEngine(*trt_network.get())); + ORT_ENFORCE(trt_engine != nullptr); + + // Build TensorRT context + auto trt_context = unique_pointer(trt_engine->createExecutionContext()); + ORT_ENFORCE(trt_context != nullptr); + + // Get input shape and binding index + int num_inputs = trt_network->getNbInputs(); + input_indexes.resize(num_inputs); + input_dim_sizes.resize(num_inputs); + for (int i = 0; i < num_inputs; ++i) { + const std::string& name = trt_network->getInput(i)->getName(); + size_t bindingIndex = trt_engine->getBindingIndex(name.c_str()); + nvinfer1::Dims dimensions = trt_engine->getBindingDimensions(static_cast(bindingIndex)); + auto iter = input_map.find(name); + if (iter != input_map.end()) { + input_indexes[bindingIndex] = iter->second; + } + size_t dim_size = 1; + for (int j = 0, end = dimensions.nbDims; j < end; ++j) { + dim_size *= dimensions.d[j]; + } + input_dim_sizes[bindingIndex] = dim_size; + } + + // Get output shape and binding index + int num_outputs = trt_network->getNbOutputs(); + output_indexes.resize(num_outputs); + output_dim_sizes.resize(num_outputs); + output_shapes.resize(num_outputs); + for (int i = 0; i < num_outputs; ++i) { + const std::string& name = trt_network->getOutput(i)->getName(); + size_t bindingIndex = trt_engine->getBindingIndex(name.c_str()); + nvinfer1::Dims dimensions = trt_engine->getBindingDimensions(static_cast(bindingIndex)); + bindingIndex -= num_inputs; + auto iter = output_map.find(name); + if (iter != output_map.end()) { + output_indexes[bindingIndex] = iter->second; + } + size_t dim_size = 1; + for (int j = 0, end = dimensions.nbDims; j < end; ++j) { + output_shapes[bindingIndex].push_back(dimensions.d[j]); + dim_size *= dimensions.d[j]; + } + output_dim_sizes[bindingIndex] = dim_size; + } + + ORT_ENFORCE(trt_engine->getNbBindings() == (num_inputs + num_outputs)); + + // Save engine, context and input/output info to map + parsers_.emplace(fused_node->Name(), std::move(trt_parser)); + engines_.emplace(fused_node->Name(), std::move(trt_engine)); + contexts_.emplace(fused_node->Name(), std::move(trt_context)); + input_info_[fused_node->Name()].push_back(input_indexes); + input_info_[fused_node->Name()].push_back(input_dim_sizes); + output_info_[fused_node->Name()].push_back(output_indexes); + output_info_[fused_node->Name()].push_back(output_dim_sizes); + output_shapes_[fused_node->Name()] = output_shapes; + + // Create function state + // TODO: remove default capture + NodeComputeInfo compute_info; + compute_info.create_state_func = [=](ComputeContext* context, FunctionState* state) { + std::unique_ptr p = std::make_unique(); + *p = {context->allocate_func, context->release_func, context->allocator_handle, parsers_[context->node_name].get(), engines_[context->node_name].get(), contexts_[context->node_name].get(), + input_info_[context->node_name], output_info_[context->node_name], output_shapes_[context->node_name]}; + *state = p.release(); + return 0; + }; + + // Release function state + compute_info.release_state_func = [](FunctionState state) { + if (state) + delete static_cast(state); + }; + + // Create compute function + compute_info.compute_func = [](FunctionState state, ONNXRunTimeTensor* input_tensors, size_t num_inputs, ONNXRunTimeTensor* output_tensors, size_t num_outputs) { + ORT_UNUSED_PARAMETER(num_inputs); + ORT_UNUSED_PARAMETER(num_outputs); + TensorrtFuncState* trt_state = reinterpret_cast(state); + const std::vector& input_indexes = (trt_state->input_info)[0]; + const std::vector& input_dim_sizes = (trt_state->input_info)[1]; + const std::vector& output_indexes = (trt_state->output_info)[0]; + const std::vector& output_dim_sizes = (trt_state->output_info)[1]; + std::vector> output_shapes = trt_state->output_shapes; + int num_binding_inputs = input_indexes.size(); + int num_binding_outputs = output_indexes.size(); + int total_bindings = num_binding_inputs + num_binding_outputs; + cudaStream_t stream; + CHECK_CUDA(cudaStreamCreate(&stream)); + std::vector buffers(total_bindings); + int batch_size = 1; + + // Get batch size and allocate cuda memory for inputs + for (int i = 0, end = num_binding_inputs; i < end; ++i) { + const auto& tensor_input = input_tensors[input_indexes[i]]; + const auto& tensor_shape = tensor_input.shape; + const int input_batch_size = tensor_shape[0]; + if (i > 0 && batch_size != input_batch_size) { + ORT_THROW("Input batch size is inconsistent"); + } + batch_size = input_batch_size; + + const float* input = static_cast(tensor_input.data); + CHECK_CUDA(cudaMalloc(&buffers[i], input_batch_size * input_dim_sizes[i] * sizeof(float))); + CHECK_CUDA(cudaMemcpy(buffers[i], input, input_batch_size * input_dim_sizes[i] * sizeof(float), cudaMemcpyHostToDevice)); + } + + // Allocate cuda memory for outputs + for (int i = 0, end = num_binding_outputs; i < end; ++i) { + CHECK_CUDA(cudaMalloc(&buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float))); + } + + // Run TRT inference + trt_state->context->enqueue(batch_size, &buffers[0], stream, nullptr); + + // Copy TRT outputs to output tensors + for (int i = 0, end = num_binding_outputs; i < end; ++i) { + // Setup output tensor property + int output_index = output_indexes[i]; + output_shapes[i].insert(output_shapes[i].begin(), batch_size); + output_tensors[output_index].dtype = input_tensors[0].dtype; + // TODO: shape inference + const auto& shape_size = output_shapes[i].size(); + output_tensors[output_index].ndim = shape_size; + output_tensors[output_index].shape = new int64_t[shape_size]; + memcpy(output_tensors[output_index].shape, &output_shapes[i][0], sizeof(int64_t) * shape_size); + output_tensors[output_index].data = (*(trt_state->test_allocate_func))(trt_state->allocator, 64, sizeof(double) * batch_size * output_dim_sizes[i]); + + CHECK_CUDA(cudaMemcpy(output_tensors[output_index].data, buffers[i + num_binding_inputs], batch_size * output_dim_sizes[i] * sizeof(float), cudaMemcpyDeviceToHost)); + } + + // Sync stream + cudaStreamSynchronize(stream); + + // Free CUDA memory + cudaStreamDestroy(stream); + + for (int i = 0, end = total_bindings; i < end; ++i) { + CHECK_CUDA(cudaFree(buffers[i])); + } + + return 0; + }; + + node_compute_funcs.push_back(compute_info); + } + + return Status::OK(); +} +} // namespace onnxruntime + diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h new file mode 100755 index 0000000000..9f724c9305 --- /dev/null +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include +#include "core/common/logging/logging.h" +#include "core/framework/op_kernel.h" +#include "NvInfer.h" +#include "NvOnnxParser.h" + +namespace onnxruntime { + +static const int kMaxBatchSize = 1; +static const int kMaxWorkSpaceSize = 1 << 30; + +class TensorrtLogger : public nvinfer1::ILogger { + nvinfer1::ILogger::Severity verbosity_; +public: + TensorrtLogger(Severity verbosity=Severity::kWARNING) + : verbosity_(verbosity) {} + void log(Severity severity, const char* msg) override { + if( severity <= verbosity_ ) { + time_t rawtime = std::time(0); + char buf[256]; + strftime(&buf[0], 256, + "%Y-%m-%d %H:%M:%S", + std::gmtime(&rawtime)); + const char* sevstr = (severity == Severity::kINTERNAL_ERROR ? " BUG" : + severity == Severity::kERROR ? " ERROR" : + severity == Severity::kWARNING ? "WARNING" : + severity == Severity::kINFO ? " INFO" : + "UNKNOWN"); + LOGS_DEFAULT(WARNING) << "[" << buf << " " << sevstr << "] " << msg; + } + } +}; + +// Information needed to construct trt execution providers. +struct TensorrtExecutionProviderInfo { + int device_id{0}; +}; + +// Information to construct kernel function state. +struct TensorrtFuncState { + AllocateFunc test_allocate_func = nullptr; + DestroyFunc test_release_func = nullptr; + AllocatorHandle allocator = nullptr; + nvonnxparser::IParser* parser = nullptr; + nvinfer1::ICudaEngine* engine = nullptr; + nvinfer1::IExecutionContext* context = nullptr; + std::vector> input_info; + std::vector> output_info; + std::vector> output_shapes; +}; + +// Logical device representation. +class TensorrtExecutionProvider : public IExecutionProvider { + public: + TensorrtExecutionProvider(); + virtual ~TensorrtExecutionProvider(); + + std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& /*kernel_registries*/) const override; + + common::Status Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) override; + + Status CopyTensor(const Tensor& src, Tensor& dst) const override; + + const void* GetExecutionHandle() const noexcept override { + return nullptr; + } + + std::shared_ptr GetKernelRegistry() const override; + + private: + struct InferDeleter { + template + void operator()(T* obj) const { + if (obj) { + obj->destroy(); + } + } + }; + + template + using unique_pointer = std::unique_ptr; + + int device_id_; + std::unordered_map> parsers_; + std::unordered_map> engines_; + std::unordered_map> contexts_; + std::unordered_map>> input_info_; + std::unordered_map>> output_info_; + std::unordered_map>> output_shapes_; +}; + +} // namespace onnxruntime + diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc new file mode 100755 index 0000000000..435c9eea4d --- /dev/null +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/tensorrt/tensorrt_provider_factory.h" +#include +#include "tensorrt_execution_provider.h" +#include "core/session/abi_session_options_impl.h" + +using namespace onnxruntime; + +namespace onnxruntime { + +struct TensorrtProviderFactory : IExecutionProviderFactory { + TensorrtProviderFactory() {} + ~TensorrtProviderFactory() override {} + + std::unique_ptr CreateProvider() override; +}; + +std::unique_ptr TensorrtProviderFactory::CreateProvider() { + return std::make_unique(); +} + +std::shared_ptr CreateExecutionProviderFactory_Tensorrt() { + return std::make_shared(); +} +} // namespace onnxruntime + +ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options) { + options->provider_factories.push_back(onnxruntime::CreateExecutionProviderFactory_Tensorrt()); + return nullptr; +} + diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index b44ac46f59..d5a28fe8a6 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -49,6 +49,9 @@ #ifdef USE_CUDA #include "core/providers/cuda/cuda_provider_factory.h" #endif +#ifdef USE_TENSORRT +#include "core/providers/tensorrt/tensorrt_provider_factory.h" +#endif #ifdef USE_MKLDNN #include "core/providers/mkldnn/mkldnn_provider_factory.h" #endif @@ -59,6 +62,7 @@ namespace onnxruntime { std::shared_ptr CreateExecutionProviderFactory_CPU(int use_arena); std::shared_ptr CreateExecutionProviderFactory_CUDA(int device_id); +std::shared_ptr CreateExecutionProviderFactory_Tensorrt(); std::shared_ptr CreateExecutionProviderFactory_Mkldnn(int use_arena); std::shared_ptr CreateExecutionProviderFactory_Nuphar(int device_id, const char*); std::shared_ptr CreateExecutionProviderFactory_BrainSlice(int ip, int, int, bool, const char*, const char*, const char*); @@ -193,6 +197,12 @@ inline void RegisterExecutionProvider(InferenceSession* sess, onnxruntime::IExec void InitializeSession(InferenceSession* sess) { onnxruntime::common::Status status; +#ifdef USE_TENSORRT + { + RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_Tensorrt()); + } +#endif + #ifdef USE_CUDA { RegisterExecutionProvider(sess, *onnxruntime::CreateExecutionProviderFactory_CUDA(0)); diff --git a/onnxruntime/test/framework/test_utils.cc b/onnxruntime/test/framework/test_utils.cc index f666507679..ff4393f2e6 100644 --- a/onnxruntime/test/framework/test_utils.cc +++ b/onnxruntime/test/framework/test_utils.cc @@ -18,9 +18,9 @@ IExecutionProvider* TestCudaExecutionProvider() { } #endif -#ifdef USE_TRT -IExecutionProvider* TestTRTExecutionProvider() { - static TRTExecutionProvider trt_provider; +#ifdef USE_TENSORRT +IExecutionProvider* TestTensorrtExecutionProvider() { + static TensorrtExecutionProvider trt_provider; return &trt_provider; } #endif diff --git a/onnxruntime/test/framework/test_utils.h b/onnxruntime/test/framework/test_utils.h index f9fc530c0f..2609f9acb6 100644 --- a/onnxruntime/test/framework/test_utils.h +++ b/onnxruntime/test/framework/test_utils.h @@ -9,8 +9,8 @@ #ifdef USE_CUDA #include "core/providers/cuda/cuda_execution_provider.h" #endif -#ifdef USE_TRT -#include "core/providers/trt/trt_execution_provider.h" +#ifdef USE_TENSORRT +#include "core/providers/tensorrt/tensorrt_execution_provider.h" #endif namespace onnxruntime { @@ -21,8 +21,8 @@ IExecutionProvider* TestCPUExecutionProvider(); IExecutionProvider* TestCudaExecutionProvider(); #endif -#ifdef USE_TRT -IExecutionProvider* TestTRTExecutionProvider(); +#ifdef USE_TENSORRT +IExecutionProvider* TestTensorrtExecutionProvider(); #endif template diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index cb92684234..a755431b1e 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -32,7 +32,7 @@ void usage() { "\t-r [repeat]: Specifies the number of times to repeat\n" "\t-v: verbose\n" "\t-n [test_case_name]: Specifies a single test case to run.\n" - "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda' or 'mkldnn'. Default: 'cpu'.\n" + "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'mkldnn' or 'tensorrt'. Default: 'cpu'.\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" "\t-h: help\n"); } @@ -81,7 +81,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { bool enable_cuda = false; bool enable_mkl = false; bool enable_nuphar = false; - bool enable_trt = false; + bool enable_tensorrt = false; OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING; { int ch; @@ -131,8 +131,8 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { enable_mkl = true; } else if (!CompareCString(optarg, ORT_TSTR("nuphar"))) { enable_nuphar = true; - } else if (!CompareCString(optarg, ORT_TSTR("trt"))) { - enable_trt = true; + } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) { + enable_tensorrt = true; } else { usage(); return -1; @@ -188,6 +188,15 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { sf.EnableSequentialExecution(); else sf.DisableSequentialExecution(); + if (enable_tensorrt) { +#ifdef USE_TENSORRT + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf)); + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0)); +#else + fprintf(stderr, "TensorRT is not supported in this build"); + return -1; +#endif + } if (enable_cuda) { #ifdef USE_CUDA ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0)); @@ -210,14 +219,6 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { #else fprintf(stderr, "MKL-DNN is not supported in this build"); return -1; -#endif - } - if (enable_trt) { -#ifdef USE_TRT - ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_TRT(sf)); -#else - fprintf(stderr, "TensorRT is not supported in this build"); - return -1; #endif } TestEnv args(tests, stat, sf); @@ -249,7 +250,7 @@ int real_main(int argc, char* argv[], OrtEnv** p_env) { {"BatchNorm3d_eval", "disable reason"}, {"BatchNorm3d_momentum_eval", "disable reason"}, {"constantofshape_float_ones", "test data bug"}, - {"constantofshape_int_zeros", "test data bug"}, + {"constantofshape_int_zeros", "test data bug"}, {"GLU", "disable reason"}, {"GLU_dim", "disable reason"}, {"Linear", "disable reason"}, diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index fed6f2abee..bc6ed2e5e6 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -28,7 +28,7 @@ namespace perftest { "Options:\n" "\t-m [test_mode]: Specifies the test mode. Value coulde be 'duration' or 'times'.\n" "\t\tProvide 'duration' to run the test for a fix duration, and 'times' to repeated for a certain times. Default:'duration'.\n" - "\t-e [cpu|cuda|mkldnn]: Specifies the provider 'cpu','cuda','mkldnn'. Default:'cpu'.\n" + "\t-e [cpu|cuda|mkldnn|tensorrt]: Specifies the provider 'cpu','cuda','mkldnn' or 'tensorrt'. Default:'cpu'.\n" "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" "\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n" "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n" @@ -63,8 +63,8 @@ namespace perftest { test_config.machine_config.provider_type_name = onnxruntime::kMklDnnExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("brainslice"))) { test_config.machine_config.provider_type_name = onnxruntime::kBrainSliceExecutionProvider; - } else if (!CompareCString(optarg, ORT_TSTR("trt"))) { - test_config.machine_config.provider_type_name = onnxruntime::kTRTExecutionProvider; + } else if (!CompareCString(optarg, ORT_TSTR("tensorrt"))) { + test_config.machine_config.provider_type_name = onnxruntime::kTensorrtExecutionProvider; } else { return false; } diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index ea95721962..df3734677d 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -122,9 +122,10 @@ bool PerformanceRunner::Initialize() { fprintf(stderr, "Nuphar is not supported in this build"); return false; #endif - } else if (provider_name == onnxruntime::kTRTExecutionProvider) { -#ifdef USE_TRT - ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_TRT(sf)); + } else if (provider_name == onnxruntime::kTensorrtExecutionProvider) { +#ifdef USE_TENSORRT + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(sf)); + ORT_THROW_ON_ERROR(OrtSessionOptionsAppendExecutionProvider_CUDA(sf, 0)); #else fprintf(stderr, "TensorRT is not supported in this build"); return false; diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 617b34a99f..a29f8100bf 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -408,6 +408,7 @@ void OpTester::Run(ExpectResult expect_result, kMklDnnExecutionProvider, kNupharExecutionProvider, kBrainSliceExecutionProvider, + kTensorrtExecutionProvider, }; bool has_run = false; @@ -446,6 +447,8 @@ void OpTester::Run(ExpectResult expect_result, execution_provider = DefaultNupharExecutionProvider(); else if (provider_type == onnxruntime::kBrainSliceExecutionProvider) execution_provider = DefaultBrainSliceExecutionProvider(); + else if (provider_type == onnxruntime::kTensorrtExecutionProvider) + execution_provider = DefaultTensorrtExecutionProvider(); // skip if execution provider is disabled if (execution_provider == nullptr) continue; diff --git a/onnxruntime/test/providers/trt/trt_basic_test.cc b/onnxruntime/test/providers/trt/trt_basic_test.cc new file mode 100644 index 0000000000..8acc3d87ff --- /dev/null +++ b/onnxruntime/test/providers/trt/trt_basic_test.cc @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/inference_session.h" +#include "test/providers/provider_test_utils.h" +#include "test/framework/test_utils.h" +#include "gtest/gtest.h" +#include "core/providers/trt/trt_execution_provider.h" + +using namespace std; +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::logging; + +namespace onnxruntime { + +namespace test { +void VerifyOutputs(const std::vector& fetches, + const std::vector& expected_dims, + const std::vector& expected_values) { + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(expected_dims); + ASSERT_EQ(expected_shape, rtensor.Shape()); + const std::vector found(rtensor.template Data(), rtensor.template Data() + expected_values.size()); + ASSERT_EQ(expected_values, found); +} + +TEST(TensorrtExecutionProviderTest, FunctionTest) { + onnxruntime::Model model("graph_1"); + auto& graph = model.MainGraph(); + std::vector inputs; + std::vector outputs; + + // FLOAT tensor. + ONNX_NAMESPACE::TypeProto float_tensor; + float_tensor.mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(1); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(3); + float_tensor.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(2); + + auto& input_arg_1 = graph.GetOrCreateNodeArg("X", &float_tensor); + auto& input_arg_2 = graph.GetOrCreateNodeArg("Y", &float_tensor); + inputs.push_back(&input_arg_1); + inputs.push_back(&input_arg_2); + auto& output_arg = graph.GetOrCreateNodeArg("node_1_out_1", &float_tensor); + outputs.push_back(&output_arg); + graph.AddNode("node_1", "Add", "node 1.", inputs, outputs); + + auto& input_arg_3 = graph.GetOrCreateNodeArg("Z", &float_tensor); + inputs.clear(); + inputs.push_back(&output_arg); + inputs.push_back(&input_arg_3); + auto& output_arg_2 = graph.GetOrCreateNodeArg("M", &float_tensor); + outputs.clear(); + outputs.push_back(&output_arg_2); + graph.AddNode("node_2", "Add", "node 2.", inputs, outputs); + + auto status = graph.Resolve(); + ASSERT_TRUE(status.IsOK()); + std::string model_file_name = "trt_execution_provider_test_graph.onnx"; + status = onnxruntime::Model::Save(model, model_file_name); + + std::vector dims_mul_x = {1, 3, 2}; + std::vector values_mul_x = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + MLValue ml_value_x; + CreateMLValue(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_x); + MLValue ml_value_y; + CreateMLValue(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_y); + MLValue ml_value_z; + CreateMLValue(TestTensorrtExecutionProvider()->GetAllocator(0, OrtMemTypeCPU), dims_mul_x, values_mul_x, &ml_value_z); + NameMLValMap feeds; + feeds.insert(std::make_pair("X", ml_value_x)); + feeds.insert(std::make_pair("Y", ml_value_y)); + feeds.insert(std::make_pair("Z", ml_value_z)); + + // prepare outputs + std::vector output_names; + output_names.push_back("M"); + std::vector fetches; + + // prepare expected inputs and outputs + std::vector expected_dims_mul_m = {1, 3, 2}; + std::vector expected_values_mul_m = {3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}; + + SessionOptions so; + so.session_logid = "TensorrtExecutionProviderTest.FunctionTest"; + RunOptions run_options; + run_options.run_tag = so.session_logid; + + InferenceSession session_object{so}; + session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime::TensorrtExecutionProvider>()); + status = session_object.Load(model_file_name); + ASSERT_TRUE(status.IsOK()); + status = session_object.Initialize(); + ASSERT_TRUE(status.IsOK()); + + // Now run + status = session_object.Run(run_options, feeds, output_names, &fetches); + ASSERT_TRUE(status.IsOK()); + VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); +} +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index cc6eb52628..bdb7457853 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -13,7 +13,7 @@ std::shared_ptr CreateExecutionProviderFactory_CUDA(i std::shared_ptr CreateExecutionProviderFactory_Mkldnn(int use_arena); std::shared_ptr CreateExecutionProviderFactory_Nuphar(int device_id, const char*); std::shared_ptr CreateExecutionProviderFactory_BrainSlice(uint32_t ip, int, int, bool, const char*, const char*, const char*); -std::shared_ptr CreateExecutionProviderFactory_TRT(); +std::shared_ptr CreateExecutionProviderFactory_Tensorrt(); namespace test { @@ -21,6 +21,14 @@ std::unique_ptr DefaultCpuExecutionProvider(bool enable_aren return CreateExecutionProviderFactory_CPU(enable_arena)->CreateProvider(); } +std::unique_ptr DefaultTensorrtExecutionProvider() { +#ifdef USE_TENSORRT + return CreateExecutionProviderFactory_Tensorrt()->CreateProvider(); +#else + return nullptr; +#endif +} + std::unique_ptr DefaultCudaExecutionProvider() { #ifdef USE_CUDA return CreateExecutionProviderFactory_CUDA(0)->CreateProvider(); @@ -54,13 +62,5 @@ std::unique_ptr DefaultBrainSliceExecutionProvider() { #endif } -std::unique_ptr DefaultTRTExecutionProvider() { -#ifdef USE_TRT - return CreateExecutionProviderFactory_TRT()->CreateProvider(); -#else - return nullptr; -#endif -} - } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 5c4f23a298..4687b8f6b4 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -12,7 +12,7 @@ std::unique_ptr DefaultCudaExecutionProvider(); std::unique_ptr DefaultMkldnnExecutionProvider(bool enable_arena = true); std::unique_ptr DefaultNupharExecutionProvider(); std::unique_ptr DefaultBrainSliceExecutionProvider(); -std::unique_ptr DefaultTRTExecutionProvider(); +std::unique_ptr DefaultTensorrtExecutionProvider(); } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/providers.h b/onnxruntime/test/util/include/providers.h index 048d8332f9..e6eab2aa54 100644 --- a/onnxruntime/test/util/include/providers.h +++ b/onnxruntime/test/util/include/providers.h @@ -16,6 +16,6 @@ #if USE_BRAINSLICE #include "core/providers/brainslice/brainslice_provider_factory.h" #endif -#if USE_TRT -#include "core/providers/trt/trt_provider_factory.h" +#ifdef USE_TENSORRT +#include "core/providers/tensorrt/tensorrt_provider_factory.h" #endif diff --git a/onnxruntime/test/win_getopt/mb/getopt.cc b/onnxruntime/test/win_getopt/mb/getopt.cc new file mode 100644 index 0000000000..ddd31ce911 --- /dev/null +++ b/onnxruntime/test/win_getopt/mb/getopt.cc @@ -0,0 +1,117 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +int opterr = 1, /* if error message should be printed */ + optind = 1, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +char* optarg; /* argument associated with option */ + +#define BADCH (int)'?' +#define BADARG (int)':' +#define EMSG "" + +/* + * getopt -- + * Parse argc/argv argument vector. + */ +int getopt(int nargc, char* const nargv[], const char* ostr) { + static char* place = EMSG; /* option letter processing */ + char* oli; /* option letter list index */ + + if (optreset || *place == 0) { /* update scanning pointer */ + optreset = 0; + place = nargv[optind]; + if (optind >= nargc || *place++ != '-') { + /* Argument is absent or is not an option */ + place = EMSG; + return (-1); + } + optopt = *place++; + if (optopt == '-' && *place == 0) { + /* "--" => end of options */ + ++optind; + place = EMSG; + return (-1); + } + if (optopt == 0) { + /* Solitary '-', treat as a '-' option + if the program (eg su) is looking for it. */ + place = EMSG; + if (strchr(ostr, '-') == NULL) return (-1); + optopt = '-'; + } + } else + optopt = *place++; + + /* See if option letter is one the caller wanted... */ + if (optopt == ':' || (oli = (char*)strchr(ostr, optopt)) == NULL) { + if (*place == 0) ++optind; + if (opterr && *ostr != ':') (void)fprintf(stderr, "illegal option -- %c\n", optopt); + return (BADCH); + } + + /* Does this option need an argument? */ + if (oli[1] != ':') { + /* don't need argument */ + optarg = NULL; + if (*place == 0) ++optind; + } else { + /* Option-argument is either the rest of this argument or the + entire next argument. */ + if (*place) + optarg = place; + else if (oli[2] == ':') + /* + * GNU Extension, for optional arguments if the rest of + * the argument is empty, we return NULL + */ + optarg = NULL; + else if (nargc > ++optind) + optarg = nargv[optind]; + else { + /* option-argument absent */ + place = EMSG; + if (*ostr == ':') return (BADARG); + if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt); + return (BADCH); + } + place = EMSG; + ++optind; + } + return (optopt); /* return option letter */ +} \ No newline at end of file diff --git a/onnxruntime/test/win_getopt/mb/include/getopt.h b/onnxruntime/test/win_getopt/mb/include/getopt.h new file mode 100644 index 0000000000..6477ee96e9 --- /dev/null +++ b/onnxruntime/test/win_getopt/mb/include/getopt.h @@ -0,0 +1,42 @@ +/* $NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $ */ + +/*- + * SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#pragma once + +extern int opterr, /* if error message should be printed */ + optind, /* index into parent argv vector */ + optopt, /* character checked for validity */ + optreset; /* reset getopt */ +extern char* optarg; /* argument associated with option */ + +int getopt(int nargc, char* const nargv[], const char* ostr); diff --git a/onnxruntime/test/win_getopt/mb/include/unistd.h b/onnxruntime/test/win_getopt/mb/include/unistd.h new file mode 100644 index 0000000000..0a1da04007 --- /dev/null +++ b/onnxruntime/test/win_getopt/mb/include/unistd.h @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "getopt.h" diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 1ed2728c02..8d3f2c24a1 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -128,8 +128,8 @@ Use the individual flags to only run the specified stages. parser.add_argument("--brain_slice_package_name", help="Name of brain slice packages") parser.add_argument("--brain_slice_client_package_name", help="Name of brainslice client package") parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") - parser.add_argument("--use_trt", action='store_true', help="Build with trt") - parser.add_argument("--trt_path", action='store_true', help="Path to trt dir") + parser.add_argument("--use_tensorrt", action='store_true', help="Build with TensorRT") + parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") return parser.parse_args() def resolve_executable_path(command_or_path): @@ -281,7 +281,7 @@ def setup_test_data(build_dir, configs, test_data_url, test_data_checksum, azure log.debug("creating shortcut %s -> %s" % (src_model_dir, dest_model_dir)) run_subprocess(['mklink', '/D', '/J', dest_model_dir, src_model_dir], shell=True) -def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args): +def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, tensorrt_home, pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args): log.info("Generating CMake build tree") cmake_dir = os.path.join(source_dir, "cmake") # TODO: fix jemalloc build so it does not conflict with onnxruntime shared lib builds. (e.g. onnxuntime_pybind) @@ -294,6 +294,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_CUDA=" + ("ON" if args.use_cuda else "OFF"), "-Donnxruntime_USE_NSYNC=" + ("OFF" if is_windows() or not args.use_nsync else "ON"), "-Donnxruntime_CUDNN_HOME=" + (cudnn_home if args.use_cuda else ""), + "-Donnxruntime_CUDA_HOME=" + (cuda_home if args.use_cuda else ""), "-Donnxruntime_USE_JEMALLOC=" + ("ON" if args.use_jemalloc else "OFF"), "-Donnxruntime_ENABLE_PYTHON=" + ("ON" if args.enable_pybind else "OFF"), "-Donnxruntime_BUILD_CSHARP=" + ("ON" if args.build_csharp else "OFF"), @@ -309,7 +310,8 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_BRAINSLICE=" + ("ON" if args.use_brainslice else "OFF"), "-Donnxruntime_USE_NUPHAR=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_USE_EIGEN_THREADPOOL=" + ("ON" if args.use_eigenthreadpool else "OFF"), - "-Donnxruntime_USE_TRT=" + ("ON" if args.use_trt else "OFF"), + "-Donnxruntime_USE_TENSORRT=" + ("ON" if args.use_tensorrt else "OFF"), + "-Donnxruntime_TENSORRT_HOME=" + (tensorrt_home if args.use_tensorrt else ""), # By default - we currently support only cross compiling for ARM/ARM64 (no native compilation supported through this script) "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), "-Donnxruntime_BUILD_x86=" + ("ON" if args.x86 else "OFF"), @@ -322,9 +324,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_BS_CLIENT_PACKAGE=%s/%s" % (args.brain_slice_package_path, args.brain_slice_client_package_name), "-Donnxruntime_BRAINSLICE_dynamic_lib_PATH=%s/%s" % (args.brain_slice_package_path, bs_shared_lib_name)] - if args.use_trt: - cmake_args += ["-DTENSORRT_ROOT=%s" % args.trt_path] - if args.use_llvm: cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path] @@ -457,6 +456,22 @@ def setup_cuda_vars(args): return cuda_home, cudnn_home +def setup_tensorrt_vars(args): + + tensorrt_home = "" + + if (args.use_tensorrt): + tensorrt_home = args.tensorrt_home if args.tensorrt_home else os.getenv("TENSORRT_HOME") + + tensorrt_home_valid = (tensorrt_home != None and os.path.exists(tensorrt_home)) + + if (not tensorrt_home_valid): + raise BuildError("tensorrt_home paths must be specified and valid.", + "tensorrt_home='{}' valid={}." + .format(tensorrt_home, tensorrt_home_valid)) + + return tensorrt_home + def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, enable_python_tests, enable_tvm = False): for config in configs: log.info("Running tests for %s configuration", config) @@ -509,6 +524,8 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para cmd += ["-j", str(num_parallel_models)] if config != 'Debug' and os.path.exists(model_dir): + if provider == 'tensorrt': + model_dir = os.path.join(model_dir, "opset8") cmd.append(model_dir) if os.path.exists(onnx_test_data_dir): cmd.append(onnx_test_data_dir) @@ -521,7 +538,10 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para else: run_subprocess([exe,'-x'] + cmd, cwd=cwd) else: - run_subprocess([exe] + cmd, cwd=cwd) + if provider == 'tensorrt': + run_subprocess([exe, '-c', '1'] + cmd, cwd=cwd) + else: + run_subprocess([exe] + cmd, cwd=cwd) def build_python_wheel(source_dir, build_dir, configs, use_cuda): for config in configs: @@ -578,6 +598,9 @@ def main(): else: args.test = True + if args.use_tensorrt: + args.use_cuda = True + if args.build_wheel: args.enable_pybind = True @@ -596,6 +619,9 @@ def main(): # if using cuda, setup cuda paths and env vars cuda_home, cudnn_home = setup_cuda_vars(args) + # if using tensorrt, setup tensorrt paths + tensorrt_home = setup_tensorrt_vars(args) + os.makedirs(build_dir, exist_ok=True) log.info("Build started") @@ -649,7 +675,7 @@ def main(): elif args.arm or args.arm64: path_to_protoc_exe = os.path.join(build_dir, 'host_protoc', 'Release', 'protoc.exe') - generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines, + generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, tensorrt_home, args.pb_home, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args) if (args.clean): @@ -666,7 +692,10 @@ def main(): onnx_test_data_dir = '/data/onnx' if is_windows() or not os.path.exists(onnx_test_data_dir): onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data") - if args.use_cuda: + if args.use_tensorrt: + onnx_test_data_dir = '' + run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'tensorrt', False, 1) + elif args.use_cuda: run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False, 2) elif args.x86 or platform.system() == 'Darwin': run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True, 1) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml new file mode 100644 index 0000000000..f1e1be7865 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -0,0 +1,16 @@ +jobs: +- job: Linux_CI_GPU_TENSORRT_Dev + pool: Linux-GPU-CUDA10 + steps: + - template: templates/set-test-data-variables-step.yml + + # There are some tests in 20190130.zip that TensorRT can't run. Instead here use 20181210 opset8 for TensorRT test. + - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d tensorrt -r $(Build.BinariesDirectory) -x "--test_data_url https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip --test_data_checksum a966def7447f4ff04f5665bca235b3f3"' + + displayName: 'Command Line Script' + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + + - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml new file mode 100644 index 0000000000..6043397d69 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -0,0 +1,144 @@ +jobs: +- job: Windows_CI_GPU_Dev + pool: Win-GPU-CUDA10 + variables: + buildDirectory: '$(Build.BinariesDirectory)' + CUDA_VERSION: '10.0' + steps: + - template: templates/set-test-data-variables-step.yml + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + feedsToUse: config + nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config' + restoreDirectory: '$(Build.SourcesDirectory)\csharp' + - task: UniversalPackages@0 + displayName: 'Download python' + inputs: + command: download + vstsFeed: '$(System.TeamProject)' + vstsFeedPackage: 'miniconda3_win64' + vstsPackageVersion: '4.5.11' + downloadDirectory: '$(Build.BinariesDirectory)\python' + - task: CmdLine@1 + displayName: 'Run python installer' + inputs: + filename: '$(Build.BinariesDirectory)\python\installer.exe' + arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' + timeoutInMinutes: 10 + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + - task: CmdLine@1 + displayName: 'Install conda modules' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe' + arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy' + timeoutInMinutes: 10 + + + - task: CmdLine@1 + displayName: 'Download cmake' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' + + - task: BatchScript@1 + displayName: 'Setup VS2017 env vars' + inputs: + filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat' + arguments: 'amd64 -vcvars_ver=14.11' + modifyEnvironment: true + + - task: CmdLine@1 + displayName: 'Download test data and generate cmake config' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11' + workingDirectory: "$(Build.BinariesDirectory)" + + - task: VSBuild@1 + displayName: 'Build Debug' + inputs: + solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln' + platform: 'x64' + configuration: 'Debug' + msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' + msbuildArchitecture: 'x64' + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\Debug' + - task: BatchScript@1 + displayName: 'Test Debug' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + workingFolder: '$(Build.BinariesDirectory)' + - task: VSBuild@1 + displayName: 'Build C# Debug' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: 'Debug' + platform: 'any cpu' + restoreNugetPackages: false + msbuildArchitecture: 'x64' + workingFolder: '$(Build.SourcesDirectory)\csharp' + msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)' + + - task: VSTest@2 + displayName: 'VsTest - C# Debug' + inputs: + testAssemblyVer2: '**\bin\Debug\**\*Tests.dll' + searchFolder: '$(Build.SourcesDirectory)\csharp\test' + runInParallel: true + configuration: Debug + + - task: VSBuild@1 + displayName: 'Build Release' + inputs: + solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln' + platform: 'x64' + configuration: 'Release' + msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' + msbuildArchitecture: 'x64' + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\Release' + + - task: BatchScript@1 + displayName: 'Test Release' + inputs: + filename: '$(Build.BinariesDirectory)\packages\python\python.exe' + arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --use_tensorrt --tensorrt_home="C:\local\TensorRT-5.0.4.3" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' + workingFolder: "$(Build.BinariesDirectory)" + + - task: VSBuild@1 + displayName: 'Build c# Release' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + platform: 'any cpu' + configuration: 'Release' + msbuildArchitecture: 'x64' + restoreNugetPackages: false + workingFolder: '$(Build.SourcesDirectory)\csharp' + msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)' + + - task: VSTest@2 + displayName: 'VsTest - C# Release' + inputs: + testAssemblyVer2: '**\bin\Release\**\*Tests.dll' + searchFolder: '$(Build.SourcesDirectory)\csharp\test' + runInParallel: true + configuration: Release + + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**\*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + - template: templates/clean-agent-build-directory-step.yml + diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt new file mode 100644 index 0000000000..7771c33b26 --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt @@ -0,0 +1,46 @@ +# Tag: nvcr.io/nvidia/tensorrt:19.02-py3 +# Label: com.nvidia.cuda.version: 10.0.130 +# Label: com.nvidia.cudnn.version: 7.4.2 +# Ubuntu 16.04 +FROM nvcr.io/nvidia/tensorrt:19.02-py3 + +ARG PYTHON_VERSION=3.5 + +ADD scripts /tmp/scripts +ENV PATH="/opt/cmake/bin:${PATH}" +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + +WORKDIR /root + +# Allow configure to pick up GDK and CuDNN where it expects it. +# (Note: $CUDNN_VERSION is defined by NVidia's base image) +RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \ + mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \ + ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \ + mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \ + ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \ + ln -s /usr/local/cudnn{-$_CUDNN_VERSION,} + +# Build and Install LLVM +ARG LLVM_VERSION=6.0.1 +RUN cd /tmp && \ + wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \ + xz -d llvm-$LLVM_VERSION.src.tar.xz && \ + tar xvf llvm-$LLVM_VERSION.src.tar && \ + cd llvm-$LLVM_VERSION.src && \ + mkdir -p build && \ + cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release && \ + cmake --build . -- -j$(nproc) && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \ + cd /tmp && \ + rm -rf llvm* + +ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH + +ARG BUILD_USER=onnxruntimedev +ARG BUILD_UID=1000 +WORKDIR /home/$BUILD_USER +RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +USER $BUILD_USER + diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 4713591024..11fbb8faad 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -62,6 +62,7 @@ if [ $PYTHON_VER!="3.5" ]; then update-alternatives --set python3 /usr/bin/python${PYTHON_VER} fi +/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0 rm -rf /var/lib/apt/lists/* diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index 8a0592e47d..c021f12ad5 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -23,6 +23,16 @@ if [ $BUILD_DEVICE = "gpu" ]; then --cuda_home /usr/local/cuda \ --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda --build_shared_lib $BUILD_EXTR_PAR /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx +elif [ $BUILD_DEVICE = "tensorrt" ]; then + _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) + python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \ + --config Release \ + --enable_onnx_tests \ + --parallel --build_shared_lib \ + --use_tensorrt --tensorrt_home /workspace/tensorrt \ + --use_openmp \ + --cuda_home /usr/local/cuda \ + --cudnn_home /usr/local/cuda --build_shared_lib $BUILD_EXTR_PAR else python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \ --config Debug Release --build_shared_lib \ diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 124f62e3e2..91cabbbc0d 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -10,7 +10,7 @@ do case "${parameter_Option}" in #ubuntu16.04 o) BUILD_OS=${OPTARG};; -#cpu, gpu +#cpu, gpu, tensorrt d) BUILD_DEVICE=${OPTARG};; r) BUILD_DIR=${OPTARG};; #python version: 3.6 3.7 (absence means default 3.5) @@ -35,6 +35,10 @@ if [ $BUILD_DEVICE = "gpu" ]; then DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9 fi docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . +elif [ $BUILD_DEVICE = "tensorrt" ]; then + IMAGE="ubuntu16.04-$CUDA_VER" + DOCKER_FILE=Dockerfile.ubuntu_tensorrt + docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . else IMAGE="ubuntu16.04" if [ $BUILD_ARCH = "x86" ]; then @@ -57,6 +61,17 @@ if [ $BUILD_DEVICE = "cpu" ]; then "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" & +elif [ $BUILD_DEVICE = "gpu" ]; then + docker rm -f "onnxruntime-$BUILD_DEVICE" || true + nvidia-docker run --rm -h $HOSTNAME \ + --rm \ + --name "onnxruntime-$BUILD_DEVICE" \ + --volume "$SOURCE_ROOT:/onnxruntime_src" \ + --volume "$BUILD_DIR:/home/onnxruntimedev" \ + --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \ + "onnxruntime-$IMAGE" \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ + -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" & else docker rm -f "onnxruntime-$BUILD_DEVICE" || true nvidia-docker run --rm -h $HOSTNAME \