From 77c69a032538d1884ebfaa8892443589c64001fc Mon Sep 17 00:00:00 2001 From: stevenlix <38092805+stevenlix@users.noreply.github.com> Date: Fri, 7 Aug 2020 17:43:56 -0700 Subject: [PATCH] Upgrade TensorRT to v7.1.3.4 (#4704) * upgrade to TensorRT 7.1.3.4 * Upgrade onnx-tensorrt parser for TensorRT 7.1.3.4 * fix format issue * fix format issue * fix format issue * Update tensorrt_execution_provider.cc * change cmake version to 3.14 * Remove --msvc_toolset 14.16 * change to onnxruntime::make_unique * use onnxruntime::make_unique * disable some tests for TensorRT * disable some tests for TensorRT * Update upsample_op_test.cc * Update tile_op_test.cc * disable some tests for TensorRT * Update constant_of_shape_test.cc * update parser * Update Dockerfile.ubuntu_tensorrt --- .gitmodules | 7 +- cmake/external/onnx-tensorrt | 2 +- dockerfiles/Dockerfile.tensorrt | 2 +- dockerfiles/README.md | 2 +- .../TensorRT-ExecutionProvider.md | 2 +- .../tensorrt/tensorrt_execution_provider.cc | 131 +++++++----------- .../test/contrib_ops/activation_op_test.cc | 2 +- .../contrib_ops/crop_and_resize_op_test.cc | 22 +-- .../cpu/generator/constant_of_shape_test.cc | 9 +- .../cpu/reduction/reduction_ops_test.cc | 4 +- .../providers/cpu/tensor/resize_op_test.cc | 4 +- .../test/providers/cpu/tensor/tile_op_test.cc | 5 +- .../providers/cpu/tensor/upsample_op_test.cc | 4 +- .../win-gpu-tensorrt-ci-pipeline.yml | 4 +- .../linux/docker/Dockerfile.ubuntu_tensorrt | 10 +- .../ci_build/github/linux/run_dockerbuild.sh | 4 +- .../ci_build/github/windows/setup_env_trt.bat | 2 +- 17 files changed, 96 insertions(+), 120 deletions(-) diff --git a/.gitmodules b/.gitmodules index f9be021588..193f1dda73 100644 --- a/.gitmodules +++ b/.gitmodules @@ -56,9 +56,10 @@ [submodule "cmake/external/SafeInt/safeint"] path = cmake/external/SafeInt/safeint url = https://github.com/dcleblanc/SafeInt.git -[submodule "cmake/external/onnx-tensorrt"] - path = cmake/external/onnx-tensorrt - url = https://github.com/stevenlix/onnx-tensorrt.git [submodule "cmake/external/libprotobuf-mutator"] path = cmake/external/libprotobuf-mutator url = https://github.com/google/libprotobuf-mutator.git +[submodule "cmake/external/onnx-tensorrt"] + path = cmake/external/onnx-tensorrt + url = https://github.com/stevenlix/onnx-tensorrt.git + branch = trt71 diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt index 77a8d522e5..9d962f4510 160000 --- a/cmake/external/onnx-tensorrt +++ b/cmake/external/onnx-tensorrt @@ -1 +1 @@ -Subproject commit 77a8d522e582efaac3166e912e96215acbf3f129 +Subproject commit 9d962f4510102e296944b9be046047a953be0ee1 diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index f4b5670a43..fff01610c5 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -5,7 +5,7 @@ # Dockerfile to run ONNXRuntime with TensorRT integration # nVidia TensorRT Base Image -FROM nvcr.io/nvidia/tensorrt:20.01-py3 +FROM nvcr.io/nvidia/tensorrt:20.07.1-py3 MAINTAINER Vinitra Swamy "viswamy@microsoft.com" ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime diff --git a/dockerfiles/README.md b/dockerfiles/README.md index c284336c55..822b05bd91 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -95,7 +95,7 @@ Therefore, ONNX RT Execution Provider for **nGraph** will be deprecated starting ``` ## TensorRT -**Ubuntu 18.04, CUDA 10.2, TensorRT 7.0.0** +**Ubuntu 18.04, CUDA 11.0, TensorRT 7.1.3.4** 1. Build the docker image from the Dockerfile in this repository. ``` diff --git a/docs/execution_providers/TensorRT-ExecutionProvider.md b/docs/execution_providers/TensorRT-ExecutionProvider.md index 54bbced09e..d390c42dbd 100644 --- a/docs/execution_providers/TensorRT-ExecutionProvider.md +++ b/docs/execution_providers/TensorRT-ExecutionProvider.md @@ -7,7 +7,7 @@ With the TensorRT execution provider, the ONNX Runtime delivers better inferenci ## Build For build instructions, please see the [BUILD page](../../BUILD.md#tensorrt). -The TensorRT execution provider for ONNX Runtime is built and tested with TensorRT 7.0.0.11. +The TensorRT execution provider for ONNX Runtime is built and tested with TensorRT 7.1.3.4. ## Using the TensorRT execution provider ### C/C++ diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 7e7785cb55..609a181533 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -230,54 +230,6 @@ bool FindCycleHelper(int i, const std::list* adjacency_map, return false; } -// Remove nodes with empty shape (for example [1, 0]) because TensorRT 7.0 doens't support empty shape -SubGraphCollection_t RemoveEmptyShapeNodes(const onnxruntime::GraphViewer& graph) { - // Here only NonZero, NonMaxSuppression and TopK related empty shape nodes are removed, particularly for RCNN models. - // TODO: Remove the code if TensorRT fixed the issue in the future release, or find a better generic way here to work around - const std::vector& node_index = graph.GetNodesInTopologicalOrder(); - const std::vector exclude_dim_names{"NonZero", "NonMaxSuppression", "TopK"}; - SubGraphCollection_t parser_nodes_vector = {{{}, false}}; - std::vector nodes_vector(node_index.size()); - std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0); - for (const auto& index : nodes_vector) { - // Check if node has empty input shape - const auto& node = graph.GetNode(node_index[index]); - bool exclude_node = false; - for (const auto& input : node->InputDefs()) { - const auto& input_shape = input->Shape(); - if (input_shape) { - for (const auto& dim : input_shape->dim()) { - std::string dim_name = dim.dim_param(); - if (!dim_name.empty()) { - for (const auto& exclude : exclude_dim_names) { - if (dim_name.find(exclude) != std::string::npos) { - exclude_node = true; - break; - } - } - if (exclude_node) { - break; - } - } - } - } - - if (exclude_node) { - break; - } - } - - // Remove the node with empty input shape - if (!exclude_node) { - parser_nodes_vector.back().first.push_back(index); - } else if (!parser_nodes_vector.back().first.empty()) { - parser_nodes_vector.push_back({{}, false}); - } - } - - return parser_nodes_vector; -} - std::unique_ptr TensorrtExecutionProvider::GetSubGraph(SubGraph_t graph_nodes_index, int& kernels_index, const onnxruntime::GraphViewer& graph) const { const std::vector& node_index = graph.GetNodesInTopologicalOrder(); std::unordered_set node_set; @@ -427,10 +379,10 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect if (group.second) { nodes_list_output.push_back(group); } else { - onnxruntime::Model model_build(graph.Name(), true, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap(), - std::vector(), *GetLogger()); - onnxruntime::Graph& graph_build = model_build.MainGraph(); + std::unique_ptr model_build = onnxruntime::make_unique(graph.Name(), true, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), graph.DomainToVersionMap(), + std::vector(), *GetLogger()); + onnxruntime::Graph& graph_build = model_build->MainGraph(); // Add node and node args // If node output is also parent graph output, the output will be added to the @@ -442,6 +394,23 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect for (auto input : node->InputDefs()) { auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto()); inputs.push_back(&n_input); + const ONNX_NAMESPACE::TensorProto* initializer = nullptr; + if (graph.GetInitializedTensor(input->Name(), initializer)) { + const ONNX_NAMESPACE::TensorProto* subgraph_initializer = nullptr; + if (!graph_build.GetInitializedTensor(input->Name(), subgraph_initializer)) { + graph_build.AddInitializedTensor(*(initializer)); + } + } + } + + for (auto input : node->ImplicitInputDefs()) { + const ONNX_NAMESPACE::TensorProto* initializer = nullptr; + if (graph.GetInitializedTensor(input->Name(), initializer)) { + const ONNX_NAMESPACE::TensorProto* subgraph_initializer = nullptr; + if (!graph_build.GetInitializedTensor(input->Name(), subgraph_initializer)) { + graph_build.AddInitializedTensor(*(initializer)); + } + } } for (auto output : node->OutputDefs()) { auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); @@ -454,12 +423,6 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect graph_build.AddNode(node->Name(), node->OpType(), node->Description(), inputs, outputs, &node->GetAttributes(), node->Domain()); } - // Add initializers to the subgraph - const auto& init_tensors = graph.GetAllInitializedTensors(); - for (const auto& tensor : init_tensors) { - graph_build.AddInitializedTensor(*(tensor.second)); - } - ORT_ENFORCE(graph_build.Resolve().IsOK()); // Add parent graph output to the subgraph @@ -490,12 +453,12 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect } // Serialize modelproto to string - const onnxruntime::GraphViewer graph_viewer(graph_build); + const onnxruntime::GraphViewer graph_viewer(std::move(graph_build)); - onnxruntime::Model model(graph_viewer.Name(), true, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), graph_viewer.DomainToVersionMap(), - std::vector(), *GetLogger()); - ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); + std::unique_ptr model = onnxruntime::make_unique(graph_viewer.Name(), true, ModelMetaData(), PathString(), + IOnnxRuntimeOpSchemaRegistryList(), graph_viewer.DomainToVersionMap(), + std::vector(), *GetLogger()); + ONNX_NAMESPACE::ModelProto model_proto = model->ToProto(); ToGraphProtoInternal(graph_viewer, *(model_proto.mutable_graph())); model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); @@ -629,10 +592,13 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t& for (int i = 0; i < static_cast(cycles.size()); ++i) { auto loc = index_to_node_map.find(cycles[i]); if (loc != index_to_node_map.end() && loc->second.find("TRTKernel") != std::string::npos) { - int trt_node_index = std::stoi(loc->second.substr(10)); - supported_nodes_vector.erase(supported_nodes_vector.begin() + trt_node_index); - trt_cycle = true; - break; + std::size_t found = loc->second.rfind("_"); + if (found != std::string::npos) { + int trt_node_index = std::stoi(loc->second.substr(found + 1)); + supported_nodes_vector.erase(supported_nodes_vector.begin() + trt_node_index); + trt_cycle = true; + break; + } } } } @@ -646,11 +612,10 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t& std::vector> TensorrtExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, const std::vector& /*kernel_registries*/) const { - // Remove nodes with empty shape - SubGraphCollection_t parser_nodes_vector = RemoveEmptyShapeNodes(graph); - // Get supported node list from TensorRT parser - SubGraphCollection_t supported_nodes_vector; + std::vector nodes_vector(graph.NumberOfNodes()); + std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0); + SubGraphCollection_t supported_nodes_vector, parser_nodes_vector = {{nodes_vector, false}}; bool early_termination = false; supported_nodes_vector = GetSupportedList(parser_nodes_vector, 0, max_partition_iterations_, graph, &early_termination); if (early_termination) { @@ -772,10 +737,10 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorName(); if (fp16_enable_ && trt_builder->platformHasFastFp16()) { trt_config->setFlag(nvinfer1::BuilderFlag::kFP16); - trt_node_name_with_precision += "_fp16"; - LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] FP16 mode is enabled."; + trt_node_name_with_precision += "_fp16"; + LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] FP16 mode is enabled."; } - + // Build TRT engine here if the graph doesn't have dynamic shape input. Otherwise engine will // be built at runtime tensorrt_ptr::unique_pointer trt_engine; @@ -790,12 +755,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(runtime_->deserializeCudaEngine(engine_buf.get(), engine_size, nullptr)); - LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] DeSerialized " + GetEnginePath(engine_cache_path_, trt_node_name_with_precision); + LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Deserialized " + GetEnginePath(engine_cache_path_, trt_node_name_with_precision); + if (trt_engine == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, + "TensorRT EP could not deserialize engine from " + trt_node_name_with_precision + ".engine"); + } } else { trt_engine = tensorrt_ptr::unique_pointer(trt_builder->buildEngineWithConfig(*trt_network, *trt_config)); if (trt_engine == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "TensorRT EP could not build Engine for fused node: " + fused_node->Name()); + "TensorRT EP could not build engine for fused node: " + fused_node->Name()); } if (engine_cache_enable_) { @@ -809,7 +778,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(trt_engine->createExecutionContext()); if (trt_context == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "TensorRT EP could not build Execution Context for fused node: " + fused_node->Name()); + "TensorRT EP could not build execution context for fused node: " + fused_node->Name()); } } @@ -830,7 +799,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorsecond; - } + } const auto& tensor_type = graph_output[i].type().tensor_type(); output_types[output_name] = tensor_type.elem_type(); } @@ -1044,14 +1013,14 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorbuildEngineWithConfig(*trt_state->network, *trt_config)); if (trt_state->engine->get() == nullptr) { - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Build Engine."); + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to build engine."); } trt_engine = trt_state->engine->get(); *(trt_state->context) = tensorrt_ptr::unique_pointer( trt_state->engine->get()->createExecutionContext()); if (trt_state->context->get() == nullptr) { - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP Failed to Create Context."); + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context."); } trt_context = trt_state->context->get(); } @@ -1296,7 +1265,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector ("X", {1, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f}); @@ -14,7 +16,7 @@ TEST(CropAndResizeTest, CropAndResize_1122) { test1.AddInput ("batch_indices", {3}, {0, 0, 0}); test1.AddInput ("crop_size", {2}, {1, 1}); test1.AddOutput ("output", {3, 1, 1, 1}, {2.75f, 1.925f, 2.2f}); - test1.Run(); + test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test2 ("CropAndResize", 1, onnxruntime::kMSDomain); test2.AddInput ("X", {1, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f}); @@ -22,7 +24,7 @@ TEST(CropAndResizeTest, CropAndResize_1122) { test2.AddInput ("batch_indices", {3}, {0, 0, 0}); test2.AddInput ("crop_size", {2}, {2, 2}); test2.AddOutput ("output", {3, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f, 1.1f, 1.65f, 2.2f, 2.75f, 1.1f, 2.2f, 2.2f, 3.3f}); - test2.Run(); + test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test3 ("CropAndResize", 1, onnxruntime::kMSDomain); test3.AddInput ("X", {1, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f}); @@ -31,7 +33,7 @@ TEST(CropAndResizeTest, CropAndResize_1122) { test3.AddInput ("crop_size", {2}, {2, 2}); test3.AddAttribute("extrapolation_value", (float)5.5); test3.AddOutput ("output", {2, 1, 2, 2}, {1.1f, 5.5f, 5.5f, 5.5f, 1.925f, 2.2f, 3.025f, 3.3f}); - test3.Run(); + test3.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(CropAndResizeTest, CropAndResize_2122) { @@ -41,7 +43,7 @@ TEST(CropAndResizeTest, CropAndResize_2122) { test1.AddInput ("batch_indices", {3}, {0, 1, 1}); test1.AddInput ("crop_size", {2}, {1, 1}); test1.AddOutput ("output", {3, 1, 1, 1}, {2.75f, 6.325f, 6.6f}); - test1.Run(); + test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test2 ("CropAndResize", 1, onnxruntime::kMSDomain); test2.AddInput ("X", {2, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f}); @@ -49,7 +51,7 @@ TEST(CropAndResizeTest, CropAndResize_2122) { test2.AddInput ("batch_indices", {3}, {0, 1, 1}); test2.AddInput ("crop_size", {2}, {2, 2}); test2.AddOutput ("output", {3, 1, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.05f, 6.6f, 7.15f, 5.5f, 6.6f, 6.6f, 7.7f}); - test2.Run(); + test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(CropAndResizeTest, CropAndResize_1222) { @@ -59,7 +61,7 @@ TEST(CropAndResizeTest, CropAndResize_1222) { test1.AddInput ("batch_indices", {3}, {0, 0, 0}); test1.AddInput ("crop_size", {2}, {1, 1}); test1.AddOutput ("output", {3, 2, 1, 1}, {2.75f, 7.15f, 1.925f, 6.325f, 2.2f, 6.6f}); - test1.Run(); + test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test2 ("CropAndResize", 1, onnxruntime::kMSDomain); test2.AddInput ("X", {1, 2, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f}); @@ -68,7 +70,7 @@ TEST(CropAndResizeTest, CropAndResize_1222) { test2.AddInput ("crop_size", {2}, {2, 2}); test2.AddOutput ("output", {3, 2, 2, 2}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f, 1.1f, 1.65f, 2.2f, 2.75f, \ 5.5f, 6.05f, 6.6f, 7.15f, 1.1f, 2.2f, 2.2f, 3.3f, 5.5f, 6.6f, 6.6f, 7.7f}); - test2.Run(); + test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(CropAndResizeTest, CropAndResize_1133) { @@ -78,7 +80,7 @@ TEST(CropAndResizeTest, CropAndResize_1133) { test1.AddInput ("batch_indices", {3}, {0, 0, 0}); test1.AddInput ("crop_size", {2}, {1, 1}); test1.AddOutput ("output", {3, 1, 1, 1}, {5.5f, 3.3f, 3.85f}); - test1.Run(); + test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test2("CropAndResize", 1, onnxruntime::kMSDomain); test2.AddInput ("X", {1, 1, 3, 3}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f, 9.9f}); @@ -86,7 +88,7 @@ TEST(CropAndResizeTest, CropAndResize_1133) { test2.AddInput ("batch_indices", {3}, {0, 0, 0}); test2.AddInput ("crop_size", {2}, {2, 2}); test2.AddOutput ("output", {3, 1, 2, 2}, {1.1f, 3.3f, 7.7f, 9.9f, 1.1f, 2.2f, 4.4f, 5.5f, 1.1f, 3.3f, 4.4f, 6.6f}); - test2.Run(); + test2.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); OpTester test3("CropAndResize", 1, onnxruntime::kMSDomain); test3.AddInput ("X", {1, 1, 3, 3}, {1.1f, 2.2f, 3.3f, 4.4f, 5.5f, 6.6f, 7.7f, 8.8f, 9.9f}); @@ -95,7 +97,7 @@ TEST(CropAndResizeTest, CropAndResize_1133) { test3.AddInput ("crop_size", {2}, {2, 2}); test3.AddAttribute("mode", "nearest"); test3.AddOutput ("output", {3, 1, 2, 2}, {1.1f, 3.3f, 7.7f, 9.9f, 1.1f, 2.2f, 4.4f, 5.5f, 1.1f, 3.3f, 4.4f, 6.6f}); - test3.Run(); + test3.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/generator/constant_of_shape_test.cc b/onnxruntime/test/providers/cpu/generator/constant_of_shape_test.cc index 0bfe62536d..16a84002ad 100644 --- a/onnxruntime/test/providers/cpu/generator/constant_of_shape_test.cc +++ b/onnxruntime/test/providers/cpu/generator/constant_of_shape_test.cc @@ -7,6 +7,7 @@ using namespace ONNX_NAMESPACE; namespace onnxruntime { +// Disable TensorRT in the tests due to lack of support of the operator. namespace test { TEST(ConstantOfShape, Float_Ones) { OpTester test("ConstantOfShape", 9); @@ -29,7 +30,7 @@ TEST(ConstantOfShape, Float_Ones) { std::fill_n(output.begin(), 4 * 3 * 2, 1.f); test.AddOutput("output", output_dims, output); - test.Run(OpTester::ExpectResult::kExpectSuccess); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(ConstantOfShape, Int32_Zeros) { @@ -51,7 +52,7 @@ TEST(ConstantOfShape, Int32_Zeros) { std::fill_n(output.begin(), output.size(), 0); test.AddOutput("output", output_dims, output); - test.Run(OpTester::ExpectResult::kExpectSuccess); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(ConstantOfShape, DefaultValue) { @@ -68,7 +69,7 @@ TEST(ConstantOfShape, DefaultValue) { std::fill_n(output.begin(), output.size(), 0.f); test.AddOutput("output", output_dims, output); - test.Run(OpTester::ExpectResult::kExpectSuccess); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } inline void SetValue(TensorProto& t_proto, float value) { @@ -128,7 +129,7 @@ void RunTypedTest(TensorProto::DataType dt, T value) { std::fill_n(output.begin(), output.size(), value); test.AddOutput("output", output_dims, output); - test.Run(OpTester::ExpectResult::kExpectSuccess); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(ConstantOfShape, TypeTests) { diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index 47287269eb..4042fbcf5a 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -1614,7 +1614,7 @@ TEST(ReductionOpTest, ArgMax_do_not_keepdims_2) { {1.0f, 2.0f, 3.0f}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: node1: at least 2 dimensions are required for input } TEST(ReductionOpTest, ArgMax_int32) { @@ -1774,7 +1774,7 @@ TEST(ReductionOpTest, ArgMin_do_not_keepdims_2) { test.AddInput("data", {3}, {1.0f, 2.0f, 3.0f}); test.AddOutput("reduced", {}, {0}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: node1: at least 2 dimensions are required for input } TEST(ReductionOpTest, ArgMin_do_not_keepdims_2_select_last) { diff --git a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc index a808485bba..7f8c9b8c6d 100644 --- a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc @@ -135,7 +135,7 @@ TEST(ResizeOpTest, ResizeOpLineartDownSampleTest_2DBilinear_pytorch_half_pixel) std::vector Y = {1.6666666f, 7.0f, 12.333333f}; test.AddOutput("Y", {sizes[0], sizes[1]}, Y); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: results mismatch } TEST(ResizeOpTest, ResizeOpLineartUpSampleTest_4DBilinear_asymmetric) { @@ -513,7 +513,7 @@ TEST(ResizeOpTest, ResizeOpNearestUpSample_Nearest2xOptimization_Sizes) { 3.0f, 3.0f, 4.0f, 4.0f}; test.AddOutput("Y", {N, C, sizes[2], sizes[3]}, Y); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: results mismatch }; run_test(false); diff --git a/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc index a29dab9dd0..f1d634a248 100644 --- a/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc @@ -18,7 +18,10 @@ void RunTest(std::initializer_list input, test.AddInput("input", input_dims, input); test.AddInput("repeats", repeat_dims, repeat); test.AddOutput("output", output_dims, output); - test.Run(); + if (std::is_same::value) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT reports error: Assertion Error in makePaddedScale: 0 (regionRanges != nullptr) + else + test.Run(); } template diff --git a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc index 1df62e773c..4ae11b07e1 100644 --- a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc @@ -292,7 +292,7 @@ TEST(UpsampleOpTest, UpsampleOp4DBilinearTest) { 7.0f, 7.5f, 8.0f, 8.5f, 9.0f, 9.0f, 9.0f, 9.0f}; test.AddOutput("Y", {N, C, (int64_t)(H * scales[2]), (int64_t)(W * scales[3])}, Y); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: results mismatch } TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) { @@ -315,7 +315,7 @@ TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) { 3.0f, 3.5f, 4.0f, 4.5f, 5.0f, 5.0f, 5.0f, 5.0f}; test.AddOutput("Y", {(int64_t)(H * scales[0]), (int64_t)(W * scales[1])}, Y); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: results mismatch } TEST(UpsampleOpTest, UpsampleOp4DBilinearTest_ScalesNoOp) { diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 60329537e7..b504c99ded 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -51,7 +51,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.0.0.11.cuda-10.2.cudnn7.6\TensorRT-7.0.0.11" --cuda_version=10.2 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" --cudnn_home="C:\local\cudnn-10.2-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.1.3.4.cuda-11.0.cudnn8.0\TensorRT-7.1.3.4" --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -84,7 +84,7 @@ jobs: del wheel_filename_file python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --msvc_toolset 14.16 --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.0.0.11.cuda-10.2.cudnn7.6\TensorRT-7.0.0.11" --cuda_version=10.2 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" --cudnn_home="C:\local\cudnn-10.2-windows10-x64-v7.6.5.32\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-7.1.3.4.cuda-11.0.cudnn8.0\TensorRT-7.1.3.4" --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt index 6a50922ad7..7e9d8cbe22 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt @@ -1,14 +1,14 @@ -# Tag: nvcr.io/nvidia/tensorrt:20.01-py3 -# Label: com.nvidia.cuda.version: 10.2.89 -# Label: com.nvidia.cudnn.version: 7.6.5 +# Tag: nvcr.io/nvidia/tensorrt:20.07.1-py3 +# Label: com.nvidia.cuda.version: 11.0.194 +# Label: com.nvidia.cudnn.version: 8.0.1 # Ubuntu 18.04 -FROM nvcr.io/nvidia/tensorrt:20.01-py3 +FROM nvcr.io/nvidia/tensorrt:20.07.1-py3 ARG PYTHON_VERSION=3.6 ADD scripts /tmp/scripts RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts \ - && rm /usr/local/bin/cmake && rm /usr/local/bin/ctest && rm -r /usr/local/share/cmake-3.12 + && rm /usr/local/bin/cmake && rm /usr/local/bin/ctest && rm -r /usr/local/share/cmake-3.14 WORKDIR /root diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index a45f40c957..3960ec3ac0 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -76,8 +76,8 @@ else fi $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg BUILD_EXTR_PAR="${BUILD_EXTR_PAR}" -f $DOCKER_FILE . elif [ $BUILD_DEVICE = "tensorrt" ]; then - # TensorRT container release 20.01 - IMAGE="$BUILD_OS-cuda10.2-cudnn7.6-tensorrt7.0" + # TensorRT container release 20.07 + IMAGE="$BUILD_OS-cuda11.0-cudnn8.0-tensorrt7.1" DOCKER_FILE=Dockerfile.ubuntu_tensorrt $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_DEVICE = "openvino" ]; then diff --git a/tools/ci_build/github/windows/setup_env_trt.bat b/tools/ci_build/github/windows/setup_env_trt.bat index 0ae9569f7d..1ab8e7ee73 100644 --- a/tools/ci_build/github/windows/setup_env_trt.bat +++ b/tools/ci_build/github/windows/setup_env_trt.bat @@ -1,2 +1,2 @@ -set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin;C:\local\cudnn-10.2-windows10-x64-v7.6.5.32\cuda\bin;%PATH% +set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda\bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false