diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index b858caba87..85a5a9275c 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -63,9 +63,11 @@ TEST_P(ModelTest, Run) { } std::unique_ptr model_info = std::make_unique(model_path.c_str()); - if (model_info->GetONNXOpSetVersion() != 8 && provider_name == "tensorrt") { + if (model_info->GetONNXOpSetVersion() != 14 && model_info->GetONNXOpSetVersion() != 15 && provider_name == "tensorrt") { // TensorRT can run most of the model tests, but only part of // them is enabled here to save CI build time. + // Besides saving CI build time, TRT isn’t able to support full ONNX ops spec and therefore some testcases will fail. + // That's one of reasons we skip those testcases and only test latest ONNX opsets. return; } if (model_info->GetONNXOpSetVersion() == 10 && provider_name == "dnnl") { @@ -196,6 +198,11 @@ TEST_P(ModelTest, Run) { #endif {"mask_rcnn_keras", "this model currently has an invalid contrib op version set to 10", {}}}; + // Some EPs may fail to pass some specific testcases. + // For example TenosrRT EP may fail on FLOAT16 related testcases if GPU doesn't support float16. + // Instead of list all these testcases, we can use following keyword set to filter out testcases wchich contain specific keyword. + std::set broken_tests_keyword_set = {}; + if (provider_name == "nuphar") { // https://msdata.visualstudio.com/Vienna/_workitems/edit/1000703 broken_tests.insert({"fp16_test_tiny_yolov2", "Computed value is off by a bit more than tol."}); @@ -363,6 +370,31 @@ TEST_P(ModelTest, Run) { broken_tests.insert({"softmax_cross_entropy_sum_log_prob_expanded", "Shape mismatch"}); } + if (provider_name == "tensorrt") { + broken_tests.insert({"convtranspose_with_kernel", "It causes segmentation fault"}); + broken_tests.insert({"convtranspose_pad", "It causes segmentation fault"}); + broken_tests.insert({"convtranspose_kernel_shape", "It causes segmentation fault"}); + broken_tests.insert({"dynamicquantizelinear_expanded", "It causes segmentation fault"}); + broken_tests.insert({"dynamicquantizelinear_min_adjusted_expanded", "It causes segmentation fault"}); + broken_tests.insert({"dynamicquantizelinear_max_adjusted_expanded", "It causes segmentation fault"}); + + broken_tests.insert({"basic_conv_with_padding", + "Cannot set more than one input unless network has Q/DQ layers. TensorRT EP could not build engine for fused node"}); + broken_tests.insert({"basic_conv_without_padding", + "Cannot set more than one input unless network has Q/DQ layers. TensorRT EP could not build engine for fused node"}); + broken_tests.insert({"conv_with_strides_no_padding", + "Cannot set more than one input unless network has Q/DQ layers. TensorRT EP could not build engine for fused node"}); + + broken_tests.insert({"conv_with_autopad_same", "Internal Error (node_of_y: Cannot set more than one input unless network has Q/DQ layers.)"}); + + // sce op is not supported + broken_tests_keyword_set.insert({"sce"}); + + // TensorRT EP CI uses Nvidia Tesla M60 which doesn't support fp16. + broken_tests_keyword_set.insert({"FLOAT16"}); + + } + if (provider_name == "dml") { broken_tests.insert({"tinyyolov3", "The parameter is incorrect"}); broken_tests.insert({"PixelShuffle", "Test requires 6D Reshape, which isn't supported by DirectML"}); @@ -513,6 +545,13 @@ TEST_P(ModelTest, Run) { iter->broken_versions_.find(model_version) != iter->broken_versions_.end())) { return; } + + for (auto iter2 = broken_tests_keyword_set.begin(); iter2 != broken_tests_keyword_set.end(); ++iter2) { + std::string keyword = *iter2; + if (ToMBString(test_case_name).find(keyword) != std::string::npos) { + return; + } + } } bool is_single_node = !model_info->GetNodeName().empty(); std::vector execution_modes = {ExecutionMode::ORT_SEQUENTIAL}; @@ -552,7 +591,31 @@ TEST_P(ModelTest, Run) { } else if (provider_name == "nuphar") { ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultNupharExecutionProvider())); } else if (provider_name == "tensorrt") { - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider())); + if (test_case_name.find(ORT_TSTR("FLOAT16")) != std::string::npos) { + OrtTensorRTProviderOptions params{ + 0, + 0, + nullptr, + 1000, + 1, + 1 << 30, + 1, // enable fp16 + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + nullptr, + 0, + nullptr, + 0}; + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(TensorrtExecutionProviderWithOptions(¶ms))); + } else { + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider())); + } + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); } else if (provider_name == "migraphx") { ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultMIGraphXExecutionProvider())); } else if (provider_name == "openvino") { @@ -883,7 +946,7 @@ TEST_P(ModelTest, Run) { #endif // TENSORRT/OpenVino has too many test failures in the single node tests -#if !defined(_WIN32) && !defined(USE_TENSORRT) && !defined(USE_OPENVINO) +#if !defined(_WIN32) && !defined(USE_OPENVINO) paths.push_back("/data/onnx"); #endif while (!paths.empty()) { diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index 78e2f24df8..c841f28f79 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -170,14 +170,14 @@ TEST(ReductionOpTest, ReduceL1_int32) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceL10DTensor) { OpTester test("ReduceL1"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) { OpTester test("ReduceL2"); @@ -289,14 +289,14 @@ TEST(ReductionOpTest, ReduceL2_int32) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Int32 not allowed as input to this layer } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceL20DTensor) { OpTester test("ReduceL2"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceLogSum) { OpTester test("ReduceLogSum"); @@ -362,14 +362,14 @@ TEST(ReductionOpTest, ReduceLogSumAxes01) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceLogSum0DTensor) { OpTester test("ReduceLogSum"); test.AddInput("data", {}, {2.f}); test.AddOutput("reduced", {}, {0.693147f}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceLogSumExp_default_axes_keepdims) { OpTester test("ReduceLogSumExp"); @@ -593,21 +593,21 @@ TEST(ReductionOpTest, ReduceLogSumExp_int32) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceLogSumExp0DTensor) { OpTester test("ReduceLogSumExp"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(ReductionOpTest, ReduceLogSumExp0DTensor_double) { OpTester test("ReduceLogSumExp"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceMax_default_axes_keepdims) { OpTester test("ReduceMax"); @@ -823,14 +823,14 @@ TEST(ReductionOpTest, ReduceMax_uint8) { #endif } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceMax0DTensor) { OpTester test("ReduceMax"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceMean_default_axes_keepdims) { OpTester test("ReduceMean"); @@ -1066,21 +1066,21 @@ TEST(ReductionOpTest, ReduceMean_int32) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceMean0DTensor) { OpTester test("ReduceMean"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(ReductionOpTest, ReduceMean0DTensor_double) { OpTester test("ReduceMean"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceMin_default_axes_keepdims) { OpTester test("ReduceMin"); @@ -1289,14 +1289,14 @@ TEST(ReductionOpTest, ReduceMin_uint8) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceMin0DTensor) { OpTester test("ReduceMin"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceSum) { OpTester test("ReduceSum"); @@ -1708,14 +1708,14 @@ TEST(ReductionOpTest, ReduceSum_noop_axes_input_initializer) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceSum0DTensor) { OpTester test("ReduceSum"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceSumSquare) { OpTester test("ReduceSumSquare"); @@ -1844,14 +1844,14 @@ TEST(ReductionOpTest, ReduceSumSquare_keepdims) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceSumSquare0DTensor) { OpTester test("ReduceSumSquare"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {4}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // !(defined USE_TENSORRT) && !(defined USE_TVM) +#endif // !(defined USE_TVM) TEST(ReductionOpTest, ReduceProd_default_axes_keepdims) { OpTester test("ReduceProd"); @@ -1976,14 +1976,14 @@ TEST(ReductionOpTest, ReduceProd_int64) { test.Run(); } -#if !(defined USE_TENSORRT) && !(defined USE_TVM) +#if !(defined USE_TVM) TEST(ReductionOpTest, ReduceProd0DTensor) { OpTester test("ReduceProd"); test.AddInput("data", {}, {2}); test.AddOutput("reduced", {}, {2}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#endif // (!defined USE_TENSORRT) && (!defined USE_TVM) +#endif // (!defined USE_TVM) TEST(ReductionOpTest, ArgMax) { OpTester test("ArgMax"); diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index efd6aacfe7..64a3fabd18 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -47,6 +47,16 @@ std::unique_ptr DefaultTensorrtExecutionProvider() { return nullptr; } +std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params) { +#ifdef USE_TENSORRT + if (auto factory = CreateExecutionProviderFactory_Tensorrt(params)) + return factory->CreateProvider(); +#else + ORT_UNUSED_PARAMETER(params); +#endif + return nullptr; +} + std::unique_ptr DefaultMIGraphXExecutionProvider() { #ifdef USE_MIGRAPHX return CreateExecutionProviderFactory_MIGraphX(0)->CreateProvider(); diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index 45c86f0b91..9fed5a15c9 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -33,6 +33,7 @@ std::unique_ptr DefaultCudaExecutionProvider(); std::unique_ptr DefaultDnnlExecutionProvider(bool enable_arena = true); std::unique_ptr DefaultNupharExecutionProvider(bool allow_unaligned_buffers = true); std::unique_ptr DefaultTensorrtExecutionProvider(); +std::unique_ptr TensorrtExecutionProviderWithOptions(const OrtTensorRTProviderOptions* params); std::unique_ptr DefaultMIGraphXExecutionProvider(); std::unique_ptr DefaultOpenVINOExecutionProvider(); std::unique_ptr DefaultNnapiExecutionProvider();