From 20e6a2b6da536f865523d9aff0bd81466d8fabf0 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 24 Oct 2019 11:37:09 -0700 Subject: [PATCH] Disable optimizers for OpTester operator unit tests (#2237) * Disable optimizers for operator unit tests as they're intended to test the operator directly rather than something that could have been modified by an optimizer. Disable TensorRT for Scan9 unit tests that fails when optimizers are enabled. Bug 525222 tracks that. * Disable TRT for the lenient shape inferencing test as it uses Unsqueeze and TRT doesn't cope with that op. --- .../test/framework/inference_session_test.cc | 5 ++-- .../providers/cpu/controlflow/scan_test.cc | 23 +++++++++++++------ .../test/providers/provider_test_utils.cc | 5 +--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index 0ff4ae63ec..5988d63971 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -1486,7 +1486,7 @@ TEST(InferenceSessionTests, TestLenientShapeInferencing) { "Mismatch between number of source and target dimensions. Source=1 Target=2"); // older opset should allow the mismatch with a warning. - // we also need for the output to be valid so OpTester doesn't throw so add an Unsqueeze after the Shape. + // we also need for the output to be valid so OpTester doesn't throw so add an Unsqueeze after the Shape. // This should result in a warning log message but successful run. class OpTesterWithReshape : public OpTester { public: @@ -1518,7 +1518,8 @@ TEST(InferenceSessionTests, TestLenientShapeInferencing) { old_opset.AddInput("data", input_shape, input_data); old_opset.AddOutput("output", invalid_output_shape, output_data); - old_opset.Run(); + // TensorRT doesn't handle Unsqueeze + old_opset.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } #ifdef USE_CUDA diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc index 9b40aff358..9663333c14 100644 --- a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -24,6 +24,8 @@ struct RunOptions { bool scalar_loop_state_value = false; bool add_bad_shape = false; bool mixed_execution_providers = false; + // Disable TensorRT because its parser fails, and it can't handle unknown dimensions + std::unordered_set excluded_provider_types{kTensorrtExecutionProvider}; }; static void CreateSubgraph(Graph& graph, RunOptions& options, const std::string& failure_message = ""); @@ -310,7 +312,7 @@ static void RunTest_v8(const std::string test_name, int64_t batch_size, int64_t test.AddOutput("scan_output_2", output_shape, output_2); test.AddOutput("scan_output_3", output_shape, output_3); - test.Run(expect_result, failure_message, {kTensorrtExecutionProvider}); // Disable TensorRT because its parser failed + test.Run(expect_result, failure_message, options.excluded_provider_types); } static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_t input_size, @@ -403,9 +405,9 @@ static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_ execution_providers.push_back(DefaultCudaExecutionProvider()); execution_providers.push_back(DefaultCpuExecutionProvider()); - test.Run(expect_result, failure_message, {kTensorrtExecutionProvider}, nullptr, &execution_providers); + test.Run(expect_result, failure_message, options.excluded_provider_types, nullptr, &execution_providers); } else { - test.Run(expect_result, failure_message, {kTensorrtExecutionProvider}); // Disable TensorRT because its parser failed + test.Run(expect_result, failure_message, options.excluded_provider_types); } } @@ -534,6 +536,12 @@ static void OuterScopeAccess_ShapeInMainGraph_NoTypeAndShapeInSubgraph(bool is_v options.include_outer_scope_add = true; + // Scan9.OuterScopeAccess_ShapeInMainGraph_NoTypeAndShapeInSubgraph fails with nuphar. See Bug 525222. + // Remove this once that is fixed. + if (is_v8 == false) { + options.excluded_provider_types.insert(kNupharExecutionProvider); + } + ShortSequenceOneInBatchOneLoopStateVar(options); } @@ -887,7 +895,7 @@ TEST(Scan9, TransposeOutputDim2) { test.AddInput("scan_input_1", input_shape, {1.0, 2.0}); test.AddOutput("scan_output_1", output_shape, {1.0, 2.0}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // Disable TensorRT on supported data types + test.Run(OpTester::ExpectResult::kExpectSuccess, "", RunOptions().excluded_provider_types); } static void InvalidInput(bool is_v8) { @@ -1079,7 +1087,7 @@ void MixedTypeInputs(bool is_v8) { test.AddOutput("scan_output_1", seq_shape, {0.0, 1.0, 2.0}); test.AddOutput("scan_output_2", seq_shape, {0, 1, 2}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // Disable TensorRT on unsupported data types + test.Run(OpTester::ExpectResult::kExpectSuccess, "", RunOptions().excluded_provider_types); } TEST_8_AND_9(MixedTypeInputs); @@ -1151,9 +1159,10 @@ void UnknownDimInSubgraphOutput(bool is_v8, bool mixed_execution_providers = fal execution_providers.push_back(DefaultCudaExecutionProvider()); execution_providers.push_back(DefaultCpuExecutionProvider()); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}, nullptr, &execution_providers); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", RunOptions().excluded_provider_types, nullptr, + &execution_providers); } else { - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //Disable TensorRT on unknown dimension tests + test.Run(OpTester::ExpectResult::kExpectSuccess, "", RunOptions().excluded_provider_types); } } diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 3efa70fdf3..67439838d3 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -514,10 +514,7 @@ void OpTester::Run(ExpectResult expect_result, so.session_logid = op_; so.session_log_verbosity_level = 1; so.execution_mode = execution_mode; - // TODO: Optimizers should be off by default so we test the operator as is, however currently - // Scan9.OuterScopeAccess_ShapeInMainGraph_NoTypeAndShapeInSubgraph fails with nuphar. See Bug 525222. - // Uncomment this line once that is addressed. - // so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off + so.graph_optimization_level = TransformerLevel::Default; // 'Default' == off Run(so, expect_result, expected_failure_string, excluded_provider_types, run_options, execution_providers); }