From 1a66dfc0f96206be47b1ee295972fec714ba815c Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Tue, 17 Nov 2020 00:26:06 -0800 Subject: [PATCH] Enable Squeeze Opset 13 for NNAPI (#5717) * Add copy sparse model in minimal CI * Add squeeze 13 support * fix small typo * Add ut for squeeze in NNAPI * Fix some issue in the UT and code * Modify based on the master change * Fix build break --- .../nnapi_builtin/builders/op_builder.cc | 36 ++++++++++- .../builders/op_support_checker.cc | 15 +++-- .../test/framework/ort_model_only_test.cc | 2 +- .../providers/cpu/tensor/squeeze_op_test.cc | 59 +++++++++++++++--- .../sparse_initializer_handling.onnx | Bin .../build_minimal_ort_and_run_tests.sh | 1 + 6 files changed, 96 insertions(+), 17 deletions(-) rename onnxruntime/test/testdata/{ => ort_minimal_test_models}/sparse_initializer_handling.onnx (100%) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 06e77f4bba..e8d1dc7908 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -1677,10 +1677,44 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const #pragma region op_squeeze class SqueezeOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const override; + private: Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const override ORT_MUST_USE_RESULT; + static vector GetAxes(ModelBuilder& model_builder, const Node& node); }; +void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const { + if (node.SinceVersion() > 12 && node.InputDefs().size() > 1) { + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + } +} + +/* static */ vector SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder, const Node& node) { + vector axes; + // Squeeze opset 13 use input as axes + if (node.SinceVersion() > 12) { + // If axes is not supplied, return an empty axes as default to squeeze all + if (node.InputDefs().size() > 1) { + const auto& initializers(model_builder.GetInitializerTensors()); + const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); + const int64_t* raw_axes = GetTensorInt64Data(axes_tensor); + const auto size = SafeInt(axes_tensor.dims()[0]); + axes.resize(size); + for (uint32_t i = 0; i < size; i++) { + // it is unlikely we have a axis value overflow for int32 + axes[i] = static_cast(raw_axes[i]); + } + } + } else { + NodeAttrHelper helper(node); + axes = helper.Get("axes", vector()); + } + + return axes; +} + Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); @@ -1693,7 +1727,7 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons } NodeAttrHelper helper(node); - vector axes = helper.Get("axes", vector()); + vector axes = GetAxes(model_builder, node); const auto& input_shape(shaper[input]); auto input_dims = input_shape.size(); for (auto& axis : axes) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 5503dca5ae..ceab7c409c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -793,13 +793,9 @@ class SqueezeOpSupportChecker : public BaseOpSupportChecker { int32_t GetMinSupportedSdkVer(const Node& /* node */, const OpSupportCheckParams& /* params */) const override { return 28; } - - // Squeeze opset 13+ uses input for axes, which is not supported yet - // TODO add support for squeeze opset 13+ - int GetMaxSupportedOpSet(const Node& /* node */) const override { return 12; } }; -bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const Node& node, +bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const Node& node, const OpSupportCheckParams& /* params */) const { Shape input_shape; if (!GetShape(*node.InputDefs()[0], input_shape)) @@ -812,6 +808,15 @@ bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* i return false; } + // Squeeze opset 13 use input 1 as axes, if we have input 1 then it need to be an initializer + if (node.SinceVersion() > 12 && node.InputDefs().size() > 1) { + const auto& axes_name = node.InputDefs()[1]->Name(); + if (!Contains(initializers, axes_name)) { + LOGS_DEFAULT(VERBOSE) << "Input axes of Squeeze must be known"; + return false; + } + } + return true; } diff --git a/onnxruntime/test/framework/ort_model_only_test.cc b/onnxruntime/test/framework/ort_model_only_test.cc index 7eda4b4323..3510cbb5f2 100644 --- a/onnxruntime/test/framework/ort_model_only_test.cc +++ b/onnxruntime/test/framework/ort_model_only_test.cc @@ -267,7 +267,7 @@ TEST(OrtModelOnlyTests, SerializeToOrtFormat) { TEST(OrtModelOnlyTests, SparseInitializerHandling) { const std::basic_string ort_file = ORT_TSTR("sparse_initializer_handling.onnx.ort"); - SaveAndCompareModels("testdata/sparse_initializer_handling.onnx", ort_file); + SaveAndCompareModels("testdata/ort_minimal_test_models/sparse_initializer_handling.onnx", ort_file); SessionOptions so; so.session_logid = "LoadOrtFormat"; diff --git a/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc b/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc index 20d041032a..fe9ee6fb3b 100644 --- a/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/squeeze_op_test.cc @@ -15,7 +15,7 @@ TEST(SqueezeOpTest, Squeeze_1) { test.AddAttribute("axes", std::vector{0}); test.AddInput("data", {1, 3, 4, 5}, std::vector(60, 1.0f)); test.AddOutput("squeezed", {3, 4, 5}, std::vector(60, 1.0f)); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed } TEST(SqueezeOpTest, Squeeze_Empty_Axes_1) { @@ -23,7 +23,7 @@ TEST(SqueezeOpTest, Squeeze_Empty_Axes_1) { test.AddInput("data", {1, 1, 4, 1}, std::vector(4, 1.0f)); test.AddOutput("squeezed", {4}, std::vector(4, 1.0f)); // TensorRT doesn't seem to support missing 'axes' - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(SqueezeOpTest, Squeeze_Empty_Axes_2) { @@ -40,7 +40,7 @@ TEST(SqueezeOpTest, Squeeze_1_int32) { test.AddAttribute("axes", std::vector{0}); test.AddInput("data", {1, 3, 4, 5}, std::vector(60, 1)); test.AddOutput("squeezed", {3, 4, 5}, std::vector(60, 1)); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed } TEST(SqueezeOpTest, Squeeze_string) { @@ -58,7 +58,7 @@ TEST(SqueezeOpTest, Squeeze_2) { std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); test.AddOutput("squeezed", {4, 2}, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed } TEST(SqueezeOpTest, UnsortedAxes) { @@ -69,7 +69,7 @@ TEST(SqueezeOpTest, UnsortedAxes) { std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); test.AddOutput("squeezed", {4, 2}, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed } TEST(SqueezeOpTest, DuplicateAxes) { @@ -80,7 +80,7 @@ TEST(SqueezeOpTest, DuplicateAxes) { std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); test.AddOutput("squeezed", {4, 2}, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); //Incorrect precision. Will be re-enabled after it's fixed } TEST(SqueezeOpTest, BadAxes) { @@ -105,7 +105,7 @@ TEST(SqueezeOpTest, SqueezeNegAxis_2) { // nGraph does not support neg axis. // OpenVINO EP Incorrect precision. Will be re-enabled after its fixed - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kOpenVINOExecutionProvider}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kOpenVINOExecutionProvider}); } TEST(SqueezeOpTest, Squeeze_2_axes_input) { @@ -117,7 +117,7 @@ TEST(SqueezeOpTest, Squeeze_2_axes_input) { std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); // Incorrect precision for OpenVINO EP. Will be re-enabled after it's fixed // TensorRT and OpenVINO dont support "axes" input in opset 13, re-enable after - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); } TEST(SqueezeOpTest, Squeeze_Empty_Axes_opset13) { @@ -125,7 +125,7 @@ TEST(SqueezeOpTest, Squeeze_Empty_Axes_opset13) { test.AddInput("data", {1, 1, 4, 1}, std::vector(4, 1.0f)); test.AddOutput("squeezed", {4}, std::vector(4, 1.0f)); // TensorRT doesn't seem to support missing 'axes' - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } TEST(SqueezeOpTest, SqueezeNegAxis_axes_input) { @@ -133,7 +133,7 @@ TEST(SqueezeOpTest, SqueezeNegAxis_axes_input) { test.AddInput("data", {1, 4, 1, 1, 2}, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); - test.AddInput("axes",{3} ,std::vector{0, -3, -2}); + test.AddInput("axes", {3}, std::vector{0, -3, -2}); test.AddOutput("squeezed", {4, 2}, std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); @@ -143,5 +143,44 @@ TEST(SqueezeOpTest, SqueezeNegAxis_axes_input) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); } +// Add 4d input shape test, since NNAPI supports up to 4d input shape +TEST(SqueezeOpTest, Squeeze_4d_2_axes_input) { + auto run_test = [](bool axes_is_initializer) { + OpTester test("Squeeze", 13); + test.AddInput("data", {1, 4, 1, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + test.AddInput("axes", {2}, std::vector{0, 2}, axes_is_initializer); + test.AddOutput("squeezed", {4, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + // Incorrect precision for OpenVINO EP. Will be re-enabled after it's fixed + // TensorRT and OpenVINO dont support "axes" input in opset 13, re-enable after + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); + }; + + run_test(false); + run_test(true); // NNAPI EP will need axes as an initializer +} + +// Add 4d input shape test, since NNAPI supports up to 4d input shape +TEST(SqueezeOpTest, Squeeze_4d_NegAxis_axes_input) { + auto run_test = [](bool axes_is_initializer) { + OpTester test("Squeeze", 13); + test.AddInput("data", {1, 4, 1, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + + test.AddInput("axes", {2}, std::vector{0, -2}, axes_is_initializer); + test.AddOutput("squeezed", {4, 2}, + std::vector{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f}); + + // nGraph does not support neg axis. + // OpenVINO EP Incorrect precision. Will be re-enabled after its fixed + // TensorRT and OpenVINO dont support "axes" input in opset 13, re-enable after + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNGraphExecutionProvider, kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); + }; + + run_test(false); + run_test(true); // NNAPI EP will need axes as an initializer +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/sparse_initializer_handling.onnx b/onnxruntime/test/testdata/ort_minimal_test_models/sparse_initializer_handling.onnx similarity index 100% rename from onnxruntime/test/testdata/sparse_initializer_handling.onnx rename to onnxruntime/test/testdata/ort_minimal_test_models/sparse_initializer_handling.onnx diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh index 78ee61d16e..1b9ef4d11c 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -16,6 +16,7 @@ rm -rf /build/Debug # which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. mkdir -p /home/onnxruntimedev/.test_data/models_to_include cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_include +cp /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_test_models/*.onnx /home/onnxruntimedev/.test_data/models_to_include # Build a minimal build with included ops and models # then run ORT minimal UTs