From f27f5afd8a9e7099177826befb113640280e108a Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Fri, 9 Apr 2021 13:54:16 -0700 Subject: [PATCH] NCHWc: Support "sizes" argument for Resize transform (#7290) --- .../core/optimizer/nchwc_transformer.cc | 91 ++++++++++++++----- .../test/optimizer/nchwc_optimizer_test.cc | 61 +++++++++---- 2 files changed, 109 insertions(+), 43 deletions(-) diff --git a/onnxruntime/core/optimizer/nchwc_transformer.cc b/onnxruntime/core/optimizer/nchwc_transformer.cc index fe37e6579f..87d931576d 100644 --- a/onnxruntime/core/optimizer/nchwc_transformer.cc +++ b/onnxruntime/core/optimizer/nchwc_transformer.cc @@ -608,6 +608,7 @@ void NchwcTransformerImpl::TransformBinary(Node& node, bool add_node) { !utils::HasDimParam(input_n_dim) || (input_0_dim.dim_param() != input_n_dim.dim_param())) { all_shapes_match = false; + break; } } } @@ -951,13 +952,15 @@ void NchwcTransformerImpl::TransformResize(Node& node) { } } - NodeArg* scales_arg; + NodeArg* sizes_arg = nullptr; + NodeArg* scales_arg = nullptr; + if (node.SinceVersion() >= 11) { - // Bail out if Resize has the optional "sizes" tensor. - if (input_defs.size() == 3) { + if (input_defs.size() >= 4) { + sizes_arg = input_defs[3]; + } + if (sizes_arg == nullptr) { scales_arg = input_defs[2]; - } else { - return; } // Only support the asymmetric coordinate transformation mode. @@ -979,28 +982,68 @@ void NchwcTransformerImpl::TransformResize(Node& node) { scales_arg = input_defs[1]; } - // Require that the scales tensor be static. - const ONNX_NAMESPACE::TensorProto* scales_tensor_proto = nullptr; - if (!graph_utils::NodeArgIsConstant(graph_, *scales_arg) || - !graph_.GetInitializedTensor(scales_arg->Name(), scales_tensor_proto) || - (scales_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) || - (scales_tensor_proto->dims_size() != 1) || - (scales_tensor_proto->dims(0) != 4)) { - return; - } - - Initializer scales{*scales_tensor_proto, graph_.ModelPath()}; - auto* scales_data = scales.template data(); - - // Cast the scales to integers and verify that the scales are positive and - // round trip back to floating point. std::vector scales_attr(4); - for (size_t n = 0; n < 4; n++) { - int64_t scale_value = static_cast(scales_data[n]); - if (scale_value <= 0 || static_cast(scale_value) != scales_data[n]) { + + if (sizes_arg != nullptr) { + // Require that the sizes tensor be static. + const auto* sizes_tensor_proto = graph_utils::GetConstantInitializer(graph_, sizes_arg->Name()); + if ((sizes_tensor_proto == nullptr) || + (sizes_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT64) || + (sizes_tensor_proto->dims_size() != 1) || + (sizes_tensor_proto->dims(0) != 4)) { return; } - scales_attr[n] = scale_value; + + const auto* input_shape = input_defs[0]->Shape(); + if (input_shape == nullptr) { + return; + } + + Initializer sizes{*sizes_tensor_proto, graph_.ModelPath()}; + auto* sizes_data = sizes.template data(); + + // The sizes data can only be used if the input shape is static and the + // effective scaling must be an integer. + for (int i = 0; i < 4; i++) { + const auto& dim = input_shape->dim(i); + if (!utils::HasDimValue(dim)) { + return; + } + int64_t dim_value = dim.dim_value(); + if (dim_value <= 0) { + return; + } + scales_attr[i] = sizes_data[i] / dim_value; + if (scales_attr[i] * dim_value != sizes_data[i]) { + return; + } + } + + } else if (scales_arg != nullptr) { + // Require that the scales tensor be static. + const auto* scales_tensor_proto = graph_utils::GetConstantInitializer(graph_, scales_arg->Name()); + if ((scales_tensor_proto == nullptr) || + (scales_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) || + (scales_tensor_proto->dims_size() != 1) || + (scales_tensor_proto->dims(0) != 4)) { + return; + } + + Initializer scales{*scales_tensor_proto, graph_.ModelPath()}; + auto* scales_data = scales.template data(); + + // Cast the scales to integers and verify that the scales are positive and + // round trip back to floating point. + for (size_t n = 0; n < 4; n++) { + int64_t scale_value = static_cast(scales_data[n]); + if (scale_value <= 0 || static_cast(scale_value) != scales_data[n]) { + return; + } + scales_attr[n] = scale_value; + } + + } else { + return; } // Only support spatial scaling at this time (batch and channel are unscaled). diff --git a/onnxruntime/test/optimizer/nchwc_optimizer_test.cc b/onnxruntime/test/optimizer/nchwc_optimizer_test.cc index 6958735a15..d1b495e536 100644 --- a/onnxruntime/test/optimizer/nchwc_optimizer_test.cc +++ b/onnxruntime/test/optimizer/nchwc_optimizer_test.cc @@ -10,6 +10,7 @@ #include "test/test_environment.h" #include "test/framework/test_utils.h" #include "test/util/include/inference_session_wrapper.h" +#include #include "gtest/gtest.h" @@ -54,29 +55,29 @@ struct NchwcTestHelper { return &graph_.GetOrCreateNodeArg(name, nullptr); } - NodeArg* MakeInitializer(const std::vector& shape, const std::vector& data) { + template + NodeArg* MakeInitializer(const std::vector& shape, const std::vector& data) { std::string name = graph_.GenerateNodeArgName("constant"); ONNX_NAMESPACE::TensorProto tensor_proto; tensor_proto.set_name(name); - tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + tensor_proto.set_data_type(utils::ToTensorProtoElementType()); + tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T)); for (auto& dim : shape) { tensor_proto.add_dims(dim); } - tensor_proto.mutable_float_data()->Resize(static_cast(data.size()), 0.f); - std::copy_n(data.data(), data.size(), tensor_proto.mutable_float_data()->mutable_data()); - graph_.AddInitializedTensor(tensor_proto); return &graph_.GetOrCreateNodeArg(name, nullptr); } NodeArg* MakeInitializer(const std::vector& shape) { - return MakeInitializer(shape, FillRandomData(shape)); + return MakeInitializer(shape, FillRandomData(shape)); } - NodeArg* Make1DInitializer(const std::vector& data) { + template + NodeArg* Make1DInitializer(const std::vector& data) { return MakeInitializer({static_cast(data.size())}, data); } @@ -104,8 +105,8 @@ struct NchwcTestHelper { int opset_version = graph_.DomainToVersionMap().find(kOnnxDomain)->second; std::vector input_args{input_arg}; if (opset_version >= 11) { - input_args.push_back(Make1DInitializer({min})); - input_args.push_back(Make1DInitializer({max})); + input_args.push_back(Make1DInitializer({min})); + input_args.push_back(Make1DInitializer({max})); } auto& node = AddNode("Clip", input_args, {output_arg}); if (opset_version < 11) { @@ -1185,21 +1186,31 @@ TEST(NchwcOptimizerTests, ConvReorderOutputCnhw) { } TEST(NchwcOptimizerTests, Upsample) { - auto test_case = [&](int opset_version, float scale_h, float scale_w) { + auto test_case = [&](int opset_version, float scale_h, float scale_w, bool use_sizes_arg) { auto build_test_case = [&](NchwcTestHelper& helper) { auto* input_arg = helper.MakeInput({3, 16, 27, 15}); auto* conv_output_arg = helper.MakeIntermediate(); auto* output_arg = helper.MakeOutput(); - helper.AddConvNode(input_arg, conv_output_arg, {132, 16, 1, 1}); + helper.AddConvNode(input_arg, conv_output_arg, {42, 16, 1, 1}); std::string op_name = opset_version >= 10 ? "Resize" : "Upsample"; std::vector input_args; input_args.push_back(conv_output_arg); if (opset_version >= 11) { - input_args.push_back(helper.Make1DInitializer({0.f, 0.f, 0.f, 0.f, 1.f, 1.f, 1.f, 1.f})); + input_args.push_back(helper.Make1DInitializer({0.f, 0.f, 0.f, 0.f, 1.f, 1.f, 1.f, 1.f})); + } + if (use_sizes_arg) { + std::vector sizes_shape(4); + sizes_shape[0] = 3; + sizes_shape[1] = 42; + sizes_shape[2] = static_cast(scale_h * 27); + sizes_shape[3] = static_cast(scale_w * 15); + input_args.push_back(helper.Make1DInitializer({})); + input_args.push_back(helper.Make1DInitializer(sizes_shape)); + } else { + input_args.push_back(helper.Make1DInitializer({1.f, 1.f, scale_h, scale_w})); } - input_args.push_back(helper.Make1DInitializer({1.f, 1.f, scale_h, scale_w})); Node& resize_node = helper.AddNode(op_name, input_args, {output_arg}); if (opset_version >= 11) { resize_node.AddAttribute("coordinate_transformation_mode", "asymmetric"); @@ -1215,8 +1226,13 @@ TEST(NchwcOptimizerTests, Upsample) { EXPECT_EQ(op_to_count["com.microsoft.nchwc.Conv"], 1); EXPECT_EQ(op_to_count["com.microsoft.nchwc.ReorderInput"], 1); EXPECT_EQ(op_to_count["com.microsoft.nchwc.ReorderOutput"], 1); - EXPECT_EQ(op_to_count["com.microsoft.nchwc.Upsample"], 1); - EXPECT_EQ(op_to_count["Resize"] + op_to_count["Upsample"], 0); + if (scale_h == std::round(scale_h) && scale_w == std::round(scale_w)) { + EXPECT_EQ(op_to_count["com.microsoft.nchwc.Upsample"], 1); + EXPECT_EQ(op_to_count["Resize"] + op_to_count["Upsample"], 0); + } else { + EXPECT_EQ(op_to_count["com.microsoft.nchwc.Upsample"], 0); + EXPECT_EQ(op_to_count["Resize"] + op_to_count["Upsample"], 1); + } }; NchwcOptimizerTester(build_test_case, check_nchwc_graph, opset_version); @@ -1224,12 +1240,19 @@ TEST(NchwcOptimizerTests, Upsample) { // Verify that upsample nodes can be converted to the NCHWc format for // various versions of the operator. - static const int opset_versions[] = {9, 10, 11, 12}; + static const int opset_versions[] = {9, 10, 11, 13}; for (auto opset_version : opset_versions) { - test_case(opset_version, 1.f, 1.f); - test_case(opset_version, 2.f, 2.f); - test_case(opset_version, 3.f, 5.f); + test_case(opset_version, 1.f, 1.f, false); + test_case(opset_version, 2.f, 2.f, false); + test_case(opset_version, 3.f, 5.f, false); + if (opset_version >= 11) { + test_case(opset_version, 2.f, 2.f, true); + test_case(opset_version, 5.f, 3.f, true); + } } + // Verify that non-integral scales are not converted to the NCHWc format. + test_case(13, 2.2f, 2.8f, false); + test_case(13, 2.2f, 2.8f, true); } TEST(NchwcOptimizerTests, Activation) {