From bf54fe481e5b57a178329737ce41b0121591cb21 Mon Sep 17 00:00:00 2001 From: Rachel Guo <35738743+YUNQIUGUO@users.noreply.github.com> Date: Mon, 19 Jul 2021 17:26:22 -0700 Subject: [PATCH] [CoreML EP] Support 1D Conv for coreml ep (#8398) * initial conv 1d * wip * clean up and add comments * refine * elimnate some of the redundant copies of code * update UT tests * update with the new creatennlayer * minor refine * address cr comments * refine * refine * address comments * address comments * fix * fix Co-authored-by: rachguo --- .../coreml/builders/impl/conv_op_builder.cc | 75 ++++++++++++++++--- .../test/providers/cpu/nn/conv_op_test.cc | 12 +++ 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/onnxruntime/core/providers/coreml/builders/impl/conv_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/conv_op_builder.cc index 43fc93eda3..394acf1a9d 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/conv_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/conv_op_builder.cc @@ -46,18 +46,59 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N std::unique_ptr layer = CreateNNLayer(model_builder, node); const auto& input_defs = node.InputDefs(); + const auto& output_defs = node.OutputDefs(); + const auto& input_name = input_defs[0]->Name(); + const auto& output_name = output_defs[0]->Name(); const auto& weight_tensor = *model_builder.GetInitializerTensors().at(input_defs[1]->Name()); - const auto& weight_shape = weight_tensor.dims(); + std::vector weight_shape = {weight_tensor.dims().cbegin(), weight_tensor.dims().cend()}; + + const bool is_1d_conv = (weight_shape.size() == 3); + + if (is_1d_conv) { + // weight_shape needs to be expanded from MXCXH->MXCXHx1 + weight_shape.push_back(1); + } NodeAttrHelper helper(node); - const auto strides = helper.Get("strides", std::vector{1, 1}); - const auto onnx_pads = helper.Get("pads", std::vector{0, 0, 0, 0}); - const auto dilations = helper.Get("dilations", std::vector{1, 1}); + auto strides = helper.Get("strides", std::vector{1, 1}); + auto dilations = helper.Get("dilations", std::vector{1, 1}); + auto onnx_pads = helper.Get("pads", std::vector{0, 0, 0, 0}); + // Strides/dilations for 1d conv is normally of length 1. Expand them by 1 + // to meet the required length 2 (for 2d conv it's normally 2) + // Similarly 1d conv normally has a length 2 padding. Expand it to length 4 by adding additional zeros. + if (is_1d_conv) { + if (strides.size() < 2) { + ORT_RETURN_IF_NOT(strides.size() == 1, "strides size does not equal 1 for Conv 1d"); + strides.push_back(1); + } + if (dilations.size() < 2) { + ORT_RETURN_IF_NOT(dilations.size() == 1, "dilations size does not equal 1 for Conv 1d"); + dilations.push_back(1); + } + if (onnx_pads.size() < 4) { + ORT_RETURN_IF_NOT(onnx_pads.size() == 2, "onnx_pads size does not equal 2 for Conv 1d"); + onnx_pads.insert(onnx_pads.begin() + 1, 0); + onnx_pads.push_back(0); + } + } const auto group = helper.Get("group", static_cast(1)); auto* coreml_conv = layer->mutable_convolution(); + std::string expand_output_name = model_builder.GetUniqueName(node.Name() + "_expandDims"); + + if (is_1d_conv) { + const auto expand_layer_name = model_builder.GetUniqueName(MakeString(node.Name(), "_Conv_expand")); + std::unique_ptr expand_layer = CreateNNLayer(expand_layer_name); + // Add an expanddims layer here. CoreML only supports 2d convolution, so for 1d Conv case + // we need to add an additional dimension here to the input to make it "2d Conv" like. + // NxCxH -> NxCxHx1 + expand_layer->mutable_expanddims()->add_axes(-1); + *expand_layer->mutable_input()->Add() = input_name; + *expand_layer->mutable_output()->Add() = expand_output_name; + model_builder.AddLayer(std::move(expand_layer)); + } coreml_conv->set_outputchannels(weight_shape[0]); // M coreml_conv->set_kernelchannels(weight_shape[1]); // C/Group coreml_conv->add_kernelsize(weight_shape[2]); // H @@ -107,10 +148,26 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N CreateCoreMLWeight(*coreml_conv->mutable_bias(), bias_tensor); } - *layer->mutable_input()->Add() = input_defs[0]->Name(); - *layer->mutable_output()->Add() = node.OutputDefs()[0]->Name(); + if (is_1d_conv) { + std::string conv_output_name = model_builder.GetUniqueName(node.Name() + "_conv_output"); + *layer->mutable_input()->Add() = expand_output_name; + *layer->mutable_output()->Add() = conv_output_name; + model_builder.AddLayer(std::move(layer)); + + // Add a squeeze layer here. Since CoreML only supports 2d conv and we expanded the dimension by 1 before, + // we need to squeeze it back from NxCxHx1->NxCxH. + const auto squeeze_layer_name = model_builder.GetUniqueName(MakeString(node.Name(), "_Conv_squeeze")); + std::unique_ptr squeeze_layer = CreateNNLayer(squeeze_layer_name); + squeeze_layer->mutable_squeeze()->add_axes(-1); + *squeeze_layer->mutable_input()->Add() = conv_output_name; + *squeeze_layer->mutable_output()->Add() = output_name; + model_builder.AddLayer(std::move(squeeze_layer)); + } else { + *layer->mutable_input()->Add() = input_name; + *layer->mutable_output()->Add() = output_name; + model_builder.AddLayer(std::move(layer)); + } - model_builder.AddLayer(std::move(layer)); return Status::OK(); } @@ -125,9 +182,9 @@ bool ConvOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPara const auto& initializers = input_params.graph_viewer.GetAllInitializedTensors(); if (Contains(initializers, weight_name)) { const auto& tensor = *initializers.at(weight_name); - if (tensor.dims().size() != 4) { + if (tensor.dims().size() != 4 && tensor.dims().size() != 3) { LOGS(logger, VERBOSE) << "Conv [" << name << "] dimension: " << tensor.dims().size() - << " Only conv 2d is supported."; + << " Only conv 2d and conv 1d are supported."; return false; } } else { diff --git a/onnxruntime/test/providers/cpu/nn/conv_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc index 104705487a..890bccd4f1 100644 --- a/onnxruntime/test/providers/cpu/nn/conv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc @@ -87,6 +87,9 @@ TEST(ConvTest, Conv1D_1) { -0.012766072526574135f, 0.07113571465015411f, 0.061429332941770554f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // CoreML EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } TEST(ConvTest, Conv1D_1_DefaultStridesAndDilations) { @@ -110,6 +113,9 @@ TEST(ConvTest, Conv1D_1_DefaultStridesAndDilations) { -0.012766072526574135f, 0.07113571465015411f, 0.061429332941770554f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // CoreML EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } // Conv3 @@ -144,6 +150,9 @@ TEST(ConvTest, Conv1D_2) { -0.18779152631759644f, -0.11083387583494186f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // CoreML EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } // Conv1 @@ -176,6 +185,9 @@ TEST(ConvTest, Conv1D_Bias) { auto expected_vals = {0.37892162799835205f, 0.4625728130340576f, 0.4934738576412201f, 0.44801419973373413f, 0.37892162799835205f, 0.2499445676803589f, 0.31682088971138f, 0.32773756980895996f}; TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); + + // CoreML EP requires weight to be an initializer + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true); } // Conv47