[CoreML EP] Support 1D Conv for coreml ep (#8398)

* initial conv 1d

* wip

* clean up and add comments

* refine

* elimnate some of the redundant copies of code

* update UT tests

* update with the new creatennlayer

* minor refine

* address cr comments

* refine

* refine

* address comments

* address comments

* fix

* fix

Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net>

onnxruntime/core/providers/coreml/builders/impl/conv_op_builder.cc

@@ -46,18 +46,59 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
   std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> layer = CreateNNLayer(model_builder, node);
 
   const auto& input_defs = node.InputDefs();
+  const auto& output_defs = node.OutputDefs();
+  const auto& input_name = input_defs[0]->Name();
+  const auto& output_name = output_defs[0]->Name();
 
   const auto& weight_tensor = *model_builder.GetInitializerTensors().at(input_defs[1]->Name());
-  const auto& weight_shape = weight_tensor.dims();
+  std::vector<int64_t> weight_shape = {weight_tensor.dims().cbegin(), weight_tensor.dims().cend()};
+
+  const bool is_1d_conv = (weight_shape.size() == 3);
+
+  if (is_1d_conv) {
+    // weight_shape needs to be expanded from MXCXH->MXCXHx1
+    weight_shape.push_back(1);
+  }
 
   NodeAttrHelper helper(node);
-  const auto strides = helper.Get("strides", std::vector<int64_t>{1, 1});
-  const auto onnx_pads = helper.Get("pads", std::vector<int64_t>{0, 0, 0, 0});
-  const auto dilations = helper.Get("dilations", std::vector<int64_t>{1, 1});
+  auto strides = helper.Get("strides", std::vector<int64_t>{1, 1});
+  auto dilations = helper.Get("dilations", std::vector<int64_t>{1, 1});
+  auto onnx_pads = helper.Get("pads", std::vector<int64_t>{0, 0, 0, 0});
+  // Strides/dilations for 1d conv is normally of length 1. Expand them by 1
+  // to meet the required length 2 (for 2d conv it's normally 2)
+  // Similarly 1d conv normally has a length 2 padding. Expand it to length 4 by adding additional zeros.
+  if (is_1d_conv) {
+    if (strides.size() < 2) {
+      ORT_RETURN_IF_NOT(strides.size() == 1, "strides size does not equal 1 for Conv 1d");
+      strides.push_back(1);
+    }
+    if (dilations.size() < 2) {
+      ORT_RETURN_IF_NOT(dilations.size() == 1, "dilations size does not equal 1 for Conv 1d");
+      dilations.push_back(1);
+    }
+    if (onnx_pads.size() < 4) {
+      ORT_RETURN_IF_NOT(onnx_pads.size() == 2, "onnx_pads size does not equal 2 for Conv 1d");
+      onnx_pads.insert(onnx_pads.begin() + 1, 0);
+      onnx_pads.push_back(0);
+    }
+  }
   const auto group = helper.Get("group", static_cast<int64_t>(1));
 
   auto* coreml_conv = layer->mutable_convolution();
 
+  std::string expand_output_name = model_builder.GetUniqueName(node.Name() + "_expandDims");
+
+  if (is_1d_conv) {
+    const auto expand_layer_name = model_builder.GetUniqueName(MakeString(node.Name(), "_Conv_expand"));
+    std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> expand_layer = CreateNNLayer(expand_layer_name);
+    // Add an expanddims layer here. CoreML only supports 2d convolution, so for 1d Conv case
+    // we need to add an additional dimension here to the input to make it "2d Conv" like.
+    // NxCxH -> NxCxHx1
+    expand_layer->mutable_expanddims()->add_axes(-1);
+    *expand_layer->mutable_input()->Add() = input_name;
+    *expand_layer->mutable_output()->Add() = expand_output_name;
+    model_builder.AddLayer(std::move(expand_layer));
+  }
   coreml_conv->set_outputchannels(weight_shape[0]);  // M
   coreml_conv->set_kernelchannels(weight_shape[1]);  // C/Group
   coreml_conv->add_kernelsize(weight_shape[2]);      // H
@@ -107,10 +148,26 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
     CreateCoreMLWeight(*coreml_conv->mutable_bias(), bias_tensor);
   }
 
-  *layer->mutable_input()->Add() = input_defs[0]->Name();
-  *layer->mutable_output()->Add() = node.OutputDefs()[0]->Name();
+  if (is_1d_conv) {
+    std::string conv_output_name = model_builder.GetUniqueName(node.Name() + "_conv_output");
+    *layer->mutable_input()->Add() = expand_output_name;
+    *layer->mutable_output()->Add() = conv_output_name;
+    model_builder.AddLayer(std::move(layer));
+
+    // Add a squeeze layer here. Since CoreML only supports 2d conv and we expanded the dimension by 1 before,
+    // we need to squeeze it back from NxCxHx1->NxCxH.
+    const auto squeeze_layer_name = model_builder.GetUniqueName(MakeString(node.Name(), "_Conv_squeeze"));
+    std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> squeeze_layer = CreateNNLayer(squeeze_layer_name);
+    squeeze_layer->mutable_squeeze()->add_axes(-1);
+    *squeeze_layer->mutable_input()->Add() = conv_output_name;
+    *squeeze_layer->mutable_output()->Add() = output_name;
+    model_builder.AddLayer(std::move(squeeze_layer));
+  } else {
+    *layer->mutable_input()->Add() = input_name;
+    *layer->mutable_output()->Add() = output_name;
+    model_builder.AddLayer(std::move(layer));
+  }
 
-  model_builder.AddLayer(std::move(layer));
   return Status::OK();
 }
 
@@ -125,9 +182,9 @@ bool ConvOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPara
   const auto& initializers = input_params.graph_viewer.GetAllInitializedTensors();
   if (Contains(initializers, weight_name)) {
     const auto& tensor = *initializers.at(weight_name);
-    if (tensor.dims().size() != 4) {
+    if (tensor.dims().size() != 4 && tensor.dims().size() != 3) {
       LOGS(logger, VERBOSE) << "Conv [" << name << "] dimension: " << tensor.dims().size()
-                            << " Only conv 2d is supported.";
+                            << " Only conv 2d and conv 1d are supported.";
       return false;
     }
   } else {

onnxruntime/test/providers/cpu/nn/conv_op_test.cc

@@ -87,6 +87,9 @@ TEST(ConvTest, Conv1D_1) {
                         -0.012766072526574135f, 0.07113571465015411f, 0.061429332941770554f};
 
   TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
+
+  // CoreML EP requires weight to be an initializer
+  TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
 }
 
 TEST(ConvTest, Conv1D_1_DefaultStridesAndDilations) {
@@ -110,6 +113,9 @@ TEST(ConvTest, Conv1D_1_DefaultStridesAndDilations) {
                         -0.012766072526574135f, 0.07113571465015411f, 0.061429332941770554f};
 
   TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
+  
+  // CoreML EP requires weight to be an initializer
+  TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
 }
 
 // Conv3
@@ -144,6 +150,9 @@ TEST(ConvTest, Conv1D_2) {
                         -0.18779152631759644f, -0.11083387583494186f};
 
   TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape);
+  
+  // CoreML EP requires weight to be an initializer
+  TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true);
 }
 
 // Conv1
@@ -176,6 +185,9 @@ TEST(ConvTest, Conv1D_Bias) {
   auto expected_vals = {0.37892162799835205f, 0.4625728130340576f, 0.4934738576412201f, 0.44801419973373413f,
                         0.37892162799835205f, 0.2499445676803589f, 0.31682088971138f, 0.32773756980895996f};
   TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape);
+
+  // CoreML EP requires weight to be an initializer
+  TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true);
 }
 
 // Conv47