saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-23 22:13:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[CoreML EP] Add Pad op support (#14946)

Browse source 
### Description
<!-- Describe your changes. -->

As title.

- Only support constant mode Pad for CoreML EP for now.
- Enable Pad tests for CoreML with inputs as initializer types.

CoreML Spec for reference:

https://apple.github.io/coremltools/mlmodel/Format/NeuralNetwork.html#paddinglayerparams


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

Fill operator gaps for ClipChamp models.

---------

Co-authored-by: rachguo <rachguo@rachguos-Mac-mini.local>
Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net>
This commit is contained in:
Rachel Guo 2023-03-22 21:54:55 -07:00 committed by GitHub
parent 7bec80d92a
commit fc3a2a3771
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 377 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

227
onnxruntime/core/providers/coreml/builders/impl/pad_op_builder.cc Normal file
View file

@ -0,0 +1,227 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/common.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/tensor_shape.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
#ifdef __APPLE__
#include "core/providers/coreml/builders/model_builder.h"
#endif
#include "core/providers/coreml/builders/op_builder_factory.h"
#include "base_op_builder.h"
namespace onnxruntime {
namespace coreml {
class PadOpBuilder : public BaseOpBuilder {
// Add operator related
#ifdef __APPLE__
public:
void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const override;
private:
Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node,
const logging::Logger& logger) const override;
#endif
// Operator support related
private:
bool IsOpSupportedImpl(const Node& node, const OpBuilderInputParams& input_params,
const logging::Logger& logger) const override;
int GetMinSupportedOpSet(const Node& /* node */) const override {
// Note: before Pad-11, inputs `pads` and `constant_value` were attributes
return 11;
}
};
// Helper function
// Use axes initializer data if `axes` input provided or create default axes vector.
static InlinedVector<int64_t> GetPaddingAxesData(const InitializedTensorSet& initializers,
const Node& node, int64_t input_rank) {
InlinedVector<int64_t> axes_tensor_data;
const auto& input_defs = node.InputDefs();
if (input_defs.size() > 3) {
// optional input axes is provided, use axes initializer data
const ONNX_NAMESPACE::TensorProto& axes_tensor = *initializers.at(input_defs[3]->Name());
Initializer axes_initializer(axes_tensor);
const auto axes_data_span = axes_initializer.DataAsSpan<int64_t>();
std::transform(
axes_data_span.begin(), axes_data_span.end(), std::back_inserter(axes_tensor_data),
[input_rank](int64_t axis) { return HandleNegativeAxis(axis, input_rank); });
} else {
// if not provided, make a default axes as [0, 1, ..., input_rank - 1]
InlinedVector<int64_t> default_axes(input_rank);
std::iota(std::begin(default_axes), std::end(default_axes), 0);
axes_tensor_data = std::move(default_axes);
}
return axes_tensor_data;
}
// Add operator related
#ifdef __APPLE__
void PadOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const {
model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // pads
model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // constant_value
if (node.InputDefs().size() > 3) {
model_builder.AddInitializerToSkip(node.InputDefs()[3]->Name()); // axes
}
}
Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
const Node& node,
const logging::Logger& logger) const {
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> layer = CreateNNLayer(model_builder, node);
auto* coreml_pad = layer->mutable_padding();
auto* constant_padding_type = coreml_pad->mutable_constant(); // CoreML::Specification::PaddingLayerParams_PaddingConstant
const auto& input_defs = node.InputDefs();
std::vector<int64_t> input_shape;
GetShape(*input_defs[0], input_shape, logger);
const auto input_rank = onnxruntime::narrow<int64_t>(input_shape.size());
const auto& pads_tensor = *model_builder.GetInitializerTensors().at(input_defs[1]->Name()); // pads
const auto& constant_value_tensor = *model_builder.GetInitializerTensors().at(input_defs[2]->Name()); // constant_value
Initializer constant_value_initializer(constant_value_tensor);
float constant_value = constant_value_initializer.DataAsSpan<float>()[0];
constant_padding_type->set_value(constant_value);
Initializer pads_initializer(pads_tensor);
auto pads_span = pads_initializer.DataAsSpan<int64_t>();
InlinedVector<int64_t> axes_tensor_data = GetPaddingAxesData(model_builder.GetInitializerTensors(), node, input_rank);
int64_t num_axes = axes_tensor_data.size();
// Add padding
auto* height_border = coreml_pad->mutable_paddingamounts()->add_borderamounts();
auto* width_border = coreml_pad->mutable_paddingamounts()->add_borderamounts();
for (int64_t i = 0; i < num_axes; i++) {
if (axes_tensor_data[i] == input_rank - 2) {
height_border->set_startedgesize(pads_span[i]);
height_border->set_endedgesize(pads_span[i + num_axes]);
}
if (axes_tensor_data[i] == input_rank - 1) {
width_border->set_startedgesize(pads_span[i]);
width_border->set_endedgesize(pads_span[i + num_axes]);
}
}
*layer->mutable_input()->Add() = input_defs[0]->Name();
*layer->mutable_output()->Add() = node.OutputDefs()[0]->Name();
model_builder.AddLayer(std::move(layer));
return Status::OK();
}
#endif
// Operator support related
bool PadOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputParams& input_params,
const logging::Logger& logger) const {
const auto& input_defs = node.InputDefs();
const auto& initializers = input_params.graph_viewer.GetAllInitializedTensors();
std::vector<int64_t> input_shape;
if (!GetShape(*input_defs[0], input_shape, logger))
return false;
if (input_shape.empty() || input_shape.size() < 2) {
LOGS(logger, VERBOSE) << "Pad requires input shape to be at least 2d, input is "
<< input_shape.size() << "d shape";
return false;
}
const TensorShape shape(input_shape);
if (shape.Size() == 0) {
LOGS(logger, VERBOSE) << "Cases that input data being empty due to a dimension with value of 0 is not supported";
return false;
}
{
NodeAttrHelper helper(node);
const auto mode = helper.Get("mode", "constant");
if (mode != "constant") {
LOGS(logger, VERBOSE) << "Only `constant` mode Pad is currently supported for now, mode: " << mode;
return false;
}
if (input_defs.size() < 3) {
LOGS(logger, VERBOSE) << "`constant_value` input is required for constant mode Pad op.";
return false;
}
// only support if `constant_value` input is a constant initializer
if (!Contains(initializers, input_defs[2]->Name())) {
LOGS(logger, VERBOSE) << "constant_value must be a constant initializer.";
return false;
}
}
{
// only support if `pads` input is known and does not contain negative values and only applies padding values
// for last two dimensions.
const auto pads_initializer_it = initializers.find(input_defs[1]->Name());
if (pads_initializer_it == initializers.end()) {
LOGS(logger, VERBOSE) << "pads must be a constant initializer.";
return false;
}
const ONNX_NAMESPACE::TensorProto& pads_initializer = *pads_initializer_it->second;
Initializer unpacked_tensor(pads_initializer);
auto pads_tensor_data = unpacked_tensor.DataAsSpan<int64_t>();
for (int64_t i = 0; i < unpacked_tensor.size(); i++) {
if (pads_tensor_data[i] < 0) {
LOGS(logger, VERBOSE) << "Negative pad value is not supported: pads["
<< i << "] = " << pads_tensor_data[i];
return false;
}
}
// Check if provided, `axes` input must be a constant initializer
if (input_defs.size() > 3) {
const auto axes_initializer_it = initializers.find(input_defs[3]->Name());
if (axes_initializer_it == initializers.end()) {
LOGS(logger, VERBOSE) << "if provided, `axes` input is required to a constant initializer";
return false;
}
}
// Check that only supports padding on last two dimensions - [H,W].
// CoreML PaddinglayerParams: https://apple.github.io/coremltools/mlmodel/Format/NeuralNetwork.html#paddinglayerparams
const auto input_rank = onnxruntime::narrow<int64_t>(input_shape.size());
InlinedVector<int64_t> axes_tensor_data = GetPaddingAxesData(initializers, node, input_rank);
int64_t num_axes = axes_tensor_data.size();
for (int64_t i = 0; i < num_axes; i++) {
if (axes_tensor_data[i] < input_rank - 2) {
if (pads_tensor_data[i] != 0 || pads_tensor_data[i + num_axes] != 0) {
// for axis specified that is not the last two dimension, padding is not supported. i.e.
// non-zero value appears in `pads` input for corresponding non-last two dimensions.
LOGS(logger, VERBOSE) << "CoreML only supports padding on last two dimensions.";
return false;
}
}
}
}
return true;
}
void CreatePadOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations) {
op_registrations.builders.push_back(std::make_unique<PadOpBuilder>());
op_registrations.op_builder_map.emplace(op_type, op_registrations.builders.back().get());
}
} // namespace coreml
} // namespace onnxruntime

4
onnxruntime/core/providers/coreml/builders/op_builder_factory.cc
View file

@ -90,6 +90,10 @@ static OpBuilderRegistrations CreateOpBuilderRegistrations() {
CreateLRNOpBuilder("LRN", op_registrations);
}
{ // Pad
CreatePadOpBuilder("Pad", op_registrations);
}
return op_registrations;
}

1
onnxruntime/core/providers/coreml/builders/op_builder_factory.h
View file

@ -34,6 +34,7 @@ void CreateArgMaxOpBuilder(const std::string& op_type, OpBuilderRegistrations& o
void CreateCastOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateFlattenOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateLRNOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreatePadOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
} // namespace coreml
} // namespace onnxruntime

162
onnxruntime/test/providers/cpu/tensor/pad_test.cc
View file

@ -75,11 +75,11 @@ static void RunAllOpsetAllDomainPadTests(
};
const std::vector<TestParams> all_test_params {
{false, false},
#if defined(USE_NNAPI) && defined(__ANDROID__)
// only enable when building NNAPI EP on Android
// test runs out of memory in QEMU aarch64 environment, so don't enable otherwise
// TODO try to enable when we move from QEMU to arm64 CI machines
{true, true},
#if (defined(USE_NNAPI) && defined(__ANDROID__)) || (defined(USE_COREML) && defined(__APPLE__))
// only enable when building NNAPI EP on Android or building CoreML EP for Apple environment
// test runs out of memory in QEMU aarch64 environment, so don't enable otherwise
// TODO try to enable when we move from QEMU to arm64 CI machines
{true, true},
#endif
};
for (const auto& test_params : all_test_params) {
@ -809,24 +809,152 @@ TEST(PadOpTest, ConstantPadAxes) {
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<int32_t>("data", {1, 2, 2, 2},
{
1, 1,
1, 1,
1, 1,
1, 1});
{1, 1,
1, 1,
1, 1,
1, 1});
test.AddInput<int64_t>("pads", {4}, {0, 1, 0, 1});
test.AddInput<int32_t>("value", {1}, {0});
test.AddInput<int32_t>("axes", {2}, {1, 3});
test.AddOutput<int32_t>("output", {1, 2, 2, 4},
{
0, 1, 1, 0,
0, 1, 1, 0,
0, 1, 1, 0,
0, 1, 1, 0
}
);
{0, 1, 1, 0,
0, 1, 1, 0,
0, 1, 1, 0,
0, 1, 1, 0});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
}
// CoreML EP only supports padding on last two dimensions and requires axes to be an initializer if provided,
// added the following test cases (can be taken by CoreML):
TEST(PadOpTest, ConstantPadAxesTest1) {
// Specified axes with last two dimensions and have non-zero padding values with one of them
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {4}, {0, 1, 0, 1}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {2}, {2, 3}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 2, 4},
{0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f});
// Note: exclude nnapi ep here, as int64_t type axes input is invalid for NNAPI. Similar for below tests.
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
TEST(PadOpTest, ConstantPadAxesTest2) {
// Specified axes with last two dimensions and have non-zero padding values on both of them
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {4}, {1, 1, 1, 1}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {2}, {2, 3}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 4, 4},
{0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
TEST(PadOpTest, ConstantPadAxesTest3) {
// Specified axes with 0's in pad values other than the last two dimensions
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {8}, {0, 0, 0, 1, 0, 0, 0, 1}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {4}, {0, 1, 2, 3}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 2, 4},
{0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
TEST(PadOpTest, ConstantPadAxesOutOfOrder) {
// Specified out of order axes values
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {4}, {1, 0, 1, 0}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {2}, {3, 2}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 2, 4},
{0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
TEST(PadOpTest, ConstantPadAxesWithOneDimensionSpecified) {
// Specified axes and non-zero padding values for only one of the last two dimensions
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {2}, {1, 1}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {1}, {3}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 2, 4},
{0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
/*
Note: Disable the Negative Axes test for ConstantPad for now until onnx shape inferencing
add support for handling negative axes.
Issue link to the bug: https://github.com/onnx/onnx/issues/5003
*/
TEST(PadOpTest, DISABLED_ConstantPadNegativeAxes) {
// Specified negative axes value
OpTester test("Pad", 18);
test.AddAttribute("mode", "constant");
test.AddInput<float>("data", {1, 2, 2, 2},
{1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f,
1.0f, 1.0f});
test.AddInput<int64_t>("pads", {2}, {1, 1}, true /* pads_is_initializer */);
test.AddInput<float>("value", {1}, {0.0f}, true /* value_is_initializer */);
test.AddInput<int64_t>("axes", {1}, {-1}, true /* axes_is_initializer */);
test.AddOutput<float>("output", {1, 2, 2, 4},
{0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f,
0.0f, 1.0f, 1.0f, 0.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNnapiExecutionProvider});
}
} // namespace test
} // namespace onnxruntime