saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[CoreML EP] add clip support (#7666)

Browse source
This commit is contained in:
Guoyu Wang 2021-05-13 10:41:30 -07:00 committed by GitHub
parent 46246f1bbd
commit 333318af04
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 222 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
onnxruntime/core/providers/coreml/builders/helper.cc
View file

@ -33,19 +33,6 @@ bool GetShape(const NodeArg& node_arg, std::vector<int64_t>& shape, const loggin
return true;
}
// TODO, move this to shared_library
bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger) {
type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED;
const auto* type_proto = node_arg.TypeAsProto();
if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) {
LOGS(logger, WARNING) << "NodeArg [" << node_arg.Name() << "] has no input type";
return false;
}
type = type_proto->tensor_type().elem_type();
return true;
}
bool IsNodeSupported(const Node& node, const GraphViewer& graph_viewer, const logging::Logger& logger) {
const auto& op_builders = GetOpBuilders();
if (Contains(op_builders, node.OpType())) {

3
onnxruntime/core/providers/coreml/builders/helper.h
View file

@ -19,9 +19,6 @@ namespace coreml {
bool GetShape(const NodeArg& node_arg, std::vector<int64_t>& shape, const logging::Logger& logger);
// TODO, move this to shared_library
bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger);
bool IsInputSupported(const NodeArg& node_arg, const std::string& parent_name, const logging::Logger& logger);
// Get a list of groups of supported nodes, each group represents a subgraph supported by CoreML EP

1
onnxruntime/core/providers/coreml/builders/impl/base_op_builder.cc
View file

@ -5,6 +5,7 @@
#include "core/providers/coreml/builders/model_builder.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/shared/utils/utils.h"
#include "base_op_builder.h"

133
onnxruntime/core/providers/coreml/builders/impl/clip_op_builder.cc Normal file
View file

@ -0,0 +1,133 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/coreml/builders/model_builder.h"
#include "core/providers/coreml/builders/op_builder_factory.h"
#include "core/providers/shared/utils/utils.h"
#include "base_op_builder.h"
namespace onnxruntime {
namespace coreml {
class ClipOpBuilder : public BaseOpBuilder {
// Add operator related
public:
void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const override;
private:
Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node,
const logging::Logger& logger) const override ORT_MUST_USE_RESULT;
// Operator support related
private:
bool IsOpSupportedImpl(const InitializedTensorSet& initializers, const Node& node,
const logging::Logger& logger) const override;
};
// Add operator related
void ClipOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const {
// Both min and max values will be injected into the layer, no need to add to the model
if (node.SinceVersion() >= 11) {
if (node.InputDefs().size() > 1)
model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name());
if (node.InputDefs().size() > 2)
model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name());
}
}
Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
const Node& node,
const logging::Logger& logger) const {
const auto& node_name = node.Name();
const auto& input_name = node.InputDefs()[0]->Name();
const auto& output_name = node.OutputDefs()[0]->Name();
float min, max;
ORT_RETURN_IF_NOT(GetClipMinMax(model_builder.GetInitializerTensors(), node, min, max, logger), "GetClipMinMax failed");
bool has_min = min != std::numeric_limits<float>::lowest();
bool has_max = max != std::numeric_limits<float>::max();
if (!has_min && !has_max) {
// Clip without min/max is an identity node
// In CoreML we don't have identity, use ActivationLinear instead
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> layer = CreateNNLayer(node);
layer->mutable_activation()->mutable_linear()->set_alpha(1.0f);
*layer->mutable_input()->Add() = input_name;
*layer->mutable_output()->Add() = output_name;
model_builder.AddLayer(std::move(layer));
} else {
// The implementation of clip(min, max) is done by
// 1. Clipping at min -> max(input, min) is handled by
// min_output = threshold(input, min)
// 2. Clipping at max -> min(min_output, max) is handled by
// output = -1 * (threshold(-min_output, -max))
// Now we have at least one or min or max is not default value
// Clipping at max will need take the output of clipping at min, or the node input, if min value is default
// If max value is default, the output of clipping at min will be the output of the node
std::string min_output_name = output_name;
if (has_max) {
min_output_name = has_min
? model_builder.GetUniqueName(node_name + "min_output")
: input_name;
}
// Handle clipping at min first
if (has_min) {
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> min_layer = CreateNNLayer(node);
if (min == 0.0f) { // If min is 0. then this min will be handled by relu
min_layer->mutable_activation()->mutable_relu();
} else { // otherwise, min will be handled by unary->threshold
min_layer->mutable_unary()->set_alpha(min);
min_layer->mutable_unary()->set_type(COREML_SPEC::UnaryFunctionLayerParams::THRESHOLD);
}
*min_layer->mutable_input()->Add() = input_name;
*min_layer->mutable_output()->Add() = min_output_name;
model_builder.AddLayer(std::move(min_layer));
}
// Clipping at max is handled by -1 * (threshold (-min_output, -max))
if (has_max) {
const auto threshold_output_name = model_builder.GetUniqueName(node_name + "threshold_output");
{ // Add threshold layer, which is actually max( -1 * min_output, -max)
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> threshold_layer = CreateNNLayer(node);
threshold_layer->mutable_unary()->set_alpha(-max);
threshold_layer->mutable_unary()->set_scale(-1.0f);
threshold_layer->mutable_unary()->set_type(COREML_SPEC::UnaryFunctionLayerParams::THRESHOLD);
*threshold_layer->mutable_input()->Add() = min_output_name;
*threshold_layer->mutable_output()->Add() = threshold_output_name;
model_builder.AddLayer(std::move(threshold_layer));
}
{ // Add linear activation layer -1 * threshold_output
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> linear_layer = CreateNNLayer(node);
linear_layer->mutable_activation()->mutable_linear()->set_alpha(-1.0f);
*linear_layer->mutable_input()->Add() = threshold_output_name;
*linear_layer->mutable_output()->Add() = output_name;
model_builder.AddLayer(std::move(linear_layer));
}
}
}
return Status::OK();
}
// Operator support related
bool ClipOpBuilder::IsOpSupportedImpl(const InitializedTensorSet& initializers, const Node& node,
const logging::Logger& logger) const {
float min, max;
return GetClipMinMax(initializers, node, min, max, logger);
}
void CreateClipOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations) {
op_registrations.builders.push_back(std::make_unique<ClipOpBuilder>());
op_registrations.op_builder_map.emplace(op_type, op_registrations.builders.back().get());
}
} // namespace coreml
} // namespace onnxruntime

4
onnxruntime/core/providers/coreml/builders/op_builder_factory.cc
View file

@ -61,6 +61,10 @@ static OpBuilderRegistrations CreateOpBuilderRegistrations() {
CreateGemmOpBuilder("MatMul", op_registrations);
}
{ // Clip
CreateClipOpBuilder("Clip", op_registrations);
}
return op_registrations;
}

1
onnxruntime/core/providers/coreml/builders/op_builder_factory.h
View file

@ -25,6 +25,7 @@ void CreateBatchNormalizationOpBuilder(const std::string& op_type, OpBuilderRegi
void CreateReshapeOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateResizeOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateConcatOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateClipOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreateActivationOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);
void CreatePoolOpBuilder(const std::string& op_type, OpBuilderRegistrations& op_registrations);

4
onnxruntime/core/providers/coreml/coreml_execution_provider.cc
View file

@ -173,9 +173,9 @@ CoreMLExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_vie
// If the graph is partitioned in multiple subgraphs, and this may impact performance,
// we want to give users a summary message at warning level.
if (num_of_partitions > 1) {
LOGS_DEFAULT(WARNING) << summary_msg;
LOGS(logger, WARNING) << summary_msg;
} else {
LOGS_DEFAULT(INFO) << summary_msg;
LOGS(logger, INFO) << summary_msg;
}
return result;

1
onnxruntime/core/providers/coreml/model/model.mm
View file

@ -219,7 +219,6 @@
[[error localizedDescription] cStringUsingEncoding:NSUTF8StringEncoding]);
}
// NSSet<NSString*>* output_feature_names = [output_feature featureNames];
for (auto& output : outputs) {
NSString* output_name = [NSString stringWithCString:output.first.c_str()
encoding:[NSString defaultCStringEncoding]];

30
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc
View file

@ -354,36 +354,6 @@ bool GetType(const NodeArg& node_arg, int32_t& type) {
return true;
}
bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, float& min, float& max) {
min = std::numeric_limits<float>::lowest();
max = std::numeric_limits<float>::max();
if (node.SinceVersion() < 11) { // Clip opset 1, 6 is using attributes for min/max
NodeAttrHelper helper(node);
min = helper.Get("min", std::numeric_limits<float>::lowest());
max = helper.Get("max", std::numeric_limits<float>::max());
} else {
if (node.InputDefs().size() > 1) { // we have input min
const auto& min_name = node.InputDefs()[1]->Name();
if (!Contains(initializers, min_name)) {
LOGS_DEFAULT(VERBOSE) << "Input min of Clip must be known";
return false;
}
min = GetTensorFloatData(*initializers.at(min_name))[0];
}
if (node.InputDefs().size() > 2) { // we have input max
const auto& max_name = node.InputDefs()[2]->Name();
if (!Contains(initializers, max_name)) {
LOGS_DEFAULT(VERBOSE) << "Input max of Clip must be known";
return false;
}
max = GetTensorFloatData(*initializers.at(max_name))[0];
}
}
return true;
}
void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) {
int32_t rank = static_cast<int>(input_shape.size());
NodeAttrHelper helper(node);

5
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h
View file

@ -116,13 +116,10 @@ common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers
const Node& node, size_t idx, int32_t& zero_point) ORT_MUST_USE_RESULT;
// Get Shape/Type of a NodeArg
// TODO, move to shared_utils
bool GetShape(const NodeArg& node_arg, Shape& shape);
bool GetType(const NodeArg& node_arg, int32_t& type);
// Get the min/max value from Clip op
// If the min/max are inputs be not initializers (value not preset), will return false
bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, float& min, float& max);
// Get the output shape of Flatten Op
void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2);

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -135,7 +135,7 @@ void ModelBuilder::PreprocessActivations() {
activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU);
} else if (op_type == "Clip") { // Relu1 or Relu6
float min, max;
if (!GetClipMinMax(GetInitializerTensors(), *node, min, max))
if (!GetClipMinMax(GetInitializerTensors(), *node, min, max, logging::LoggingManager::DefaultLogger()))
continue;
if (min == -1.0f && max == 1.0f) {

4
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

@ -2310,7 +2310,7 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
}
float min, max;
GetClipMinMax(model_builder.GetInitializerTensors(), node, min, max);
GetClipMinMax(model_builder.GetInitializerTensors(), node, min, max, logging::LoggingManager::DefaultLogger());
int32_t op_code;
if (min == 0.0f && max == 6.0f)
@ -2544,7 +2544,7 @@ Status MinMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
// This is for ops with dedicated OpBuilder
#define NNAPI_EP_ADD_SINGLE_OP_BUILDER(OP_TYPE, BUILDER_NAME) \
{ \
op_registrations.builders.push_back(std::make_unique<BUILDER_NAME>()); \
op_registrations.builders.push_back(std::make_unique<BUILDER_NAME>()); \
op_registrations.op_builder_map.emplace(OP_TYPE, op_registrations.builders.back().get()); \
}

4
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc
View file

@ -1374,7 +1374,7 @@ class ClipOpSupportChecker : public BaseOpSupportChecker {
bool ClipOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const Node& node,
const OpSupportCheckParams& /* params */) const {
float min, max;
if (!GetClipMinMax(initializers, node, min, max))
if (!GetClipMinMax(initializers, node, min, max, logging::LoggingManager::DefaultLogger()))
return false;
// We only supoort relu6 or relu1
@ -1636,7 +1636,7 @@ bool MinMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* in
// This is for ops with dedicated OpSupportChecker
#define NNAPI_EP_ADD_SINGLE_OP_SUPPORT_CHECKER(OP_TYPE, SUPPORT_CHECKER_NAME) \
{ \
op_registrations.support_checkers.push_back(std::make_unique<SUPPORT_CHECKER_NAME>()); \
op_registrations.support_checkers.push_back(std::make_unique<SUPPORT_CHECKER_NAME>()); \
op_registrations.op_support_checker_map.emplace(OP_TYPE, op_registrations.support_checkers.back().get()); \
}

55
onnxruntime/core/providers/shared/utils/utils.cc
View file

@ -27,6 +27,61 @@ GET_TENSOR_DATA(Int64Data, int64_t, int64_data)
#undef GET_TENSOR_DATA
bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger) {
type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED;
const auto* type_proto = node_arg.TypeAsProto();
if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) {
LOGS(logger, WARNING) << "NodeArg [" << node_arg.Name() << "] has no input type";
return false;
}
type = type_proto->tensor_type().elem_type();
return true;
}
bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
float& min, float& max, const logging::Logger& logger) {
const auto& node_name = node.Name();
int32_t input_type;
if (!GetType(*node.InputDefs()[0], input_type, logger))
return false;
if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
LOGS(logger, VERBOSE) << "GetClipMinMax() only support Clip node with float inputs for now. "
<< "The node [" << node_name << "] has input 0 type: " << input_type;
return false;
}
min = std::numeric_limits<float>::lowest();
max = std::numeric_limits<float>::max();
if (node.SinceVersion() < 11) { // Clip opset 1, 6 is using attributes for min/max
NodeAttrHelper helper(node);
min = helper.Get("min", std::numeric_limits<float>::lowest());
max = helper.Get("max", std::numeric_limits<float>::max());
} else {
if (node.InputDefs().size() > 1) { // we have input min
const auto& min_name = node.InputDefs()[1]->Name();
if (!Contains(initializers, min_name)) {
LOGS(logger, VERBOSE) << "Input min of Clip must be known";
return false;
}
min = GetTensorFloatData(*initializers.at(min_name))[0];
}
if (node.InputDefs().size() > 2) { // we have input max
const auto& max_name = node.InputDefs()[2]->Name();
if (!Contains(initializers, max_name)) {
LOGS(logger, VERBOSE) << "Input max of Clip must be known";
return false;
}
max = GetTensorFloatData(*initializers.at(max_name))[0];
}
}
return true;
}
NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node)
: node_attributes_(node.GetAttributes()) {}

16
onnxruntime/core/providers/shared/utils/utils.h
View file

@ -11,7 +11,12 @@
namespace onnxruntime {
namespace logging {
class Logger;
}
class Node;
class NodeArg;
// Get initialize tensort float/int32/int64 data without unpacking
// NOTE!!! This will not work when the initializer has external data
@ -19,6 +24,17 @@ const float* GetTensorFloatData(const ONNX_NAMESPACE::TensorProto& tensor);
const int32_t* GetTensorInt32Data(const ONNX_NAMESPACE::TensorProto& tensor);
const int64_t* GetTensorInt64Data(const ONNX_NAMESPACE::TensorProto& tensor);
// Get the min/max of a Clip operator.
// If min/max are not known initializer tensors, will return false
// For now we only support getting float min/max,
// since in most cases, Clip(0,6)[Relu6] will be fused by quantization tool
bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
float& min, float& max, const logging::Logger& logger);
// Get the type of the given NodeArg
// Will return false if the given NodeArg has no type
bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger);
/**
* Wrapping onnxruntime::Node for retrieving attribute values
*/

6
onnxruntime/test/providers/cpu/math/clip_test.cc
View file

@ -204,7 +204,8 @@ TEST(MathOpTest, ClipDimWithZero) {
test.AddOutput<float>("Y", dims, {});
// Tensorrt does not support Clip opset 11 yet.
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
// CoreML EP does not support empty inputs
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kCoreMLExecutionProvider});
OpTester test1("Clip"); //
test1.AddInput<float>("X", dims, {});
@ -212,7 +213,8 @@ TEST(MathOpTest, ClipDimWithZero) {
test1.AddAttribute("max", 10.0f);
test1.AddOutput<float>("Y", dims, {});
// TRT doesn't handle this
test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
// CoreML EP does not support empty inputs
test1.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kCoreMLExecutionProvider});
}
} // namespace test