saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-24 19:43:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[CoreML/NNAPI EPs] Move direct use of initializer data to unpacked tensor data (#8780)

Browse source
This commit is contained in:
Guoyu Wang 2021-08-21 14:58:41 -07:00 committed by GitHub
parent 0c5a305742
commit 89656bb712
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 190 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc
View file

@ -3,13 +3,14 @@
#ifdef __APPLE__
#include <core/common/safeint.h>
#include <core/providers/common.h>
#include "core/providers/coreml/builders/impl/builder_utils.h"
#include "core/common/safeint.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/shared/utils/utils.h"
#include "builder_utils.h"
#include "coreml/NeuralNetwork.pb.h"
#include "core/providers/coreml/builders/helper.h"
namespace onnxruntime {
namespace coreml {
@ -93,9 +94,10 @@ common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight,
const ONNX_NAMESPACE::TensorProto& tensor) {
auto data_type = tensor.data_type();
if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
const float* data = GetTensorFloatData(tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor));
auto num_elements = SafeInt<size_t>(Product(tensor.dims()));
CreateCoreMLWeight(weight, data, num_elements);
CreateCoreMLWeight(weight, reinterpret_cast<const float*>(unpacked_tensor.data()), num_elements);
} else {
// TODO: support other type
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,

1
onnxruntime/core/providers/coreml/builders/impl/builder_utils.h
View file

@ -10,6 +10,7 @@
#include <unordered_map>
#include "core/common/status.h"
#include "core/graph/basic_types.h"
#include "core/providers/common.h"
namespace CoreML {
namespace Specification {

18
onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc
View file

@ -2,6 +2,7 @@
// Licensed under the MIT License.
#include <core/common/safeint.h>
#include <core/framework/tensorprotoutils.h>
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/coreml/builders/helper.h"
@ -49,19 +50,22 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod
// This is an internal function, requires input tensor to be 2d float tensor
// TODO, add support of other data types
static std::vector<float> GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor) {
const float* src_data = GetTensorFloatData(tensor);
static Status GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor,
std::vector<float>& transposed_data) {
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor));
const float* src_data = reinterpret_cast<const float*>(unpacked_tensor.data());
const auto& tensor_shape = tensor.dims();
auto x_t = SafeInt<size_t>(tensor_shape[0]);
auto y_t = SafeInt<size_t>(tensor_shape[1]);
std::vector<float> transposed_data(x_t * y_t);
transposed_data.resize(x_t * y_t);
for (size_t x = 0; x < x_t; x++) {
for (size_t y = 0; y < y_t; y++) {
transposed_data[y * x_t + x] = src_data[x * y_t + y];
}
}
return transposed_data;
return Status::OK();
}
Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node,
@ -82,7 +86,8 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
coreml_inner_product->set_inputchannels(b_shape[0]);
coreml_inner_product->set_outputchannels(b_shape[1]);
// Add weight (b of MatMul)
const auto b_transposed = GetTensorFloatDataTransposed(b_tensor);
std::vector<float> b_transposed;
ORT_RETURN_IF_ERROR(GetTensorFloatDataTransposed(b_tensor, b_transposed));
CreateCoreMLWeight(*coreml_inner_product->mutable_weights(), b_transposed.data(), b_transposed.size());
} else { // Gemm
NodeAttrHelper helper(node);
@ -90,7 +95,8 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
if (transB == 0) {
coreml_inner_product->set_inputchannels(b_shape[0]);
coreml_inner_product->set_outputchannels(b_shape[1]);
const auto b_transposed = GetTensorFloatDataTransposed(b_tensor);
std::vector<float> b_transposed;
ORT_RETURN_IF_ERROR(GetTensorFloatDataTransposed(b_tensor, b_transposed));
CreateCoreMLWeight(*coreml_inner_product->mutable_weights(), b_transposed.data(), b_transposed.size());
} else {
coreml_inner_product->set_inputchannels(b_shape[1]);

10
onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc
View file

@ -2,6 +2,7 @@
// Licensed under the MIT License.
#include "core/providers/common.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/cpu/tensor/reshape_helper.h"
#include "core/providers/shared/utils/utils.h"
@ -82,7 +83,14 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputP
}
const auto& perm_tensor = *initializers.at(perm_name);
const int64_t* raw_perm = GetTensorInt64Data(perm_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* raw_perm = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto& perm_dims = perm_tensor.dims();
if (perm_dims.empty() || perm_dims[0] == 0) {
LOGS(logger, VERBOSE) << "New shape of reshape cannot be empty";

40
onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc
View file

@ -4,10 +4,11 @@
#include <math.h>
#include "core/providers/common.h"
#include "core/providers/cpu/tensor/reshape_helper.h"
#include "core/providers/shared/utils/utils.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/cpu/tensor/reshape_helper.h"
#include "core/providers/shared/utils/utils.h"
#ifdef __APPLE__
#include "core/providers/coreml/builders/model_builder.h"
#endif
@ -40,7 +41,9 @@ class ResizeOpBuilder : public BaseOpBuilder {
};
// Helper functions
bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node, std::vector<float>& scales) {
bool GetResizeScales(const InitializedTensorSet& initializers,
const Node& node, std::vector<float>& scales,
const logging::Logger& logger) {
const auto& input_defs = node.InputDefs();
if (input_defs.size() < 3)
return false;
@ -49,12 +52,20 @@ bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node,
if (scales_tensor.dims_size() != 1 || scales_tensor.dims()[0] != 4)
return false;
const float* scales_data = GetTensorFloatData(scales_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage();
return false;
}
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
scales = std::vector<float>{scales_data, scales_data + 4};
return true;
}
bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node& node, std::vector<int64_t>& sizes) {
bool GetResizeOutputSizes(const InitializedTensorSet& initializers,
const Node& node, std::vector<int64_t>& sizes,
const logging::Logger& logger) {
const auto& input_defs = node.InputDefs();
if (input_defs.size() < 4)
return false;
@ -63,7 +74,13 @@ bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node&
if (sizes_tensor.dims_size() != 1 || sizes_tensor.dims()[0] != 4)
return false;
const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
sizes = std::vector<int64_t>{sizes_data, sizes_data + 4};
return true;
}
@ -106,14 +123,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
if (input_defs.size() == 3) { // use scales
std::vector<float> scales;
ORT_RETURN_IF_NOT(GetResizeScales(initializers, node, scales), "Error getting resize scales");
ORT_RETURN_IF_NOT(GetResizeScales(initializers, node, scales, logger), "Error getting resize scales");
coreml_upsample->add_scalingfactor(static_cast<int64_t>(scales[2]));
coreml_upsample->add_scalingfactor(static_cast<int64_t>(scales[3]));
} else { // we already checked number of inputs in IsOpSupportedImpl
std::vector<int64_t> input_shape;
ORT_RETURN_IF_NOT(GetShape(*input_defs[0], input_shape, logger), "Error getting input shape");
std::vector<int64_t> output_sizes;
ORT_RETURN_IF_NOT(GetResizeOutputSizes(initializers, node, output_sizes), "Error getting resize output_sizes");
ORT_RETURN_IF_NOT(GetResizeOutputSizes(initializers, node, output_sizes, logger),
"Error getting resize output_sizes");
coreml_upsample->add_scalingfactor(static_cast<int64_t>(output_sizes[2] / input_shape[2]));
coreml_upsample->add_scalingfactor(static_cast<int64_t>(output_sizes[3] / input_shape[3]));
}
@ -205,7 +223,7 @@ bool ResizeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPa
// We want to check if the scales or sizes are not trying to resize on N/C channels here
if (input_defs.size() == 3) { // we are using scales
std::vector<float> scales;
if (!GetResizeScales(initializers, node, scales))
if (!GetResizeScales(initializers, node, scales, logger))
return false;
float scale_n = scales[0];
@ -235,7 +253,7 @@ bool ResizeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPa
} else {
// we are using sizes
std::vector<int64_t> output_sizes;
if (!GetResizeOutputSizes(initializers, node, output_sizes))
if (!GetResizeOutputSizes(initializers, node, output_sizes, logger))
return false;
auto output_size_n = output_sizes[0];

14
onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc
View file

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <core/common/safeint.h>
#include "core/framework/tensorprotoutils.h"
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#ifdef __APPLE__
@ -40,15 +40,16 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
}
}
/* static */ std::vector<int64_t> GetAxes(ModelBuilder& model_builder, const Node& node) {
std::vector<int64_t> axes;
/* static */ Status GetAxes(ModelBuilder& model_builder, const Node& node, std::vector<int64_t>& axes) {
// Squeeze opset 13 use input as axes
if (node.SinceVersion() > 12) {
// If axes is not provided, return an empty axes as default to squeeze all
if (node.InputDefs().size() > 1) {
const auto& initializers(model_builder.GetInitializerTensors());
const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name());
const int64_t* raw_axes = GetTensorInt64Data(axes_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor));
const int64_t* raw_axes = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto size = SafeInt<size_t>(axes_tensor.dims()[0]);
axes.resize(size);
for (size_t i = 0; i < size; i++) {
@ -60,7 +61,7 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
axes = helper.Get("axes", std::vector<int64_t>());
}
return axes;
return Status::OK();
}
Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
@ -69,7 +70,8 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
std::unique_ptr<COREML_SPEC::NeuralNetworkLayer> layer = CreateNNLayer(model_builder, node);
auto* coreml_squeeze = layer->mutable_squeeze();
std::vector<int64_t> axes = GetAxes(model_builder, node);
std::vector<int64_t> axes;
ORT_RETURN_IF_ERROR(GetAxes(model_builder, node, axes));
if (axes.empty()) {
coreml_squeeze->set_squeezeall(true);
} else {

16
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc
View file

@ -304,9 +304,19 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co
return true;
}
float GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, size_t idx) {
const auto& scale_tensor = *initializers.at(node.InputDefs()[idx]->Name());
return GetTensorFloatData(scale_tensor)[0];
common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node,
size_t idx, float& scale) {
std::vector<uint8_t> unpacked_tensor;
const auto& name = node.InputDefs()[idx]->Name();
const auto& scale_tensor = *initializers.at(name);
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(scale_tensor, node.ModelPath(), unpacked_tensor));
// The scale should be one or more floats
ORT_RETURN_IF(unpacked_tensor.size() < 4, "The initializer [", name, "] should have one or more floats ",
"with size no less than 4, actual size: ", unpacked_tensor.size());
scale = reinterpret_cast<const float*>(unpacked_tensor.data())[0];
return Status::OK();
}
common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers,

3
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h
View file

@ -110,7 +110,8 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const
bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node,
const std::vector<size_t>& indices);
float GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, size_t idx);
common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node,
size_t idx, float& scale);
common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers,
const Node& node, size_t idx, int32_t& zero_point) ORT_MUST_USE_RESULT;

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -284,8 +284,6 @@ Status ModelBuilder::RegisterInitializers() {
std::vector<uint8_t> unpacked_tensor;
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
src = reinterpret_cast<const uint8_t*>(GetTensorFloatData(tensor));
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, graph_viewer_.ModelPath(), unpacked_tensor));

89
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

@ -301,8 +301,6 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder,
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
src = reinterpret_cast<const uint8_t*>(GetTensorFloatData(tensor));
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
ORT_RETURN_IF_ERROR(
@ -391,8 +389,6 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder,
std::vector<uint8_t> unpacked_tensor;
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
src = reinterpret_cast<const uint8_t*>(GetTensorFloatData(tensor));
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
ORT_RETURN_IF_ERROR(
@ -525,10 +521,9 @@ static Status GetBinaryOpQuantizationScaleAndZeroPoint(
float& a_scale, float& b_scale, float& y_scale,
int32_t& a_zero_point, int32_t& b_zero_point, int32_t& y_zero_point) {
const auto& initializers = model_builder.GetInitializerTensors();
a_scale = GetQuantizationScale(initializers, node, 1);
b_scale = GetQuantizationScale(initializers, node, 4);
y_scale = GetQuantizationScale(initializers, node, 6);
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, a_scale));
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 4, b_scale));
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 6, y_scale));
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, a_zero_point));
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 5, b_zero_point));
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 7, y_zero_point));
@ -591,10 +586,11 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint(
w_zero_point = 0;
// We need to copy the 1d scales array for per-channel quantization
const auto* scales = GetTensorFloatData(scale_tensor);
size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0];
vector<float> scales_vec(scales_size, 0.0f);
memcpy(scales_vec.data(), scales, sizeof(float) * scales_size);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor));
const float* scales = reinterpret_cast<const float*>(unpacked_tensor.data());
const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0];
vector<float> scales_vec(scales, scales + scales_size);
w_scales = onnxruntime::make_optional(std::move(scales_vec));
return Status::OK();
}
@ -699,7 +695,7 @@ Status GetQuantizedInputScaleAndZeroPoint(const InitializedTensorSet& initialize
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported op: ", op_type);
}
scale = GetQuantizationScale(initializers, node, scale_idx);
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, scale_idx, scale));
zero_point = 0;
if (node.InputDefs().size() > zero_point_idx) {
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, zero_point_idx, zero_point));
@ -1055,7 +1051,9 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
}
const auto& shape_tensor = *initializers.at(node.InputDefs()[1]->Name());
const int64_t* raw_shape = GetTensorInt64Data(shape_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor));
const int64_t* raw_shape = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto size = SafeInt<uint32_t>(shape_tensor.dims()[0]);
Shape input_shape = shaper[input];
@ -1111,10 +1109,21 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu
a.reserve(size);
b.reserve(size);
const float* scale_data = GetTensorFloatData(scale_tensor);
const float* bias_data = GetTensorFloatData(bias_tensor);
const float* mean_data = GetTensorFloatData(mean_tensor);
const float* var_data = GetTensorFloatData(var_tensor);
std::vector<uint8_t> unpacked_scale_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_scale_tensor));
const float* scale_data = reinterpret_cast<const float*>(unpacked_scale_tensor.data());
std::vector<uint8_t> unpacked_bias_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_bias_tensor));
const float* bias_data = reinterpret_cast<const float*>(unpacked_bias_tensor.data());
std::vector<uint8_t> unpacked_mean_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(mean_tensor, unpacked_mean_tensor));
const float* mean_data = reinterpret_cast<const float*>(unpacked_mean_tensor.data());
std::vector<uint8_t> unpacked_var_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(var_tensor, unpacked_var_tensor));
const float* var_data = reinterpret_cast<const float*>(unpacked_var_tensor.data());
for (int64_t i = 0; i < size; i++) {
a.push_back(scale_data[i] / sqrt(var_data[i] + eps));
@ -1279,14 +1288,15 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
int32_t y_zero_point = 0;
if (is_qlinear_average_pool) {
const auto& initializers = model_builder.GetInitializerTensors();
float x_scale = GetQuantizationScale(initializers, node, 1 /* idx */);
float x_scale = 0.0f;
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1 /* idx */, x_scale));
int32_t x_zero_point = 0;
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2 /* idx */, x_zero_point));
// Verify if the scale and zero point values from onnx input and nnapi input match
ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point));
y_scale = GetQuantizationScale(initializers, node, 3 /* idx */);
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 3 /* idx */, y_scale));
if (node.InputDefs().size() > 4)
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 4 /* idx */, y_zero_point));
}
@ -1506,9 +1516,11 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
for (auto dim : bias_tensor.dims())
bias_dimen.push_back(SafeInt<uint32_t>(dim));
const void* buffer = GetTensorInt32Data(bias_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor));
OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale);
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer, bias_operand_type));
ORT_RETURN_IF_ERROR(
model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type));
}
const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET"));
@ -1952,7 +1964,8 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
int32_t y_zero_point = 0;
if (is_qlinear_sigmoid) {
const auto& initializers = model_builder.GetInitializerTensors();
float x_scale = GetQuantizationScale(initializers, node, 1);
float x_scale = 0.0f;
ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, x_scale));
int32_t x_zero_point = 0;
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, x_zero_point));
@ -2074,7 +2087,7 @@ class SqueezeOpBuilder : public BaseOpBuilder {
private:
Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const override ORT_MUST_USE_RESULT;
static vector<int32_t> GetAxes(ModelBuilder& model_builder, const Node& node);
static Status GetAxes(ModelBuilder& model_builder, const Node& node, vector<int32_t>& axes);
};
void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const {
@ -2083,15 +2096,17 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
}
}
/* static */ vector<int32_t> SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder, const Node& node) {
vector<int32_t> axes;
/* static */ Status SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder,
const Node& node, vector<int32_t>& axes) {
// Squeeze opset 13 use input as axes
if (node.SinceVersion() > 12) {
// If axes is not supplied, return an empty axes as default to squeeze all
if (node.InputDefs().size() > 1) {
const auto& initializers(model_builder.GetInitializerTensors());
const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name());
const int64_t* raw_axes = GetTensorInt64Data(axes_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor));
const int64_t* raw_axes = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto size = SafeInt<uint32_t>(axes_tensor.dims()[0]);
axes.resize(size);
for (uint32_t i = 0; i < size; i++) {
@ -2104,7 +2119,7 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
axes = helper.Get("axes", vector<int32_t>());
}
return axes;
return Status::OK();
}
Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const {
@ -2114,7 +2129,9 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input));
}
return AddSqueezeOp(model_builder, node.Name(), input, node.OutputDefs()[0]->Name(), GetAxes(model_builder, node));
vector<int32_t> axes;
ORT_RETURN_IF_ERROR(GetAxes(model_builder, node, axes));
return AddSqueezeOp(model_builder, node.Name(), input, node.OutputDefs()[0]->Name(), axes);
}
#pragma endregion
@ -2147,7 +2164,8 @@ Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builde
const auto& output = node.OutputDefs()[0]->Name();
bool output_is_nhwc = model_builder.IsOperandNHWC(input);
float scale = GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1);
float scale = 0.0f;
ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale));
int32_t zero_point = 0;
Type output_type = Type::TENSOR_QUANT8_ASYMM;
@ -2194,7 +2212,8 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil
const auto& output = node.OutputDefs()[0]->Name();
bool output_is_nhwc = model_builder.IsOperandNHWC(input);
float scale = GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1);
float scale = 0.0;
ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale));
int32_t zero_point = 0;
if (input_defs.size() == 3) { // Get zero point
ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(model_builder.GetInitializerTensors(), node, 2, zero_point));
@ -2386,7 +2405,9 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
if (input_defs.size() == 3) { // we are using scales
const auto& scales_name = input_defs[2]->Name();
const auto& scales_tensor = *initializers.at(scales_name);
const float* scales_data = GetTensorFloatData(scales_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor));
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
float scale_h = scales_data[2];
float scale_w = scales_data[3];
ORT_RETURN_IF_ERROR(
@ -2394,7 +2415,9 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
} else { // we are using sizes
const auto& sizes_name = input_defs[3]->Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor));
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
ORT_RETURN_IF_ERROR(
shaper.ResizeUsingOutputSizes(input, SafeInt<uint32_t>(sizes_data[2]), SafeInt<uint32_t>(sizes_data[3]), use_nchw, output));
}

59
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc
View file

@ -3,7 +3,7 @@
#include <core/common/logging/logging.h>
#include <core/common/safeint.h>
#include <core/framework/tensorprotoutils.h>
#include <core/graph/graph.h>
#include "core/providers/common.h"
@ -392,7 +392,13 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init
}
const auto& perm_tensor = *initializers.at(perm_name);
const int64_t* raw_perm = GetTensorInt64Data(perm_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* raw_perm = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto perm_size = SafeInt<uint32_t>(perm_tensor.dims()[0]);
NodeAttrHelper helper(node);
@ -590,8 +596,24 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
}
// NNAPI requires Quantized Average Pool has same scale and zero point for both input and output
auto input_scale = GetQuantizationScale(initializers, node, 1);
auto output_scale = GetQuantizationScale(initializers, node, 3);
float input_scale = 0.0f;
auto status = GetQuantizationScale(initializers, node, 1, input_scale);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale for input_scale failed, message: "
<< status.ErrorMessage();
return false;
}
float output_scale = 0.0f;
status = GetQuantizationScale(initializers, node, 3, output_scale);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale for output_scale failed, message: "
<< status.ErrorMessage();
return false;
}
if (input_scale != output_scale) {
LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name
<< "] has different input_scale: " << input_scale
@ -601,7 +623,7 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
int32_t input_zp = 0;
int32_t output_zp = 0;
auto status = GetQuantizationZeroPoint(initializers, node, 2, input_zp);
status = GetQuantizationZeroPoint(initializers, node, 2, input_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationZeroPoint for input_zp failed, message: "
@ -1144,7 +1166,14 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const Node& node) const {
// NNAPI requires the scale be 1.f/256 and zero point to be 0
// See https://android.googlesource.com/platform/frameworks/ml/+/refs/heads/android10-c2f2-release/nn/common/operations/Activation.cpp#180
auto output_scale = GetQuantizationScale(initializers, node, 3);
float output_scale = 0.0f;
auto status = GetQuantizationScale(initializers, node, 3, output_scale);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale failed, message: " << status.ErrorMessage();
return false;
}
if (output_scale != 1.f / 256) {
LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name
<< "] output scale can only be 1.f/256, actual scale: " << output_scale;
@ -1153,7 +1182,7 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const Node& node) const {
int32_t output_zp;
if (has_output_zp) {
auto status = GetQuantizationZeroPoint(initializers, node, 4, output_zp);
status = GetQuantizationZeroPoint(initializers, node, 4, output_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationZeroPoint failed, message: " << status.ErrorMessage();
@ -1500,7 +1529,13 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
// We want to check if the scales or sizes are not trying to resize on N/C channels here
if (input_defs.size() == 3) { // we are using scales
const auto& scales_tensor = *initializers.at(input_defs[2]->Name());
const float* scales_data = GetTensorFloatData(scales_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage();
return false;
}
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
float scale_n = scales_data[0];
float scale_c = scales_data[1];
if (scale_n != 1.0f || scale_c != 1.0f) {
@ -1513,7 +1548,13 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
// we are using sizes
const auto& sizes_name = input_defs[3]->Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
uint32_t size_n = SafeInt<uint32_t>(sizes_data[0]);
uint32_t size_c = SafeInt<uint32_t>(sizes_data[1]);
if (size_n != input_shape[0] || size_c != input_shape[1]) {

20
onnxruntime/core/providers/shared/utils/utils.cc
View file

@ -11,22 +11,6 @@
namespace onnxruntime {
#define GET_TENSOR_DATA(FUNC_NAME, ELEMENT_TYPE, DATA) \
const ELEMENT_TYPE* GetTensor##FUNC_NAME(const ONNX_NAMESPACE::TensorProto& tensor) { \
bool has_external_data = tensor.has_data_location() && \
tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL; \
ORT_ENFORCE(!has_external_data, "tensor: ", tensor.name(), " has external data"); \
return tensor.DATA().empty() \
? reinterpret_cast<const ELEMENT_TYPE*>(tensor.raw_data().data()) \
: tensor.DATA().data(); \
}
GET_TENSOR_DATA(FloatData, float, float_data)
GET_TENSOR_DATA(Int32Data, int32_t, int32_data)
GET_TENSOR_DATA(Int64Data, int64_t, int64_data)
#undef GET_TENSOR_DATA
bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger) {
type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED;
const auto* type_proto = node_arg.TypeAsProto();
@ -69,7 +53,7 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(min_name), unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpack min tensor: " << status.ErrorMessage();
LOGS(logger, ERROR) << "Error while unpacking min tensor: " << status.ErrorMessage();
return false;
}
min = reinterpret_cast<float*>(unpacked_tensor.data())[0];
@ -84,7 +68,7 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(max_name), unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpack max tensor: " << status.ErrorMessage();
LOGS(logger, ERROR) << "Error while unpacking max tensor: " << status.ErrorMessage();
return false;
}
max = reinterpret_cast<float*>(unpacked_tensor.data())[0];

6
onnxruntime/core/providers/shared/utils/utils.h
View file

@ -18,12 +18,6 @@ class Logger;
class Node;
class NodeArg;
// Get initialize tensort float/int32/int64 data without unpacking
// NOTE!!! This will not work when the initializer has external data
const float* GetTensorFloatData(const ONNX_NAMESPACE::TensorProto& tensor);
const int32_t* GetTensorInt32Data(const ONNX_NAMESPACE::TensorProto& tensor);
const int64_t* GetTensorInt64Data(const ONNX_NAMESPACE::TensorProto& tensor);
// Get the min/max of a Clip operator.
// If min/max are not known initializer tensors, will return false
// For now we only support getting float min/max,