diff --git a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc index 0e82adc45a..fb755b53f7 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc @@ -3,13 +3,14 @@ #ifdef __APPLE__ -#include -#include +#include "core/providers/coreml/builders/impl/builder_utils.h" + +#include "core/common/safeint.h" +#include "core/framework/tensorprotoutils.h" +#include "core/providers/coreml/builders/helper.h" #include "core/providers/shared/utils/utils.h" -#include "builder_utils.h" #include "coreml/NeuralNetwork.pb.h" -#include "core/providers/coreml/builders/helper.h" namespace onnxruntime { namespace coreml { @@ -93,9 +94,10 @@ common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight, const ONNX_NAMESPACE::TensorProto& tensor) { auto data_type = tensor.data_type(); if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - const float* data = GetTensorFloatData(tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); auto num_elements = SafeInt(Product(tensor.dims())); - CreateCoreMLWeight(weight, data, num_elements); + CreateCoreMLWeight(weight, reinterpret_cast(unpacked_tensor.data()), num_elements); } else { // TODO: support other type return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, diff --git a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.h b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.h index e34edc4a78..0f9ffa4fa0 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.h +++ b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.h @@ -10,6 +10,7 @@ #include #include "core/common/status.h" #include "core/graph/basic_types.h" +#include "core/providers/common.h" namespace CoreML { namespace Specification { diff --git a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc index 84b941270b..e23429c1b9 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/coreml/builders/helper.h" @@ -49,19 +50,22 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod // This is an internal function, requires input tensor to be 2d float tensor // TODO, add support of other data types -static std::vector GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor) { - const float* src_data = GetTensorFloatData(tensor); +static Status GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor, + std::vector& transposed_data) { + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); + const float* src_data = reinterpret_cast(unpacked_tensor.data()); const auto& tensor_shape = tensor.dims(); auto x_t = SafeInt(tensor_shape[0]); auto y_t = SafeInt(tensor_shape[1]); - std::vector transposed_data(x_t * y_t); + transposed_data.resize(x_t * y_t); for (size_t x = 0; x < x_t; x++) { for (size_t y = 0; y < y_t; y++) { transposed_data[y * x_t + x] = src_data[x * y_t + y]; } } - return transposed_data; + return Status::OK(); } Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node, @@ -82,7 +86,8 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N coreml_inner_product->set_inputchannels(b_shape[0]); coreml_inner_product->set_outputchannels(b_shape[1]); // Add weight (b of MatMul) - const auto b_transposed = GetTensorFloatDataTransposed(b_tensor); + std::vector b_transposed; + ORT_RETURN_IF_ERROR(GetTensorFloatDataTransposed(b_tensor, b_transposed)); CreateCoreMLWeight(*coreml_inner_product->mutable_weights(), b_transposed.data(), b_transposed.size()); } else { // Gemm NodeAttrHelper helper(node); @@ -90,7 +95,8 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (transB == 0) { coreml_inner_product->set_inputchannels(b_shape[0]); coreml_inner_product->set_outputchannels(b_shape[1]); - const auto b_transposed = GetTensorFloatDataTransposed(b_tensor); + std::vector b_transposed; + ORT_RETURN_IF_ERROR(GetTensorFloatDataTransposed(b_tensor, b_transposed)); CreateCoreMLWeight(*coreml_inner_product->mutable_weights(), b_transposed.data(), b_transposed.size()); } else { coreml_inner_product->set_inputchannels(b_shape[1]); diff --git a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc index 056d1f0574..7d1dded377 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "core/providers/common.h" +#include "core/framework/tensorprotoutils.h" #include "core/providers/cpu/tensor/reshape_helper.h" #include "core/providers/shared/utils/utils.h" @@ -82,7 +83,14 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputP } const auto& perm_tensor = *initializers.at(perm_name); - const int64_t* raw_perm = GetTensorInt64Data(perm_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS(logger, ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); + return false; + } + + const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); const auto& perm_dims = perm_tensor.dims(); if (perm_dims.empty() || perm_dims[0] == 0) { LOGS(logger, VERBOSE) << "New shape of reshape cannot be empty"; diff --git a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc index b6f39eabe0..eb52bbf7b2 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc @@ -4,10 +4,11 @@ #include #include "core/providers/common.h" -#include "core/providers/cpu/tensor/reshape_helper.h" - -#include "core/providers/shared/utils/utils.h" +#include "core/framework/tensorprotoutils.h" #include "core/providers/coreml/builders/helper.h" +#include "core/providers/cpu/tensor/reshape_helper.h" +#include "core/providers/shared/utils/utils.h" + #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" #endif @@ -40,7 +41,9 @@ class ResizeOpBuilder : public BaseOpBuilder { }; // Helper functions -bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node, std::vector& scales) { +bool GetResizeScales(const InitializedTensorSet& initializers, + const Node& node, std::vector& scales, + const logging::Logger& logger) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 3) return false; @@ -49,12 +52,20 @@ bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node, if (scales_tensor.dims_size() != 1 || scales_tensor.dims()[0] != 4) return false; - const float* scales_data = GetTensorFloatData(scales_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS(logger, ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); + return false; + } + const float* scales_data = reinterpret_cast(unpacked_tensor.data()); scales = std::vector{scales_data, scales_data + 4}; return true; } -bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node& node, std::vector& sizes) { +bool GetResizeOutputSizes(const InitializedTensorSet& initializers, + const Node& node, std::vector& sizes, + const logging::Logger& logger) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 4) return false; @@ -63,7 +74,13 @@ bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node& if (sizes_tensor.dims_size() != 1 || sizes_tensor.dims()[0] != 4) return false; - const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS(logger, ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); + return false; + } + const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); sizes = std::vector{sizes_data, sizes_data + 4}; return true; } @@ -106,14 +123,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, if (input_defs.size() == 3) { // use scales std::vector scales; - ORT_RETURN_IF_NOT(GetResizeScales(initializers, node, scales), "Error getting resize scales"); + ORT_RETURN_IF_NOT(GetResizeScales(initializers, node, scales, logger), "Error getting resize scales"); coreml_upsample->add_scalingfactor(static_cast(scales[2])); coreml_upsample->add_scalingfactor(static_cast(scales[3])); } else { // we already checked number of inputs in IsOpSupportedImpl std::vector input_shape; ORT_RETURN_IF_NOT(GetShape(*input_defs[0], input_shape, logger), "Error getting input shape"); std::vector output_sizes; - ORT_RETURN_IF_NOT(GetResizeOutputSizes(initializers, node, output_sizes), "Error getting resize output_sizes"); + ORT_RETURN_IF_NOT(GetResizeOutputSizes(initializers, node, output_sizes, logger), + "Error getting resize output_sizes"); coreml_upsample->add_scalingfactor(static_cast(output_sizes[2] / input_shape[2])); coreml_upsample->add_scalingfactor(static_cast(output_sizes[3] / input_shape[3])); } @@ -205,7 +223,7 @@ bool ResizeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPa // We want to check if the scales or sizes are not trying to resize on N/C channels here if (input_defs.size() == 3) { // we are using scales std::vector scales; - if (!GetResizeScales(initializers, node, scales)) + if (!GetResizeScales(initializers, node, scales, logger)) return false; float scale_n = scales[0]; @@ -235,7 +253,7 @@ bool ResizeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputPa } else { // we are using sizes std::vector output_sizes; - if (!GetResizeOutputSizes(initializers, node, output_sizes)) + if (!GetResizeOutputSizes(initializers, node, output_sizes, logger)) return false; auto output_size_n = output_sizes[0]; diff --git a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc index b907c3a8b5..2c5325daf9 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include - +#include "core/framework/tensorprotoutils.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" #ifdef __APPLE__ @@ -40,15 +40,16 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const } } -/* static */ std::vector GetAxes(ModelBuilder& model_builder, const Node& node) { - std::vector axes; +/* static */ Status GetAxes(ModelBuilder& model_builder, const Node& node, std::vector& axes) { // Squeeze opset 13 use input as axes if (node.SinceVersion() > 12) { // If axes is not provided, return an empty axes as default to squeeze all if (node.InputDefs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); - const int64_t* raw_axes = GetTensorInt64Data(axes_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); + const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); const auto size = SafeInt(axes_tensor.dims()[0]); axes.resize(size); for (size_t i = 0; i < size; i++) { @@ -60,7 +61,7 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const axes = helper.Get("axes", std::vector()); } - return axes; + return Status::OK(); } Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, @@ -69,7 +70,8 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, std::unique_ptr layer = CreateNNLayer(model_builder, node); auto* coreml_squeeze = layer->mutable_squeeze(); - std::vector axes = GetAxes(model_builder, node); + std::vector axes; + ORT_RETURN_IF_ERROR(GetAxes(model_builder, node, axes)); if (axes.empty()) { coreml_squeeze->set_squeezeall(true); } else { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index 1111be32bc..1108318835 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -304,9 +304,19 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co return true; } -float GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, size_t idx) { - const auto& scale_tensor = *initializers.at(node.InputDefs()[idx]->Name()); - return GetTensorFloatData(scale_tensor)[0]; +common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, + size_t idx, float& scale) { + std::vector unpacked_tensor; + const auto& name = node.InputDefs()[idx]->Name(); + const auto& scale_tensor = *initializers.at(name); + ORT_RETURN_IF_ERROR( + onnxruntime::utils::UnpackInitializerData(scale_tensor, node.ModelPath(), unpacked_tensor)); + + // The scale should be one or more floats + ORT_RETURN_IF(unpacked_tensor.size() < 4, "The initializer [", name, "] should have one or more floats ", + "with size no less than 4, actual size: ", unpacked_tensor.size()); + scale = reinterpret_cast(unpacked_tensor.data())[0]; + return Status::OK(); } common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers, diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index f73bf7f2d2..915460f091 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -110,7 +110,8 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node, const std::vector& indices); -float GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, size_t idx); +common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node, + size_t idx, float& scale); common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers, const Node& node, size_t idx, int32_t& zero_point) ORT_MUST_USE_RESULT; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 11fb49411f..8c7004d793 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -284,8 +284,6 @@ Status ModelBuilder::RegisterInitializers() { std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - src = reinterpret_cast(GetTensorFloatData(tensor)); - break; case ONNX_NAMESPACE::TensorProto_DataType_UINT8: ORT_RETURN_IF_ERROR( onnxruntime::utils::UnpackInitializerData(tensor, graph_viewer_.ModelPath(), unpacked_tensor)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 4424db590e..319ebe26c5 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -301,8 +301,6 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - src = reinterpret_cast(GetTensorFloatData(tensor)); - break; case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { ORT_RETURN_IF_ERROR( @@ -391,8 +389,6 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder, std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - src = reinterpret_cast(GetTensorFloatData(tensor)); - break; case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { ORT_RETURN_IF_ERROR( @@ -525,10 +521,9 @@ static Status GetBinaryOpQuantizationScaleAndZeroPoint( float& a_scale, float& b_scale, float& y_scale, int32_t& a_zero_point, int32_t& b_zero_point, int32_t& y_zero_point) { const auto& initializers = model_builder.GetInitializerTensors(); - a_scale = GetQuantizationScale(initializers, node, 1); - b_scale = GetQuantizationScale(initializers, node, 4); - y_scale = GetQuantizationScale(initializers, node, 6); - + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, a_scale)); + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 4, b_scale)); + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 6, y_scale)); ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, a_zero_point)); ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 5, b_zero_point)); ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 7, y_zero_point)); @@ -591,10 +586,11 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( w_zero_point = 0; // We need to copy the 1d scales array for per-channel quantization - const auto* scales = GetTensorFloatData(scale_tensor); - size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - vector scales_vec(scales_size, 0.0f); - memcpy(scales_vec.data(), scales, sizeof(float) * scales_size); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor)); + const float* scales = reinterpret_cast(unpacked_tensor.data()); + const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; + vector scales_vec(scales, scales + scales_size); w_scales = onnxruntime::make_optional(std::move(scales_vec)); return Status::OK(); } @@ -699,7 +695,7 @@ Status GetQuantizedInputScaleAndZeroPoint(const InitializedTensorSet& initialize return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported op: ", op_type); } - scale = GetQuantizationScale(initializers, node, scale_idx); + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, scale_idx, scale)); zero_point = 0; if (node.InputDefs().size() > zero_point_idx) { ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, zero_point_idx, zero_point)); @@ -1055,7 +1051,9 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons } const auto& shape_tensor = *initializers.at(node.InputDefs()[1]->Name()); - const int64_t* raw_shape = GetTensorInt64Data(shape_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor)); + const int64_t* raw_shape = reinterpret_cast(unpacked_tensor.data()); const auto size = SafeInt(shape_tensor.dims()[0]); Shape input_shape = shaper[input]; @@ -1111,10 +1109,21 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu a.reserve(size); b.reserve(size); - const float* scale_data = GetTensorFloatData(scale_tensor); - const float* bias_data = GetTensorFloatData(bias_tensor); - const float* mean_data = GetTensorFloatData(mean_tensor); - const float* var_data = GetTensorFloatData(var_tensor); + std::vector unpacked_scale_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_scale_tensor)); + const float* scale_data = reinterpret_cast(unpacked_scale_tensor.data()); + + std::vector unpacked_bias_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_bias_tensor)); + const float* bias_data = reinterpret_cast(unpacked_bias_tensor.data()); + + std::vector unpacked_mean_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(mean_tensor, unpacked_mean_tensor)); + const float* mean_data = reinterpret_cast(unpacked_mean_tensor.data()); + + std::vector unpacked_var_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(var_tensor, unpacked_var_tensor)); + const float* var_data = reinterpret_cast(unpacked_var_tensor.data()); for (int64_t i = 0; i < size; i++) { a.push_back(scale_data[i] / sqrt(var_data[i] + eps)); @@ -1279,14 +1288,15 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N int32_t y_zero_point = 0; if (is_qlinear_average_pool) { const auto& initializers = model_builder.GetInitializerTensors(); - float x_scale = GetQuantizationScale(initializers, node, 1 /* idx */); + float x_scale = 0.0f; + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1 /* idx */, x_scale)); int32_t x_zero_point = 0; ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2 /* idx */, x_zero_point)); // Verify if the scale and zero point values from onnx input and nnapi input match ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point)); - y_scale = GetQuantizationScale(initializers, node, 3 /* idx */); + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 3 /* idx */, y_scale)); if (node.InputDefs().size() > 4) ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 4 /* idx */, y_zero_point)); } @@ -1506,9 +1516,11 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N for (auto dim : bias_tensor.dims()) bias_dimen.push_back(SafeInt(dim)); - const void* buffer = GetTensorInt32Data(bias_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor)); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); - ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer, bias_operand_type)); + ORT_RETURN_IF_ERROR( + model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); } const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); @@ -1952,7 +1964,8 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int32_t y_zero_point = 0; if (is_qlinear_sigmoid) { const auto& initializers = model_builder.GetInitializerTensors(); - float x_scale = GetQuantizationScale(initializers, node, 1); + float x_scale = 0.0f; + ORT_RETURN_IF_ERROR(GetQuantizationScale(initializers, node, 1, x_scale)); int32_t x_zero_point = 0; ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(initializers, node, 2, x_zero_point)); @@ -2074,7 +2087,7 @@ class SqueezeOpBuilder : public BaseOpBuilder { private: Status AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const override ORT_MUST_USE_RESULT; - static vector GetAxes(ModelBuilder& model_builder, const Node& node); + static Status GetAxes(ModelBuilder& model_builder, const Node& node, vector& axes); }; void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) const { @@ -2083,15 +2096,17 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const } } -/* static */ vector SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder, const Node& node) { - vector axes; +/* static */ Status SqueezeOpBuilder::GetAxes(ModelBuilder& model_builder, + const Node& node, vector& axes) { // Squeeze opset 13 use input as axes if (node.SinceVersion() > 12) { // If axes is not supplied, return an empty axes as default to squeeze all if (node.InputDefs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); - const int64_t* raw_axes = GetTensorInt64Data(axes_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); + const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); const auto size = SafeInt(axes_tensor.dims()[0]); axes.resize(size); for (uint32_t i = 0; i < size; i++) { @@ -2104,7 +2119,7 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const axes = helper.Get("axes", vector()); } - return axes; + return Status::OK(); } Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) const { @@ -2114,7 +2129,9 @@ Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons ORT_RETURN_IF_ERROR(GetNCHWInput(model_builder, node, 0, input)); } - return AddSqueezeOp(model_builder, node.Name(), input, node.OutputDefs()[0]->Name(), GetAxes(model_builder, node)); + vector axes; + ORT_RETURN_IF_ERROR(GetAxes(model_builder, node, axes)); + return AddSqueezeOp(model_builder, node.Name(), input, node.OutputDefs()[0]->Name(), axes); } #pragma endregion @@ -2147,7 +2164,8 @@ Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builde const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); - float scale = GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1); + float scale = 0.0f; + ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale)); int32_t zero_point = 0; Type output_type = Type::TENSOR_QUANT8_ASYMM; @@ -2194,7 +2212,8 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); - float scale = GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1); + float scale = 0.0; + ORT_RETURN_IF_ERROR(GetQuantizationScale(model_builder.GetInitializerTensors(), node, 1, scale)); int32_t zero_point = 0; if (input_defs.size() == 3) { // Get zero point ORT_RETURN_IF_ERROR(GetQuantizationZeroPoint(model_builder.GetInitializerTensors(), node, 2, zero_point)); @@ -2386,7 +2405,9 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (input_defs.size() == 3) { // we are using scales const auto& scales_name = input_defs[2]->Name(); const auto& scales_tensor = *initializers.at(scales_name); - const float* scales_data = GetTensorFloatData(scales_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor)); + const float* scales_data = reinterpret_cast(unpacked_tensor.data()); float scale_h = scales_data[2]; float scale_w = scales_data[3]; ORT_RETURN_IF_ERROR( @@ -2394,7 +2415,9 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const } else { // we are using sizes const auto& sizes_name = input_defs[3]->Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor); + std::vector unpacked_tensor; + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor)); + const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); ORT_RETURN_IF_ERROR( shaper.ResizeUsingOutputSizes(input, SafeInt(sizes_data[2]), SafeInt(sizes_data[3]), use_nchw, output)); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index dd8f7b68e3..8e02e75f68 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -3,7 +3,7 @@ #include #include - +#include #include #include "core/providers/common.h" @@ -392,7 +392,13 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init } const auto& perm_tensor = *initializers.at(perm_name); - const int64_t* raw_perm = GetTensorInt64Data(perm_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); + return false; + } + const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); const auto perm_size = SafeInt(perm_tensor.dims()[0]); NodeAttrHelper helper(node); @@ -590,8 +596,24 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial } // NNAPI requires Quantized Average Pool has same scale and zero point for both input and output - auto input_scale = GetQuantizationScale(initializers, node, 1); - auto output_scale = GetQuantizationScale(initializers, node, 3); + float input_scale = 0.0f; + auto status = GetQuantizationScale(initializers, node, 1, input_scale); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name + << "] GetQuantizationScale for input_scale failed, message: " + << status.ErrorMessage(); + return false; + } + + float output_scale = 0.0f; + status = GetQuantizationScale(initializers, node, 3, output_scale); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name + << "] GetQuantizationScale for output_scale failed, message: " + << status.ErrorMessage(); + return false; + } + if (input_scale != output_scale) { LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name << "] has different input_scale: " << input_scale @@ -601,7 +623,7 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial int32_t input_zp = 0; int32_t output_zp = 0; - auto status = GetQuantizationZeroPoint(initializers, node, 2, input_zp); + status = GetQuantizationZeroPoint(initializers, node, 2, input_zp); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name << "] GetQuantizationZeroPoint for input_zp failed, message: " @@ -1144,7 +1166,14 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const Node& node) const { // NNAPI requires the scale be 1.f/256 and zero point to be 0 // See https://android.googlesource.com/platform/frameworks/ml/+/refs/heads/android10-c2f2-release/nn/common/operations/Activation.cpp#180 - auto output_scale = GetQuantizationScale(initializers, node, 3); + float output_scale = 0.0f; + auto status = GetQuantizationScale(initializers, node, 3, output_scale); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name + << "] GetQuantizationScale failed, message: " << status.ErrorMessage(); + return false; + } + if (output_scale != 1.f / 256) { LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name << "] output scale can only be 1.f/256, actual scale: " << output_scale; @@ -1153,7 +1182,7 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const Node& node) const { int32_t output_zp; if (has_output_zp) { - auto status = GetQuantizationZeroPoint(initializers, node, 4, output_zp); + status = GetQuantizationZeroPoint(initializers, node, 4, output_zp); if (!status.IsOK()) { LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name << "] GetQuantizationZeroPoint failed, message: " << status.ErrorMessage(); @@ -1500,7 +1529,13 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // We want to check if the scales or sizes are not trying to resize on N/C channels here if (input_defs.size() == 3) { // we are using scales const auto& scales_tensor = *initializers.at(input_defs[2]->Name()); - const float* scales_data = GetTensorFloatData(scales_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); + return false; + } + const float* scales_data = reinterpret_cast(unpacked_tensor.data()); float scale_n = scales_data[0]; float scale_c = scales_data[1]; if (scale_n != 1.0f || scale_c != 1.0f) { @@ -1513,7 +1548,13 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // we are using sizes const auto& sizes_name = input_defs[3]->Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - const int64_t* sizes_data = GetTensorInt64Data(sizes_tensor); + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); + return false; + } + const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); uint32_t size_n = SafeInt(sizes_data[0]); uint32_t size_c = SafeInt(sizes_data[1]); if (size_n != input_shape[0] || size_c != input_shape[1]) { diff --git a/onnxruntime/core/providers/shared/utils/utils.cc b/onnxruntime/core/providers/shared/utils/utils.cc index 81c053bb4b..6f38a8e368 100644 --- a/onnxruntime/core/providers/shared/utils/utils.cc +++ b/onnxruntime/core/providers/shared/utils/utils.cc @@ -11,22 +11,6 @@ namespace onnxruntime { -#define GET_TENSOR_DATA(FUNC_NAME, ELEMENT_TYPE, DATA) \ - const ELEMENT_TYPE* GetTensor##FUNC_NAME(const ONNX_NAMESPACE::TensorProto& tensor) { \ - bool has_external_data = tensor.has_data_location() && \ - tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL; \ - ORT_ENFORCE(!has_external_data, "tensor: ", tensor.name(), " has external data"); \ - return tensor.DATA().empty() \ - ? reinterpret_cast(tensor.raw_data().data()) \ - : tensor.DATA().data(); \ - } - -GET_TENSOR_DATA(FloatData, float, float_data) -GET_TENSOR_DATA(Int32Data, int32_t, int32_data) -GET_TENSOR_DATA(Int64Data, int64_t, int64_data) - -#undef GET_TENSOR_DATA - bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logger) { type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; const auto* type_proto = node_arg.TypeAsProto(); @@ -69,7 +53,7 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, std::vector unpacked_tensor; auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(min_name), unpacked_tensor); if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpack min tensor: " << status.ErrorMessage(); + LOGS(logger, ERROR) << "Error while unpacking min tensor: " << status.ErrorMessage(); return false; } min = reinterpret_cast(unpacked_tensor.data())[0]; @@ -84,7 +68,7 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, std::vector unpacked_tensor; auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(max_name), unpacked_tensor); if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpack max tensor: " << status.ErrorMessage(); + LOGS(logger, ERROR) << "Error while unpacking max tensor: " << status.ErrorMessage(); return false; } max = reinterpret_cast(unpacked_tensor.data())[0]; diff --git a/onnxruntime/core/providers/shared/utils/utils.h b/onnxruntime/core/providers/shared/utils/utils.h index 2462920182..925df731fc 100644 --- a/onnxruntime/core/providers/shared/utils/utils.h +++ b/onnxruntime/core/providers/shared/utils/utils.h @@ -18,12 +18,6 @@ class Logger; class Node; class NodeArg; -// Get initialize tensort float/int32/int64 data without unpacking -// NOTE!!! This will not work when the initializer has external data -const float* GetTensorFloatData(const ONNX_NAMESPACE::TensorProto& tensor); -const int32_t* GetTensorInt32Data(const ONNX_NAMESPACE::TensorProto& tensor); -const int64_t* GetTensorInt64Data(const ONNX_NAMESPACE::TensorProto& tensor); - // Get the min/max of a Clip operator. // If min/max are not known initializer tensors, will return false // For now we only support getting float min/max,