saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[xnnpack] Have Initializer in Mobile related EPs in Minimal_build and creating EP specific dynamic-schema (#12555)

Browse source 
* Remove the dependence of Qlinearsoftmax schema

* refactor initializerview &&  create shared schema

* Dynamic Create EP specific schema

* Have Initializer in minimal_build

* address comments

* remove CancelFuseSubGraph
This commit is contained in:
Cheng 2022-09-06 14:32:15 +08:00 committed by GitHub
parent ac4f1bf960
commit 8cedafe250
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 250 additions and 363 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
include/onnxruntime/core/graph/graph.h
View file

@ -18,6 +18,7 @@
#if !defined(ORT_MINIMAL_BUILD)
#include "onnx/defs/schema.h"
#include "core/common/inlined_containers.h"
#else
#include "onnx/defs/data_type_utils.h"
#endif
@ -1092,13 +1093,6 @@ class Graph {
*/
Node& BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name);
/**
If we have BeginFuseSubGraph, but somehow hit errors, such as Compile of an EP failed on thesub_graph.
We can call CancelFuseSubGraph to undo the changes of BeginFuseSubGraph
@param fused_node The fused node and it's function body to be removed from the graph
*/
void CancelFuseSubGraph(const Node& fused_node);
void FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node);
#endif
@ -1605,6 +1599,9 @@ class Graph {
// for the fused kernel. I really don't like it. but for short-term solution, let's host
// those schemas here.
InlinedVector<std::unique_ptr<ONNX_NAMESPACE::OpSchema>> fused_schemas_containers_;
// in some case, a fused sub-graph will happens multiple times in one model, we use a map
// to store reusable-schema in lookup.
InlinedHashMap<std::string, std::reference_wrapper<ONNX_NAMESPACE::OpSchema>> reusable_fused_schema_map_;
#endif // !defined(ORT_MINIMAL_BUILD)
// Graph nodes.

12
include/onnxruntime/core/graph/indexed_sub_graph.h
View file

@ -52,9 +52,17 @@ struct IndexedSubGraph {
/** Nodes covered by this subgraph. The NodeIndex values are from the parent Graph.*/
std::vector<onnxruntime::NodeIndex> nodes;
// use an existing schema instead of generating one when fusing nodes using the MetaDef.
enum class SourceOfSchema : uint8_t {
CREATE, /// create new schema from info in IndexedSubGraph instance.
/// schema instance will not be re-usable.
REUSE_OR_CREATE, /// re-use existing dynamically created schema with matching domain+name.
/// create re-usable schema if one is not found.
EXISTING, /// use existing statically registered schema.
/// e.g. domain+name matches ONNX or contrib op domain+op_type+opset.
};
// Either using an existing schema or generating reusable one when fusing nodes using the MetaDef.
// MetaDef.domain + MetaDef.name => the domain.op_type that a schema must exist for with a valid since_version.
bool use_existing_schema{false};
SourceOfSchema schema_source{SourceOfSchema::CREATE};
/** Set the meta definition needed to represent this subgraph as a FunctionProto
It's needed IF AND ONLY IF there are multiple indexes contained in #nodes. */

5
onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc
View file

@ -80,12 +80,15 @@ void OpSet_Internal_NHWC_ONNX::ForEachSchema(const std::function<void(ONNX_NAMES
REGISTER_NHWC_SCHEMA_FROM_MSDOMAIN(fn, QLinearAveragePool, 1);
// TODO: Add other layout sensitive ops when needed. Those are:
// QLinearConv,
// BatchNormalization,
// AveragePool, GlobalAveragePool, GlobalMaxPool,
// LRN,
// GridSample
// DepthToSpace, SpaceToDepth
// not all schema are registered here. For part of layout insensitive ops
// we will use onnx schema directly, for others, like fused-node/qdq-group
// we may leverage internal schema or create on the fly.
}
} // namespace internal_nhwc_onnx

49
onnxruntime/core/graph/function_utils.cc
View file

@ -26,12 +26,13 @@ static int GetVersionForDomain(const std::string& domain, const M<std::string, i
return it->second;
}
std::unique_ptr<ONNX_NAMESPACE::OpSchema>
CreateSchema(const Graph& graph,
const IndexedSubGraph& nodes_to_fuse) {
std::unique_ptr<ONNX_NAMESPACE::OpSchema> CreateSchema(
const Graph& graph,
const IndexedSubGraph& nodes_to_fuse, bool allow_aggregated_tensor_type) {
const auto* meta_def = nodes_to_fuse.GetMetaDef();
auto op_schema = std::make_unique<ONNX_NAMESPACE::OpSchema>();
op_schema->SetName(meta_def->name);
using ONNX_NAMESPACE::OpSchema;
auto op_schema = std::make_unique<OpSchema>(meta_def->name, __FILE__, __LINE__);
op_schema->SetDomain(meta_def->domain);
op_schema->SetDoc(meta_def->doc_string);
op_schema->SinceVersion(meta_def->since_version);
@ -40,20 +41,34 @@ CreateSchema(const Graph& graph,
op_schema->TypeAndShapeInferenceFunction(meta_def->type_and_shape_inference_function);
}
int i = 0;
if (allow_aggregated_tensor_type) {
// The generated schema will use the same type constraint for all inputs and outputs,
// and that type constraint will match all tensor types.
// Due to this, a user of this style of schema must manually check whether any applicable type constraints
// for each input or output are satisfied prior to creating a node that uses this schema
//
op_schema->TypeConstraint("TAggregatedTypes", ONNX_NAMESPACE::OpSchema::all_tensor_types_with_bfloat(),
"all_tensor_types_with_bfloat");
}
for (auto& input : meta_def->inputs) {
auto input_arg = graph.GetNodeArg(input);
int i = 0;
for (const auto& input : meta_def->inputs) {
const auto *input_arg = graph.GetNodeArg(input);
// inputs must have a type. can be inferred for outputs.
ORT_ENFORCE(input_arg->Type() != nullptr);
op_schema->Input(i, input, "", *input_arg->Type());
++i;
op_schema->Input(i, input, "",
allow_aggregated_tensor_type ? "TAggregatedTypes" : *input_arg->Type(),
OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type);
i++;
}
i = 0;
for (auto& output : meta_def->outputs) {
auto output_arg = graph.GetNodeArg(output);
op_schema->Output(i, output, "", *output_arg->Type());
++i;
for (const auto& output : meta_def->outputs) {
const auto* output_arg = graph.GetNodeArg(output);
op_schema->Output(i, output, "",
allow_aggregated_tensor_type ? "TAggregatedTypes" : *output_arg->Type(),
OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type);
i++;
}
op_schema->Finalize();
@ -278,7 +293,7 @@ std::unique_ptr<ONNX_NAMESPACE::OpSchema> CreateSchema(const std::string& functi
}
class Inliner {
private:
private:
std::string prefix;
const onnxruntime::NodeAttributes& attr_map;
std::vector<InlinedHashMap<std::string, std::string>> rename_scopes;
@ -372,7 +387,7 @@ private:
if (attr.has_g()) {
transform(*attr.mutable_g());
}
for (auto& graph: *attr.mutable_graphs())
for (auto& graph : *attr.mutable_graphs())
transform(graph);
++attr_iter;
}
@ -392,7 +407,7 @@ private:
rename_scopes.pop_back();
}
public:
public:
// The main specialization method: specialize a FunctionProto for a particular call-site.
static void specialize(const NodeProto& callnode, FunctionProto& callee, const onnxruntime::NodeAttributes& attr_map, std::string unique_prefix) {
Inliner inliner(unique_prefix, attr_map);

16
onnxruntime/core/graph/function_utils.h
View file

@ -14,13 +14,17 @@
namespace onnxruntime {
namespace function_utils {
/** Create a OpSchema given a subgraph EP whant to fuse.
* This is used when EP return fusion in GetCapability implementation.
* @param graph The graph which host the subgraph.
* @param nodes_to_fuse The metadata for the subgraph that EP want to fuse.
*/
/** Create a OpSchema given a subgraph EP want to fuse.
* This is used when EP return fusion in GetCapability implementation.
* @param graph The graph which host the subgraph.
* @param nodes_to_fuse The metadata for the subgraph that EP want to fuse.
* @param allow_aggregated_tensor_type if true, it will use a type constraint called
* TAggregatedTypes for all inputs and outputs,
* and that this will match all tensor types in the all_tensor_types_with_bfloat list.
*/
std::unique_ptr<ONNX_NAMESPACE::OpSchema> CreateSchema(const Graph& graph,
const IndexedSubGraph& nodes_to_fuse);
const IndexedSubGraph& nodes_to_fuse,
bool allow_aggregated_tensor_type = false);
/** Create a OpSchema given from a local function in onnx model.
* @param function_domain The domain of the function.

50
onnxruntime/core/graph/graph.cc
View file

@ -10,6 +10,7 @@
#include <stack>
#include <queue>
#include "core/common/common.h"
#include "gsl/gsl"
#include "core/common/logging/logging.h"
#include "core/common/inlined_containers.h"
@ -168,6 +169,12 @@ static TypeProto TypeProtoFromTensorProto(const TensorProto& tensor) {
return t;
}
static std::string GenerateSchemaKey(const IndexedSubGraph& subgraph_ptr) {
return MakeString(subgraph_ptr.GetMetaDef()->domain, "_",
subgraph_ptr.GetMetaDef()->name, "_",
subgraph_ptr.GetMetaDef()->since_version);
}
#endif // !defined(ORT_MINIMAL_BUILD)
#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@ -3795,13 +3802,13 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std
std::unordered_map<std::string, int> output_indexes;
int cur_idx = 0;
for (auto& arg_name : func_meta_def->inputs) {
for (const auto& arg_name : func_meta_def->inputs) {
input_args.push_back(GetNodeArg(arg_name));
input_indexes[arg_name] = cur_idx++;
}
cur_idx = 0;
for (auto& arg_name : func_meta_def->outputs) {
for (const auto& arg_name : func_meta_def->outputs) {
output_args.push_back(GetNodeArg(arg_name));
output_indexes[arg_name] = cur_idx++;
}
@ -3820,12 +3827,20 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std
// kernel lookup works as per usual, if not using an existing schema.
// in an extended minimal build we do the lookup via a hash so don't need a schema.
fused_node.SetSinceVersion(func_meta_def->since_version);
if (sub_graph.use_existing_schema) {
if (sub_graph.schema_source == IndexedSubGraph::SourceOfSchema::EXISTING) {
ORT_ENFORCE(SetOpSchemaFromRegistryForNode(fused_node),
"Schema was not found for fused node. Domain:", fused_node.Domain(), " OpType:", fused_node.OpType());
} else if (IndexedSubGraph::SourceOfSchema::REUSE_OR_CREATE == sub_graph.schema_source) {
auto schema_key = GenerateSchemaKey(sub_graph);
if (!reusable_fused_schema_map_.contains(schema_key)) {
fused_schemas_containers_.push_back(
function_utils::CreateSchema(*this, sub_graph, /*allow_aggregated_tensor_type=*/true));
reusable_fused_schema_map_.emplace(schema_key, *fused_schemas_containers_.back());
}
fused_node.op_ = &(reusable_fused_schema_map_.at(schema_key).get());
} else {
auto temp_schema_ptr = function_utils::CreateSchema(*this, sub_graph);
fused_schemas_containers_.push_back(std::move(temp_schema_ptr));
fused_schemas_containers_.push_back(function_utils::CreateSchema(*this, sub_graph));
fused_node.op_ = fused_schemas_containers_.back().get();
}
#endif
@ -3838,31 +3853,6 @@ Node& Graph::BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::stri
return node;
}
void Graph::CancelFuseSubGraph(const Node& fused_node) {
auto node_idx = fused_node.Index();
if (!GetNode(node_idx))
return;
if (fused_node.NodeType() != Node::Type::Fused)
return;
#if !defined(ORT_MINIMAL_BUILD)
// Remove the tempoary schema from schema container container
auto* temp_schema_ptr = fused_node.Op();
auto it = std::find_if(
fused_schemas_containers_.begin(), fused_schemas_containers_.end(),
[temp_schema_ptr](const std::unique_ptr<ONNX_NAMESPACE::OpSchema>& schema) {
return schema.get() == temp_schema_ptr;
});
if (it != fused_schemas_containers_.end()) {
fused_schemas_containers_.erase(it);
}
#endif
// Remove the fused_node
RemoveNode(node_idx);
}
void Graph::FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node) {
const auto* func_meta_def = sub_graph.GetMetaDef();
ORT_ENFORCE(nullptr != func_meta_def);

3
onnxruntime/core/optimizer/initializer.cc
View file

@ -44,6 +44,7 @@ Initializer::Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const
data_ = std::move(w);
}
#if !defined(ORT_EXTENDED_MINIMAL_BUILD)
namespace {
template <typename T>
struct ToFp16;
@ -318,5 +319,5 @@ void Initializer::scale_by_axis(const Initializer& scalers, int axis) {
utils::MLTypeCallDispatcher<MLFloat16, BFloat16, float, double, int32_t, int64_t> t_disp(data_.GetElementType());
t_disp.Invoke<ScaleByAxis>(data_, scalers.data_, block_size, num_blocks);
}
#endif // ORT_EXTENDED_MINIMAL_BUILD
} // namespace onnxruntime

20
onnxruntime/core/optimizer/initializer.h
View file

@ -27,18 +27,18 @@ class Initializer final {
gsl::span<const int64_t> dims);
Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto,
const Path& model_path);
const Path& model_path = {});
~Initializer() = default;
void ToProto(ONNX_NAMESPACE::TensorProto& tensor_proto) const {
tensor_proto = utils::TensorToTensorProto(data_, name_);
}
#if !defined(ORT_EXTENDED_MINIMAL_BUILD)
ONNX_NAMESPACE::TensorProto ToFP16(const std::string& name) const;
ONNX_NAMESPACE::TensorProto ToBFloat16(const std::string& name) const;
#endif // ORT_EXTENDED_MINIMAL_BUILD
int data_type() const {
return data_.GetElementType();
}
@ -57,8 +57,13 @@ class Initializer final {
return data_.Data<T>();
}
const int8_t* raw_data() const {
return reinterpret_cast<const int8_t*>(data_.DataRaw());
template <typename T>
auto DataAsSpan() const {
return data_.DataAsSpan<T>();
}
gsl::span<const uint8_t> DataAsByteSpan() const {
return gsl::make_span(reinterpret_cast<const uint8_t*>(data_.DataRaw()), data_.SizeInBytes());
}
gsl::span<const int64_t> dims() const {
@ -67,12 +72,13 @@ class Initializer final {
int64_t size() const { return data_.Shape().Size(); }
#if !defined(ORT_EXTENDED_MINIMAL_BUILD)
Initializer& add(float value);
Initializer& add(const Initializer& other);
Initializer& sub(const Initializer& other);
Initializer& mul(const Initializer& other);
Initializer& div(const Initializer& other);
@ -80,7 +86,7 @@ class Initializer final {
Initializer& sqrt();
void scale_by_axis(const Initializer& other, int axis);
#endif // ORT_EXTENDED_MINIMAL_BUILD
private:
std::string name_;

2
onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc
View file

@ -54,7 +54,7 @@ bool IsQDQPairSupported(
Initializer dq_scale(*dq_scale_tensor_proto, model_path);
return q_zp.data_type() == dq_zp.data_type() &&
*q_zp.raw_data() == *dq_zp.raw_data() &&
q_zp.DataAsByteSpan() == dq_zp.DataAsByteSpan() &&
*q_scale.data<float>() == *dq_scale.data<float>();
}

7
onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc
View file

@ -10,6 +10,7 @@
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/coreml/builders/impl/base_op_builder.h"
#include "core/providers/coreml/builders/op_builder_factory.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
namespace coreml {
@ -67,10 +68,8 @@ Status AddPReluWeight(ModelBuilder& model_builder, const Node& node,
// assume X has 3 or 4 dimensions, that was checked in IsPReluOpSupported()
const auto num_channels = x_shape[x_shape.size() - 3];
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(slope_tensor, unpacked_tensor));
float value;
std::memcpy(&value, unpacked_tensor.data(), sizeof(value));
Initializer unpacked_tensor(slope_tensor);
float value = unpacked_tensor.DataAsSpan<float>()[0];
auto& weight_values = *prelu.mutable_alpha()->mutable_floatvalue();
weight_values.Clear();

6
onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc
View file

@ -9,6 +9,7 @@
#include "core/framework/tensorprotoutils.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
#include "coreml/NeuralNetwork.pb.h"
@ -94,10 +95,9 @@ common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight,
const ONNX_NAMESPACE::TensorProto& tensor) {
auto data_type = tensor.data_type();
if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor));
Initializer unpacked_tensor(tensor);
auto num_elements = SafeInt<size_t>(Product(tensor.dims()));
CreateCoreMLWeight(weight, reinterpret_cast<const float*>(unpacked_tensor.data()), num_elements);
CreateCoreMLWeight(weight, unpacked_tensor.data<float>(), num_elements);
} else {
// TODO: support other type
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,

7
onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc
View file

@ -7,6 +7,8 @@
#include "core/providers/shared/utils/utils.h"
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/coreml/builders/op_builder_factory.h"
#include "core/optimizer/initializer.h"
#ifdef __APPLE__
#include "core/providers/coreml/builders/model_builder.h"
#include "builder_utils.h"
@ -52,9 +54,8 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod
// TODO, add support of other data types
static Status GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor,
std::vector<float>& transposed_data) {
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor));
const float* src_data = reinterpret_cast<const float*>(unpacked_tensor.data());
Initializer unpacked_tensor(tensor);
auto src_data = unpacked_tensor.DataAsSpan<float>();
const auto& tensor_shape = tensor.dims();
auto x_t = SafeInt<size_t>(tensor_shape[0]);
auto y_t = SafeInt<size_t>(tensor_shape[1]);

11
onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc
View file

@ -4,6 +4,7 @@
#include "core/providers/common.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/cpu/tensor/reshape_helper.h"
#include "core/optimizer/initializer.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/coreml/builders/helper.h"
@ -83,14 +84,8 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputP
}
const auto& perm_tensor = *initializers.at(perm_name);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* raw_perm = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(perm_tensor);
auto raw_perm = unpacked_tensor.DataAsSpan<int64_t>();
const auto& perm_dims = perm_tensor.dims();
if (perm_dims.empty() || perm_dims[0] == 0) {
LOGS(logger, VERBOSE) << "New shape of reshape cannot be empty";

29
onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc
View file

@ -8,6 +8,7 @@
#include "core/providers/coreml/builders/helper.h"
#include "core/providers/cpu/tensor/reshape_helper.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
#ifdef __APPLE__
#include "core/providers/coreml/builders/model_builder.h"
@ -43,7 +44,7 @@ class ResizeOpBuilder : public BaseOpBuilder {
// Helper functions
bool GetResizeScales(const InitializedTensorSet& initializers,
const Node& node, std::vector<float>& scales,
const logging::Logger& logger) {
const logging::Logger&) {
const auto& input_defs = node.InputDefs();
if (input_defs.size() < 3)
return false;
@ -51,21 +52,15 @@ bool GetResizeScales(const InitializedTensorSet& initializers,
const auto& scales_tensor = *initializers.at(input_defs[2]->Name());
if (scales_tensor.dims_size() != 1 || scales_tensor.dims()[0] != 4)
return false;
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage();
return false;
}
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
scales = std::vector<float>{scales_data, scales_data + 4};
Initializer unpacked_tensor(scales_tensor);
auto scales_data = unpacked_tensor.DataAsSpan<float>();
scales = std::vector<float>{scales_data.begin(), scales_data.end()};
return true;
}
bool GetResizeOutputSizes(const InitializedTensorSet& initializers,
const Node& node, std::vector<int64_t>& sizes,
const logging::Logger& logger) {
const logging::Logger&) {
const auto& input_defs = node.InputDefs();
if (input_defs.size() < 4)
return false;
@ -73,15 +68,9 @@ bool GetResizeOutputSizes(const InitializedTensorSet& initializers,
const auto& sizes_tensor = *initializers.at(input_defs[3]->Name());
if (sizes_tensor.dims_size() != 1 || sizes_tensor.dims()[0] != 4)
return false;
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
sizes = std::vector<int64_t>{sizes_data, sizes_data + 4};
Initializer unpacked_tensor(sizes_tensor);
auto sizes_data = unpacked_tensor.DataAsSpan<int64_t>();
sizes = std::vector<int64_t>{sizes_data.begin(), sizes_data.end()};
return true;
}

13
onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc
View file

@ -4,6 +4,8 @@
#include "core/framework/tensorprotoutils.h"
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
#ifdef __APPLE__
#include "core/providers/coreml/builders/model_builder.h"
#endif
@ -47,14 +49,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
if (node.InputDefs().size() > 1) {
const auto& initializers(model_builder.GetInitializerTensors());
const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name());
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor));
const int64_t* raw_axes = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(axes_tensor);
auto raw_axes = unpacked_tensor.DataAsSpan<int64_t>();
const auto size = SafeInt<size_t>(axes_tensor.dims()[0]);
axes.resize(size);
for (size_t i = 0; i < size; i++) {
axes[i] = raw_axes[i];
}
axes.reserve(size);
axes.insert(axes.end(), raw_axes.begin(), raw_axes.end());
}
} else {
NodeAttrHelper helper(node);

23
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc
View file

@ -18,6 +18,7 @@
#include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
namespace nnapi {
@ -193,33 +194,19 @@ common::Status GetQuantizationScaleAndZeroPoint(
"NodeArg: ", io_def.node_arg.Name(), " is not quantized");
}
const auto unpack_tensor = [&model_path](const InitializedTensorSet& initializers,
const std::string& name, std::vector<uint8_t>& unpacked_tensor) {
const auto& tensor = *initializers.at(name);
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, model_path, unpacked_tensor));
return Status::OK();
};
const auto& quant_param = *io_def.quant_param;
{ // get the scale
std::vector<uint8_t> unpacked_tensor;
const auto& name = quant_param.scale.Name();
ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor));
Initializer unpacked_tensor(*initializers.at(name), model_path);
// The scale should be one or more floats
ORT_RETURN_IF(unpacked_tensor.size() < 4,
"The initializer [", name, "] should have one or more floats ",
"with size no less than 4, actual size: ", unpacked_tensor.size());
scale = reinterpret_cast<const float*>(unpacked_tensor.data())[0];
scale = unpacked_tensor.DataAsSpan<float>()[0];
}
if (quant_param.zero_point) { // get the zero point if it's there
std::vector<uint8_t> unpacked_tensor;
const auto& name = quant_param.zero_point->Name();
ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor));
ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty");
Initializer unpacked_tensor(*initializers.at(name), model_path);
// Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI
zero_point = static_cast<int32_t>(unpacked_tensor[0]);
zero_point = static_cast<int32_t>(unpacked_tensor.DataAsByteSpan()[0]);
}
return Status::OK();

28
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -13,6 +13,7 @@
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "core/optimizer/initializer.h"
#include "helper.h"
#include "op_builder.h"
@ -285,26 +286,15 @@ Status ModelBuilder::RegisterInitializers() {
if (Contains(skipped_initializers_, tensor.name()))
continue;
uint32_t index;
size_t size, padded_size;
std::tie(index, size, padded_size) = initializers[i++];
auto [index, size, padded_size] = initializers[i++];
const uint8_t* src = nullptr;
// uint8_t data need unpack, need a holder for free memory after copy
std::vector<uint8_t> unpacked_tensor;
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, graph_viewer_.ModelPath(), unpacked_tensor));
ORT_RETURN_IF_NOT(size == unpacked_tensor.size(),
"initializer tensor: ", tensor.name(), "'s size: ", unpacked_tensor.size(),
" should match the calculated size: ", size);
src = unpacked_tensor.data();
break;
// default:
// We should not get anything else here since we already checked in the 1st pass
}
// TensorProto_DataType_UINT8 or TensorProto_DataType_FLOAT:
Initializer unpacked_tensor(tensor, graph_viewer_.ModelPath());
size_t size_in_bytes = unpacked_tensor.DataAsByteSpan().size();
ORT_RETURN_IF_NOT(size == size_in_bytes,
"initializer tensor: ", tensor.name(), "'s size: ",
size_in_bytes, " should match the calculated size: ", size);
src = unpacked_tensor.DataAsByteSpan().data();
uint8_t* dest = nnapi_model_->mem_initializers_->GetDataPtr() + offset;
memcpy(dest, src, size);
ORT_RETURN_IF_ERROR(SetOperandValue(index, nnapi_model_->mem_initializers_.get(), size, offset));

155
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

@ -4,6 +4,7 @@
#include "op_builder.h"
#include <onnx/onnx_pb.h>
#include <algorithm>
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
@ -17,6 +18,7 @@
#include "helper.h"
#include "model_builder.h"
#include "op_support_checker.h"
#include "core/optimizer/initializer.h"
using namespace android::nn::wrapper;
@ -126,9 +128,8 @@ static Status GetAxesForSqueezeAndUnSqueeze(ModelBuilder& model_builder, const N
if (node_unit.Inputs().size() > 1) {
const auto& initializers(model_builder.GetInitializerTensors());
const auto& axes_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name());
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor));
const int64_t* raw_axes = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(axes_tensor);
auto raw_axes = unpacked_tensor.DataAsSpan<int64_t>();
const auto size = SafeInt<uint32_t>(axes_tensor.dims()[0]);
axes.resize(size);
for (uint32_t i = 0; i < size; i++) {
@ -163,16 +164,11 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder,
// TODO support other data types
const uint8_t* src = nullptr;
std::vector<uint8_t> unpacked_tensor;
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(),
unpacked_tensor));
src = unpacked_tensor.data();
break;
}
default:
@ -180,7 +176,8 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder,
"The initializer of graph ", name,
" doesn't have valid type: ", tensor.data_type());
}
Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath());
src = unpacked_tensor.DataAsByteSpan().data();
const auto out_t = shape[0], in_t = shape[1],
h_t = shape[2], w_t = shape[3];
Shape dest_shape;
@ -248,15 +245,10 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder,
// TODO support other data types
const uint8_t* src = nullptr;
std::vector<uint8_t> unpacked_tensor;
switch (tensor.data_type()) {
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(),
unpacked_tensor));
src = unpacked_tensor.data();
break;
}
default:
@ -264,7 +256,9 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder,
"The initializer of graph ", name,
" doesn't have valid type: ", tensor.data_type());
}
Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath());
// could be float/u8/s8, so we have to use raw data here.
src = unpacked_tensor.DataAsByteSpan().data();
const auto x_t = shape[0], y_t = shape[1];
Shape dest_shape = {y_t, x_t};
OperandType operand_type = source_operand_type;
@ -422,11 +416,10 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint(
w_zero_point = 0;
// We need to copy the 1d scales array for per-channel quantization
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor));
const float* scales = reinterpret_cast<const float*>(unpacked_tensor.data());
Initializer unpacked_tensor(scale_tensor);
auto scales = unpacked_tensor.DataAsSpan<float>();
const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0];
std::vector<float> scales_vec(scales, scales + scales_size);
std::vector<float> scales_vec(scales.begin(), scales.begin() + scales_size);
w_scales = onnxruntime::make_optional(std::move(scales_vec));
return Status::OK();
}
@ -892,9 +885,8 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
auto input = node_unit.Inputs()[0].node_arg.Name();
const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name());
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor));
const int64_t* raw_shape = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(shape_tensor);
auto raw_shape = unpacked_tensor.DataAsSpan<int64_t>();
const auto size = SafeInt<uint32_t>(shape_tensor.dims()[0]);
Shape input_shape = shaper[input];
@ -1006,21 +998,15 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu
a.reserve(size);
b.reserve(size);
std::vector<uint8_t> unpacked_scale_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_scale_tensor));
const float* scale_data = reinterpret_cast<const float*>(unpacked_scale_tensor.data());
Initializer unpacked_scale_tensor(scale_tensor);
Initializer unpacked_bias_tensor(bias_tensor);
Initializer unpacked_mean_tensor(mean_tensor);
Initializer unpacked_var_tensor(var_tensor);
std::vector<uint8_t> unpacked_bias_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_bias_tensor));
const float* bias_data = reinterpret_cast<const float*>(unpacked_bias_tensor.data());
std::vector<uint8_t> unpacked_mean_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(mean_tensor, unpacked_mean_tensor));
const float* mean_data = reinterpret_cast<const float*>(unpacked_mean_tensor.data());
std::vector<uint8_t> unpacked_var_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(var_tensor, unpacked_var_tensor));
const float* var_data = reinterpret_cast<const float*>(unpacked_var_tensor.data());
auto scale_data = unpacked_scale_tensor.DataAsSpan<float>();
auto bias_data = unpacked_bias_tensor.DataAsSpan<float>();
auto mean_data = unpacked_mean_tensor.DataAsSpan<float>();
auto var_data = unpacked_var_tensor.DataAsSpan<float>();
for (int64_t i = 0; i < size; i++) {
a.push_back(scale_data[i] / sqrt(var_data[i] + eps));
@ -1390,12 +1376,10 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
Shape bias_dimen;
for (auto dim : bias_tensor.dims())
bias_dimen.push_back(SafeInt<uint32_t>(dim));
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor));
Initializer unpacked_tensor(bias_tensor);
OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale);
ORT_RETURN_IF_ERROR(
model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type));
model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data<int32_t>(), bias_operand_type));
}
const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET"));
@ -1819,11 +1803,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
Shape bias_dimen;
for (auto dim : bias_tensor.dims())
bias_dimen.push_back(SafeInt<uint32_t>(dim));
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor));
Initializer unpacked_tensor(bias_tensor);
OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, a_scale * b_scale);
ORT_RETURN_IF_ERROR(
model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type));
model_builder.AddOperandFromPersistMemoryBuffer(
bias,
unpacked_tensor.data<int32_t>(), bias_operand_type));
bias_idx = operand_indices.at(bias);
}
@ -2381,17 +2366,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
if (inputs.size() == 3) { // we are using scales
const auto& scales_name = inputs[2].node_arg.Name();
const auto& scales_tensor = *initializers.at(scales_name);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor));
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
Initializer unpacked_tensor(scales_tensor);
auto scales_data = unpacked_tensor.DataAsSpan<float>();
ORT_RETURN_IF_ERROR(
shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output));
} else { // we are using sizes
const auto& sizes_name = inputs[3].node_arg.Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor));
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(sizes_tensor);
auto sizes_data = unpacked_tensor.DataAsSpan<int64_t>();
ORT_RETURN_IF_ERROR(
shaper.ResizeUsingOutputSizes(input, SafeInt<uint32_t>(sizes_data[h_idx]), SafeInt<uint32_t>(sizes_data[w_idx]), use_nchw, output));
}
@ -2502,33 +2485,25 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
indices_data_type != ONNX_NAMESPACE::TensorProto_DataType_INT32) {
// Add indices operand into nnapi
const auto& indices_tensor = *initializers.at(input2);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(indices_tensor, unpacked_tensor));
Initializer unpacked_tensor(indices_tensor);
const auto data_type = indices_tensor.data_type();
const auto indices_shape = indices_tensor.dims();
uint32_t size = 1;
Shape indices_dimen;
indices_dimen.reserve(indices_tensor.dims_size());
for (auto i = 0; i < indices_tensor.dims_size(); i++) {
size *= SafeInt<uint32_t>(indices_shape[i]);
indices_dimen.push_back(static_cast<uint32_t>(indices_shape[i]));
}
std::vector<int32_t> indices(size);
// see https://gist.github.com/shafik/848ae25ee209f698763cffee272a58f8#type-punning-arrays for the usage of memcpy here
std::vector<int32_t> indices(unpacked_tensor.size());
if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) {
for (uint32_t i = 0; i < size; i++) {
int64_t index_i64;
memcpy(&index_i64, unpacked_tensor.data() + i * sizeof(int64_t), sizeof(int64_t));
indices[i] = SafeInt<int32_t>(index_i64);
}
auto indice_span = unpacked_tensor.DataAsSpan<int64_t>();
std::transform(indice_span.begin(), indice_span.end(), indices.begin(),
[](int64_t indice_n) -> int32_t { return SafeInt<int32_t>(indice_n); });
} else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) {
for (uint32_t i = 0; i < size; i++) {
int32_t index;
memcpy(&index, unpacked_tensor.data() + i * sizeof(int32_t), sizeof(int32_t));
indices[i] = SafeInt<int32_t>(index);
}
auto indice_span = unpacked_tensor.DataAsSpan<int32_t>();
indices.assign(indice_span.begin(), indice_span.end());
}
OperandType indices_operand_type(Type::TENSOR_INT32, indices_dimen);
@ -2684,20 +2659,14 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
const auto& initializers(model_builder.GetInitializerTensors());
const auto& tensor = *initializers.at(input_name);
std::vector<uint8_t> unpacked_tensor;
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(),
unpacked_tensor));
size_t tensor_byte_size = unpacked_tensor.size();
Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath());
const auto data_type = tensor.data_type();
if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) {
const int64_t* tensor_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
size_t size = tensor_byte_size / sizeof(int64_t);
data.insert(data.end(), tensor_data, tensor_data + size);
auto tensor_data = unpacked_tensor.DataAsSpan<int64_t>();
data.insert(data.end(), tensor_data.begin(), tensor_data.end());
} else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) {
const int32_t* tensor_data = reinterpret_cast<const int32_t*>(unpacked_tensor.data());
size_t size = tensor_byte_size / sizeof(int32_t);
data.insert(data.end(), tensor_data, tensor_data + size);
auto tensor_data = unpacked_tensor.DataAsSpan<int32_t>();
data.insert(data.end(), tensor_data.begin(), tensor_data.end());
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"Data type for starts and ends inputs' is not supported in this build. Got ",
@ -2843,29 +2812,14 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
const auto* pads_initializer = model_builder.GetConstantInitializer(pads);
ORT_RETURN_IF_NOT(pads_initializer, "pads must be a constant");
std::vector<uint8_t> pads_initializer_raw_data{};
ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*pads_initializer, pads_initializer_raw_data));
Initializer pads_initializer_raw_data(*pads_initializer);
// assume pads_initializer has int64 data, per ONNX spec
ORT_RETURN_IF_NOT(pads_initializer_raw_data.size() == 2 * data_rank * sizeof(int64_t),
"Expected pads initializer size in bytes: ", 2 * data_rank * sizeof(int64_t),
", actual: ", pads_initializer_raw_data.size());
std::vector<int32_t> converted_pads_data{};
converted_pads_data.reserve(2 * data_rank);
auto copy_and_convert = [](const void* raw_i64_src,
std::back_insert_iterator<decltype(converted_pads_data)> i32_dst) {
int64_t i64;
memcpy(&i64, raw_i64_src, sizeof(i64));
*i32_dst = SafeInt<int32_t>(i64);
};
auto pads_span = pads_initializer_raw_data.DataAsSpan<int64_t>();
for (size_t i = 0; i < data_rank; ++i) {
copy_and_convert(&pads_initializer_raw_data[i * sizeof(int64_t)],
std::back_inserter(converted_pads_data));
copy_and_convert(&pads_initializer_raw_data[(i + data_rank) * sizeof(int64_t)],
std::back_inserter(converted_pads_data));
converted_pads_data.push_back(SafeInt<int32_t>(pads_span[i]));
converted_pads_data.push_back(SafeInt<int32_t>(pads_span[i + data_rank]));
}
const Shape converted_pads_shape{data_rank, 2};
@ -2880,15 +2834,8 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
const auto& constant_value = inputs[2].node_arg.Name();
const auto* constant_value_initializer = model_builder.GetConstantInitializer(constant_value);
ORT_RETURN_IF_NOT(constant_value_initializer, "constant_value must be a constant");
std::vector<uint8_t> pad_value_raw_data{};
ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*constant_value_initializer, pad_value_raw_data));
// assume constant_value_initializer has float data
// ONNX spec says it matches `data` input type, and op support checker limits that to float
ORT_RETURN_IF_NOT(pad_value_raw_data.size() == sizeof(float),
"Expected constant_value initializer size in bytes: ", sizeof(float),
", actual size: ", pad_value_raw_data.size());
memcpy(&pad_value, pad_value_raw_data.data(), sizeof(float));
Initializer pad_value_raw_data_init(*constant_value_initializer);
pad_value = pad_value_raw_data_init.DataAsSpan<float>()[0];
}
ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value);

62
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc
View file

@ -12,7 +12,7 @@
#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
namespace nnapi {
@ -184,18 +184,10 @@ static bool IsQuantizationZeroPointSupported(const InitializedTensorSet& initial
<< " zero point dimension " << zero_dim;
return false;
}
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, model_path, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name
<< ", error msg: " << status.ErrorMessage();
return false;
}
Initializer unpacked_tensor(zero_tensor, model_path);
// Verify all onnx weight zero point(s) are 0(s)
const int8_t* zero_points = reinterpret_cast<const int8_t*>(unpacked_tensor.data());
for (size_t i = 0; i < unpacked_tensor.size(); i++) {
auto zero_points = unpacked_tensor.DataAsSpan<int8_t>();
for (int64_t i = 0; i < unpacked_tensor.size(); i++) {
if (zero_points[i] != 0) {
LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, "
<< "zero_points[" << i << "] has value: " << zero_points[i];
@ -731,13 +723,8 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init
}
const auto& perm_tensor = *initializers.at(perm_name);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage();
return false;
}
const int64_t* raw_perm = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
Initializer unpacked_tensor(perm_tensor);
auto raw_perm = unpacked_tensor.DataAsSpan<int64_t>();
const auto perm_size = SafeInt<uint32_t>(perm_tensor.dims()[0]);
NodeAttrHelper helper(node_unit);
@ -1996,13 +1983,8 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
// We want to check if the scales or sizes are not trying to resize on N/C channels here
if (inputs.size() == 3) { // we are using scales
const auto& scales_tensor = *initializers.at(inputs[2].node_arg.Name());
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage();
return false;
}
const float* scales_data = reinterpret_cast<const float*>(unpacked_tensor.data());
Initializer unpacked_tensor(scales_tensor);
auto scales_data = unpacked_tensor.DataAsSpan<float>();
float scale_n = scales_data[0];
float scale_c = IsNodeLayoutNHWC(node_unit) ? scales_data[3] : scales_data[1];
if (scale_n != 1.0f || scale_c != 1.0f) {
@ -2015,15 +1997,9 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
// we are using sizes
const auto& sizes_name = inputs[3].node_arg.Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage();
return false;
}
Initializer unpacked_tensor(sizes_tensor);
int channel_idx = IsNodeLayoutNHWC(node_unit) ? 3 : 1;
const int64_t* sizes_data = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
auto sizes_data = unpacked_tensor.DataAsSpan<int64_t>();
uint32_t size_n = SafeInt<uint32_t>(sizes_data[0]);
uint32_t size_c = SafeInt<uint32_t>(sizes_data[channel_idx]);
if (size_n != input_shape[0] || size_c != input_shape[channel_idx]) {
@ -2352,20 +2328,12 @@ bool PadOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initiali
}
const ONNX_NAMESPACE::TensorProto& pads_initializer = *pads_initializer_it->second;
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(pads_initializer, unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Error while unpacking pads initializer: " << status.ErrorMessage();
return false;
}
int64_t pad_value;
ORT_ENFORCE(unpacked_tensor.size() % sizeof(pad_value) == 0);
for (size_t i = 0; i < unpacked_tensor.size(); i += sizeof(pad_value)) {
memcpy(&pad_value, &unpacked_tensor[i], sizeof(pad_value));
if (pad_value < 0) {
Initializer unpacked_tensor(pads_initializer);
auto tensor_data = unpacked_tensor.DataAsSpan<int64_t>();
for (int64_t i = 0; i < unpacked_tensor.size(); i++) {
if (tensor_data[i] < 0) {
LOGS_DEFAULT(VERBOSE) << "Negative pad value is not supported: pads["
<< i / sizeof(pad_value) << "] = " << pad_value;
<< i << "] = " << tensor_data[i];
return false;
}
}

19
onnxruntime/core/providers/shared/utils/utils.cc
View file

@ -9,6 +9,7 @@
#include <core/graph/graph.h>
#include <core/providers/common.h>
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
@ -51,13 +52,8 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
LOGS(logger, VERBOSE) << "Input min of Clip must be known";
return false;
}
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(min_name), unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking min tensor: " << status.ErrorMessage();
return false;
}
min = reinterpret_cast<float*>(unpacked_tensor.data())[0];
Initializer unpacked_tensor(*initializers.at(min_name));
min = unpacked_tensor.DataAsSpan<float>()[0];
}
if (node.InputDefs().size() > 2) { // we have input max
@ -66,13 +62,8 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
LOGS(logger, VERBOSE) << "Input max of Clip must be known";
return false;
}
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(max_name), unpacked_tensor);
if (!status.IsOK()) {
LOGS(logger, ERROR) << "Error while unpacking max tensor: " << status.ErrorMessage();
return false;
}
max = reinterpret_cast<float*>(unpacked_tensor.data())[0];
Initializer unpacked_tensor(*initializers.at(max_name));
max = unpacked_tensor.DataAsSpan<float>()[0];
}
}

35
onnxruntime/core/providers/xnnpack/detail/utils.cc
View file

@ -2,10 +2,10 @@
// Licensed under the MIT License.
#include "utils.h"
#include <stdint.h>
#include <unordered_map>
#include <vector>
#include "core/common/common.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/indexed_sub_graph.h"
#include "core/graph/node_attr_utils.h"
@ -14,6 +14,7 @@
#include "onnx/defs/attr_proto_util.h"
#include "core/common/safeint.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
namespace xnnpack {
@ -160,7 +161,8 @@ std::unique_ptr<IndexedSubGraph::MetaDef> FuseQDQGroup(const NodeUnit& node_unit
def.inputs.push_back(y_quant_param.scale.Name());
def.inputs.push_back(y_quant_param.zero_point ? y_quant_param.zero_point->Name() : "");
if (qtype == QuantizedOpType::QDQSoftmax) {
def.domain = kMSDomain; // reuse Qlinearsoftmax in kMSDomain, we need to define our own schema
def.domain = kDynamicDomainByCreate;
def.since_version = 1;
def.attributes.emplace("opset", utils::MakeAttribute(std::string("opset"), int64_t(node_unit.SinceVersion())));
}
} else if (qtype == QuantizedOpType::QDQMaxPool) {
@ -183,10 +185,11 @@ std::unique_ptr<IndexedSubGraph::MetaDef> FuseQDQGroup(const NodeUnit& node_unit
}
// Fuse activation with node_unit.
std::unique_ptr<IndexedSubGraph::MetaDef> FuseActivation(const NodeUnit& node_unit, const Node& activation,
std::unique_ptr<IndexedSubGraph::MetaDef> FuseActivation(const NodeUnit& node_unit, const NodeUnit& activation_unit,
const GraphViewer& graph) {
std::unique_ptr<IndexedSubGraph::MetaDef> metadef = std::make_unique<IndexedSubGraph::MetaDef>();
IndexedSubGraph::MetaDef& def = *metadef;
const Node& activation = activation_unit.GetNode();
// we use the op type/domain to match the static xnnpack Conv or MaxPool kernel
// registration
@ -260,20 +263,6 @@ std::unique_ptr<IndexedSubGraph::MetaDef> FuseActivation(const NodeUnit& node_un
return metadef;
}
const onnx::TensorProto* GetQuantizationScale(const InitializedTensorSet& initializers,
const NodeUnitIODef& io_def) {
if (io_def.quant_param.has_value() == false) {
return nullptr;
}
const auto scale_name = io_def.quant_param->scale.Name();
auto it = initializers.find(scale_name);
if (it == initializers.cend()) {
return nullptr;
}
return it->second;
}
std::pair<const onnx::TensorProto*, const onnx::TensorProto*>
GetQuantizationZeroPointAndScale(const GraphViewer& graphview,
const NodeUnitIODef& io_def) {
@ -318,7 +307,6 @@ TensorQuantType GetTensorQuantType(const NodeUnit& node_unit, int32_t io_index,
return TensorTypeInvalid;
}
std::vector<uint8_t> unpacked_tensor;
// we have processed float-type in the beginning
// we do not handle u8s8
switch (input_type) {
@ -347,14 +335,9 @@ TensorQuantType GetTensorQuantType(const NodeUnit& node_unit, int32_t io_index,
} else if (scales_dim == tensor_shape[0]) {
// default 0 for zero-point if zero_dim == 0
if (zero_tensor != nullptr) {
auto status = utils::UnpackInitializerData(*zero_tensor, node_unit.ModelPath(), unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "error when unpack zero tensor: "
<< ", error msg: " << status.ErrorMessage();
break;
}
const int8_t* zero_points = reinterpret_cast<const int8_t*>(unpacked_tensor.data());
for (size_t i = 0; i < unpacked_tensor.size(); i++) {
Initializer zp_val(*zero_tensor, node_unit.ModelPath());
auto zero_points = zp_val.DataAsSpan<int8_t>();
for (int64_t i = 0; i < zp_val.size(); i++) {
if (zero_points[i] != 0) {
LOGS_DEFAULT(VERBOSE) << "only support 0 as zero point for per-channel quantization, "
<< "zero_points[" << i << "] has value: " << zero_points[i];

3
onnxruntime/core/providers/xnnpack/detail/utils.h
View file

@ -21,6 +21,7 @@ namespace onnxruntime {
class GraphViewer;
class NodeUnit;
namespace xnnpack {
constexpr const char* kDynamicDomainByCreate = "xnnpack";
enum OpComputeType : uint8_t {
op_compute_type_invalid = 0,
@ -75,7 +76,7 @@ bool IsPaddingTypeSupported(AutoPadType auto_pad);
using XnnpackOperator = std::unique_ptr<struct xnn_operator, XnnpackOperatorDeleter>;
std::unique_ptr<IndexedSubGraph::MetaDef> FuseActivation(const NodeUnit& conv_unit, const Node& activation,
std::unique_ptr<IndexedSubGraph::MetaDef> FuseActivation(const NodeUnit& conv_unit, const NodeUnit& activation,
const GraphViewer& graph);
std::unique_ptr<IndexedSubGraph::MetaDef> FuseQDQGroup(const NodeUnit& unit_node);

11
onnxruntime/core/providers/xnnpack/nn/softmax.cc
View file

@ -9,7 +9,6 @@
#include "core/providers/cpu/math/softmax_shared.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime {
namespace xnnpack {
std::pair<const onnx::TensorProto*, const onnx::TensorProto*>
@ -31,12 +30,12 @@ bool IsQuantSoftmaxSupported(const NodeUnit& node_unit, const GraphViewer& graph
// qdq models converted from other framework
auto [scale_tensor, zero_tensor] = GetQuantizationZeroPointAndScale(graph, node_unit.Outputs()[0]);
Initializer q_scale(*scale_tensor, node_unit.ModelPath());
if (fabs(*q_scale.data<float>() - 1.0f / 256.0f) > 0.0001f) {
if (fabs(q_scale.DataAsSpan<float>()[0] - 1.0f / 256.0f) > 0.0001f) {
break;
}
if (scale_tensor) {
if (zero_tensor) {
Initializer q_zp(*zero_tensor, node_unit.ModelPath());
if (*q_zp.raw_data() != 0) {
if (q_zp.DataAsSpan<uint8_t>()[0] != 0) {
break;
}
}
@ -231,8 +230,8 @@ ONNX_OPERATOR_KERNEL_EX(Softmax, kOnnxDomain, 13, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Softmax);
ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kMSDomain, 1, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<uint8_t>()),
ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kDynamicDomainByCreate, 1, kXnnpackExecutionProvider,
KernelDefBuilder(), // dynamic schema
Softmax);
} // namespace xnnpack

36
onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc
View file

@ -1,10 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include "core/graph/function_utils.h"
#include "xnnpack_execution_provider.h"
#include "detail/utils.h"
#include "detail/node_support_checker.h"
@ -33,7 +35,7 @@ KernelCreateInfo BuildKernelCreateInfo<void>() {
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, Start, Op)>
#define KERNEL_CREATE_INFO_TYPED(Start, type, Op) \
BuildKernelCreateInfo< \
BuildKernelCreateInfo< \
ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, Start, type, Op)>
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv);
@ -47,7 +49,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSIntern
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, int8_t, QLinearConv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, QLinearAveragePool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider,
kMSDomain, 1, QLinearSoftmax);
kDynamicDomainByCreate, 1, QLinearSoftmax);
std::unique_ptr<KernelRegistry> RegisterKernels() {
auto kernel_registry = std::make_unique<onnxruntime::KernelRegistry>();
@ -69,7 +71,7 @@ std::unique_ptr<KernelRegistry> RegisterKernels() {
KERNEL_CREATE_INFO_TYPED(10, int8_t, QLinearConv),
KERNEL_CREATE_INFO(1, QLinearAveragePool),
BuildKernelCreateInfo<
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSDomain, 1, QLinearSoftmax)>,
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kDynamicDomainByCreate, 1, QLinearSoftmax)>,
};
for (auto& function_table_entry : function_table) {
@ -124,12 +126,21 @@ void XnnpackExecutionProvider::RegisterAllocator(AllocatorManager& allocator_man
}
}
// For ops are not lay-out sensitive and does not defined in
// onnx-domain, it will be created dynamicly
static bool RequestDynamicSchema(const NodeUnit& node_unit) {
static const InlinedHashSet<std::string_view> dynamic_schema_set = {"QLinearSoftmax"};
std::string key = node_unit.UnitType() == NodeUnit::Type::QDQGroup
? "QLinear" + node_unit.OpType() : node_unit.OpType();
return dynamic_schema_set.contains(key);
}
// Add Compute Capability for the second call. All target nodes have the tag of "XnnpackExecutionProvider"
// after the first call. So we are going to do QDQ fusion in the second call
// All nodes was collected in one sub_graph
static void AddComputeCapabilityForNodeUnit(const NodeUnit& node_unit,
const std::function<void(std::unique_ptr<IndexedSubGraph>)>& adder,
std::unordered_map<const Node*, const NodeUnit*>& supported_map) {
const std::function<void(std::unique_ptr<IndexedSubGraph>)>& adder,
std::unordered_map<const Node*, const NodeUnit*>& supported_map) {
std::unique_ptr<IndexedSubGraph> sub_graph = std::make_unique<IndexedSubGraph>();
auto process_node = [&sub_graph, &supported_map, &node_unit](const Node& node) {
sub_graph->nodes.push_back(node.Index());
@ -141,10 +152,13 @@ static void AddComputeCapabilityForNodeUnit(const NodeUnit& node_unit,
process_node(*node_i);
}
sub_graph->SetMetaDef(FuseQDQGroup(node_unit));
sub_graph->use_existing_schema = true;
} else {
process_node(node_unit.GetNode());
}
sub_graph->schema_source = RequestDynamicSchema(node_unit)
? IndexedSubGraph::SourceOfSchema::REUSE_OR_CREATE
: IndexedSubGraph::SourceOfSchema::EXISTING;
adder(std::move(sub_graph));
}
@ -190,10 +204,10 @@ std::vector<std::unique_ptr<ComputeCapability>> XnnpackExecutionProvider::GetCap
if (n == nullptr) {
continue;
}
const NodeUnit& node_unit = *node_unit_map[n];
// if node is part of a QDQ group,
// we will mark it compatible in the first call as long as we support the target node.
const Node& node = *n;
const NodeUnit& node_unit = *node_unit_map[n];
bool request_node = false;
// any node in NodeUnit will trigger IsNodeSupported, so we just check once.
if (node_unit_supported_result.count(&node_unit)) {
@ -221,9 +235,9 @@ std::vector<std::unique_ptr<ComputeCapability>> XnnpackExecutionProvider::GetCap
// GraphPartitioner will match the statically registered xnnpack NHWC Conv kernel instead of
// calling IExecutionProvider::Compile
ComputeCapability& capability = *iter->second;
capability.sub_graph->SetMetaDef(FuseActivation(*fuse_with, node, graph));
capability.sub_graph->nodes.push_back(node.Index());
capability.sub_graph->use_existing_schema = true;
capability.sub_graph->SetMetaDef(FuseActivation(*fuse_with, node_unit, graph));
capability.sub_graph->nodes.push_back(node_unit.Index());
capability.sub_graph->schema_source = IndexedSubGraph::SourceOfSchema::EXISTING;
}
}
} else if (node_unit.GetNode().GetExecutionProviderType() == Type()) {