From 8cedafe250ee3b5baaaab854d6cb5b3e05e0254c Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 6 Sep 2022 14:32:15 +0800 Subject: [PATCH] [xnnpack] Have `Initializer` in Mobile related EPs in Minimal_build and creating EP specific dynamic-schema (#12555) * Remove the dependence of Qlinearsoftmax schema * refactor initializerview && create shared schema * Dynamic Create EP specific schema * Have Initializer in minimal_build * address comments * remove CancelFuseSubGraph --- include/onnxruntime/core/graph/graph.h | 11 +- .../core/graph/indexed_sub_graph.h | 12 +- .../contrib_ops/internal_nhwc_onnx_opset.cc | 5 +- onnxruntime/core/graph/function_utils.cc | 49 ++++-- onnxruntime/core/graph/function_utils.h | 16 +- onnxruntime/core/graph/graph.cc | 50 +++--- onnxruntime/core/optimizer/initializer.cc | 3 +- onnxruntime/core/optimizer/initializer.h | 20 ++- .../optimizer/qdq_transformer/qdq_util.cc | 2 +- .../builders/impl/activation_op_builder.cc | 7 +- .../coreml/builders/impl/builder_utils.cc | 6 +- .../coreml/builders/impl/gemm_op_builder.cc | 7 +- .../builders/impl/reshape_op_builder.cc | 11 +- .../coreml/builders/impl/resize_op_builder.cc | 29 +--- .../builders/impl/squeeze_op_builder.cc | 13 +- .../nnapi/nnapi_builtin/builders/helper.cc | 23 +-- .../nnapi_builtin/builders/model_builder.cc | 28 +--- .../nnapi_builtin/builders/op_builder.cc | 155 ++++++------------ .../builders/op_support_checker.cc | 62 ++----- .../core/providers/shared/utils/utils.cc | 19 +-- .../core/providers/xnnpack/detail/utils.cc | 35 +--- .../core/providers/xnnpack/detail/utils.h | 3 +- .../core/providers/xnnpack/nn/softmax.cc | 11 +- .../xnnpack/xnnpack_execution_provider.cc | 36 ++-- 24 files changed, 250 insertions(+), 363 deletions(-) diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index 8f81ce14dd..8e930ce5a8 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -18,6 +18,7 @@ #if !defined(ORT_MINIMAL_BUILD) #include "onnx/defs/schema.h" +#include "core/common/inlined_containers.h" #else #include "onnx/defs/data_type_utils.h" #endif @@ -1092,13 +1093,6 @@ class Graph { */ Node& BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name); - /** - If we have BeginFuseSubGraph, but somehow hit errors, such as Compile of an EP failed on thesub_graph. - We can call CancelFuseSubGraph to undo the changes of BeginFuseSubGraph - @param fused_node The fused node and it's function body to be removed from the graph - */ - void CancelFuseSubGraph(const Node& fused_node); - void FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node); #endif @@ -1605,6 +1599,9 @@ class Graph { // for the fused kernel. I really don't like it. but for short-term solution, let's host // those schemas here. InlinedVector> fused_schemas_containers_; + // in some case, a fused sub-graph will happens multiple times in one model, we use a map + // to store reusable-schema in lookup. + InlinedHashMap> reusable_fused_schema_map_; #endif // !defined(ORT_MINIMAL_BUILD) // Graph nodes. diff --git a/include/onnxruntime/core/graph/indexed_sub_graph.h b/include/onnxruntime/core/graph/indexed_sub_graph.h index 639384f6cc..935ada88c6 100644 --- a/include/onnxruntime/core/graph/indexed_sub_graph.h +++ b/include/onnxruntime/core/graph/indexed_sub_graph.h @@ -52,9 +52,17 @@ struct IndexedSubGraph { /** Nodes covered by this subgraph. The NodeIndex values are from the parent Graph.*/ std::vector nodes; - // use an existing schema instead of generating one when fusing nodes using the MetaDef. + enum class SourceOfSchema : uint8_t { + CREATE, /// create new schema from info in IndexedSubGraph instance. + /// schema instance will not be re-usable. + REUSE_OR_CREATE, /// re-use existing dynamically created schema with matching domain+name. + /// create re-usable schema if one is not found. + EXISTING, /// use existing statically registered schema. + /// e.g. domain+name matches ONNX or contrib op domain+op_type+opset. + }; + // Either using an existing schema or generating reusable one when fusing nodes using the MetaDef. // MetaDef.domain + MetaDef.name => the domain.op_type that a schema must exist for with a valid since_version. - bool use_existing_schema{false}; + SourceOfSchema schema_source{SourceOfSchema::CREATE}; /** Set the meta definition needed to represent this subgraph as a FunctionProto It's needed IF AND ONLY IF there are multiple indexes contained in #nodes. */ diff --git a/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc b/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc index b45757efb9..9b043a4ea7 100644 --- a/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc +++ b/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc @@ -80,12 +80,15 @@ void OpSet_Internal_NHWC_ONNX::ForEachSchema(const std::functionsecond; } -std::unique_ptr -CreateSchema(const Graph& graph, - const IndexedSubGraph& nodes_to_fuse) { +std::unique_ptr CreateSchema( + const Graph& graph, + const IndexedSubGraph& nodes_to_fuse, bool allow_aggregated_tensor_type) { const auto* meta_def = nodes_to_fuse.GetMetaDef(); - auto op_schema = std::make_unique(); - op_schema->SetName(meta_def->name); + + using ONNX_NAMESPACE::OpSchema; + auto op_schema = std::make_unique(meta_def->name, __FILE__, __LINE__); op_schema->SetDomain(meta_def->domain); op_schema->SetDoc(meta_def->doc_string); op_schema->SinceVersion(meta_def->since_version); @@ -40,20 +41,34 @@ CreateSchema(const Graph& graph, op_schema->TypeAndShapeInferenceFunction(meta_def->type_and_shape_inference_function); } - int i = 0; + if (allow_aggregated_tensor_type) { + // The generated schema will use the same type constraint for all inputs and outputs, + // and that type constraint will match all tensor types. + // Due to this, a user of this style of schema must manually check whether any applicable type constraints + // for each input or output are satisfied prior to creating a node that uses this schema + // + op_schema->TypeConstraint("TAggregatedTypes", ONNX_NAMESPACE::OpSchema::all_tensor_types_with_bfloat(), + "all_tensor_types_with_bfloat"); + } - for (auto& input : meta_def->inputs) { - auto input_arg = graph.GetNodeArg(input); + int i = 0; + for (const auto& input : meta_def->inputs) { + const auto *input_arg = graph.GetNodeArg(input); // inputs must have a type. can be inferred for outputs. ORT_ENFORCE(input_arg->Type() != nullptr); - op_schema->Input(i, input, "", *input_arg->Type()); - ++i; + op_schema->Input(i, input, "", + allow_aggregated_tensor_type ? "TAggregatedTypes" : *input_arg->Type(), + OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type); + i++; } + i = 0; - for (auto& output : meta_def->outputs) { - auto output_arg = graph.GetNodeArg(output); - op_schema->Output(i, output, "", *output_arg->Type()); - ++i; + for (const auto& output : meta_def->outputs) { + const auto* output_arg = graph.GetNodeArg(output); + op_schema->Output(i, output, "", + allow_aggregated_tensor_type ? "TAggregatedTypes" : *output_arg->Type(), + OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type); + i++; } op_schema->Finalize(); @@ -278,7 +293,7 @@ std::unique_ptr CreateSchema(const std::string& functi } class Inliner { -private: + private: std::string prefix; const onnxruntime::NodeAttributes& attr_map; std::vector> rename_scopes; @@ -372,7 +387,7 @@ private: if (attr.has_g()) { transform(*attr.mutable_g()); } - for (auto& graph: *attr.mutable_graphs()) + for (auto& graph : *attr.mutable_graphs()) transform(graph); ++attr_iter; } @@ -392,7 +407,7 @@ private: rename_scopes.pop_back(); } -public: + public: // The main specialization method: specialize a FunctionProto for a particular call-site. static void specialize(const NodeProto& callnode, FunctionProto& callee, const onnxruntime::NodeAttributes& attr_map, std::string unique_prefix) { Inliner inliner(unique_prefix, attr_map); diff --git a/onnxruntime/core/graph/function_utils.h b/onnxruntime/core/graph/function_utils.h index 59c665d92f..fb67bc2948 100644 --- a/onnxruntime/core/graph/function_utils.h +++ b/onnxruntime/core/graph/function_utils.h @@ -14,13 +14,17 @@ namespace onnxruntime { namespace function_utils { -/** Create a OpSchema given a subgraph EP whant to fuse. -* This is used when EP return fusion in GetCapability implementation. -* @param graph The graph which host the subgraph. -* @param nodes_to_fuse The metadata for the subgraph that EP want to fuse. -*/ +/** Create a OpSchema given a subgraph EP want to fuse. + * This is used when EP return fusion in GetCapability implementation. + * @param graph The graph which host the subgraph. + * @param nodes_to_fuse The metadata for the subgraph that EP want to fuse. + * @param allow_aggregated_tensor_type if true, it will use a type constraint called + * TAggregatedTypes for all inputs and outputs, + * and that this will match all tensor types in the all_tensor_types_with_bfloat list. + */ std::unique_ptr CreateSchema(const Graph& graph, - const IndexedSubGraph& nodes_to_fuse); + const IndexedSubGraph& nodes_to_fuse, + bool allow_aggregated_tensor_type = false); /** Create a OpSchema given from a local function in onnx model. * @param function_domain The domain of the function. diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 4b8939f237..11433346b1 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -10,6 +10,7 @@ #include #include +#include "core/common/common.h" #include "gsl/gsl" #include "core/common/logging/logging.h" #include "core/common/inlined_containers.h" @@ -168,6 +169,12 @@ static TypeProto TypeProtoFromTensorProto(const TensorProto& tensor) { return t; } + +static std::string GenerateSchemaKey(const IndexedSubGraph& subgraph_ptr) { + return MakeString(subgraph_ptr.GetMetaDef()->domain, "_", + subgraph_ptr.GetMetaDef()->name, "_", + subgraph_ptr.GetMetaDef()->since_version); +} #endif // !defined(ORT_MINIMAL_BUILD) #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) @@ -3795,13 +3802,13 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std std::unordered_map output_indexes; int cur_idx = 0; - for (auto& arg_name : func_meta_def->inputs) { + for (const auto& arg_name : func_meta_def->inputs) { input_args.push_back(GetNodeArg(arg_name)); input_indexes[arg_name] = cur_idx++; } cur_idx = 0; - for (auto& arg_name : func_meta_def->outputs) { + for (const auto& arg_name : func_meta_def->outputs) { output_args.push_back(GetNodeArg(arg_name)); output_indexes[arg_name] = cur_idx++; } @@ -3820,12 +3827,20 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std // kernel lookup works as per usual, if not using an existing schema. // in an extended minimal build we do the lookup via a hash so don't need a schema. fused_node.SetSinceVersion(func_meta_def->since_version); - if (sub_graph.use_existing_schema) { + if (sub_graph.schema_source == IndexedSubGraph::SourceOfSchema::EXISTING) { ORT_ENFORCE(SetOpSchemaFromRegistryForNode(fused_node), "Schema was not found for fused node. Domain:", fused_node.Domain(), " OpType:", fused_node.OpType()); + } else if (IndexedSubGraph::SourceOfSchema::REUSE_OR_CREATE == sub_graph.schema_source) { + auto schema_key = GenerateSchemaKey(sub_graph); + if (!reusable_fused_schema_map_.contains(schema_key)) { + fused_schemas_containers_.push_back( + function_utils::CreateSchema(*this, sub_graph, /*allow_aggregated_tensor_type=*/true)); + reusable_fused_schema_map_.emplace(schema_key, *fused_schemas_containers_.back()); + } + + fused_node.op_ = &(reusable_fused_schema_map_.at(schema_key).get()); } else { - auto temp_schema_ptr = function_utils::CreateSchema(*this, sub_graph); - fused_schemas_containers_.push_back(std::move(temp_schema_ptr)); + fused_schemas_containers_.push_back(function_utils::CreateSchema(*this, sub_graph)); fused_node.op_ = fused_schemas_containers_.back().get(); } #endif @@ -3838,31 +3853,6 @@ Node& Graph::BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::stri return node; } -void Graph::CancelFuseSubGraph(const Node& fused_node) { - auto node_idx = fused_node.Index(); - if (!GetNode(node_idx)) - return; - - if (fused_node.NodeType() != Node::Type::Fused) - return; - -#if !defined(ORT_MINIMAL_BUILD) - // Remove the tempoary schema from schema container container - auto* temp_schema_ptr = fused_node.Op(); - auto it = std::find_if( - fused_schemas_containers_.begin(), fused_schemas_containers_.end(), - [temp_schema_ptr](const std::unique_ptr& schema) { - return schema.get() == temp_schema_ptr; - }); - if (it != fused_schemas_containers_.end()) { - fused_schemas_containers_.erase(it); - } -#endif - - // Remove the fused_node - RemoveNode(node_idx); -} - void Graph::FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node) { const auto* func_meta_def = sub_graph.GetMetaDef(); ORT_ENFORCE(nullptr != func_meta_def); diff --git a/onnxruntime/core/optimizer/initializer.cc b/onnxruntime/core/optimizer/initializer.cc index 726c1ff318..50092d2fb8 100644 --- a/onnxruntime/core/optimizer/initializer.cc +++ b/onnxruntime/core/optimizer/initializer.cc @@ -44,6 +44,7 @@ Initializer::Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const data_ = std::move(w); } +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) namespace { template struct ToFp16; @@ -318,5 +319,5 @@ void Initializer::scale_by_axis(const Initializer& scalers, int axis) { utils::MLTypeCallDispatcher t_disp(data_.GetElementType()); t_disp.Invoke(data_, scalers.data_, block_size, num_blocks); } - +#endif // ORT_EXTENDED_MINIMAL_BUILD } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/initializer.h b/onnxruntime/core/optimizer/initializer.h index f5ee267370..47ade74d2e 100644 --- a/onnxruntime/core/optimizer/initializer.h +++ b/onnxruntime/core/optimizer/initializer.h @@ -27,18 +27,18 @@ class Initializer final { gsl::span dims); Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, - const Path& model_path); + const Path& model_path = {}); ~Initializer() = default; void ToProto(ONNX_NAMESPACE::TensorProto& tensor_proto) const { tensor_proto = utils::TensorToTensorProto(data_, name_); } - +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) ONNX_NAMESPACE::TensorProto ToFP16(const std::string& name) const; ONNX_NAMESPACE::TensorProto ToBFloat16(const std::string& name) const; - +#endif // ORT_EXTENDED_MINIMAL_BUILD int data_type() const { return data_.GetElementType(); } @@ -57,8 +57,13 @@ class Initializer final { return data_.Data(); } - const int8_t* raw_data() const { - return reinterpret_cast(data_.DataRaw()); + template + auto DataAsSpan() const { + return data_.DataAsSpan(); + } + + gsl::span DataAsByteSpan() const { + return gsl::make_span(reinterpret_cast(data_.DataRaw()), data_.SizeInBytes()); } gsl::span dims() const { @@ -67,12 +72,13 @@ class Initializer final { int64_t size() const { return data_.Shape().Size(); } +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) Initializer& add(float value); Initializer& add(const Initializer& other); Initializer& sub(const Initializer& other); - + Initializer& mul(const Initializer& other); Initializer& div(const Initializer& other); @@ -80,7 +86,7 @@ class Initializer final { Initializer& sqrt(); void scale_by_axis(const Initializer& other, int axis); - +#endif // ORT_EXTENDED_MINIMAL_BUILD private: std::string name_; diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc index d595d9e35e..1898b6b290 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc @@ -54,7 +54,7 @@ bool IsQDQPairSupported( Initializer dq_scale(*dq_scale_tensor_proto, model_path); return q_zp.data_type() == dq_zp.data_type() && - *q_zp.raw_data() == *dq_zp.raw_data() && + q_zp.DataAsByteSpan() == dq_zp.DataAsByteSpan() && *q_scale.data() == *dq_scale.data(); } diff --git a/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc index 37993f3e8c..f15618f821 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc @@ -10,6 +10,7 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/coreml/builders/impl/base_op_builder.h" #include "core/providers/coreml/builders/op_builder_factory.h" +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace coreml { @@ -67,10 +68,8 @@ Status AddPReluWeight(ModelBuilder& model_builder, const Node& node, // assume X has 3 or 4 dimensions, that was checked in IsPReluOpSupported() const auto num_channels = x_shape[x_shape.size() - 3]; - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(slope_tensor, unpacked_tensor)); - float value; - std::memcpy(&value, unpacked_tensor.data(), sizeof(value)); + Initializer unpacked_tensor(slope_tensor); + float value = unpacked_tensor.DataAsSpan()[0]; auto& weight_values = *prelu.mutable_alpha()->mutable_floatvalue(); weight_values.Clear(); diff --git a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc index 9374082390..d14c6ff041 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc @@ -9,6 +9,7 @@ #include "core/framework/tensorprotoutils.h" #include "core/providers/coreml/builders/helper.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" #include "coreml/NeuralNetwork.pb.h" @@ -94,10 +95,9 @@ common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight, const ONNX_NAMESPACE::TensorProto& tensor) { auto data_type = tensor.data_type(); if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); + Initializer unpacked_tensor(tensor); auto num_elements = SafeInt(Product(tensor.dims())); - CreateCoreMLWeight(weight, reinterpret_cast(unpacked_tensor.data()), num_elements); + CreateCoreMLWeight(weight, unpacked_tensor.data(), num_elements); } else { // TODO: support other type return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, diff --git a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc index a2f5d47265..8d675050fa 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc @@ -7,6 +7,8 @@ #include "core/providers/shared/utils/utils.h" #include "core/providers/coreml/builders/helper.h" #include "core/providers/coreml/builders/op_builder_factory.h" +#include "core/optimizer/initializer.h" + #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" #include "builder_utils.h" @@ -52,9 +54,8 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod // TODO, add support of other data types static Status GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor, std::vector& transposed_data) { - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); - const float* src_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(tensor); + auto src_data = unpacked_tensor.DataAsSpan(); const auto& tensor_shape = tensor.dims(); auto x_t = SafeInt(tensor_shape[0]); auto y_t = SafeInt(tensor_shape[1]); diff --git a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc index 19bff2e97d..86318e8b2b 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc @@ -4,6 +4,7 @@ #include "core/providers/common.h" #include "core/framework/tensorprotoutils.h" #include "core/providers/cpu/tensor/reshape_helper.h" +#include "core/optimizer/initializer.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/coreml/builders/helper.h" @@ -83,14 +84,8 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputP } const auto& perm_tensor = *initializers.at(perm_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); - return false; - } - - const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(perm_tensor); + auto raw_perm = unpacked_tensor.DataAsSpan(); const auto& perm_dims = perm_tensor.dims(); if (perm_dims.empty() || perm_dims[0] == 0) { LOGS(logger, VERBOSE) << "New shape of reshape cannot be empty"; diff --git a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc index eb52bbf7b2..d35ef21e27 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc @@ -8,6 +8,7 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/cpu/tensor/reshape_helper.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" @@ -43,7 +44,7 @@ class ResizeOpBuilder : public BaseOpBuilder { // Helper functions bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node, std::vector& scales, - const logging::Logger& logger) { + const logging::Logger&) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 3) return false; @@ -51,21 +52,15 @@ bool GetResizeScales(const InitializedTensorSet& initializers, const auto& scales_tensor = *initializers.at(input_defs[2]->Name()); if (scales_tensor.dims_size() != 1 || scales_tensor.dims()[0] != 4) return false; - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); - return false; - } - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); - scales = std::vector{scales_data, scales_data + 4}; + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); + scales = std::vector{scales_data.begin(), scales_data.end()}; return true; } bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node& node, std::vector& sizes, - const logging::Logger& logger) { + const logging::Logger&) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 4) return false; @@ -73,15 +68,9 @@ bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const auto& sizes_tensor = *initializers.at(input_defs[3]->Name()); if (sizes_tensor.dims_size() != 1 || sizes_tensor.dims()[0] != 4) return false; - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); - return false; - } - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); - sizes = std::vector{sizes_data, sizes_data + 4}; + Initializer unpacked_tensor(sizes_tensor); + auto sizes_data = unpacked_tensor.DataAsSpan(); + sizes = std::vector{sizes_data.begin(), sizes_data.end()}; return true; } diff --git a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc index 2c5325daf9..0665d8e334 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc @@ -4,6 +4,8 @@ #include "core/framework/tensorprotoutils.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" + #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" #endif @@ -47,14 +49,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const if (node.InputDefs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); - const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(axes_tensor); + auto raw_axes = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(axes_tensor.dims()[0]); - axes.resize(size); - for (size_t i = 0; i < size; i++) { - axes[i] = raw_axes[i]; - } + axes.reserve(size); + axes.insert(axes.end(), raw_axes.begin(), raw_axes.end()); } } else { NodeAttrHelper helper(node); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index c5ddfce065..09058eaad1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -18,6 +18,7 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace nnapi { @@ -193,33 +194,19 @@ common::Status GetQuantizationScaleAndZeroPoint( "NodeArg: ", io_def.node_arg.Name(), " is not quantized"); } - const auto unpack_tensor = [&model_path](const InitializedTensorSet& initializers, - const std::string& name, std::vector& unpacked_tensor) { - const auto& tensor = *initializers.at(name); - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_path, unpacked_tensor)); - return Status::OK(); - }; - const auto& quant_param = *io_def.quant_param; { // get the scale - std::vector unpacked_tensor; const auto& name = quant_param.scale.Name(); - ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); + Initializer unpacked_tensor(*initializers.at(name), model_path); // The scale should be one or more floats - ORT_RETURN_IF(unpacked_tensor.size() < 4, - "The initializer [", name, "] should have one or more floats ", - "with size no less than 4, actual size: ", unpacked_tensor.size()); - scale = reinterpret_cast(unpacked_tensor.data())[0]; + scale = unpacked_tensor.DataAsSpan()[0]; } if (quant_param.zero_point) { // get the zero point if it's there - std::vector unpacked_tensor; const auto& name = quant_param.zero_point->Name(); - ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); - ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty"); + Initializer unpacked_tensor(*initializers.at(name), model_path); // Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI - zero_point = static_cast(unpacked_tensor[0]); + zero_point = static_cast(unpacked_tensor.DataAsByteSpan()[0]); } return Status::OK(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 40597c0cfe..6164bf9743 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -13,6 +13,7 @@ #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" +#include "core/optimizer/initializer.h" #include "helper.h" #include "op_builder.h" @@ -285,26 +286,15 @@ Status ModelBuilder::RegisterInitializers() { if (Contains(skipped_initializers_, tensor.name())) continue; - uint32_t index; - size_t size, padded_size; - std::tie(index, size, padded_size) = initializers[i++]; + auto [index, size, padded_size] = initializers[i++]; const uint8_t* src = nullptr; - // uint8_t data need unpack, need a holder for free memory after copy - std::vector unpacked_tensor; - switch (tensor.data_type()) { - case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - case ONNX_NAMESPACE::TensorProto_DataType_UINT8: - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, graph_viewer_.ModelPath(), unpacked_tensor)); - ORT_RETURN_IF_NOT(size == unpacked_tensor.size(), - "initializer tensor: ", tensor.name(), "'s size: ", unpacked_tensor.size(), - " should match the calculated size: ", size); - src = unpacked_tensor.data(); - break; - // default: - // We should not get anything else here since we already checked in the 1st pass - } - + // TensorProto_DataType_UINT8 or TensorProto_DataType_FLOAT: + Initializer unpacked_tensor(tensor, graph_viewer_.ModelPath()); + size_t size_in_bytes = unpacked_tensor.DataAsByteSpan().size(); + ORT_RETURN_IF_NOT(size == size_in_bytes, + "initializer tensor: ", tensor.name(), "'s size: ", + size_in_bytes, " should match the calculated size: ", size); + src = unpacked_tensor.DataAsByteSpan().data(); uint8_t* dest = nnapi_model_->mem_initializers_->GetDataPtr() + offset; memcpy(dest, src, size); ORT_RETURN_IF_ERROR(SetOperandValue(index, nnapi_model_->mem_initializers_.get(), size, offset)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 0354b1aeb0..a3b0ea05c9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -4,6 +4,7 @@ #include "op_builder.h" #include +#include #include "core/common/logging/logging.h" #include "core/common/safeint.h" @@ -17,6 +18,7 @@ #include "helper.h" #include "model_builder.h" #include "op_support_checker.h" +#include "core/optimizer/initializer.h" using namespace android::nn::wrapper; @@ -126,9 +128,8 @@ static Status GetAxesForSqueezeAndUnSqueeze(ModelBuilder& model_builder, const N if (node_unit.Inputs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); - const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(axes_tensor); + auto raw_axes = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(axes_tensor.dims()[0]); axes.resize(size); for (uint32_t i = 0; i < size; i++) { @@ -163,16 +164,11 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, // TODO support other data types const uint8_t* src = nullptr; - std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - src = unpacked_tensor.data(); break; } default: @@ -180,7 +176,8 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, "The initializer of graph ", name, " doesn't have valid type: ", tensor.data_type()); } - + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); + src = unpacked_tensor.DataAsByteSpan().data(); const auto out_t = shape[0], in_t = shape[1], h_t = shape[2], w_t = shape[3]; Shape dest_shape; @@ -248,15 +245,10 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder, // TODO support other data types const uint8_t* src = nullptr; - std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - src = unpacked_tensor.data(); break; } default: @@ -264,7 +256,9 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder, "The initializer of graph ", name, " doesn't have valid type: ", tensor.data_type()); } - + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); + // could be float/u8/s8, so we have to use raw data here. + src = unpacked_tensor.DataAsByteSpan().data(); const auto x_t = shape[0], y_t = shape[1]; Shape dest_shape = {y_t, x_t}; OperandType operand_type = source_operand_type; @@ -422,11 +416,10 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( w_zero_point = 0; // We need to copy the 1d scales array for per-channel quantization - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor)); - const float* scales = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scale_tensor); + auto scales = unpacked_tensor.DataAsSpan(); const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - std::vector scales_vec(scales, scales + scales_size); + std::vector scales_vec(scales.begin(), scales.begin() + scales_size); w_scales = onnxruntime::make_optional(std::move(scales_vec)); return Status::OK(); } @@ -892,9 +885,8 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons auto input = node_unit.Inputs()[0].node_arg.Name(); const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor)); - const int64_t* raw_shape = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(shape_tensor); + auto raw_shape = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(shape_tensor.dims()[0]); Shape input_shape = shaper[input]; @@ -1006,21 +998,15 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu a.reserve(size); b.reserve(size); - std::vector unpacked_scale_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_scale_tensor)); - const float* scale_data = reinterpret_cast(unpacked_scale_tensor.data()); + Initializer unpacked_scale_tensor(scale_tensor); + Initializer unpacked_bias_tensor(bias_tensor); + Initializer unpacked_mean_tensor(mean_tensor); + Initializer unpacked_var_tensor(var_tensor); - std::vector unpacked_bias_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_bias_tensor)); - const float* bias_data = reinterpret_cast(unpacked_bias_tensor.data()); - - std::vector unpacked_mean_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(mean_tensor, unpacked_mean_tensor)); - const float* mean_data = reinterpret_cast(unpacked_mean_tensor.data()); - - std::vector unpacked_var_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(var_tensor, unpacked_var_tensor)); - const float* var_data = reinterpret_cast(unpacked_var_tensor.data()); + auto scale_data = unpacked_scale_tensor.DataAsSpan(); + auto bias_data = unpacked_bias_tensor.DataAsSpan(); + auto mean_data = unpacked_mean_tensor.DataAsSpan(); + auto var_data = unpacked_var_tensor.DataAsSpan(); for (int64_t i = 0; i < size; i++) { a.push_back(scale_data[i] / sqrt(var_data[i] + eps)); @@ -1390,12 +1376,10 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N Shape bias_dimen; for (auto dim : bias_tensor.dims()) bias_dimen.push_back(SafeInt(dim)); - - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor)); + Initializer unpacked_tensor(bias_tensor); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); ORT_RETURN_IF_ERROR( - model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); + model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); } const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); @@ -1819,11 +1803,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N Shape bias_dimen; for (auto dim : bias_tensor.dims()) bias_dimen.push_back(SafeInt(dim)); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor)); + Initializer unpacked_tensor(bias_tensor); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, a_scale * b_scale); ORT_RETURN_IF_ERROR( - model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); + model_builder.AddOperandFromPersistMemoryBuffer( + bias, + unpacked_tensor.data(), bias_operand_type)); bias_idx = operand_indices.at(bias); } @@ -2381,17 +2366,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (inputs.size() == 3) { // we are using scales const auto& scales_name = inputs[2].node_arg.Name(); const auto& scales_tensor = *initializers.at(scales_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor)); - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); ORT_RETURN_IF_ERROR( shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output)); } else { // we are using sizes const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor)); - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(sizes_tensor); + auto sizes_data = unpacked_tensor.DataAsSpan(); ORT_RETURN_IF_ERROR( shaper.ResizeUsingOutputSizes(input, SafeInt(sizes_data[h_idx]), SafeInt(sizes_data[w_idx]), use_nchw, output)); } @@ -2502,33 +2485,25 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const indices_data_type != ONNX_NAMESPACE::TensorProto_DataType_INT32) { // Add indices operand into nnapi const auto& indices_tensor = *initializers.at(input2); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(indices_tensor, unpacked_tensor)); + Initializer unpacked_tensor(indices_tensor); const auto data_type = indices_tensor.data_type(); const auto indices_shape = indices_tensor.dims(); - uint32_t size = 1; Shape indices_dimen; indices_dimen.reserve(indices_tensor.dims_size()); for (auto i = 0; i < indices_tensor.dims_size(); i++) { - size *= SafeInt(indices_shape[i]); indices_dimen.push_back(static_cast(indices_shape[i])); } - std::vector indices(size); - // see https://gist.github.com/shafik/848ae25ee209f698763cffee272a58f8#type-punning-arrays for the usage of memcpy here + std::vector indices(unpacked_tensor.size()); + if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) { - for (uint32_t i = 0; i < size; i++) { - int64_t index_i64; - memcpy(&index_i64, unpacked_tensor.data() + i * sizeof(int64_t), sizeof(int64_t)); - indices[i] = SafeInt(index_i64); - } + auto indice_span = unpacked_tensor.DataAsSpan(); + std::transform(indice_span.begin(), indice_span.end(), indices.begin(), + [](int64_t indice_n) -> int32_t { return SafeInt(indice_n); }); } else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) { - for (uint32_t i = 0; i < size; i++) { - int32_t index; - memcpy(&index, unpacked_tensor.data() + i * sizeof(int32_t), sizeof(int32_t)); - indices[i] = SafeInt(index); - } + auto indice_span = unpacked_tensor.DataAsSpan(); + indices.assign(indice_span.begin(), indice_span.end()); } OperandType indices_operand_type(Type::TENSOR_INT32, indices_dimen); @@ -2684,20 +2659,14 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& initializers(model_builder.GetInitializerTensors()); const auto& tensor = *initializers.at(input_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - size_t tensor_byte_size = unpacked_tensor.size(); + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); const auto data_type = tensor.data_type(); if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) { - const int64_t* tensor_data = reinterpret_cast(unpacked_tensor.data()); - size_t size = tensor_byte_size / sizeof(int64_t); - data.insert(data.end(), tensor_data, tensor_data + size); + auto tensor_data = unpacked_tensor.DataAsSpan(); + data.insert(data.end(), tensor_data.begin(), tensor_data.end()); } else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) { - const int32_t* tensor_data = reinterpret_cast(unpacked_tensor.data()); - size_t size = tensor_byte_size / sizeof(int32_t); - data.insert(data.end(), tensor_data, tensor_data + size); + auto tensor_data = unpacked_tensor.DataAsSpan(); + data.insert(data.end(), tensor_data.begin(), tensor_data.end()); } else { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Data type for starts and ends inputs' is not supported in this build. Got ", @@ -2843,29 +2812,14 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto* pads_initializer = model_builder.GetConstantInitializer(pads); ORT_RETURN_IF_NOT(pads_initializer, "pads must be a constant"); - std::vector pads_initializer_raw_data{}; - ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*pads_initializer, pads_initializer_raw_data)); + Initializer pads_initializer_raw_data(*pads_initializer); // assume pads_initializer has int64 data, per ONNX spec - ORT_RETURN_IF_NOT(pads_initializer_raw_data.size() == 2 * data_rank * sizeof(int64_t), - "Expected pads initializer size in bytes: ", 2 * data_rank * sizeof(int64_t), - ", actual: ", pads_initializer_raw_data.size()); - std::vector converted_pads_data{}; converted_pads_data.reserve(2 * data_rank); - - auto copy_and_convert = [](const void* raw_i64_src, - std::back_insert_iterator i32_dst) { - int64_t i64; - memcpy(&i64, raw_i64_src, sizeof(i64)); - *i32_dst = SafeInt(i64); - }; - + auto pads_span = pads_initializer_raw_data.DataAsSpan(); for (size_t i = 0; i < data_rank; ++i) { - copy_and_convert(&pads_initializer_raw_data[i * sizeof(int64_t)], - std::back_inserter(converted_pads_data)); - - copy_and_convert(&pads_initializer_raw_data[(i + data_rank) * sizeof(int64_t)], - std::back_inserter(converted_pads_data)); + converted_pads_data.push_back(SafeInt(pads_span[i])); + converted_pads_data.push_back(SafeInt(pads_span[i + data_rank])); } const Shape converted_pads_shape{data_rank, 2}; @@ -2880,15 +2834,8 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto& constant_value = inputs[2].node_arg.Name(); const auto* constant_value_initializer = model_builder.GetConstantInitializer(constant_value); ORT_RETURN_IF_NOT(constant_value_initializer, "constant_value must be a constant"); - - std::vector pad_value_raw_data{}; - ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*constant_value_initializer, pad_value_raw_data)); - // assume constant_value_initializer has float data - // ONNX spec says it matches `data` input type, and op support checker limits that to float - ORT_RETURN_IF_NOT(pad_value_raw_data.size() == sizeof(float), - "Expected constant_value initializer size in bytes: ", sizeof(float), - ", actual size: ", pad_value_raw_data.size()); - memcpy(&pad_value, pad_value_raw_data.data(), sizeof(float)); + Initializer pad_value_raw_data_init(*constant_value_initializer); + pad_value = pad_value_raw_data_init.DataAsSpan()[0]; } ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 54c4cc5a52..aebba40590 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -12,7 +12,7 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" - +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace nnapi { @@ -184,18 +184,10 @@ static bool IsQuantizationZeroPointSupported(const InitializedTensorSet& initial << " zero point dimension " << zero_dim; return false; } - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, model_path, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name - << ", error msg: " << status.ErrorMessage(); - return false; - } - + Initializer unpacked_tensor(zero_tensor, model_path); // Verify all onnx weight zero point(s) are 0(s) - const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); - for (size_t i = 0; i < unpacked_tensor.size(); i++) { + auto zero_points = unpacked_tensor.DataAsSpan(); + for (int64_t i = 0; i < unpacked_tensor.size(); i++) { if (zero_points[i] != 0) { LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, " << "zero_points[" << i << "] has value: " << zero_points[i]; @@ -731,13 +723,8 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init } const auto& perm_tensor = *initializers.at(perm_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); - return false; - } - const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(perm_tensor); + auto raw_perm = unpacked_tensor.DataAsSpan(); const auto perm_size = SafeInt(perm_tensor.dims()[0]); NodeAttrHelper helper(node_unit); @@ -1996,13 +1983,8 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // We want to check if the scales or sizes are not trying to resize on N/C channels here if (inputs.size() == 3) { // we are using scales const auto& scales_tensor = *initializers.at(inputs[2].node_arg.Name()); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); - return false; - } - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); float scale_n = scales_data[0]; float scale_c = IsNodeLayoutNHWC(node_unit) ? scales_data[3] : scales_data[1]; if (scale_n != 1.0f || scale_c != 1.0f) { @@ -2015,15 +1997,9 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // we are using sizes const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); - return false; - } - + Initializer unpacked_tensor(sizes_tensor); int channel_idx = IsNodeLayoutNHWC(node_unit) ? 3 : 1; - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); + auto sizes_data = unpacked_tensor.DataAsSpan(); uint32_t size_n = SafeInt(sizes_data[0]); uint32_t size_c = SafeInt(sizes_data[channel_idx]); if (size_n != input_shape[0] || size_c != input_shape[channel_idx]) { @@ -2352,20 +2328,12 @@ bool PadOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initiali } const ONNX_NAMESPACE::TensorProto& pads_initializer = *pads_initializer_it->second; - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(pads_initializer, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking pads initializer: " << status.ErrorMessage(); - return false; - } - - int64_t pad_value; - ORT_ENFORCE(unpacked_tensor.size() % sizeof(pad_value) == 0); - for (size_t i = 0; i < unpacked_tensor.size(); i += sizeof(pad_value)) { - memcpy(&pad_value, &unpacked_tensor[i], sizeof(pad_value)); - if (pad_value < 0) { + Initializer unpacked_tensor(pads_initializer); + auto tensor_data = unpacked_tensor.DataAsSpan(); + for (int64_t i = 0; i < unpacked_tensor.size(); i++) { + if (tensor_data[i] < 0) { LOGS_DEFAULT(VERBOSE) << "Negative pad value is not supported: pads[" - << i / sizeof(pad_value) << "] = " << pad_value; + << i << "] = " << tensor_data[i]; return false; } } diff --git a/onnxruntime/core/providers/shared/utils/utils.cc b/onnxruntime/core/providers/shared/utils/utils.cc index d0e33062d7..89452e53a4 100644 --- a/onnxruntime/core/providers/shared/utils/utils.cc +++ b/onnxruntime/core/providers/shared/utils/utils.cc @@ -9,6 +9,7 @@ #include #include #include "core/providers/shared/node_unit/node_unit.h" +#include "core/optimizer/initializer.h" namespace onnxruntime { @@ -51,13 +52,8 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, LOGS(logger, VERBOSE) << "Input min of Clip must be known"; return false; } - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(min_name), unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking min tensor: " << status.ErrorMessage(); - return false; - } - min = reinterpret_cast(unpacked_tensor.data())[0]; + Initializer unpacked_tensor(*initializers.at(min_name)); + min = unpacked_tensor.DataAsSpan()[0]; } if (node.InputDefs().size() > 2) { // we have input max @@ -66,13 +62,8 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node, LOGS(logger, VERBOSE) << "Input max of Clip must be known"; return false; } - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(*initializers.at(max_name), unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking max tensor: " << status.ErrorMessage(); - return false; - } - max = reinterpret_cast(unpacked_tensor.data())[0]; + Initializer unpacked_tensor(*initializers.at(max_name)); + max = unpacked_tensor.DataAsSpan()[0]; } } diff --git a/onnxruntime/core/providers/xnnpack/detail/utils.cc b/onnxruntime/core/providers/xnnpack/detail/utils.cc index 497e36fdbd..3d80921694 100644 --- a/onnxruntime/core/providers/xnnpack/detail/utils.cc +++ b/onnxruntime/core/providers/xnnpack/detail/utils.cc @@ -2,10 +2,10 @@ // Licensed under the MIT License. #include "utils.h" -#include #include #include +#include "core/common/common.h" #include "core/framework/tensorprotoutils.h" #include "core/graph/indexed_sub_graph.h" #include "core/graph/node_attr_utils.h" @@ -14,6 +14,7 @@ #include "onnx/defs/attr_proto_util.h" #include "core/common/safeint.h" #include "core/optimizer/initializer.h" + namespace onnxruntime { namespace xnnpack { @@ -160,7 +161,8 @@ std::unique_ptr FuseQDQGroup(const NodeUnit& node_unit def.inputs.push_back(y_quant_param.scale.Name()); def.inputs.push_back(y_quant_param.zero_point ? y_quant_param.zero_point->Name() : ""); if (qtype == QuantizedOpType::QDQSoftmax) { - def.domain = kMSDomain; // reuse Qlinearsoftmax in kMSDomain, we need to define our own schema + def.domain = kDynamicDomainByCreate; + def.since_version = 1; def.attributes.emplace("opset", utils::MakeAttribute(std::string("opset"), int64_t(node_unit.SinceVersion()))); } } else if (qtype == QuantizedOpType::QDQMaxPool) { @@ -183,10 +185,11 @@ std::unique_ptr FuseQDQGroup(const NodeUnit& node_unit } // Fuse activation with node_unit. -std::unique_ptr FuseActivation(const NodeUnit& node_unit, const Node& activation, +std::unique_ptr FuseActivation(const NodeUnit& node_unit, const NodeUnit& activation_unit, const GraphViewer& graph) { std::unique_ptr metadef = std::make_unique(); IndexedSubGraph::MetaDef& def = *metadef; + const Node& activation = activation_unit.GetNode(); // we use the op type/domain to match the static xnnpack Conv or MaxPool kernel // registration @@ -260,20 +263,6 @@ std::unique_ptr FuseActivation(const NodeUnit& node_un return metadef; } -const onnx::TensorProto* GetQuantizationScale(const InitializedTensorSet& initializers, - const NodeUnitIODef& io_def) { - if (io_def.quant_param.has_value() == false) { - return nullptr; - } - - const auto scale_name = io_def.quant_param->scale.Name(); - auto it = initializers.find(scale_name); - if (it == initializers.cend()) { - return nullptr; - } - return it->second; -} - std::pair GetQuantizationZeroPointAndScale(const GraphViewer& graphview, const NodeUnitIODef& io_def) { @@ -318,7 +307,6 @@ TensorQuantType GetTensorQuantType(const NodeUnit& node_unit, int32_t io_index, return TensorTypeInvalid; } - std::vector unpacked_tensor; // we have processed float-type in the beginning // we do not handle u8s8 switch (input_type) { @@ -347,14 +335,9 @@ TensorQuantType GetTensorQuantType(const NodeUnit& node_unit, int32_t io_index, } else if (scales_dim == tensor_shape[0]) { // default 0 for zero-point if zero_dim == 0 if (zero_tensor != nullptr) { - auto status = utils::UnpackInitializerData(*zero_tensor, node_unit.ModelPath(), unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "error when unpack zero tensor: " - << ", error msg: " << status.ErrorMessage(); - break; - } - const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); - for (size_t i = 0; i < unpacked_tensor.size(); i++) { + Initializer zp_val(*zero_tensor, node_unit.ModelPath()); + auto zero_points = zp_val.DataAsSpan(); + for (int64_t i = 0; i < zp_val.size(); i++) { if (zero_points[i] != 0) { LOGS_DEFAULT(VERBOSE) << "only support 0 as zero point for per-channel quantization, " << "zero_points[" << i << "] has value: " << zero_points[i]; diff --git a/onnxruntime/core/providers/xnnpack/detail/utils.h b/onnxruntime/core/providers/xnnpack/detail/utils.h index 18b61a199d..752af9c537 100644 --- a/onnxruntime/core/providers/xnnpack/detail/utils.h +++ b/onnxruntime/core/providers/xnnpack/detail/utils.h @@ -21,6 +21,7 @@ namespace onnxruntime { class GraphViewer; class NodeUnit; namespace xnnpack { +constexpr const char* kDynamicDomainByCreate = "xnnpack"; enum OpComputeType : uint8_t { op_compute_type_invalid = 0, @@ -75,7 +76,7 @@ bool IsPaddingTypeSupported(AutoPadType auto_pad); using XnnpackOperator = std::unique_ptr; -std::unique_ptr FuseActivation(const NodeUnit& conv_unit, const Node& activation, +std::unique_ptr FuseActivation(const NodeUnit& conv_unit, const NodeUnit& activation, const GraphViewer& graph); std::unique_ptr FuseQDQGroup(const NodeUnit& unit_node); diff --git a/onnxruntime/core/providers/xnnpack/nn/softmax.cc b/onnxruntime/core/providers/xnnpack/nn/softmax.cc index fd66fb8ba0..4089d33345 100644 --- a/onnxruntime/core/providers/xnnpack/nn/softmax.cc +++ b/onnxruntime/core/providers/xnnpack/nn/softmax.cc @@ -9,7 +9,6 @@ #include "core/providers/cpu/math/softmax_shared.h" #include "core/optimizer/initializer.h" - namespace onnxruntime { namespace xnnpack { std::pair @@ -31,12 +30,12 @@ bool IsQuantSoftmaxSupported(const NodeUnit& node_unit, const GraphViewer& graph // qdq models converted from other framework auto [scale_tensor, zero_tensor] = GetQuantizationZeroPointAndScale(graph, node_unit.Outputs()[0]); Initializer q_scale(*scale_tensor, node_unit.ModelPath()); - if (fabs(*q_scale.data() - 1.0f / 256.0f) > 0.0001f) { + if (fabs(q_scale.DataAsSpan()[0] - 1.0f / 256.0f) > 0.0001f) { break; } - if (scale_tensor) { + if (zero_tensor) { Initializer q_zp(*zero_tensor, node_unit.ModelPath()); - if (*q_zp.raw_data() != 0) { + if (q_zp.DataAsSpan()[0] != 0) { break; } } @@ -231,8 +230,8 @@ ONNX_OPERATOR_KERNEL_EX(Softmax, kOnnxDomain, 13, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Softmax); -ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kMSDomain, 1, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), +ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kDynamicDomainByCreate, 1, kXnnpackExecutionProvider, + KernelDefBuilder(), // dynamic schema Softmax); } // namespace xnnpack diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc index 42865fe4ba..8b670e89dd 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include #include #include #include +#include "core/graph/function_utils.h" #include "xnnpack_execution_provider.h" #include "detail/utils.h" #include "detail/node_support_checker.h" @@ -33,7 +35,7 @@ KernelCreateInfo BuildKernelCreateInfo() { ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, Start, Op)> #define KERNEL_CREATE_INFO_TYPED(Start, type, Op) \ - BuildKernelCreateInfo< \ + BuildKernelCreateInfo< \ ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, Start, type, Op)> class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv); @@ -47,7 +49,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSIntern class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, int8_t, QLinearConv); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, QLinearAveragePool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, - kMSDomain, 1, QLinearSoftmax); + kDynamicDomainByCreate, 1, QLinearSoftmax); std::unique_ptr RegisterKernels() { auto kernel_registry = std::make_unique(); @@ -69,7 +71,7 @@ std::unique_ptr RegisterKernels() { KERNEL_CREATE_INFO_TYPED(10, int8_t, QLinearConv), KERNEL_CREATE_INFO(1, QLinearAveragePool), BuildKernelCreateInfo< - ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSDomain, 1, QLinearSoftmax)>, + ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kDynamicDomainByCreate, 1, QLinearSoftmax)>, }; for (auto& function_table_entry : function_table) { @@ -124,12 +126,21 @@ void XnnpackExecutionProvider::RegisterAllocator(AllocatorManager& allocator_man } } +// For ops are not lay-out sensitive and does not defined in +// onnx-domain, it will be created dynamicly +static bool RequestDynamicSchema(const NodeUnit& node_unit) { + static const InlinedHashSet dynamic_schema_set = {"QLinearSoftmax"}; + std::string key = node_unit.UnitType() == NodeUnit::Type::QDQGroup + ? "QLinear" + node_unit.OpType() : node_unit.OpType(); + return dynamic_schema_set.contains(key); +} + // Add Compute Capability for the second call. All target nodes have the tag of "XnnpackExecutionProvider" // after the first call. So we are going to do QDQ fusion in the second call // All nodes was collected in one sub_graph static void AddComputeCapabilityForNodeUnit(const NodeUnit& node_unit, - const std::function)>& adder, - std::unordered_map& supported_map) { + const std::function)>& adder, + std::unordered_map& supported_map) { std::unique_ptr sub_graph = std::make_unique(); auto process_node = [&sub_graph, &supported_map, &node_unit](const Node& node) { sub_graph->nodes.push_back(node.Index()); @@ -141,10 +152,13 @@ static void AddComputeCapabilityForNodeUnit(const NodeUnit& node_unit, process_node(*node_i); } sub_graph->SetMetaDef(FuseQDQGroup(node_unit)); - sub_graph->use_existing_schema = true; } else { process_node(node_unit.GetNode()); } + + sub_graph->schema_source = RequestDynamicSchema(node_unit) + ? IndexedSubGraph::SourceOfSchema::REUSE_OR_CREATE + : IndexedSubGraph::SourceOfSchema::EXISTING; adder(std::move(sub_graph)); } @@ -190,10 +204,10 @@ std::vector> XnnpackExecutionProvider::GetCap if (n == nullptr) { continue; } - const NodeUnit& node_unit = *node_unit_map[n]; // if node is part of a QDQ group, // we will mark it compatible in the first call as long as we support the target node. - const Node& node = *n; + const NodeUnit& node_unit = *node_unit_map[n]; + bool request_node = false; // any node in NodeUnit will trigger IsNodeSupported, so we just check once. if (node_unit_supported_result.count(&node_unit)) { @@ -221,9 +235,9 @@ std::vector> XnnpackExecutionProvider::GetCap // GraphPartitioner will match the statically registered xnnpack NHWC Conv kernel instead of // calling IExecutionProvider::Compile ComputeCapability& capability = *iter->second; - capability.sub_graph->SetMetaDef(FuseActivation(*fuse_with, node, graph)); - capability.sub_graph->nodes.push_back(node.Index()); - capability.sub_graph->use_existing_schema = true; + capability.sub_graph->SetMetaDef(FuseActivation(*fuse_with, node_unit, graph)); + capability.sub_graph->nodes.push_back(node_unit.Index()); + capability.sub_graph->schema_source = IndexedSubGraph::SourceOfSchema::EXISTING; } } } else if (node_unit.GetNode().GetExecutionProviderType() == Type()) {