From 7baf3749390589d72d9ff962059a76fbb6785d00 Mon Sep 17 00:00:00 2001 From: gwang-msft <62914304+gwang-msft@users.noreply.github.com> Date: Mon, 6 Jul 2020 18:44:04 -0700 Subject: [PATCH] Change the input to NNAPI EP ModelBuilder from ModelProto to GraphViewer (#4389) * init version to use graph instead of model_proto for IsOpSupported * move add to modelbuilder to use graph node * move the rest of model_builder to use graph instead of modelproto * remove redundant code * Clear some redundant code * merge master and some minor style changes * move check if an initializer is external to individual op instead the whole graph * Addressed comments * Change the GetType and GetShape to log waring info inside to simplify the caller, remove some redundant onnxruntime namespace * add squeeze op support, some more code style clean up * fix a bug where duplicate output can be added to a subgraph, some other minor logging changes --- .../nnapi/nnapi_builtin/builders/helper.cc | 99 +++ .../nnapi/nnapi_builtin/builders/helper.h | 47 +- .../nnapi_builtin/builders/model_builder.cc | 176 ++-- .../nnapi_builtin/builders/model_builder.h | 19 +- .../builders/node_attr_helper.cc | 104 --- .../nnapi_builtin/builders/node_attr_helper.h | 27 - .../nnapi_builtin/builders/op_builder.cc | 771 +++++++++--------- .../nnapi/nnapi_builtin/builders/op_builder.h | 17 +- .../nnapi/nnapi_builtin/builders/shaper.cc | 36 + .../nnapi/nnapi_builtin/builders/shaper.h | 35 +- .../nnapi_builtin/nnapi_execution_provider.cc | 54 +- .../nnapi_lib/NeuralNetworksWrapper.h | 2 - 12 files changed, 665 insertions(+), 722 deletions(-) create mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc delete mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc delete mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc new file mode 100644 index 0000000000..e12b480501 --- /dev/null +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -0,0 +1,99 @@ +// +// Created by daquexian on 8/3/18. +// + +#include +#include +#include +#include + +#include "helper.h" + +using std::string; +using std::vector; + +std::string GetErrorCause(int error_code) { + switch (error_code) { + case ANEURALNETWORKS_NO_ERROR: + return "ANEURALNETWORKS_NO_ERROR"; + case ANEURALNETWORKS_OUT_OF_MEMORY: + return "ANEURALNETWORKS_OUT_OF_MEMORY"; + case ANEURALNETWORKS_INCOMPLETE: + return "ANEURALNETWORKS_INCOMPLETE"; + case ANEURALNETWORKS_UNEXPECTED_NULL: + return "ANEURALNETWORKS_UNEXPECTED_NULL"; + case ANEURALNETWORKS_BAD_DATA: + return "ANEURALNETWORKS_BAD_DATA"; + case ANEURALNETWORKS_OP_FAILED: + return "ANEURALNETWORKS_OP_FAILED"; + case ANEURALNETWORKS_BAD_STATE: + return "ANEURALNETWORKS_BAD_STATE"; + case ANEURALNETWORKS_UNMAPPABLE: + return "ANEURALNETWORKS_UNMAPPABLE"; + case ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE: + return "ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE"; + case ANEURALNETWORKS_UNAVAILABLE_DEVICE: + return "ANEURALNETWORKS_UNAVAILABLE_DEVICE"; + + default: + return "Unknown error code: " + std::to_string(error_code); + } +} + +NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node) + : node_attributes_(node.GetAttributes()) {} + +float NodeAttrHelper::Get(const std::string& key, float def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).f(); + + return def_val; +} + +int32_t NodeAttrHelper::Get(const std::string& key, int32_t def_val) const { + if (HasAttr(key)) + return SafeInt(node_attributes_.at(key).i()); + + return def_val; +} + +string NodeAttrHelper::Get(const std::string& key, const string& def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).s(); + + return def_val; +} + +vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + int64_t val = attr.ints(j); + v.push_back(SafeInt(val)); + } + return v; + } + + return def_val; +} + +vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + v.push_back(attr.floats(j)); + } + + return v; + } + + return def_val; +} + +bool NodeAttrHelper::HasAttr(const std::string& key) const { + return Contains(node_attributes_, key); +} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index 4faf37b10c..605f5180aa 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -3,7 +3,7 @@ // #pragma once -#include +#include #include #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" @@ -34,30 +34,23 @@ inline bool Contains(const Map& map, const Key& key) { return map.find(key) != map.end(); } -inline std::string GetErrorCause(int error_code) { - switch (error_code) { - case ANEURALNETWORKS_NO_ERROR: - return "ANEURALNETWORKS_NO_ERROR"; - case ANEURALNETWORKS_OUT_OF_MEMORY: - return "ANEURALNETWORKS_OUT_OF_MEMORY"; - case ANEURALNETWORKS_INCOMPLETE: - return "ANEURALNETWORKS_INCOMPLETE"; - case ANEURALNETWORKS_UNEXPECTED_NULL: - return "ANEURALNETWORKS_UNEXPECTED_NULL"; - case ANEURALNETWORKS_BAD_DATA: - return "ANEURALNETWORKS_BAD_DATA"; - case ANEURALNETWORKS_OP_FAILED: - return "ANEURALNETWORKS_OP_FAILED"; - case ANEURALNETWORKS_BAD_STATE: - return "ANEURALNETWORKS_BAD_STATE"; - case ANEURALNETWORKS_UNMAPPABLE: - return "ANEURALNETWORKS_UNMAPPABLE"; - case ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE: - return "ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE"; - case ANEURALNETWORKS_UNAVAILABLE_DEVICE: - return "ANEURALNETWORKS_UNAVAILABLE_DEVICE"; +std::string GetErrorCause(int error_code); - default: - return "Unknown error code: " + std::to_string(error_code); - } -} +/** + * Wrapping onnxruntime::Node for retrieving attribute values + */ +class NodeAttrHelper { + public: + NodeAttrHelper(const onnxruntime::Node& proto); + + float Get(const std::string& key, float def_val) const; + int32_t Get(const std::string& key, int32_t def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::string Get(const std::string& key, const std::string& def_val) const; + + bool HasAttr(const std::string& key) const; + + private: + const onnxruntime::NodeAttributes& node_attributes_; +}; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 98844f9c73..c92aa2a11d 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -2,12 +2,12 @@ // Licensed under the MIT License. #include +#include -#include "core/common/safeint.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" #include "helper.h" #include "model_builder.h" -#include "node_attr_helper.h" +#include "op_builder.h" namespace onnxruntime { namespace nnapi { @@ -15,14 +15,8 @@ namespace nnapi { using namespace android::nn::wrapper; using std::vector; -const float* GetTensorFloatDataA(const ONNX_NAMESPACE::TensorProto& tensor) { - return tensor.float_data().empty() - ? reinterpret_cast(tensor.raw_data().data()) - : tensor.float_data().data(); -} - -ModelBuilder::ModelBuilder(ONNX_NAMESPACE::ModelProto& model_proto) - : nnapi_(NnApiImplementation()), model_proto_(model_proto) { +ModelBuilder::ModelBuilder(const GraphViewer& graph_view) + : nnapi_(NnApiImplementation()), graph_view_(graph_view) { GetAllInitializers(); op_builders_ = CreateOpBuilders(); } @@ -31,21 +25,20 @@ int32_t ModelBuilder::GetAndroidSdkVer() const { return nnapi_ ? nnapi_->android_sdk_version : 0; } -bool ModelBuilder::IsNodeSupported( - const ONNX_NAMESPACE::NodeProto& node) { - if (auto* opBuilder = GetOpBuilder(node)) { - return opBuilder->IsOpSupported(*this, node); +bool ModelBuilder::IsNodeSupported(const Node& node) { + if (auto* op_builder = GetOpBuilder(node)) { + return op_builder->IsOpSupported(*this, node); } else { return false; } } -bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, - const ONNX_NAMESPACE::ModelProto& model_proto) { +bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, const GraphViewer& graph_view) { if (!supported_node_vec.empty()) { if (supported_node_vec.size() == 1) { - const auto& node = model_proto.graph().node(supported_node_vec[0]); - const auto& op = node.op_type(); + const auto& node_indices = graph_view.GetNodesInTopologicalOrder(); + const auto* node(graph_view.GetNode(node_indices[supported_node_vec[0]])); + const auto& op = node->OpType(); // It is not worth it to perform a single Reshape/Dropout/Identity operator // which is only copying the data in NNAPI // If this is the case, let it fall back @@ -58,7 +51,7 @@ bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, return true; } return false; -} // namespace nnapi +} std::vector> ModelBuilder::GetSupportedNodes() { std::vector> supported_node_vecs; @@ -73,25 +66,26 @@ std::vector> ModelBuilder::GetSupportedNodes() { #endif std::vector supported_node_vec; - for (int i = 0; i < model_proto_.graph().node_size(); i++) { - const auto& node(model_proto_.graph().node(i)); - bool supported = IsNodeSupported(node); - LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node.op_type() + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + bool supported = IsNodeSupported(*node); + LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node->OpType() << "] index: [" << i - << "] name: [" << node.name() + << "] name: [" << node->Name() << "] supported: [" << supported << "]"; if (supported) { supported_node_vec.push_back(i); } else { - if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) { + if (IsValidSupportedNodesVec(supported_node_vec, graph_view_)) { supported_node_vecs.push_back(supported_node_vec); supported_node_vec.clear(); } } } - if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) + if (IsValidSupportedNodesVec(supported_node_vec, graph_view_)) supported_node_vecs.push_back(supported_node_vec); LOGS_DEFAULT(VERBOSE) << "Support vectors size is " << supported_node_vecs.size(); @@ -156,16 +150,16 @@ void ModelBuilder::GetTargetDevices() { const std::string nnapi_cpu("nnapi-reference"); uint32_t num_devices = 0; THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworks_getDeviceCount(&num_devices), - "Getting list of available devices"); + "Getting count of available devices"); for (uint32_t i = 0; i < num_devices; i++) { ANeuralNetworksDevice* device = nullptr; const char* device_name = nullptr; THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworks_getDevice(i, &device), - "Getting list of available devices"); + "Getting " + std::to_string(i) + "th device"); THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getName(device, &device_name), - "Getting list of available devices"); + "Getting " + std::to_string(i) + "th device's name"); bool device_is_cpu = nnapi_cpu == device_name; if ((target_device_option_ == TargetDeviceOption::CPU_DISABLED && !device_is_cpu) || @@ -177,27 +171,30 @@ void ModelBuilder::GetTargetDevices() { } void ModelBuilder::GetAllInitializers() { - for (const auto& tensor : model_proto_.graph().initializer()) { - initializers_.emplace(tensor.name(), tensor); + for (const auto& pair : graph_view_.GetAllInitializedTensors()) { + initializers_.emplace(pair.first, *pair.second); } } void ModelBuilder::PreprocessInitializers() { - for (const auto& node : model_proto_.graph().node()) { - if (auto* opBuilder = GetOpBuilder(node)) { - opBuilder->AddInitializersToSkip(*this, node); + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + if (auto* op_builder = GetOpBuilder(*node)) { + op_builder->AddInitializersToSkip(*this, *node); } } } void ModelBuilder::RegisterInitializers() { // First pass to get all the stats of the initializers - auto initializer_size = model_proto_.graph().initializer_size(); + auto initializer_size = initializers_.size(); std::vector> initializers(initializer_size); size_t sizeAll = 0; - for (int i = 0; i < initializer_size; ++i) { - const auto& tensor = model_proto_.graph().initializer(i); + int i = 0; + for (const auto& pair : initializers_) { + const auto& tensor = pair.second; const auto& name = tensor.name(); if (Contains(skipped_initializers_, name)) continue; @@ -226,23 +223,24 @@ void ModelBuilder::RegisterInitializers() { const size_t size = operand_type.GetOperandBlobByteSize(); const size_t padded_size = GetPaddedByteSize(size); sizeAll += padded_size; - initializers[i] = std::make_tuple(index, size, padded_size); + initializers[i++] = std::make_tuple(index, size, padded_size); } // 2nd pass copies all the initializer data into NNAPI shared memory + i = 0; nnapi_model_->mem_initializers_ = std::make_unique(nnapi_, "mem_initializers_", sizeAll); // 2nd pass to copy all the initializers into shared memory size_t offset = 0; - for (int i = 0; i < initializer_size; ++i) { - const auto& tensor = model_proto_.graph().initializer(i); + for (const auto& pair : initializers_) { + const auto& tensor = pair.second; if (Contains(skipped_initializers_, tensor.name())) continue; uint32_t index; size_t size, padded_size; - std::tie(index, size, padded_size) = initializers[i]; + std::tie(index, size, padded_size) = initializers[i++]; const char* src = nullptr; if (tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { src = tensor.float_data().empty() @@ -258,9 +256,8 @@ void ModelBuilder::RegisterInitializers() { } void ModelBuilder::RegisterModelInputs() { - for (int32_t input_idx = 0; input_idx < model_proto_.graph().input_size(); input_idx++) { - const auto& input(model_proto_.graph().input(input_idx)); - std::string input_name = input.name(); + for (const auto* node_arg : graph_view_.GetInputs()) { + const auto& input_name = node_arg->Name(); { // input should not be an initializer if (Contains(operands_, input_name)) @@ -270,15 +267,21 @@ void ModelBuilder::RegisterModelInputs() { continue; } + const auto* shape_proto = node_arg->Shape(); + ORT_ENFORCE(shape_proto != nullptr, "shape_proto cannot be null for input: " + input_name); Shaper::Shape shape; - for (const auto& dim : input.type().tensor_type().shape().dim()) { + + for (const auto& dim : shape_proto->dim()) { // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() shape.push_back(SafeInt(dim.dim_value())); } Type type = Type::TENSOR_FLOAT32; - if (input.type().tensor_type().has_elem_type()) { - switch (input.type().tensor_type().elem_type()) { + const auto* type_proto = node_arg->TypeAsProto(); + if (!type_proto || !type_proto->tensor_type().has_elem_type()) { + ORT_THROW("The input of graph doesn't have elem_type: " + input_name); + } else { + switch (type_proto->tensor_type().elem_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: type = Type::TENSOR_FLOAT32; break; @@ -286,27 +289,23 @@ void ModelBuilder::RegisterModelInputs() { // TODO: support other type ORT_THROW("The input of graph doesn't have valid type, name: " + input_name + " type: " + - std::to_string(input.type().tensor_type().elem_type())); + std::to_string(type_proto->tensor_type().elem_type())); } - } else { - ORT_THROW("The input of graph doesn't have elem_type: " + - input_name); } OperandType operand_type(type, shape); shaper_.AddShape(input_name, operand_type.dimensions); auto index = AddNewOperand(input_name, operand_type, false /* is_nhwc */); - input_index_vec_.push_back(index); nnapi_model_->AddInput(input_name, operand_type); } -} // namespace nnapi +} void ModelBuilder::RegisterModelOutputs() { - for (int32_t output_idx = 0; output_idx < model_proto_.graph().output_size(); output_idx++) { - const auto& output(model_proto_.graph().output(output_idx)); - const std::string& output_name(output.name()); + for (const auto* node_arg : graph_view_.GetOutputs()) { + const auto& output_name = node_arg->Name(); + if (!Contains(operands_, output_name)) { ORT_THROW("The output of graph is not registered" + output_name); } @@ -331,9 +330,7 @@ void ModelBuilder::RegisterModelShaper() { uint32_t ModelBuilder::AddNewOperand(const std::string& name, const OperandType& operand_type, bool is_nhwc) { - THROW_ON_ERROR(nnapi_->ANeuralNetworksModel_addOperand( - nnapi_model_->model_, &operand_type.operandType)); - auto idx = next_index_++; + auto idx = AddNewNNAPIOperand(operand_type); RegisterOperand(name, idx, operand_type, is_nhwc); return idx; } @@ -399,12 +396,13 @@ uint32_t ModelBuilder::AddOperandFromPersistMemoryBuffer( } void ModelBuilder::AddOperations() { - for (const auto& node : model_proto_.graph().node()) { - if (auto* opBuilder = GetOpBuilder(node)) { - opBuilder->AddToModelBuilder(*this, node); + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + if (auto* op_builder = GetOpBuilder(*node)) { + op_builder->AddToModelBuilder(*this, *node); } else { - throw std::invalid_argument( - "Node not supported" + node.name()); + ORT_THROW("Node [" + node->Name() + "], type [" + node->OpType() + "] is not supported"); } } } @@ -472,48 +470,44 @@ std::unique_ptr ModelBuilder::Compile() { return std::move(nnapi_model_); } -int32_t ModelBuilder::FindActivation(const std::string& output) { +int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) { int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; - const ONNX_NAMESPACE::NodeProto* activationNode{nullptr}; - std::string node_name; - for (const auto& _node : model_proto_.graph().node()) { - if (_node.op_type() == "Relu" && output == _node.input(0)) { - fuse_code = ANEURALNETWORKS_FUSED_RELU; - activationNode = &_node; + for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { + const auto& dst_node = it->GetNode(); + const auto* dst_input = dst_node.InputDefs()[it->GetDstArgIndex()]; + if (dst_node.OpType() == "Relu") { + if (&output == dst_input) { + fuse_code = ANEURALNETWORKS_FUSED_RELU; + } + } else { + // if there is any other non-relu node using the output + // will add relu separately + if (&output == dst_input) + return ANEURALNETWORKS_FUSED_NONE; } } + // if output is a graph output, will add relu separately if (fuse_code != ANEURALNETWORKS_FUSED_NONE) { - for (const auto& _node : model_proto_.graph().node()) { - if (&_node == activationNode) - continue; - - // if there is any other node using the output - // will add relu separately - for (const auto& node_input : _node.input()) { - if (output == node_input) - return ANEURALNETWORKS_FUSED_NONE; - } - } - - // if output is a graph output - // will add relu separately - for (const auto& model_output : model_proto_.graph().output()) { - if (model_output.name() == output) + for (const auto* graph_output : graph_view_.GetOutputs()) { + if (&output == graph_output) return ANEURALNETWORKS_FUSED_NONE; } - fused_activations_.insert(activationNode->name()); + LOGS_DEFAULT(VERBOSE) << "Node [" << node.Name() << "] type [" << node.OpType() + << "], fused the output [" << output.Name() << "]"; + + fused_activations_.insert(output.Name()); } return fuse_code; } -IOpBuilder* ModelBuilder::GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node) { - if (!Contains(op_builders_, node.op_type())) +IOpBuilder* ModelBuilder::GetOpBuilder(const Node& node) { + if (!Contains(op_builders_, node.OpType())) return nullptr; - return op_builders_[node.op_type()].get(); + return op_builders_[node.OpType()].get(); } std::string ModelBuilder::GetUniqueName(const std::string& base_name) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index d9ca4a1b69..2b22736106 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -5,14 +5,16 @@ #include #include +#include #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" -#include "op_builder.h" #include "shaper.h" namespace onnxruntime { namespace nnapi { +class IOpBuilder; + class ModelBuilder { public: using Shape = Shaper::Shape; @@ -26,7 +28,7 @@ class ModelBuilder { CPU_ONLY, // use CPU only }; - ModelBuilder(ONNX_NAMESPACE::ModelProto& model_proto); + ModelBuilder(const GraphViewer& graph_view); ~ModelBuilder() = default; std::vector> GetSupportedNodes(); @@ -42,7 +44,7 @@ class ModelBuilder { const std::vector& is_nhwc_vec); // Find if an output has a fuseable activation (Relu) - int32_t FindActivation(const std::string& output); + int32_t FindActivation(const Node& node, const NodeArg& output); // Add an NNAPI scalar operand uint32_t AddOperandFromScalar(bool value); @@ -89,11 +91,10 @@ class ModelBuilder { const std::unordered_set& GetFusedActivations() const { return fused_activations_; } - const std::unordered_map& + const std::unordered_map& GetInitializerTensors() const { return initializers_; } - const ONNX_NAMESPACE::ModelProto& GetOnnxModel() const { return model_proto_; } + const Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } void RegisterNHWCOperand(const std::string& name); bool IsOperandNHWC(const std::string& name); @@ -109,7 +110,7 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; - ONNX_NAMESPACE::ModelProto& model_proto_; + const GraphViewer& graph_view_; std::unique_ptr nnapi_model_; uint32_t name_token_{0}; @@ -149,7 +150,7 @@ class ModelBuilder { uint32_t next_index_ = 0; - bool IsNodeSupported(const ONNX_NAMESPACE::NodeProto& node); + bool IsNodeSupported(const Node& node); // Convert the onnx model to ANeuralNetworksModel void Prepare(); @@ -171,7 +172,7 @@ class ModelBuilder { const android::nn::wrapper::OperandType& operand_type, bool is_nhwc); - IOpBuilder* GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node); + IOpBuilder* GetOpBuilder(const Node& node); }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc deleted file mode 100644 index 7fee45cec4..0000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc +++ /dev/null @@ -1,104 +0,0 @@ -// -// Created by daquexian on 8/3/18. -// - -#include -#include -#include - -#include "core/common/safeint.h" -#include "node_attr_helper.h" - -using std::string; -using std::vector; - -NodeAttrHelper::NodeAttrHelper(const ONNX_NAMESPACE::NodeProto& proto) : node_(proto) { -} - -float NodeAttrHelper::Get(const std::string& key, float def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return attr.f(); - } - } - - return def_val; -} - -int32_t NodeAttrHelper::Get(const std::string& key, int32_t def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - int64_t val = attr.i(); - return SafeInt(val); - } - } - - return def_val; -} - -string NodeAttrHelper::Get(const std::string& key, const string& def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return attr.s(); - } - } - - return def_val; -} - -vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) { - if (!HasAttr(key)) { - return def_val; - } - - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - std::vector v; - v.reserve(static_cast(attr.ints_size())); - for (int j = 0; j < attr.ints_size(); j++) { - int64_t val = attr.ints(j); - v.push_back(SafeInt(val)); - } - return v; - } - } - - return def_val; -} - -vector NodeAttrHelper::Get(const std::string& key, - const vector& def_val) { - if (!HasAttr(key)) { - return def_val; - } - - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - std::vector v; - v.reserve(static_cast(attr.floats_size())); - for (int j = 0; j < attr.floats_size(); j++) { - v.push_back(attr.floats(j)); - } - - return v; - } - } - - return def_val; -} - -bool NodeAttrHelper::HasAttr(const std::string& key) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return true; - } - } - - return false; -} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h deleted file mode 100644 index 80594bc45a..0000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h +++ /dev/null @@ -1,27 +0,0 @@ -// -// Created by daquexian on 8/3/18. -// - -#pragma once - -#include -#include - -/** - * Wrapping onnx::NodeProto for retrieving attribute values - */ -class NodeAttrHelper { - public: - NodeAttrHelper(const ONNX_NAMESPACE::NodeProto& proto); - - float Get(const std::string& key, float def_val); - int32_t Get(const std::string& key, int32_t def_val); - std::vector Get(const std::string& key, const std::vector& def_val); - std::vector Get(const std::string& key, const std::vector& def_val); - std::string Get(const std::string& key, const std::string& def_val); - - bool HasAttr(const std::string& key); - - private: - const ONNX_NAMESPACE::NodeProto& node_; -}; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 3b69ee44a6..10864be333 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -2,12 +2,11 @@ // Licensed under the MIT License. #include +#include #include -#include "core/common/safeint.h" #include "helper.h" #include "model_builder.h" -#include "node_attr_helper.h" #include "op_builder.h" namespace onnxruntime { @@ -15,6 +14,7 @@ namespace nnapi { using namespace android::nn::wrapper; using std::vector; +using Shape = Shaper::Shape; #pragma region helpers @@ -43,7 +43,7 @@ void AddTransposeOperator(ModelBuilder& model_builder, std::vector input_indices; input_indices.push_back(operand_indices.at(input)); // input - ModelBuilder::Shape perm_dimen = {SafeInt(perm.size())}; + Shape perm_dimen = {SafeInt(perm.size())}; OperandType perm_operand_type(Type::TENSOR_INT32, perm_dimen); uint32_t perm_idx = model_builder.AddOperandFromPersistMemoryBuffer( perm_name, perm.data(), perm_operand_type); @@ -124,79 +124,32 @@ void AddBinaryOperator(int32_t op_type, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -int GetType(const ONNX_NAMESPACE::ModelProto& model_proto, - const std::string& name) { - int invalid_type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; - for (const auto& input : model_proto.graph().input()) { - if (input.name() != name) - continue; - - return input.type().tensor_type().elem_type(); +bool GetType(const NodeArg& node_arg, int32_t& type) { + type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; + const auto* type_proto = node_arg.TypeAsProto(); + if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) { + LOGS_DEFAULT(WARNING) << "NodeArg [" << node_arg.Name() << "] has no input type"; + return false; } - for (const auto& value_info : model_proto.graph().value_info()) { - if (value_info.name() != name) - continue; - - if (!value_info.has_type()) { - return invalid_type; - } else if (!value_info.type().has_tensor_type()) { - return invalid_type; - } - - return value_info.type().tensor_type().elem_type(); - } - - return invalid_type; + type = type_proto->tensor_type().elem_type(); + return true; } -Shaper::Shape GetShape(const ONNX_NAMESPACE::ModelProto& model_proto, - const std::string& name) { - Shaper::Shape empty_shape; - for (const auto& input : model_proto.graph().input()) { - if (input.name() != name) - continue; +bool GetShape(const NodeArg& node_arg, Shape& shape) { + shape.clear(); + const auto* shape_proto = node_arg.Shape(); - Shaper::Shape shape; - for (const auto& dim : input.type().tensor_type().shape().dim()) - shape.push_back(dim.dim_value()); - - return shape; + if (!shape_proto) { + LOGS_DEFAULT(WARNING) << "NodeArg [" << node_arg.Name() << "] has no shape info"; + return false; } - for (const auto& tensor : model_proto.graph().initializer()) { - if (tensor.name() != name) - continue; + // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() + for (const auto& dim : shape_proto->dim()) + shape.push_back(SafeInt(dim.dim_value())); - Shaper::Shape shape; - for (auto dim : tensor.dims()) - shape.push_back(SafeInt(dim)); - - return shape; - } - - for (const auto& value_info : model_proto.graph().value_info()) { - if (value_info.name() != name) - continue; - - if (!value_info.has_type()) { - return empty_shape; - } else if (!value_info.type().has_tensor_type()) { - return empty_shape; - } else if (!value_info.type().tensor_type().has_shape()) { - return empty_shape; - } else if (value_info.type().tensor_type().shape().dim_size() == 0) { - return empty_shape; - } - - Shaper::Shape shape; - for (const auto& dim : value_info.type().tensor_type().shape().dim()) - shape.push_back(dim.dim_value()); - - return shape; - } - - return empty_shape; + return true; } enum DataLayout { @@ -209,7 +162,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, const std::string& name, DataLayout new_layout) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - ModelBuilder::Shape shape; + Shape shape; for (auto dim : tensor.dims()) shape.push_back(SafeInt(dim)); @@ -227,7 +180,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, auto out_t = shape[0], in_t = shape[1], h_t = shape[2], w_t = shape[3]; - ModelBuilder::Shape dest_shape; + Shape dest_shape; if (new_layout == L_0231) dest_shape = {out_t, h_t, w_t, in_t}; // L_0231 else @@ -235,7 +188,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, const float* src = GetTensorFloatData(tensor); float* buffer = new float[Product(shape)]; - const OperandType operandType(type, dest_shape); + const OperandType operand_type(type, dest_shape); for (uint32_t out = 0; out < out_t; out++) { for (uint32_t in = 0; in < in_t; in++) { for (uint32_t h = 0; h < h_t; h++) { @@ -264,7 +217,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operandType); + auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); delete[] buffer; return operand_idx; } @@ -273,7 +226,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, uint32_t AddInitializerTransposed(ModelBuilder& model_builder, const std::string& name) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - ModelBuilder::Shape shape; + Shape shape; for (auto dim : tensor.dims()) shape.push_back(SafeInt(dim)); @@ -290,8 +243,8 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, } auto x_t = shape[0], y_t = shape[1]; - ModelBuilder::Shape dest_shape = {y_t, x_t}; - const OperandType operandType(type, dest_shape); + Shape dest_shape = {y_t, x_t}; + const OperandType operand_type(type, dest_shape); const float* src = GetTensorFloatData(tensor); float* buffer = new float[Product(shape)]; for (uint32_t x = 0; x < x_t; x++) { @@ -299,7 +252,7 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, buffer[y * x_t + x] = src[x * y_t + y]; } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operandType); + auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); delete[] buffer; return operand_idx; @@ -312,57 +265,56 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, class BaseOpBuilder : public IOpBuilder { public: virtual ~BaseOpBuilder() = default; - virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) override {} + virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const Node& /* node */) override {} - bool IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override final; + bool IsOpSupported(ModelBuilder& model_builder, const Node& node) override final; - void AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override final; + void AddToModelBuilder(ModelBuilder& model_builder, const Node& node) override final; protected: - virtual bool IsOpSupportedImpl( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node); - virtual int32_t GetMinSupportedSdkVer( - ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const { return 27; } + virtual int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const Node& /* node */) const { return 27; } - virtual bool HasSupportedInputs( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + virtual bool HasSupportedInputs(const Node& node); - virtual void AddToModelBuilderImpl( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + virtual void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) = 0; + + bool HasExternalInitializer(ModelBuilder& model_builder, const Node& node); }; -bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, const Node& node) { #ifdef __ANDROID__ int32_t android_sdk_ver = model_builder.GetAndroidSdkVer(); int32_t required_sdk_ver = GetMinSupportedSdkVer(model_builder, node); if (required_sdk_ver > android_sdk_ver) { LOGS_DEFAULT(VERBOSE) << "Current Android API level [" << android_sdk_ver - << "], Operator [" << node.op_type() + << "], Operator [" << node.OpType() << "] is only supported on API >" << required_sdk_ver; return false; } #endif - if (!HasSupportedInputs(model_builder, node)) + if (!HasSupportedInputs(node)) + return false; + + // We do not support external initializers for now + if (HasExternalInitializer(model_builder, node)) return false; return IsOpSupportedImpl(model_builder, node); -} // namespace nnapi +} -bool BaseOpBuilder::HasSupportedInputs( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool BaseOpBuilder::HasSupportedInputs(const Node& node) { // We only check the type of input 0 by default // specific op builder can override this - auto input_type = GetType(model_builder.GetOnnxModel(), node.input(0)); + int32_t input_type; + if (!GetType(*node.InputDefs()[0], input_type)) + return false; + if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - LOGS_DEFAULT(VERBOSE) << "[" << node.op_type() + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] Input type: [" << input_type << "] is not supported for now"; return false; @@ -371,24 +323,36 @@ bool BaseOpBuilder::HasSupportedInputs( return true; } -bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) { +bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& /* node */) { return true; } -void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const Node& node) { ORT_ENFORCE(IsOpSupported(model_builder, node), - "Unsupported operator " + node.op_type()); + "Unsupported operator " + node.OpType()); AddToModelBuilderImpl(model_builder, node); - LOGS_DEFAULT(VERBOSE) << "Operator name: [" << node.name() - << "] type: [" << node.op_type() << "] was added"; + LOGS_DEFAULT(VERBOSE) << "Operator name: [" << node.Name() + << "] type: [" << node.OpType() << "] was added"; } -void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) { - ORT_NOT_IMPLEMENTED("Unsupported operator " + node.op_type()); +bool BaseOpBuilder::HasExternalInitializer(ModelBuilder& model_builder, const Node& node) { + const auto& initializers(model_builder.GetOnnxGraph().GetAllInitializedTensors()); + for (const auto* node_arg : node.InputDefs()) { + const auto& input_name(node_arg->Name()); + if (!Contains(initializers, input_name)) + continue; + + const auto* tensor = initializers.at(input_name); + if (tensor->has_data_location() && + tensor->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name + << "] with external data location are not currently supported"; + return true; + } + } + + return false; } #pragma endregion op_base @@ -397,17 +361,14 @@ void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, class BinaryOpBuilder : public BaseOpBuilder { private: - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& node) const override; private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) const { - const auto& op(node.op_type()); +int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& node) const { + const auto& op(node.OpType()); if (op == "Sub" || op == "Div") { return 28; } @@ -415,9 +376,8 @@ int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */ return 27; } -void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op(node.op_type()); +void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + const auto& op(node.OpType()); int32_t op_code; if (op == "Add") op_code = ANEURALNETWORKS_ADD; @@ -430,8 +390,10 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, else { ORT_THROW("UnaryOpBuilder, unknown op: " + op); } - std::string input1 = node.input(0); - std::string input2 = node.input(1); + std::string input1 = node.InputDefs()[0]->Name(); + std::string input2 = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool input1_is_nhwc = model_builder.IsOperandNHWC(input1); bool input2_is_nhwc = model_builder.IsOperandNHWC(input2); bool output_is_nhwc = false; @@ -440,22 +402,21 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, output_is_nhwc = input1_is_nhwc; } else if (input1_is_nhwc) { // need transpsoe input1 back to nchw - const auto& nhwc_input = node.input(0); + const auto& nhwc_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNCHWOperand(nhwc_input, input1)) { input1 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); TransposeNHWCToNCHW(model_builder, nhwc_input, input1); } } else { // input2_is_nhwc // need transpsoe input2 back to nchw - const auto& nhwc_input = node.input(1); + const auto& nhwc_input = node.InputDefs()[1]->Name(); if (!model_builder.GetNCHWOperand(nhwc_input, input2)) { input2 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); TransposeNHWCToNCHW(model_builder, nhwc_input, input2); } } - const auto& output = node.output(0); - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); } @@ -465,24 +426,23 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ReluOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.input(0); - const auto& output = node.output(0); + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); shaper.Identity(input, output); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); // skip this relu if it is some op's fuse output - if (Contains(model_builder.GetFusedActivations(), node.name())) { + if (Contains(model_builder.GetFusedActivations(), input)) { + LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); } else { std::vector input_indices; @@ -497,24 +457,23 @@ void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class TransposeOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 28; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); - if (input_size > 4) { - LOGS_DEFAULT(VERBOSE) << "Transpose only supports up to 4d shape, input is " +bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Transpose only supports 1-4d shape, input is " << input_size << "d shape"; return false; } @@ -522,12 +481,11 @@ bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); - auto input = node.input(0); - const auto& output = node.output(0); + auto input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); NodeAttrHelper helper(node); vector perm = helper.Get("perm", vector()); auto input_dims = shaper[input].size(); @@ -547,7 +505,7 @@ void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, perm[i] = axis_nchw_to_nhwc[perm[i]]; } - std::string perm_name = model_builder.GetUniqueName(node.name() + input + "perm"); + std::string perm_name = model_builder.GetUniqueName(node.Name() + input + "perm"); // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see // any scenario will do this since onnx is nchw only, assume the output is always not nhwc @@ -562,48 +520,44 @@ void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ReshapeOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - model_builder.AddInitializerToSkip(node.input(1)); +void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } -bool ReshapeOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { const auto& initializers(model_builder.GetInitializerTensors()); - if (!Contains(initializers, node.input(1))) { + const auto& perm_name = node.InputDefs()[1]->Name(); + if (!Contains(initializers, perm_name)) { LOGS_DEFAULT(VERBOSE) << "New shape of reshape must be known"; return false; } - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); - if (input_size > 4) { - LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 4d shape, input is " - << input_size << "d shape"; + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + if (input_shape.size() > 4 || input_shape.empty()) { + LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 1-4d shape, input is " + << input_shape.size() << "d shape"; return false; } - const auto& shape_tensor = initializers.at(node.input(1)); + const auto& shape_tensor = initializers.at(perm_name); const int64_t* rawShape = GetTensorInt64Data(shape_tensor); const auto size = SafeInt(shape_tensor.dims()[0]); - const auto input_shape = GetShape(model_builder.GetOnnxModel(), node.input(0)); for (uint32_t i = 0; i < size; i++) { // NNAPI reshape does not support 0 as dimension if (rawShape[i] == 0 && i < input_shape.size() && input_shape[i] == 0) { - LOGS_DEFAULT(VERBOSE) - << "Reshape doesn't suppport 0 reshape dimension on a dynamic dimension"; + LOGS_DEFAULT(VERBOSE) << "Reshape doesn't suppport 0 reshape dimension on a dynamic dimension"; return false; } } @@ -611,33 +565,31 @@ bool ReshapeOpBuilder::IsOpSupportedImpl( return true; } -void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - auto input = node.input(0); - + auto input = node.InputDefs()[0]->Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before reshape - const auto& nhwc_input = node.input(0); + const auto& nhwc_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNCHWOperand(nhwc_input, input)) { input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); TransposeNHWCToNCHW(model_builder, nhwc_input, input); } } - const auto& output = node.output(0); + const auto& output = node.OutputDefs()[0]->Name(); std::vector input_indices; input_indices.push_back(operand_indices.at(input)); // input - const auto& shape_tensor = initializers.at(node.input(1)); + const auto& shape_tensor = initializers.at(node.InputDefs()[1]->Name()); const int64_t* rawShape = GetTensorInt64Data(shape_tensor); const auto size = SafeInt(shape_tensor.dims()[0]); - ModelBuilder::Shape input_shape = shaper[input]; + Shape input_shape = shaper[input]; std::vector shape(size); for (uint32_t i = 0; i < size; i++) { int32_t dim = SafeInt(rawShape[i]); @@ -645,8 +597,8 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, shape[i] = dim == 0 ? input_shape[i] : dim; } - ModelBuilder::Shape shape_dimen = {size}; - std::string shape_name = model_builder.GetUniqueName(node.name() + input + "newshape"); + Shape shape_dimen = {size}; + std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); uint32_t shape_idx = model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type); input_indices.push_back(shape_idx); @@ -663,39 +615,34 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class BatchNormalizationOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { // skip everything except input0 for BatchNormalization - model_builder.AddInitializerToSkip(node.input(1)); // scale - model_builder.AddInitializerToSkip(node.input(2)); // B - model_builder.AddInitializerToSkip(node.input(3)); // mean - model_builder.AddInitializerToSkip(node.input(4)); //var + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // scale + model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // B + model_builder.AddInitializerToSkip(node.InputDefs()[3]->Name()); // mean + model_builder.AddInitializerToSkip(node.InputDefs()[4]->Name()); //var } -bool BatchNormalizationOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - if (node.output_size() != 1) { +bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { + if (node.OutputDefs().size() != 1) { LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export " "it in test mode."; return false; } const auto& initializers(model_builder.GetInitializerTensors()); - const auto& scale_name = node.input(1); - const auto& b_name = node.input(2); - const auto& mean_name = node.input(3); - const auto& var_name = node.input(4); + const auto& scale_name = node.InputDefs()[1]->Name(); + const auto& b_name = node.InputDefs()[2]->Name(); + const auto& mean_name = node.InputDefs()[3]->Name(); + const auto& var_name = node.InputDefs()[4]->Name(); if (!Contains(initializers, scale_name)) { LOGS_DEFAULT(VERBOSE) << "Scale of BN must be known"; return false; @@ -716,8 +663,7 @@ bool BatchNormalizationOpBuilder::IsOpSupportedImpl( return true; } -void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); @@ -725,13 +671,13 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil // For reshape we are not really doing anything but // register a new operand with new shape - const auto& input = node.input(0); - const auto& output = node.output(0); + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); - const auto& scale_tensor = initializers.at(node.input(1)); - const auto& bias_tensor = initializers.at(node.input(2)); - const auto& mean_tensor = initializers.at(node.input(3)); - const auto& var_tensor = initializers.at(node.input(4)); + const auto& scale_tensor = initializers.at(node.InputDefs()[1]->Name()); + const auto& bias_tensor = initializers.at(node.InputDefs()[2]->Name()); + const auto& mean_tensor = initializers.at(node.InputDefs()[3]->Name()); + const auto& var_tensor = initializers.at(node.InputDefs()[4]->Name()); const auto eps = helper.Get("epsilon", 1e-5f); const auto size = SafeInt(scale_tensor.dims()[0]); @@ -750,10 +696,10 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil bias_data[i]); } - const auto tensor_a_name = model_builder.GetUniqueName(node.name() + input + "_imm_a"); - const auto tensor_b_name = model_builder.GetUniqueName(node.name() + input + "_imm_b"); - const auto tensor_imm_product_name = model_builder.GetUniqueName(node.name() + input + "_imm_mul"); - ModelBuilder::Shape tensor_a_dimen; + const auto tensor_a_name = model_builder.GetUniqueName(node.Name() + input + "_imm_a"); + const auto tensor_b_name = model_builder.GetUniqueName(node.Name() + input + "_imm_b"); + const auto tensor_imm_product_name = model_builder.GetUniqueName(node.Name() + input + "_imm_mul"); + Shape tensor_a_dimen; bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = input_is_nhwc; @@ -764,10 +710,10 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil shaper.AddShape(tensor_a_name, tensor_a_dimen); shaper.AddShape(tensor_b_name, tensor_a_dimen); - const OperandType operandType_a(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), operandType_a); - const OperandType operandType_b(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), operandType_b); + const OperandType a_operand_type(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), a_operand_type); + const OperandType b_operand_type(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), b_operand_type); // Mul AddBinaryOperator(ANEURALNETWORKS_MUL, @@ -778,7 +724,7 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil output_is_nhwc); // Add - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); AddBinaryOperator(ANEURALNETWORKS_ADD, model_builder, tensor_imm_product_name, tensor_b_name, @@ -793,22 +739,17 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil class PoolOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { - return 29; + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { + return 28; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); +bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + const auto& op = node.OpType(); if (op == "AveragePool" || op == "MaxPool") { NodeAttrHelper helper(node); @@ -845,16 +786,20 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return false; } - if (node.output_size() != 1) { + if (node.OutputDefs().size() != 1) { LOGS_DEFAULT(VERBOSE) << "Argmax in maxpooling is not supported"; return false; } } else if (op == "GlobalAveragePool" || op == "GlobalMaxPool") { - const auto input_shape = GetShape(model_builder.GetOnnxModel(), node.input(0)); - if (input_shape.size() > 4) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); + if (input_size != 4) { LOGS_DEFAULT(VERBOSE) << "GlobalAveragePool/GlobalMaxPool Only rank-4 tensor is supported in " - << node.input(0) << ", actual dim count " << input_shape.size(); + << node.InputDefs()[0]->Name() << ", actual dim count " << input_size; return false; } } @@ -862,15 +807,14 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node); - auto input = node.input(0); + auto input = node.InputDefs()[0]->Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -879,7 +823,7 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } else { output_is_nhwc = true; if (!input_is_nhwc) { - const auto& nchw_input = node.input(0); + const auto& nchw_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNHWCOperand(nchw_input, input)) { input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); TransposeNCHWToNHWC(model_builder, nchw_input, input); @@ -887,8 +831,8 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } - const auto& output = node.output(0); - const auto& op = node.op_type(); + const auto& output = node.OutputDefs()[0]->Name(); + const auto& op = node.OpType(); int32_t op_type; if (op == "AveragePool" || op == "GlobalAveragePool") @@ -912,7 +856,7 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, static_cast(shaper[input][2])}; } - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); std::vector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); @@ -940,25 +884,20 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ConvOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { // skip the weight for conv as we need to transpose - model_builder.AddInitializerToSkip(node.input(1)); + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } -bool ConvOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { NodeAttrHelper helper(node); if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; @@ -966,7 +905,7 @@ bool ConvOpBuilder::IsOpSupportedImpl( } const auto group = helper.Get("group", 1); - const auto weight_name = node.input(1); + const auto weight_name = node.InputDefs()[1]->Name(); if (Contains(model_builder.GetInitializerTensors(), weight_name)) { const auto& tensor = model_builder.GetInitializerTensors().at(weight_name); if (tensor.dims().size() != 4) { @@ -985,8 +924,7 @@ bool ConvOpBuilder::IsOpSupportedImpl( return true; } -void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -1006,7 +944,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); const auto group = helper.Get("group", 1); - auto input = node.input(0); + auto input = node.InputDefs()[0]->Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -1015,7 +953,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } else { output_is_nhwc = true; if (!input_is_nhwc) { - const auto& nchw_input = node.input(0); + const auto& nchw_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNHWCOperand(nchw_input, input)) { input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); TransposeNCHWToNHWC(model_builder, nchw_input, input); @@ -1023,8 +961,8 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } - const auto& weight = node.input(1); - const auto& output = node.output(0); + const auto& weight = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); bool conv2d = (group == 1); const auto& weight_tensor = initializers.at(weight); @@ -1041,15 +979,15 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder, weight, L_1230)); } - bool hasBias = (node.input_size() >= 3); - std::string bias = hasBias ? node.input(2) : weight + "_bias"; + bool hasBias = (node.InputDefs().size() >= 3); + std::string bias = hasBias ? node.InputDefs()[2]->Name() : weight + "_bias"; uint32_t bias_idx_val; if (hasBias) { bias_idx_val = operand_indices.at(bias); } else { const auto weight_dimen = shaper[weight]; - ModelBuilder::Shape bias_dimen; + Shape bias_dimen; if (conv2d) bias_dimen = {weight_dimen[0]}; else @@ -1061,9 +999,9 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, for (uint32_t i = 0; i < buffer.size(); i++) { buffer[i] = 0.f; } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); bias_idx_val = model_builder.AddOperandFromPersistMemoryBuffer( - bias, buffer.data(), operandType); + bias, buffer.data(), bias_operand_type); } else { ORT_THROW("Unknown weight type " + TypeToStr(weight_type)); } @@ -1080,7 +1018,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, int32_t depthwiseMultiplier = shaper[weight][3] / group; input_indices.push_back(model_builder.AddOperandFromScalar(depthwiseMultiplier)); } - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); // TODO support API 28 input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); @@ -1112,22 +1050,16 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class CastOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 29; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool CastOpBuilder::IsOpSupportedImpl( - ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) { +bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { NodeAttrHelper helper(node); auto to = helper.Get("to", 0); if (to != ONNX_NAMESPACE::TensorProto::FLOAT && @@ -1139,14 +1071,13 @@ bool CastOpBuilder::IsOpSupportedImpl( return true; } -void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); NodeAttrHelper helper(node); - const auto& input = node.input(0); - const auto& output = node.output(0); + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); auto to = helper.Get("to", 0); @@ -1177,23 +1108,21 @@ void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class SoftMaxOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 29; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool SoftMaxOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); +bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); if (input_size != 2 && input_size != 4) { LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " << input_size << "d shape"; @@ -1202,24 +1131,23 @@ bool SoftMaxOpBuilder::IsOpSupportedImpl( return true; } -void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node); - auto input = node.input(0); + auto input = node.InputDefs()[0]->Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before softmax - const auto& nhwc_input = node.input(0); + const auto& nhwc_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNCHWOperand(nhwc_input, input)) { input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); TransposeNHWCToNCHW(model_builder, nhwc_input, input); } } - const auto& output = node.output(0); + const auto& output = node.OutputDefs()[0]->Name(); float beta = 1.f; int32_t axis = helper.Get("axis", 1); std::vector input_indices; @@ -1239,20 +1167,18 @@ void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class IdentityOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { // Identity is not really going to do anything // Just register the dimension and type, with same index and new name auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input = node.input(0); - const auto& output = node.output(0); + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); std::vector input_indices; @@ -1269,35 +1195,42 @@ void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class GemmOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool GemmOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); +bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { + const auto& op = node.OpType(); const auto& initializers(model_builder.GetInitializerTensors()); - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "A must be 2D"; - return false; + Shape a_shape; + { + if (!GetShape(*node.InputDefs()[0], a_shape)) + return false; + + if (a_shape.size() != 2) { + LOGS_DEFAULT(VERBOSE) << "A must be 2D"; + return false; + } } - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "B must be 2D"; - return false; + Shape b_shape; + { + if (!GetShape(*node.InputDefs()[1], b_shape)) + return false; + + if (b_shape.size() != 2) { + LOGS_DEFAULT(VERBOSE) << "B must be 2D"; + return false; + } } if (op == "MatMul") { // Only support A*B B is an initializer - if (!Contains(initializers, node.input(1))) { + if (!Contains(initializers, node.InputDefs()[1]->Name())) { LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; return false; } @@ -1317,14 +1250,16 @@ bool GemmOpBuilder::IsOpSupportedImpl( return false; } - if (transB == 0 && !Contains(initializers, node.input(1))) { + if (transB == 0 && !Contains(initializers, node.InputDefs()[1]->Name())) { LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1"; return false; } - if (node.input_size() == 3) { - const auto b_shape = GetShape(model_builder.GetOnnxModel(), node.input(1)); - const auto c_shape = GetShape(model_builder.GetOnnxModel(), node.input(2)); + if (node.InputDefs().size() == 3) { + Shape c_shape; + if (!GetShape(*node.InputDefs()[2], c_shape)) + return false; + if (c_shape.size() != 1 || c_shape[0] != (transB == 0 ? b_shape[1] : b_shape[0])) { LOGS_DEFAULT(VERBOSE) << "C of Gemm must be a vector of b_shape[0]" @@ -1339,30 +1274,28 @@ bool GemmOpBuilder::IsOpSupportedImpl( return true; } -void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); +void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { + const auto& op = node.OpType(); if (op == "MatMul") { - model_builder.AddInitializerToSkip(node.input(1)); + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } else if (op == "Gemm") { NodeAttrHelper helper(node); const auto transB = helper.Get("transB", 0); if (transB == 0) - model_builder.AddInitializerToSkip(node.input(1)); + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } } -void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); +void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + const auto& op = node.OpType(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node); - const auto& input1 = node.input(0); - const auto& input2 = node.input(1); - const auto& output = node.output(0); + const auto& input1 = node.InputDefs()[0]->Name(); + const auto& input2 = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); const auto transB = helper.Get("transB", 0); uint32_t input_2_idx; @@ -1373,30 +1306,30 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } uint32_t bias_idx; - if (node.input_size() == 2) { - std::string bias = node.name() + op + "_bias"; + if (node.InputDefs().size() == 2) { + std::string bias = node.Name() + op + "_bias"; const auto& B_type = operand_types.at(input2).type; - ModelBuilder::Shape bias_dimen = {shaper[input2][0]}; + Shape bias_dimen = {shaper[input2][0]}; if (B_type == Type::TENSOR_FLOAT32) { float buffer[bias_dimen[0]]; for (uint32_t i = 0; i < bias_dimen[0]; i++) { buffer[i] = 0.f; } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( - bias, &buffer[0], operandType); + bias, &buffer[0], bias_operand_type); } else { ORT_THROW("Unknown weight type " + TypeToStr(B_type)); } } else { - bias_idx = operand_indices.at(node.input(2)); + bias_idx = operand_indices.at(node.InputDefs()[2]->Name()); } std::vector input_indices; input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); shaper.FC(input1, input2, output); @@ -1411,17 +1344,13 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class UnaryOpBuilder : public BaseOpBuilder { private: - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& node) const override; - void AddToModelBuilderImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) const { - const auto& op(node.op_type()); +int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& node) const { + const auto& op(node.OpType()); if (op == "Abs" || op == "Exp" || op == "Neg" || @@ -1434,41 +1363,40 @@ int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, return 27; } -void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& op(node.op_type()); + const auto& op_type(node.OpType()); - const auto& input = node.input(0); - const auto& output = node.output(0); + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); bool output_is_nhwc = model_builder.IsOperandNHWC(input); shaper.Identity(input, output); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); int32_t op_code; - if (op == "Abs") + if (op_type == "Abs") op_code = ANEURALNETWORKS_ABS; - else if (op == "Exp") + else if (op_type == "Exp") op_code = ANEURALNETWORKS_EXP; - else if (op == "Floor") + else if (op_type == "Floor") op_code = ANEURALNETWORKS_FLOOR; - else if (op == "Log") + else if (op_type == "Log") op_code = ANEURALNETWORKS_LOG; - else if (op == "Sigmoid") + else if (op_type == "Sigmoid") op_code = ANEURALNETWORKS_LOGISTIC; - else if (op == "Neg") + else if (op_type == "Neg") op_code = ANEURALNETWORKS_NEG; - else if (op == "Sin") + else if (op_type == "Sin") op_code = ANEURALNETWORKS_SIN; - else if (op == "Sqrt") + else if (op_type == "Sqrt") op_code = ANEURALNETWORKS_SQRT; - else if (op == "Tanh") + else if (op_type == "Tanh") op_code = ANEURALNETWORKS_TANH; else { - ORT_THROW("UnaryOpBuilder, unknown op: " + op); + ORT_THROW("UnaryOpBuilder, unknown op: " + op_type); } std::vector input_indices; input_indices.push_back(operand_indices.at(input)); @@ -1481,58 +1409,64 @@ void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ConcatOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() > 4) { - LOGS_DEFAULT(VERBOSE) << "Concat supports at most 4D shape"; +bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Concat only supports up to 1-4d shape, input is " + << input_size << "d shape"; return false; } return true; } -void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node); std::vector input_indices; - const auto& input0 = node.input(0); + const auto& input0 = node.InputDefs()[0]->Name(); bool all_input_have_same_layout = true; bool output_is_nhwc = false; + const auto node_input_size = node.InputDefs().size(); // First we want to see if all the input are smae layout - for (int i = 0; i < node.input_size() - 1; i++) { + for (size_t i = 0; i < node_input_size - 1; i++) { all_input_have_same_layout = all_input_have_same_layout && - model_builder.IsOperandNHWC(node.input(i)) == model_builder.IsOperandNHWC(node.input(i + 1)); + model_builder.IsOperandNHWC(node.InputDefs()[i]->Name()) == + model_builder.IsOperandNHWC(node.InputDefs()[i + 1]->Name()); } std::vector inputs; - inputs.reserve(node.input_size()); + inputs.reserve(node_input_size); if (all_input_have_same_layout) { // if all the inputs are of same layout, output will be the same layout if (model_builder.IsOperandNHWC(input0)) { output_is_nhwc = true; } - for (const auto& input : node.input()) { + for (size_t i = 0; i < node_input_size; i++) { + auto input = node.InputDefs()[i]->Name(); input_indices.push_back(operand_indices.at(input)); inputs.push_back(input); } } else { // if all the inputs are not same layout, // will need transpos those nhwc tensors back to nchw - for (auto input : node.input()) { + for (size_t i = 0; i < node_input_size; i++) { + auto input = node.InputDefs()[i]->Name(); if (model_builder.IsOperandNHWC(input)) { std::string nhwc_input = input; input = model_builder.GetUniqueName(input + "_nhwc_to_nchw"); @@ -1558,7 +1492,7 @@ void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } input_indices.push_back(model_builder.AddOperandFromScalar(axis)); - const auto& output = node.output(0); + const auto& output = node.OutputDefs()[0]->Name(); shaper.Concat(inputs, axis, output); const OperandType output_operand_type(operand_types.at(input0).type, shaper[output]); model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, {output}, @@ -1567,6 +1501,78 @@ void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, #pragma endregion +#pragma region op_squeeze + +class SqueezeOpBuilder : public BaseOpBuilder { + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { + return 28; + } + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; +}; + +bool SqueezeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Squeeze only supports 1-4d shape, input is " + << input_size << "d shape"; + return false; + } + + return true; +} + +void SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + auto input = node.InputDefs()[0]->Name(); + if (model_builder.IsOperandNHWC(input)) { + // We want to transpose nhwc operand back to nchw before squeeze + const auto& nhwc_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input)) { + input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input); + } + } + + NodeAttrHelper helper(node); + vector axes = helper.Get("axes", vector()); + auto input_dims = shaper[input].size(); + for (auto& axis : axes) { + if (axis < 0) + axis += input_dims; + } + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); // input + + if (!axes.empty()) { + const auto axes_name = model_builder.GetUniqueName(node.Name() + input + "_axes"); + Shape axes_dimen = {static_cast(axes.size())}; + shaper.AddShape(axes_name, axes_dimen); + const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type); + input_indices.push_back(operand_indices.at(axes_name)); // axes + } + + const auto& output = node.OutputDefs()[0]->Name(); + shaper.Squeeze(input, axes, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_SQUEEZE, input_indices, {output}, + {output_operand_type}, {false}); +} + +#pragma endregion + #pragma region CreateOpBuilders std::unordered_map> @@ -1619,6 +1625,7 @@ CreateOpBuilders() { } op_map.emplace("Concat", std::make_shared()); + op_map.emplace("Squeeze", std::make_shared()); return op_map; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index d0901ee1c1..d800a59b12 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -13,27 +13,22 @@ class IOpBuilder { virtual ~IOpBuilder() = default; // Check if an operator is supported - virtual bool IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; + virtual bool IsOpSupported(ModelBuilder& model_builder, const Node& node) = 0; // Check if the initializers of this operator need preprocess // which will not be copied - virtual void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; + virtual void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) = 0; // Add the operator to NNAPI model - virtual void AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; + virtual void AddToModelBuilder(ModelBuilder& model_builder, const Node& node) = 0; }; // Generate a lookup table with IOpBuilder delegates // for different onnx operators -std::unordered_map> -CreateOpBuilders(); +std::unordered_map> CreateOpBuilders(); -void TransposeNHWCToNCHW(ModelBuilder& model_builder, - const std::string& input, - const std::string& output); +// Transpose the NHWCinput to NCHW output +void TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output); } // namespace nnapi } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc index 3f9a2251d7..680930795c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc @@ -1,4 +1,5 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" + #include "helper.h" #include "shaper.h" @@ -378,6 +379,41 @@ void Shaper::Concat(const std::vector& input_names, } } +void Shaper::Squeeze(const std::string& input_name, + const std::vector& axes, + const std::string& output_name) { + std::vector input_dimen = shape_map_.at(input_name); + int32_t input_size = input_dimen.size(); + size_t axes_size = axes.size(); + std::unordered_set axes_to_be_squeezed; + if (axes_size == 0) { + for (int32_t idx = 0; idx < input_size; ++idx) { + if (input_dimen[idx] == 1) + axes_to_be_squeezed.insert(idx); + } + } else { + for (const auto& axis : axes) + axes_to_be_squeezed.insert(axis); + } + + // Make output dimensions + std::vector output_dimen; + output_dimen.reserve(input_size - axes_to_be_squeezed.size()); + for (int32_t i = 0; i < input_size; i++) { + if (!Contains(axes_to_be_squeezed, i)) + output_dimen.push_back(input_dimen[i]); + } + + shape_map_[output_name] = output_dimen; + + if (!shaper_finalized_) { + shape_ops_.push_back( + [input_name, axes, output_name](Shaper& shaper) { + shaper.Squeeze(input_name, axes, output_name); + }); + } +} + void Shaper::AddShape(const std::string& name, const Shape& shape) { shape_map_[name] = shape; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index 862c6134d9..634f02d1a4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -9,6 +9,9 @@ class Shaper { using Shape = std::vector; void AddShape(const std::string& name, const Shape& shape); + inline const Shape& operator[](const std::string& key) const { + return shape_map_.at(key); + } void Conv(const std::string& input_name, const std::string& weight_name, @@ -33,23 +36,19 @@ class Shaper { bool nchw, const std::string& output_name); - void Reshape(const std::string& input_name, - const std::vector& shape, - const std::string& output_name); - void Transpose(const std::string& input_name, - const std::vector& perm, - const std::string& output_name); - void Eltwise(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name); - void Identity(const std::string& input_name, - const std::string& output_name); - void FC(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name); + void Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name); - void Concat(const std::vector& input_names, - const int32_t axis, - const std::string& output_name); + void Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name); + + void Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + + void Identity(const std::string& input_name, const std::string& output_name); + + void FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + + void Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name); + + void Squeeze(const std::string& input, const std::vector& axes, const std::string& output); // If the shape of certain input is dynamic // Use the following 2 functions to update the particular shape @@ -61,10 +60,6 @@ class Shaper { // is converted to NNAPI void Finalize() { shaper_finalized_ = true; } - inline const Shape& operator[](const std::string& key) const { - return shape_map_.at(key); - } - void Clear(); private: diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 91201cb517..a2fe6b345e 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -43,51 +43,14 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const std::vector& /*kernel_registries*/) const { std::vector> result; - // Need access to model_path_ - for (const auto& tensor : graph_view.GetAllInitializedTensors()) { - if (tensor.second->has_data_location() && - tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { - LOGS_DEFAULT(WARNING) << "NNAPI: Initializers with external data" - " location are not currently supported"; - return result; - } - } - - // TODO, switch to use graph instead of model - // This method is based on that of TRT EP - // Construct modelproto from graph - onnxruntime::Model model(graph_view.Name(), true, ModelMetaData(), - PathString(), - IOnnxRuntimeOpSchemaRegistryList(), - graph_view.DomainToVersionMap(), - std::vector(), - *GetLogger()); std::unordered_set all_node_inputs; - onnxruntime::Graph& graph_build = model.MainGraph(); for (const auto& node : graph_view.Nodes()) { - std::vector inputs, outputs; for (auto* input : node.InputDefs()) { - auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto()); - inputs.push_back(&n_input); all_node_inputs.insert(input->Name()); } - for (auto* output : node.OutputDefs()) { - auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); - outputs.push_back(&n_output); - } - graph_build.AddNode(node.Name(), node.OpType(), node.Description(), inputs, outputs, &node.GetAttributes(), node.Domain()); - } - //Add initializer to graph - const auto& init_tensors = graph_view.GetAllInitializedTensors(); - for (const auto& tensor : init_tensors) { - graph_build.AddInitializedTensor(*(tensor.second)); } - ORT_ENFORCE(graph_build.Resolve().IsOK()); - ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); - model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); - - nnapi::ModelBuilder builder(model_proto); + nnapi::ModelBuilder builder(graph_view); const auto supported_nodes_vector = builder.GetSupportedNodes(); // Find inputs, initializers and outputs for each supported subgraph @@ -179,9 +142,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) { if (all_node_inputs.find(it->first->Name()) != all_node_inputs.end()) { outputs.insert(std::pair(it->second, it->first)); - } - - if (std::find(graph_outputs.begin(), graph_outputs.end(), it->first) != graph_outputs.end()) { + } else if (std::find(graph_outputs.begin(), graph_outputs.end(), it->first) != graph_outputs.end()) { outputs.insert(std::pair(it->second, it->first)); } } @@ -218,16 +179,11 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorBody(); - onnxruntime::Model model(graph_body.Name(), true, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), graph_body.DomainToVersionMap(), - std::vector(), *GetLogger()); - ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); - *(model_proto.mutable_graph()) = graph_body.ToGraphProto(); - model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + const Graph& graph_body = func_body->Body(); { - nnapi::ModelBuilder builder(model_proto); + onnxruntime::GraphViewer graph_viewer(graph_body); + nnapi::ModelBuilder builder(graph_viewer); builder.SetUseNCHW(false); builder.SetUseFp16(false); std::unique_ptr nnapi_model = builder.Compile(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h index 9cf99f5dae..426a10ea7f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h @@ -17,8 +17,6 @@ #ifndef ANDROID_ML_NN_RUNTIME_NEURAL_NETWORKS_WRAPPER_H #define ANDROID_ML_NN_RUNTIME_NEURAL_NETWORKS_WRAPPER_H #include "nnapi_implementation.h" -#include -#include #include #include