diff --git a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h index a0aeb5ac26..b6d6233082 100644 --- a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h +++ b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h @@ -45,6 +45,9 @@ enum NNAPIFlags { // and NNAPI_FLAG_CPU_ONLY flags are set NNAPI_FLAG_CPU_ONLY = 0x008, + // DISABLE_CPU may bring perf improvement, but may also make it initialize failed. + // A soft disabling flag is used to try disable CPU first, and if it fails, fallback to CPU again. + NNAPI_FLAG_CPU_DISABLED_SOFT = 0x010, // Keep NNAPI_FLAG_LAST at the end of the enum definition // And assign the last NNAPIFlag to it NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_ONLY, diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/LRN_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/LRN_op_builder.cc index 9b2f781ddc..8b091d4c9a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/LRN_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/LRN_op_builder.cc @@ -45,7 +45,7 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node_unit); - const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); + const auto android_feature_level = model_builder.GetFeatureLevel(); auto input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/base_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/base_op_builder.cc index b082a9af07..c5bff13e1a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/base_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/base_op_builder.cc @@ -48,7 +48,7 @@ bool HasExternalInitializer(const InitializedTensorSet& initializers, const Node Status BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const { OpSupportCheckParams params{ - model_builder.GetNNAPIFeatureLevel(), + model_builder.GetFeatureLevel(), model_builder.UseNCHW(), }; ORT_RETURN_IF_NOT(IsOpSupported(model_builder.GetInitializerTensors(), node_unit, params), diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/conv_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/conv_op_builder.cc index 466b661dd4..81e1ad9336 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/conv_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/conv_op_builder.cc @@ -245,7 +245,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N int32_t fuse_code = model_builder.FindActivation(node_unit); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); - if (model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { + if (model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); // 1. NNAPI Grouped Conv does not support dilations diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/depthtospace_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/depthtospace_op_builder.cc index e1d79efa03..8f04da8254 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/depthtospace_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/depthtospace_op_builder.cc @@ -39,7 +39,7 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); + const auto android_feature_level = model_builder.GetFeatureLevel(); NodeAttrHelper helper(node_unit); const auto& input = node_unit.Inputs()[0].node_arg.Name(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/gemm_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/gemm_op_builder.cc index 8014ebbb43..4e63b5a5ff 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/gemm_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/gemm_op_builder.cc @@ -84,7 +84,7 @@ class GemmOpBuilder : public BaseOpBuilder { // Add operator related void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetNNAPIFeatureLevel())) { + if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetFeatureLevel())) { // no initializers to skip for batch matmul return; } @@ -125,7 +125,7 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod } Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetNNAPIFeatureLevel())) { + if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetFeatureLevel())) { return op_builder_helpers::BuildBatchMatMul(model_builder, node_unit); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/pool_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/pool_op_builder.cc index adf9c7ff5f..6330dcb388 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/pool_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/pool_op_builder.cc @@ -146,7 +146,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N ADD_SCALAR_OPERAND(model_builder, input_indices, kernel_shape[0]); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); - if (model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { // nchw only supported on api 29+ + if (model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { // nchw only supported on api 29+ ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/resize_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/resize_op_builder.cc index 0190080dbe..d4037ee85f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/resize_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/resize_op_builder.cc @@ -77,7 +77,7 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& initializers(model_builder.GetInitializerTensors()); NodeAttrHelper helper(node_unit); const auto& inputs = node_unit.Inputs(); - const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); + const auto android_feature_level = model_builder.GetFeatureLevel(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); auto input = inputs[0].node_arg.Name(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/slice_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/slice_op_builder.cc index 5b36ee9a78..a75c0ff6cc 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/slice_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/slice_op_builder.cc @@ -163,7 +163,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (std::all_of(compute_metadata.steps_.cbegin(), compute_metadata.steps_.cend(), [](int64_t i) { return i == 1; }) && - model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { + model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { op_code = ANEURALNETWORKS_SLICE; // the nnapi size of the slice in this case is the output shape ORT_RETURN_IF_ERROR(AddOperand("sizes", param_dimen, compute_metadata.output_dims_)); // nnapi_sizes diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/softmax_op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/softmax_op_builder.cc index e139bb970c..a7337699a4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/softmax_op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/softmax_op_builder.cc @@ -62,7 +62,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto android_feature_level = model_builder.GetNNAPIFeatureLevel(); + const auto android_feature_level = model_builder.GetFeatureLevel(); NodeAttrHelper helper(node_unit); auto input = node_unit.Inputs()[0].node_arg.Name(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 39a65118b3..e0d1fdbed3 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "model_builder.h" +#include #include "core/common/logging/logging.h" #include "core/common/safeint.h" @@ -10,6 +11,7 @@ #include "core/framework/tensorprotoutils.h" #include "core/graph/graph_viewer.h" #include "core/providers/common.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/nnapi/nnapi_builtin/builders/helper.h" @@ -26,10 +28,6 @@ namespace nnapi { ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer) : nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer), shaper_{graph_viewer} {} -int32_t ModelBuilder::GetNNAPIFeatureLevel() const { - return nnapi_ ? static_cast(nnapi_->nnapi_runtime_feature_level) : 0; -} - // Scalar operand is copied into the model, no need to persist #define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \ Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \ @@ -54,7 +52,13 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) { Status ModelBuilder::Prepare() { RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_)); - ORT_RETURN_IF_ERROR(GetTargetDevices()); + ORT_RETURN_IF_ERROR(GetTargetDevices(nnapi_, target_device_option_, nnapi_target_devices_, + nnapi_target_devices_detail_)); + nnapi_reference_device_ = nnapi_target_devices_detail_.find("nnapi-reference") != std::string::npos + ? nnapi_target_devices_.back() + : nullptr; + nnapi_feature_level_ = GetNNAPIFeatureLevel(*this); + // SetExecutePreference(android::nn::wrapper::ExecutePreference::PREFER_SUSTAINED_SPEED); PreprocessNodeUnits(); GetAllQuantizedOpInputs(); PreprocessInitializers(); @@ -77,45 +81,6 @@ static size_t GetPaddedByteSize(size_t size) { return (size + kDefaultByteAlignmentForNNAPI - 1) & ~(kDefaultByteAlignmentForNNAPI - 1); } -Status ModelBuilder::GetTargetDevices() { - // GetTargetDevices is only supported on API 29+ - if (GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_3) - return Status::OK(); - - if (target_device_option_ == TargetDeviceOption::ALL_DEVICES) - return Status::OK(); - - const std::string nnapi_cpu("nnapi-reference"); - uint32_t num_devices = 0; - RETURN_STATUS_ON_ERROR_WITH_NOTE( - nnapi_->ANeuralNetworks_getDeviceCount(&num_devices), "Getting count of available devices"); - - for (uint32_t i = 0; i < num_devices; i++) { - ANeuralNetworksDevice* device = nullptr; - const char* device_name = nullptr; - int32_t device_type; - RETURN_STATUS_ON_ERROR_WITH_NOTE( - nnapi_->ANeuralNetworks_getDevice(i, &device), "Getting " + std::to_string(i) + "th device"); - - RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getName(device, &device_name), - "Getting " + std::to_string(i) + "th device's name"); - - RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getType(device, &device_type), - "Getting " + std::to_string(i) + "th device's type"); - - bool device_is_cpu = nnapi_cpu == device_name; - if ((target_device_option_ == TargetDeviceOption::CPU_DISABLED && !device_is_cpu) || - (target_device_option_ == TargetDeviceOption::CPU_ONLY && device_is_cpu)) { - nnapi_target_devices_.push_back(device); - const auto device_detail = MakeString("[Name: [", device_name, "], Type [", device_type, "]], "); - nnapi_target_devices_detail_ += device_detail; - LOGS_DEFAULT(VERBOSE) << "Target device " << device_detail << " is added"; - } - } - - return Status::OK(); -} - void ModelBuilder::PreprocessInitializers() { for (const auto& node_unit : node_unit_holder_) { if (const auto* op_builder = GetOpBuilder(*node_unit)) { @@ -400,10 +365,10 @@ Status ModelBuilder::AddNewNNAPIOperand(const OperandType& operand_type, uint32_ index = next_index_++; if (operand_type.channelQuant) { - if (GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_3) { + if (nnapi_feature_level_ < ANEURALNETWORKS_FEATURE_LEVEL_3) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Per-channel quantization is only supported on Android API level 29+,", - " system NNAPI feature level: ", GetNNAPIFeatureLevel()); + " system NNAPI feature level: ", nnapi_feature_level_); } RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_setOperandSymmPerChannelQuantParams( @@ -539,7 +504,7 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { "on identifyInputsAndOutputs"); // relax fp32tofp16 is only available on API 28+ - if (use_fp16_ && GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_1) { + if (use_fp16_ && nnapi_feature_level_ > ANEURALNETWORKS_FEATURE_LEVEL_1) { RETURN_STATUS_ON_ERROR_WITH_NOTE( nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16( nnapi_model_->model_, true), @@ -555,8 +520,8 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { // This is only available on API 29+, for API 28- the nnapi_target_devices_ will // be empty so we will not check API level here, see GetTargetDevices() bool use_create_for_devices = false; - if (!nnapi_target_devices_.empty()) { - std::unique_ptr supported_ops_holder = std::make_unique(num_nnapi_ops_); + std::unique_ptr supported_ops_holder = std::make_unique(num_nnapi_ops_); + if (target_device_option_ != TargetDeviceOption::ALL_DEVICES) { auto* supported_ops = supported_ops_holder.get(); RETURN_STATUS_ON_ERROR_WITH_NOTE( nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices( @@ -566,21 +531,65 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { bool all_ops_supported = std::all_of(supported_ops, supported_ops + num_nnapi_ops_, [](bool is_supported) { return is_supported; }); + // allowing fall back to cpu if it's not strict CPU_DISABLED mode if (!all_ops_supported) { - // There are some ops not supported by the list of the target devices - // Fail the Compile - // - // TODO, add some logic to not fail for some cases - // Such as, if there are some acceptable fall back to cpu (nnapi-reference) - // and cpu is not in the target devices list - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "The model cannot run using current set of target devices, ", - nnapi_target_devices_detail_); + if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) { + // There are some ops not supported by the list of the target devices + // Fail the Compile + // + // TODO, add some logic to not fail for some cases + // Such as, if there are some acceptable fall back to cpu (nnapi-reference) + // and cpu is not in the target devices list + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, + "The model cannot run using current set of target devices, ", + nnapi_target_devices_detail_); + } } else { use_create_for_devices = true; } } +#ifndef NDEBUG + if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1) || + (target_device_option_ == TargetDeviceOption::CPU_DISABLED_SOFT)) { + auto* supported_ops = supported_ops_holder.get(); + + if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) { + RETURN_STATUS_ON_ERROR_WITH_NOTE( + nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices( + nnapi_model_->model_, nnapi_target_devices_.data(), + static_cast(nnapi_target_devices_.size() - 1), supported_ops), + "on getSupportedOperationsForDevices"); + } + + std::unordered_map optype_support; + const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); + for (size_t idx = 0; idx < node_indices.size(); idx++) { + auto node_idx = node_indices[idx]; + const auto* node(graph_viewer_.GetNode(node_idx)); + if (!supported_ops[idx]) { + optype_support[node->OpType()]++; + } else { + optype_support[node->OpType()]--; + } + } + size_t total_ops = 0; + std::string fb_op_alloc_detail, nm_op_alloc_detail; + + for (const auto& [op, count] : optype_support) { + if (count > 0) { + total_ops += count; + fb_op_alloc_detail += std::to_string(count) + "x " + op + ","; + } else { + nm_op_alloc_detail += std::to_string(-count) + "x " + op + ","; + } + } + + LOGS_DEFAULT(VERBOSE) << total_ops << " Ops [" << fb_op_alloc_detail << "] out of " << num_nnapi_ops_ << " are falling-back to nnapi-reference, and [" + << nm_op_alloc_detail << "] are running in accelerators."; + } +#endif + if (use_create_for_devices) { RETURN_STATUS_ON_ERROR_WITH_NOTE( nnapi_->ANeuralNetworksCompilation_createForDevices( diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 5b4d1bd7e9..a767626207 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -9,6 +9,7 @@ #include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "shaper.h" namespace onnxruntime { @@ -28,24 +29,10 @@ class ModelBuilder { public: using Shape = Shaper::Shape; - enum class TargetDeviceOption : int8_t { - ALL_DEVICES, // use all avaliable target devices - - /* TODO support these options - PREFERRED_DEVICES, // Use one or more preferred devices (must be given) - EXCLUDED_DEVICES, // Exclude one or more devices (must be given) - */ - - CPU_DISABLED, // use all avaliable target devices except CPU - CPU_ONLY, // use CPU only - }; - ModelBuilder(const GraphViewer& graph_viewer); common::Status Compile(std::unique_ptr& model); - int32_t GetNNAPIFeatureLevel() const; - // Add an NNAPI operation (operator) common::Status AddOperation(int op, const InlinedVector& input_indices, const std::vector& output_names, @@ -90,7 +77,7 @@ class ModelBuilder { // Set NNAPI execution preference // Default preference is PREFER_SUSTAINED_SPEED - void ExecutePreference( + void SetExecutePreference( android::nn::wrapper::ExecutePreference pref) { exe_pref_ = pref; } // Accessors for members @@ -115,6 +102,11 @@ class ModelBuilder { // the given node must be in the underlying graph_viewer const NodeUnit& GetNodeUnit(const Node* node) const; + const NnApi* GetNnApi() const { return nnapi_; } + + const std::vector& GetDevices() const { return nnapi_target_devices_; } + + int32_t GetFeatureLevel() const { return nnapi_feature_level_; } private: const NnApi* nnapi_{nullptr}; const GraphViewer& graph_viewer_; @@ -158,8 +150,11 @@ class ModelBuilder { TargetDeviceOption target_device_option_{TargetDeviceOption::ALL_DEVICES}; std::vector nnapi_target_devices_; + ANeuralNetworksDevice* nnapi_reference_device_{nullptr}; std::string nnapi_target_devices_detail_; // Debug info for target devices + // feature_level, to decide if we can run this node on NNAPI + int32_t nnapi_feature_level_ = 0; // The number of nnapi operations in this model size_t num_nnapi_ops_ = 0; uint32_t next_index_ = 0; @@ -167,8 +162,6 @@ class ModelBuilder { // Convert the onnx model to ANeuralNetworksModel common::Status Prepare(); - common::Status GetTargetDevices(); - // If a NNAPI operation will use initializers directly, we will add the initializers to the skip list void PreprocessInitializers(); // Preprocess all the activation nodes (Relu/Relu1/Relu6) for easy query later diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc index 023eec26ee..489008f801 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc @@ -847,9 +847,9 @@ Status AddSqueezeOp(ModelBuilder& model_builder, const std::string& node_name, const std::string& input, const std::string& output, std::vector axes) { - if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) { + if (model_builder.GetFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) { return ORT_MAKE_STATUS( - ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel()); + ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetFeatureLevel()); } auto& shaper(model_builder.GetShaper()); @@ -1013,7 +1013,7 @@ bool CanSkipReshape(const ModelBuilder& model_builder, const NodeUnit& node_unit // Now the dest node is Gemm/Matmul, we want to make sure it is supported OpSupportCheckParams params{ - model_builder.GetNNAPIFeatureLevel(), + model_builder.GetFeatureLevel(), model_builder.UseNCHW(), }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 8d6b7ae544..a842c54fb7 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -7,6 +7,7 @@ #include "core/providers/common.h" #include "core/providers/nnapi/nnapi_builtin/builders/helper.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" +#include "nnapi_api_helper.h" #ifdef USENNAPISHAREDMEM #include @@ -89,7 +90,7 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const { bool Model::SupportsDynamicOutputShape() const { // dynamic output shape is only supported on Android API level 29+ (ANEURALNETWORKS_FEATURE_LEVEL_3) - return GetNNAPIFeatureLevel() >= ANEURALNETWORKS_FEATURE_LEVEL_3 && dynamic_output_buffer_size_ > 0; + return GetNNAPIFeatureLevel(nnapi_,{}) >= ANEURALNETWORKS_FEATURE_LEVEL_3 && dynamic_output_buffer_size_ > 0; } Status Model::PrepareForExecution(std::unique_ptr& execution) { @@ -104,10 +105,6 @@ Status Model::PrepareForExecution(std::unique_ptr& execution) { return Status::OK(); } -int32_t Model::GetNNAPIFeatureLevel() const { - return nnapi_ ? static_cast(nnapi_->nnapi_runtime_feature_level) : 0; -} - #pragma region Model::NNMemory #ifdef USENNAPISHAREDMEM diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 53884a3cb8..ad3edb0c05 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -142,8 +142,6 @@ class Model { const android::nn::wrapper::OperandType& operand_type); void AddScalarOutput(const std::string& output_name); - - int32_t GetNNAPIFeatureLevel() const; }; class Execution { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc new file mode 100644 index 0000000000..892e3c6d27 --- /dev/null +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc @@ -0,0 +1,120 @@ +#include "nnapi_api_helper.h" + +#include "core/providers/nnapi/nnapi_builtin/builders/model_builder.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" +#include "core/common/logging/logging.h" + +#ifdef __ANDROID__ +#include +#endif + +namespace onnxruntime { +namespace nnapi { + +int32_t GetNNAPIRuntimeFeatureLevel(const ::NnApi* nnapi) { + if (!nnapi) + return 0; + int32_t runtime_level = static_cast(nnapi->nnapi_runtime_feature_level); +#ifdef __ANDROID__ + int device_api_level = android_get_device_api_level(); + runtime_level = (device_api_level < __ANDROID_API_S__) ? device_api_level : runtime_level; +#endif + return runtime_level; +} + +int32_t GetDeviceFeatureLevel(const ::NnApi* nnapi, const std::vector& device_handles) { + if (!nnapi) + return 0; + + int32_t target_feature_level = GetNNAPIRuntimeFeatureLevel(nnapi); + + int64_t devices_feature_level = -1; + + for (const auto* device_handle : device_handles) { + int64_t curr_device_feature_level = 0; + if (nnapi->ANeuralNetworksDevice_getFeatureLevel(device_handle, &curr_device_feature_level) != ANEURALNETWORKS_NO_ERROR) { + continue; + } + + devices_feature_level = std::max(curr_device_feature_level, devices_feature_level); + } + + if ((devices_feature_level > 0) && (devices_feature_level < target_feature_level)) { + LOGS_DEFAULT(INFO) << "Changing NNAPI Feature Level " << target_feature_level + << " to supported by target devices: " << devices_feature_level; + + target_feature_level = static_cast(devices_feature_level); + } + return target_feature_level; +} + +[[nodiscard]] Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_option, + std::vector& nnapi_target_devices, std::string& nnapi_target_devices_detail) { + // GetTargetDevices is only supported on API 29+ + // get runtime_feature_level + if (GetNNAPIRuntimeFeatureLevel(nnapi) < ANEURALNETWORKS_FEATURE_LEVEL_3) + return Status::OK(); + + const std::string nnapi_cpu("nnapi-reference"); + uint32_t num_devices = 0; + RETURN_STATUS_ON_ERROR_WITH_NOTE( + nnapi->ANeuralNetworks_getDeviceCount(&num_devices), "Getting count of available devices"); + + int32_t cpu_index = -1; + for (uint32_t i = 0; i < num_devices; i++) { + ANeuralNetworksDevice* device = nullptr; + const char* device_name = nullptr; + int32_t device_type = 0; + RETURN_STATUS_ON_ERROR_WITH_NOTE( + nnapi->ANeuralNetworks_getDevice(i, &device), "Getting " + std::to_string(i) + "th device"); + + RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getName(device, &device_name), + "Getting " + std::to_string(i) + "th device's name"); + + RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getType(device, &device_type), + "Getting " + std::to_string(i) + "th device's type"); + bool device_is_cpu = nnapi_cpu == device_name; + if ((target_device_option == TargetDeviceOption::CPU_DISABLED || + target_device_option == TargetDeviceOption::CPU_DISABLED_SOFT) && device_is_cpu) { + continue; + } + + if (device_is_cpu) { + cpu_index = int32_t(nnapi_target_devices.size()); + } + nnapi_target_devices.push_back(device); + const auto device_detail = MakeString("[Name: [", device_name, "], Type [", device_type, "]], "); + nnapi_target_devices_detail += device_detail; + } + + // put CPU device at the end + if (cpu_index != -1 && cpu_index != int32_t(nnapi_target_devices.size()) - 1) { + std::swap(nnapi_target_devices[nnapi_target_devices.size() - 1], nnapi_target_devices[cpu_index]); + } + + return Status::OK(); +} + +int32_t GetNNAPIFeatureLevelWithDeviceTag(const ::NnApi* nnapi, TargetDeviceOption target_device_option) { + std::vector nnapi_target_devices; + std::string nnapi_target_devices_detail; + if (!GetTargetDevices(nnapi, target_device_option, nnapi_target_devices, nnapi_target_devices_detail).IsOK()) { + LOGS_DEFAULT(WARNING) << "GetTargetDevices failed"; + } + LOGS_DEFAULT(VERBOSE) << "finding devices [" << nnapi_target_devices_detail << "] in NNAPI"; + return GetDeviceFeatureLevel(nnapi, nnapi_target_devices); +} + +int32_t GetNNAPIFeatureLevel(const ModelBuilder& model_builder) { + return GetDeviceFeatureLevel(model_builder.GetNnApi(), model_builder.GetDevices()); +} + +int32_t GetNNAPIFeatureLevel(const ::NnApi* nnapi, const std::vector& device_handles) { + if (!nnapi) + return 0; + + return GetDeviceFeatureLevel(nnapi, device_handles); +} + +} // namespace nnapi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h new file mode 100644 index 0000000000..a48b63de6e --- /dev/null +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h @@ -0,0 +1,39 @@ +#pragma once +#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" + +#ifdef __ANDROID__ +#include +#endif + +struct NnApi; + +namespace onnxruntime { +namespace nnapi { + +class ModelBuilder; + +enum class TargetDeviceOption : int8_t { + ALL_DEVICES, // use all avaliable target devices + + /* TODO support these options + PREFERRED_DEVICES, // Use one or more preferred devices (must be given) + EXCLUDED_DEVICES, // Exclude one or more devices (must be given) + */ + + CPU_DISABLED, // use all available target devices except CPU + CPU_DISABLED_SOFT, // try best to use all target devices except CPU or fallback to CPU if can't + CPU_ONLY, // use CPU only +}; + +int32_t GetDeviceFeatureLevel(const ::NnApi* nnapi_, const std::vector& device_handles); + +int32_t GetNNAPIRuntimeFeatureLevel(const ::NnApi* nnapi); + +int32_t GetNNAPIFeatureLevel(const ::NnApi* nnapi, const std::vector& device_handles); +int32_t GetNNAPIFeatureLevel(const ModelBuilder& model_builder); + +Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_option, + std::vector& nnapi_target_devices, std::string& nnapi_target_devices_detail); +int32_t GetNNAPIFeatureLevelWithDeviceTag(const ::NnApi* nnapi, TargetDeviceOption target_device_option); +} // namespace nnapi +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 11f771d007..95eda10cfd 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -3,6 +3,7 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" +#include "core/common/logging/macros.h" #include "core/common/string_utils.h" #include "core/framework/allocatormgr.h" #include "core/framework/compute_capability.h" @@ -84,15 +85,19 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // If we are actually running on Android system, we can get the API level by querying the system // However, since we also allow the NNAPI EP run GetCapability for model conversion on a non-Android system, // since we cannot get the runtime system API level, we have to specify it using compile definition. - static const int32_t android_feature_level = []() { + const int32_t android_feature_level = [this]() { #ifdef __ANDROID__ - const auto* nnapi = NnApiImplementation(); - return nnapi->nnapi_runtime_feature_level; + const auto* nnapi_handle = NnApiImplementation(); + auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED + : nnapi::TargetDeviceOption::ALL_DEVICES; + flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT) ? nnapi::TargetDeviceOption::CPU_DISABLED_SOFT : flag; + return nnapi::GetNNAPIFeatureLevelWithDeviceTag(nnapi_handle, flag); #else + ORT_UNUSED_PARAMETER(nnapi_flags_); return ORT_NNAPI_MAX_SUPPORTED_API_LEVEL; #endif }(); - + LOGS_DEFAULT(VERBOSE) << "Finding Android API level: " << android_feature_level; const nnapi::OpSupportCheckParams params{ android_feature_level, !!(nnapi_flags_ & NNAPI_FLAG_USE_NCHW), @@ -270,13 +275,16 @@ common::Status NnapiExecutionProvider::Compile(const std::vector nnapi_model; @@ -370,8 +378,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector -i '| |'\n\n" - "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" + "\t [Usage]: -e -i '| |'\n" + "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n\n" "\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n" "\t [NNAPI only] [NNAPI_FLAG_USE_NCHW]: Use the NCHW layout in NNAPI EP.\n" "\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED]: Prevent NNAPI from using CPU devices.\n" + "\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED_SOFT]: Try to prevent NNAPI from using CPU devices, but allow falling-back .\n" "\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n" - "\t [Usage]: -e -i ' '\n\n" - "\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n" + "\t [Usage]: -e -i ' '\n" + "\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" "\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 5778ca5d37..22e9ec2eb8 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -473,7 +473,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device } else if (key == "rpc_control_latency") { qnn_options[key] = value; } else { - ORT_THROW(R"(Wrong key type entered. Choose from options: + ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path', 'profiling_level', 'rpc_control_latency'])"); } } @@ -546,13 +546,16 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); nnapi_flags |= NNAPI_FLAG_USE_NCHW; } else if (key == "NNAPI_FLAG_CPU_DISABLED") { nnapi_flags |= NNAPI_FLAG_CPU_DISABLED; + } else if (key == "NNAPI_FLAG_CPU_DISABLED_SOFT") { + nnapi_flags |= NNAPI_FLAG_CPU_DISABLED_SOFT; } else if (key == "NNAPI_FLAG_CPU_ONLY") { nnapi_flags |= NNAPI_FLAG_CPU_ONLY; } else if (key.empty()) { } else { - ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_ONLY'] \n"); + ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_DISABLED_SOFT', 'NNAPI_FLAG_CPU_ONLY'] \n"); } } + Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nnapi(session_options, nnapi_flags)); #else ORT_THROW("NNAPI is not supported in this build\n"); @@ -794,6 +797,10 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); + //static std::string const qq{"Who was Jim Henson?"}; + //static std::string const aa{"Jim Henson was a nice puppet"}; + //const char* const input_strings[] = {qq.c_str(), aa.c_str()}; + //input_tensor.FillStringTensor(input_strings, 1U); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); } diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index 3608b78b1c..9812c07147 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -7,6 +7,7 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h" #include "core/session/inference_session.h" #include "core/framework/tensorprotoutils.h" #include "test/common/tensor_op_test_utils.h" @@ -328,7 +329,7 @@ TEST(NnapiExecutionProviderTest, TestQDQResizeNCHW) { auto Mode = ExpectedEPNodeAssignment::None; #if defined(__ANDROID__) const auto* nnapi = NnApiImplementation(); - if (nnapi->nnapi_runtime_feature_level >= ANEURALNETWORKS_FEATURE_LEVEL_3) { + if (nnapi::GetNNAPIRuntimeFeatureLevel(nnapi) >= ANEURALNETWORKS_FEATURE_LEVEL_3) { Mode = ExpectedEPNodeAssignment::All; } #endif @@ -459,7 +460,7 @@ TEST(NnapiExecutionProviderTest, TestQDQConcat_UnsupportedInputScalesAndZp) { // starting a testing android emulator in command line. (Run an android build with emulator started) // TODO: consider to configure this and enable it to run in Android CI. const auto* nnapi = NnApiImplementation(); - if (nnapi->nnapi_runtime_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { + if (nnapi::GetNNAPIRuntimeFeatureLevel(nnapi) < ANEURALNETWORKS_FEATURE_LEVEL_3) { RunQDQModelTest(BuildQDQConcatTestCaseUnsupportedInputScaleZp(), "nnapi_qdq_test_graph_concat_unsupported", {ExpectedEPNodeAssignment::None});