saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[NNAPI] fix feature_level query

Browse source
This commit is contained in:
wejoncy 2023-02-16 15:17:03 +08:00 committed by JiCheng
parent 6891ab5bac
commit 3873a55bd3
21 changed files with 286 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h
View file

@ -45,6 +45,9 @@ enum NNAPIFlags {
// and NNAPI_FLAG_CPU_ONLY flags are set
NNAPI_FLAG_CPU_ONLY = 0x008,
// DISABLE_CPU may bring perf improvement, but may also make it initialize failed.
// A soft disabling flag is used to try disable CPU first, and if it fails, fallback to CPU again.
NNAPI_FLAG_CPU_DISABLED_SOFT = 0x010,
// Keep NNAPI_FLAG_LAST at the end of the enum definition
// And assign the last NNAPIFlag to it
NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_ONLY,

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/LRN_op_builder.cc
View file

@ -45,7 +45,7 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
NodeAttrHelper helper(node_unit);
const auto android_feature_level = model_builder.GetNNAPIFeatureLevel();
const auto android_feature_level = model_builder.GetFeatureLevel();
auto input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/base_op_builder.cc
View file

@ -48,7 +48,7 @@ bool HasExternalInitializer(const InitializedTensorSet& initializers, const Node
Status BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
OpSupportCheckParams params{
model_builder.GetNNAPIFeatureLevel(),
model_builder.GetFeatureLevel(),
model_builder.UseNCHW(),
};
ORT_RETURN_IF_NOT(IsOpSupported(model_builder.GetInitializerTensors(), node_unit, params),

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/conv_op_builder.cc
View file

@ -245,7 +245,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
int32_t fuse_code = model_builder.FindActivation(node_unit);
ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code);
if (model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) {
if (model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) {
ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw);
// 1. NNAPI Grouped Conv does not support dilations

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/depthtospace_op_builder.cc
View file

@ -39,7 +39,7 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
auto& shaper(model_builder.GetShaper());
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
const auto android_feature_level = model_builder.GetNNAPIFeatureLevel();
const auto android_feature_level = model_builder.GetFeatureLevel();
NodeAttrHelper helper(node_unit);
const auto& input = node_unit.Inputs()[0].node_arg.Name();

4
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/gemm_op_builder.cc
View file

@ -84,7 +84,7 @@ class GemmOpBuilder : public BaseOpBuilder {
// Add operator related
void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetNNAPIFeatureLevel())) {
if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetFeatureLevel())) {
// no initializers to skip for batch matmul
return;
}
@ -125,7 +125,7 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod
}
Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetNNAPIFeatureLevel())) {
if (op_builder_helpers::IsSupportedBatchMatMul(node_unit, model_builder.GetFeatureLevel())) {
return op_builder_helpers::BuildBatchMatMul(model_builder, node_unit);
}

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/pool_op_builder.cc
View file

@ -146,7 +146,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
ADD_SCALAR_OPERAND(model_builder, input_indices, kernel_shape[0]);
ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code);
if (model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { // nchw only supported on api 29+
if (model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) { // nchw only supported on api 29+
ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw);
}

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/resize_op_builder.cc
View file

@ -77,7 +77,7 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
const auto& initializers(model_builder.GetInitializerTensors());
NodeAttrHelper helper(node_unit);
const auto& inputs = node_unit.Inputs();
const auto android_feature_level = model_builder.GetNNAPIFeatureLevel();
const auto android_feature_level = model_builder.GetFeatureLevel();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
auto input = inputs[0].node_arg.Name();

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/slice_op_builder.cc
View file

@ -163,7 +163,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
if (std::all_of(compute_metadata.steps_.cbegin(),
compute_metadata.steps_.cend(),
[](int64_t i) { return i == 1; }) &&
model_builder.GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) {
model_builder.GetFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_2) {
op_code = ANEURALNETWORKS_SLICE;
// the nnapi size of the slice in this case is the output shape
ORT_RETURN_IF_ERROR(AddOperand("sizes", param_dimen, compute_metadata.output_dims_)); // nnapi_sizes

2
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/impl/softmax_op_builder.cc
View file

@ -62,7 +62,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
auto& shaper(model_builder.GetShaper());
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
const auto android_feature_level = model_builder.GetNNAPIFeatureLevel();
const auto android_feature_level = model_builder.GetFeatureLevel();
NodeAttrHelper helper(node_unit);
auto input = node_unit.Inputs()[0].node_arg.Name();

125
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -2,6 +2,7 @@
// Licensed under the MIT License.
#include "model_builder.h"
#include <unordered_map>
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
@ -10,6 +11,7 @@
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph_viewer.h"
#include "core/providers/common.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
@ -26,10 +28,6 @@ namespace nnapi {
ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer)
: nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer), shaper_{graph_viewer} {}
int32_t ModelBuilder::GetNNAPIFeatureLevel() const {
return nnapi_ ? static_cast<int32_t>(nnapi_->nnapi_runtime_feature_level) : 0;
}
// Scalar operand is copied into the model, no need to persist
#define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \
Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \
@ -54,7 +52,13 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) {
Status ModelBuilder::Prepare() {
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_));
ORT_RETURN_IF_ERROR(GetTargetDevices());
ORT_RETURN_IF_ERROR(GetTargetDevices(nnapi_, target_device_option_, nnapi_target_devices_,
nnapi_target_devices_detail_));
nnapi_reference_device_ = nnapi_target_devices_detail_.find("nnapi-reference") != std::string::npos
? nnapi_target_devices_.back()
: nullptr;
nnapi_feature_level_ = GetNNAPIFeatureLevel(*this);
// SetExecutePreference(android::nn::wrapper::ExecutePreference::PREFER_SUSTAINED_SPEED);
PreprocessNodeUnits();
GetAllQuantizedOpInputs();
PreprocessInitializers();
@ -77,45 +81,6 @@ static size_t GetPaddedByteSize(size_t size) {
return (size + kDefaultByteAlignmentForNNAPI - 1) & ~(kDefaultByteAlignmentForNNAPI - 1);
}
Status ModelBuilder::GetTargetDevices() {
// GetTargetDevices is only supported on API 29+
if (GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_3)
return Status::OK();
if (target_device_option_ == TargetDeviceOption::ALL_DEVICES)
return Status::OK();
const std::string nnapi_cpu("nnapi-reference");
uint32_t num_devices = 0;
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworks_getDeviceCount(&num_devices), "Getting count of available devices");
for (uint32_t i = 0; i < num_devices; i++) {
ANeuralNetworksDevice* device = nullptr;
const char* device_name = nullptr;
int32_t device_type;
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworks_getDevice(i, &device), "Getting " + std::to_string(i) + "th device");
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getName(device, &device_name),
"Getting " + std::to_string(i) + "th device's name");
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getType(device, &device_type),
"Getting " + std::to_string(i) + "th device's type");
bool device_is_cpu = nnapi_cpu == device_name;
if ((target_device_option_ == TargetDeviceOption::CPU_DISABLED && !device_is_cpu) ||
(target_device_option_ == TargetDeviceOption::CPU_ONLY && device_is_cpu)) {
nnapi_target_devices_.push_back(device);
const auto device_detail = MakeString("[Name: [", device_name, "], Type [", device_type, "]], ");
nnapi_target_devices_detail_ += device_detail;
LOGS_DEFAULT(VERBOSE) << "Target device " << device_detail << " is added";
}
}
return Status::OK();
}
void ModelBuilder::PreprocessInitializers() {
for (const auto& node_unit : node_unit_holder_) {
if (const auto* op_builder = GetOpBuilder(*node_unit)) {
@ -400,10 +365,10 @@ Status ModelBuilder::AddNewNNAPIOperand(const OperandType& operand_type, uint32_
index = next_index_++;
if (operand_type.channelQuant) {
if (GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_3) {
if (nnapi_feature_level_ < ANEURALNETWORKS_FEATURE_LEVEL_3) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Per-channel quantization is only supported on Android API level 29+,",
" system NNAPI feature level: ", GetNNAPIFeatureLevel());
" system NNAPI feature level: ", nnapi_feature_level_);
}
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_setOperandSymmPerChannelQuantParams(
@ -539,7 +504,7 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
"on identifyInputsAndOutputs");
// relax fp32tofp16 is only available on API 28+
if (use_fp16_ && GetNNAPIFeatureLevel() > ANEURALNETWORKS_FEATURE_LEVEL_1) {
if (use_fp16_ && nnapi_feature_level_ > ANEURALNETWORKS_FEATURE_LEVEL_1) {
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
nnapi_model_->model_, true),
@ -555,8 +520,8 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
// This is only available on API 29+, for API 28- the nnapi_target_devices_ will
// be empty so we will not check API level here, see GetTargetDevices()
bool use_create_for_devices = false;
if (!nnapi_target_devices_.empty()) {
std::unique_ptr<bool[]> supported_ops_holder = std::make_unique<bool[]>(num_nnapi_ops_);
std::unique_ptr<bool[]> supported_ops_holder = std::make_unique<bool[]>(num_nnapi_ops_);
if (target_device_option_ != TargetDeviceOption::ALL_DEVICES) {
auto* supported_ops = supported_ops_holder.get();
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices(
@ -566,21 +531,65 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
bool all_ops_supported = std::all_of(supported_ops, supported_ops + num_nnapi_ops_,
[](bool is_supported) { return is_supported; });
// allowing fall back to cpu if it's not strict CPU_DISABLED mode
if (!all_ops_supported) {
// There are some ops not supported by the list of the target devices
// Fail the Compile
//
// TODO, add some logic to not fail for some cases
// Such as, if there are some acceptable fall back to cpu (nnapi-reference)
// and cpu is not in the target devices list
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"The model cannot run using current set of target devices, ",
nnapi_target_devices_detail_);
if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) {
// There are some ops not supported by the list of the target devices
// Fail the Compile
//
// TODO, add some logic to not fail for some cases
// Such as, if there are some acceptable fall back to cpu (nnapi-reference)
// and cpu is not in the target devices list
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"The model cannot run using current set of target devices, ",
nnapi_target_devices_detail_);
}
} else {
use_create_for_devices = true;
}
}
#ifndef NDEBUG
if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1) ||
(target_device_option_ == TargetDeviceOption::CPU_DISABLED_SOFT)) {
auto* supported_ops = supported_ops_holder.get();
if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) {
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices(
nnapi_model_->model_, nnapi_target_devices_.data(),
static_cast<uint32_t>(nnapi_target_devices_.size() - 1), supported_ops),
"on getSupportedOperationsForDevices");
}
std::unordered_map<std::string, int32_t> optype_support;
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();
for (size_t idx = 0; idx < node_indices.size(); idx++) {
auto node_idx = node_indices[idx];
const auto* node(graph_viewer_.GetNode(node_idx));
if (!supported_ops[idx]) {
optype_support[node->OpType()]++;
} else {
optype_support[node->OpType()]--;
}
}
size_t total_ops = 0;
std::string fb_op_alloc_detail, nm_op_alloc_detail;
for (const auto& [op, count] : optype_support) {
if (count > 0) {
total_ops += count;
fb_op_alloc_detail += std::to_string(count) + "x " + op + ",";
} else {
nm_op_alloc_detail += std::to_string(-count) + "x " + op + ",";
}
}
LOGS_DEFAULT(VERBOSE) << total_ops << " Ops [" << fb_op_alloc_detail << "] out of " << num_nnapi_ops_ << " are falling-back to nnapi-reference, and ["
<< nm_op_alloc_detail << "] are running in accelerators.";
}
#endif
if (use_create_for_devices) {
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksCompilation_createForDevices(

27
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h
View file

@ -9,6 +9,7 @@
#include "core/graph/basic_types.h"
#include "core/providers/nnapi/nnapi_builtin/model.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h"
#include "shaper.h"
namespace onnxruntime {
@ -28,24 +29,10 @@ class ModelBuilder {
public:
using Shape = Shaper::Shape;
enum class TargetDeviceOption : int8_t {
ALL_DEVICES, // use all avaliable target devices
/* TODO support these options
PREFERRED_DEVICES, // Use one or more preferred devices (must be given)
EXCLUDED_DEVICES, // Exclude one or more devices (must be given)
*/
CPU_DISABLED, // use all avaliable target devices except CPU
CPU_ONLY, // use CPU only
};
ModelBuilder(const GraphViewer& graph_viewer);
common::Status Compile(std::unique_ptr<Model>& model);
int32_t GetNNAPIFeatureLevel() const;
// Add an NNAPI operation (operator)
common::Status AddOperation(int op, const InlinedVector<uint32_t>& input_indices,
const std::vector<std::string>& output_names,
@ -90,7 +77,7 @@ class ModelBuilder {
// Set NNAPI execution preference
// Default preference is PREFER_SUSTAINED_SPEED
void ExecutePreference(
void SetExecutePreference(
android::nn::wrapper::ExecutePreference pref) { exe_pref_ = pref; }
// Accessors for members
@ -115,6 +102,11 @@ class ModelBuilder {
// the given node must be in the underlying graph_viewer
const NodeUnit& GetNodeUnit(const Node* node) const;
const NnApi* GetNnApi() const { return nnapi_; }
const std::vector<ANeuralNetworksDevice*>& GetDevices() const { return nnapi_target_devices_; }
int32_t GetFeatureLevel() const { return nnapi_feature_level_; }
private:
const NnApi* nnapi_{nullptr};
const GraphViewer& graph_viewer_;
@ -158,8 +150,11 @@ class ModelBuilder {
TargetDeviceOption target_device_option_{TargetDeviceOption::ALL_DEVICES};
std::vector<ANeuralNetworksDevice*> nnapi_target_devices_;
ANeuralNetworksDevice* nnapi_reference_device_{nullptr};
std::string nnapi_target_devices_detail_; // Debug info for target devices
// feature_level, to decide if we can run this node on NNAPI
int32_t nnapi_feature_level_ = 0;
// The number of nnapi operations in this model
size_t num_nnapi_ops_ = 0;
uint32_t next_index_ = 0;
@ -167,8 +162,6 @@ class ModelBuilder {
// Convert the onnx model to ANeuralNetworksModel
common::Status Prepare();
common::Status GetTargetDevices();
// If a NNAPI operation will use initializers directly, we will add the initializers to the skip list
void PreprocessInitializers();
// Preprocess all the activation nodes (Relu/Relu1/Relu6) for easy query later

6
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc
View file

@ -847,9 +847,9 @@ Status AddSqueezeOp(ModelBuilder& model_builder,
const std::string& node_name,
const std::string& input, const std::string& output,
std::vector<int32_t> axes) {
if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) {
if (model_builder.GetFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) {
return ORT_MAKE_STATUS(
ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel());
ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetFeatureLevel());
}
auto& shaper(model_builder.GetShaper());
@ -1013,7 +1013,7 @@ bool CanSkipReshape(const ModelBuilder& model_builder, const NodeUnit& node_unit
// Now the dest node is Gemm/Matmul, we want to make sure it is supported
OpSupportCheckParams params{
model_builder.GetNNAPIFeatureLevel(),
model_builder.GetFeatureLevel(),
model_builder.UseNCHW(),
};

7
onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc
View file

@ -7,6 +7,7 @@
#include "core/providers/common.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "nnapi_api_helper.h"
#ifdef USENNAPISHAREDMEM
#include <sys/mman.h>
@ -89,7 +90,7 @@ size_t Model::GetMappedOutputIdx(const std::string& name) const {
bool Model::SupportsDynamicOutputShape() const {
// dynamic output shape is only supported on Android API level 29+ (ANEURALNETWORKS_FEATURE_LEVEL_3)
return GetNNAPIFeatureLevel() >= ANEURALNETWORKS_FEATURE_LEVEL_3 && dynamic_output_buffer_size_ > 0;
return GetNNAPIFeatureLevel(nnapi_,{}) >= ANEURALNETWORKS_FEATURE_LEVEL_3 && dynamic_output_buffer_size_ > 0;
}
Status Model::PrepareForExecution(std::unique_ptr<Execution>& execution) {
@ -104,10 +105,6 @@ Status Model::PrepareForExecution(std::unique_ptr<Execution>& execution) {
return Status::OK();
}
int32_t Model::GetNNAPIFeatureLevel() const {
return nnapi_ ? static_cast<int32_t>(nnapi_->nnapi_runtime_feature_level) : 0;
}
#pragma region Model::NNMemory
#ifdef USENNAPISHAREDMEM

2
onnxruntime/core/providers/nnapi/nnapi_builtin/model.h
View file

@ -142,8 +142,6 @@ class Model {
const android::nn::wrapper::OperandType& operand_type);
void AddScalarOutput(const std::string& output_name);
int32_t GetNNAPIFeatureLevel() const;
};
class Execution {

120
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc Normal file
View file

@ -0,0 +1,120 @@
#include "nnapi_api_helper.h"
#include "core/providers/nnapi/nnapi_builtin/builders/model_builder.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "core/common/logging/logging.h"
#ifdef __ANDROID__
#include <android/api-level.h>
#endif
namespace onnxruntime {
namespace nnapi {
int32_t GetNNAPIRuntimeFeatureLevel(const ::NnApi* nnapi) {
if (!nnapi)
return 0;
int32_t runtime_level = static_cast<int32_t>(nnapi->nnapi_runtime_feature_level);
#ifdef __ANDROID__
int device_api_level = android_get_device_api_level();
runtime_level = (device_api_level < __ANDROID_API_S__) ? device_api_level : runtime_level;
#endif
return runtime_level;
}
int32_t GetDeviceFeatureLevel(const ::NnApi* nnapi, const std::vector<ANeuralNetworksDevice*>& device_handles) {
if (!nnapi)
return 0;
int32_t target_feature_level = GetNNAPIRuntimeFeatureLevel(nnapi);
int64_t devices_feature_level = -1;
for (const auto* device_handle : device_handles) {
int64_t curr_device_feature_level = 0;
if (nnapi->ANeuralNetworksDevice_getFeatureLevel(device_handle, &curr_device_feature_level) != ANEURALNETWORKS_NO_ERROR) {
continue;
}
devices_feature_level = std::max(curr_device_feature_level, devices_feature_level);
}
if ((devices_feature_level > 0) && (devices_feature_level < target_feature_level)) {
LOGS_DEFAULT(INFO) << "Changing NNAPI Feature Level " << target_feature_level
<< " to supported by target devices: " << devices_feature_level;
target_feature_level = static_cast<int32_t>(devices_feature_level);
}
return target_feature_level;
}
[[nodiscard]] Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_option,
std::vector<ANeuralNetworksDevice*>& nnapi_target_devices, std::string& nnapi_target_devices_detail) {
// GetTargetDevices is only supported on API 29+
// get runtime_feature_level
if (GetNNAPIRuntimeFeatureLevel(nnapi) < ANEURALNETWORKS_FEATURE_LEVEL_3)
return Status::OK();
const std::string nnapi_cpu("nnapi-reference");
uint32_t num_devices = 0;
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi->ANeuralNetworks_getDeviceCount(&num_devices), "Getting count of available devices");
int32_t cpu_index = -1;
for (uint32_t i = 0; i < num_devices; i++) {
ANeuralNetworksDevice* device = nullptr;
const char* device_name = nullptr;
int32_t device_type = 0;
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi->ANeuralNetworks_getDevice(i, &device), "Getting " + std::to_string(i) + "th device");
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getName(device, &device_name),
"Getting " + std::to_string(i) + "th device's name");
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getType(device, &device_type),
"Getting " + std::to_string(i) + "th device's type");
bool device_is_cpu = nnapi_cpu == device_name;
if ((target_device_option == TargetDeviceOption::CPU_DISABLED ||
target_device_option == TargetDeviceOption::CPU_DISABLED_SOFT) && device_is_cpu) {
continue;
}
if (device_is_cpu) {
cpu_index = int32_t(nnapi_target_devices.size());
}
nnapi_target_devices.push_back(device);
const auto device_detail = MakeString("[Name: [", device_name, "], Type [", device_type, "]], ");
nnapi_target_devices_detail += device_detail;
}
// put CPU device at the end
if (cpu_index != -1 && cpu_index != int32_t(nnapi_target_devices.size()) - 1) {
std::swap(nnapi_target_devices[nnapi_target_devices.size() - 1], nnapi_target_devices[cpu_index]);
}
return Status::OK();
}
int32_t GetNNAPIFeatureLevelWithDeviceTag(const ::NnApi* nnapi, TargetDeviceOption target_device_option) {
std::vector<ANeuralNetworksDevice*> nnapi_target_devices;
std::string nnapi_target_devices_detail;
if (!GetTargetDevices(nnapi, target_device_option, nnapi_target_devices, nnapi_target_devices_detail).IsOK()) {
LOGS_DEFAULT(WARNING) << "GetTargetDevices failed";
}
LOGS_DEFAULT(VERBOSE) << "finding devices [" << nnapi_target_devices_detail << "] in NNAPI";
return GetDeviceFeatureLevel(nnapi, nnapi_target_devices);
}
int32_t GetNNAPIFeatureLevel(const ModelBuilder& model_builder) {
return GetDeviceFeatureLevel(model_builder.GetNnApi(), model_builder.GetDevices());
}
int32_t GetNNAPIFeatureLevel(const ::NnApi* nnapi, const std::vector<ANeuralNetworksDevice*>& device_handles) {
if (!nnapi)
return 0;
return GetDeviceFeatureLevel(nnapi, device_handles);
}
} // namespace nnapi
} // namespace onnxruntime

39
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h Normal file
View file

@ -0,0 +1,39 @@
#pragma once
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h"
#ifdef __ANDROID__
#include <android/api-level.h>
#endif
struct NnApi;
namespace onnxruntime {
namespace nnapi {
class ModelBuilder;
enum class TargetDeviceOption : int8_t {
ALL_DEVICES, // use all avaliable target devices
/* TODO support these options
PREFERRED_DEVICES, // Use one or more preferred devices (must be given)
EXCLUDED_DEVICES, // Exclude one or more devices (must be given)
*/
CPU_DISABLED, // use all available target devices except CPU
CPU_DISABLED_SOFT, // try best to use all target devices except CPU or fallback to CPU if can't
CPU_ONLY, // use CPU only
};
int32_t GetDeviceFeatureLevel(const ::NnApi* nnapi_, const std::vector<ANeuralNetworksDevice*>& device_handles);
int32_t GetNNAPIRuntimeFeatureLevel(const ::NnApi* nnapi);
int32_t GetNNAPIFeatureLevel(const ::NnApi* nnapi, const std::vector<ANeuralNetworksDevice*>& device_handles);
int32_t GetNNAPIFeatureLevel(const ModelBuilder& model_builder);
Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_option,
std::vector<ANeuralNetworksDevice*>& nnapi_target_devices, std::string& nnapi_target_devices_detail);
int32_t GetNNAPIFeatureLevelWithDeviceTag(const ::NnApi* nnapi, TargetDeviceOption target_device_option);
} // namespace nnapi
} // namespace onnxruntime

23
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -3,6 +3,7 @@
#include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h"
#include "core/common/logging/macros.h"
#include "core/common/string_utils.h"
#include "core/framework/allocatormgr.h"
#include "core/framework/compute_capability.h"
@ -84,15 +85,19 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view
// If we are actually running on Android system, we can get the API level by querying the system
// However, since we also allow the NNAPI EP run GetCapability for model conversion on a non-Android system,
// since we cannot get the runtime system API level, we have to specify it using compile definition.
static const int32_t android_feature_level = []() {
const int32_t android_feature_level = [this]() {
#ifdef __ANDROID__
const auto* nnapi = NnApiImplementation();
return nnapi->nnapi_runtime_feature_level;
const auto* nnapi_handle = NnApiImplementation();
auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED
: nnapi::TargetDeviceOption::ALL_DEVICES;
flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT) ? nnapi::TargetDeviceOption::CPU_DISABLED_SOFT : flag;
return nnapi::GetNNAPIFeatureLevelWithDeviceTag(nnapi_handle, flag);
#else
ORT_UNUSED_PARAMETER(nnapi_flags_);
return ORT_NNAPI_MAX_SUPPORTED_API_LEVEL;
#endif
}();
LOGS_DEFAULT(VERBOSE) << "Finding Android API level: " << android_feature_level;
const nnapi::OpSupportCheckParams params{
android_feature_level,
!!(nnapi_flags_ & NNAPI_FLAG_USE_NCHW),
@ -270,13 +275,16 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
builder.SetUseFp16(nnapi_flags_ & NNAPI_FLAG_USE_FP16);
bool cpu_disabled = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED;
bool cpu_disabled_soft = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT;
bool cpu_only = nnapi_flags_ & NNAPI_FLAG_CPU_ONLY;
if (cpu_disabled && cpu_only) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Both NNAPI_FLAG_CPU_DISABLED and NNAPI_FLAG_CPU_ONLY are set");
} else if (cpu_disabled) {
builder.SetTargetDeviceOption(nnapi::ModelBuilder::TargetDeviceOption::CPU_DISABLED);
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_DISABLED);
} else if (cpu_only) {
builder.SetTargetDeviceOption(nnapi::ModelBuilder::TargetDeviceOption::CPU_ONLY);
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_ONLY);
} else if (cpu_disabled_soft) {
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_DISABLED_SOFT);
}
std::unique_ptr<nnapi::Model> nnapi_model;
@ -370,8 +378,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
const void* inputBuffer = input_tensor.GetTensorRawData();
inputs.push_back({input_name, inputBuffer, std::move(input_type)});
}
}
#ifdef __ANDROID__
// From this point we will need to take the exclusive lock on the model until the Predict is

9
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -84,14 +84,15 @@ namespace perftest {
"\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"
"\t [TensorRT only] [trt_context_memory_sharing_enable]: Enable TensorRT context memory sharing between subgraphs.\n"
"\t [TensorRT only] [trt_layer_norm_fp32_fallback]: Force Pow + Reduce ops in layer norm to run in FP32 to avoid overflow.\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n\n"
"\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n"
"\t [NNAPI only] [NNAPI_FLAG_USE_NCHW]: Use the NCHW layout in NNAPI EP.\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED]: Prevent NNAPI from using CPU devices.\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED_SOFT]: Try to prevent NNAPI from using CPU devices, but allow falling-back .\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n"
"\t [Usage]: -e <provider_name> -i '<key1> <key2>'\n\n"
"\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n"
"\t [Usage]: -e <provider_name> -i '<key1> <key2>'\n"
"\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n\n"
"\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n"
"\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n"
"\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n"

11
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -473,7 +473,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
} else if (key == "rpc_control_latency") {
qnn_options[key] = value;
} else {
ORT_THROW(R"(Wrong key type entered. Choose from options:
ORT_THROW(R"(Wrong key type entered. Choose from options:
['backend_path', 'profiling_level', 'rpc_control_latency'])");
}
}
@ -546,13 +546,16 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
nnapi_flags |= NNAPI_FLAG_USE_NCHW;
} else if (key == "NNAPI_FLAG_CPU_DISABLED") {
nnapi_flags |= NNAPI_FLAG_CPU_DISABLED;
} else if (key == "NNAPI_FLAG_CPU_DISABLED_SOFT") {
nnapi_flags |= NNAPI_FLAG_CPU_DISABLED_SOFT;
} else if (key == "NNAPI_FLAG_CPU_ONLY") {
nnapi_flags |= NNAPI_FLAG_CPU_ONLY;
} else if (key.empty()) {
} else {
ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_ONLY'] \n");
ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_DISABLED_SOFT', 'NNAPI_FLAG_CPU_ONLY'] \n");
}
}
Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_Nnapi(session_options, nnapi_flags));
#else
ORT_THROW("NNAPI is not supported in this build\n");
@ -794,6 +797,10 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) {
auto allocator = Ort::AllocatorWithDefaultOptions();
Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(),
input_node_dim.size(), tensor_info.GetElementType());
//static std::string const qq{"Who was Jim Henson?"};
//static std::string const aa{"Jim Henson was a nice puppet"};
//const char* const input_strings[] = {qq.c_str(), aa.c_str()};
//input_tensor.FillStringTensor(input_strings, 1U);
InitializeTensorWithSeed(seed, input_tensor);
PreLoadTestData(0, i, std::move(input_tensor));
}

5
onnxruntime/test/providers/nnapi/nnapi_basic_test.cc
View file

@ -7,6 +7,7 @@
#include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h"
#include "core/session/inference_session.h"
#include "core/framework/tensorprotoutils.h"
#include "test/common/tensor_op_test_utils.h"
@ -328,7 +329,7 @@ TEST(NnapiExecutionProviderTest, TestQDQResizeNCHW) {
auto Mode = ExpectedEPNodeAssignment::None;
#if defined(__ANDROID__)
const auto* nnapi = NnApiImplementation();
if (nnapi->nnapi_runtime_feature_level >= ANEURALNETWORKS_FEATURE_LEVEL_3) {
if (nnapi::GetNNAPIRuntimeFeatureLevel(nnapi) >= ANEURALNETWORKS_FEATURE_LEVEL_3) {
Mode = ExpectedEPNodeAssignment::All;
}
#endif
@ -459,7 +460,7 @@ TEST(NnapiExecutionProviderTest, TestQDQConcat_UnsupportedInputScalesAndZp) {
// starting a testing android emulator in command line. (Run an android build with emulator started)
// TODO: consider to configure this and enable it to run in Android CI.
const auto* nnapi = NnApiImplementation();
if (nnapi->nnapi_runtime_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) {
if (nnapi::GetNNAPIRuntimeFeatureLevel(nnapi) < ANEURALNETWORKS_FEATURE_LEVEL_3) {
RunQDQModelTest(BuildQDQConcatTestCaseUnsupportedInputScaleZp(),
"nnapi_qdq_test_graph_concat_unsupported",
{ExpectedEPNodeAssignment::None});