saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fail early when getting device

Browse source
This commit is contained in:
wejoncy 2023-03-06 14:45:20 +08:00 committed by JiCheng
parent 82ae138143
commit adfd38edb9
5 changed files with 44 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -26,10 +26,12 @@ using namespace android::nn::wrapper;
namespace onnxruntime {
namespace nnapi {
ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle)
: nnapi_(nnapi_handle), graph_viewer_(graph_viewer), nnapi_model_{std::make_unique<Model>(nnapi_handle)}, shaper_{graph_viewer} {
ORT_THROW_IF_ERROR(
GetTargetDevices(nnapi_, target_device_option_, nnapi_target_devices_, nnapi_target_devices_detail_));
ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle,
const std::vector<ANeuralNetworksDevice*>& nnapi_target_devices,
const std::string& nnapi_target_devices_detail)
: nnapi_(nnapi_handle), graph_viewer_(graph_viewer), nnapi_model_{std::make_unique<Model>(nnapi_handle)}, shaper_{graph_viewer},
nnapi_target_devices_(nnapi_target_devices), nnapi_target_devices_detail_(nnapi_target_devices_detail) {
nnapi_reference_device_ = nnapi_target_devices_detail_.find(nnapi_cpu) != std::string::npos
? nnapi_target_devices_.back()
: nullptr;
@ -44,7 +46,7 @@ ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_h
OperandType operandType(Type::op_type, InlinedVector<uint32_t>{}); \
ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \
RETURN_STATUS_ON_ERROR_WITH_NOTE( \
nnapi_.ANeuralNetworksModel_setOperandValue( \
nnapi_.ANeuralNetworksModel_setOperandValue( \
nnapi_model_->model_, index, &value, sizeof(value)), \
"value: " + std::to_string(value)); \
return Status::OK(); \
@ -589,7 +591,8 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
<< nm_op_alloc_detail << "] are running in accelerators.";
}
#endif
// When calling ANeuralNetworksCompilation_createForDevices,
// the CPU implementation is not used to handle the failure cases for model compilation and execution.
if (use_create_for_devices) {
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_.ANeuralNetworksCompilation_createForDevices(

11
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h
View file

@ -30,7 +30,9 @@ class ModelBuilder {
public:
using Shape = Shaper::Shape;
ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle);
ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle,
const std::vector<ANeuralNetworksDevice*>& nnapi_target_devices,
const std::string& nnapi_target_devices_detail);
common::Status Compile(std::unique_ptr<Model>& model);
@ -74,8 +76,6 @@ class ModelBuilder {
// It is off by default
void SetUseFp16(bool use_fp16) { use_fp16_ = use_fp16; }
void SetTargetDeviceOption(TargetDeviceOption option) { target_device_option_ = option; }
// Set NNAPI execution preference
// Default preference is PREFER_FAST_SINGLE_ANSWER
void SetExecutePreference(
@ -145,10 +145,9 @@ class ModelBuilder {
std::unordered_set<std::string> unique_names_;
TargetDeviceOption target_device_option_{TargetDeviceOption::ALL_DEVICES};
std::vector<ANeuralNetworksDevice*> nnapi_target_devices_;
const std::vector<ANeuralNetworksDevice*>& nnapi_target_devices_;
ANeuralNetworksDevice* nnapi_reference_device_{nullptr};
std::string nnapi_target_devices_detail_; // Debug info for target devices
const std::string& nnapi_target_devices_detail_; // Debug info for target devices
// feature_level, to decide if we can run this node on NNAPI
int32_t nnapi_target_device_feature_level_ = 0;

7
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc
View file

@ -102,7 +102,8 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_handle.ANeuralNetworksDevice_getType(device, &device_type),
"Getting " + std::to_string(i) + "th device's type");
bool device_is_cpu = nnapi_cpu == device_name;
// https://developer.android.com/ndk/reference/group/neural-networks#aneuralnetworksdevice_gettype
bool device_is_cpu = device_type == ANEURALNETWORKS_DEVICE_CPU;
if ((target_device_option == TargetDeviceOption::CPU_DISABLED && device_is_cpu) ||
(target_device_option == TargetDeviceOption::CPU_ONLY && !device_is_cpu)) {
continue;
@ -119,7 +120,7 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev
// put CPU device at the end
// 1) it's helpful to accelerate nnapi compile, just assuming nnapi-reference has the lowest priority
// and nnapi internally skip the last device if it has already found one.
// 2) we can easily exclude nnapi-reference for mode nnapi_disable_cpu_soft.
// 2) we can easily exclude nnapi-reference when not strict excluding CPU.
// 3) we can easily log the detail of how op was assigned on NNAPI devices which is helpful for debugging.
if (cpu_index != -1 && cpu_index != static_cast<int32_t>(nnapi_target_devices.size()) - 1) {
std::swap(nnapi_target_devices[nnapi_target_devices.size() - 1], nnapi_target_devices[cpu_index]);
@ -128,7 +129,7 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev
return Status::OK();
}
// we Will get devices set first and then get the max feature level supported by all target devices
// Get devices-set first and then get the max feature level supported by all target devices
// return -1 if failed. It's not necessary to handle the error here, because level=-1 will refuse all ops
int32_t GetNNAPIEffectiveFeatureLevelFromTargetDeviceOption(const NnApi& nnapi_handle, TargetDeviceOption target_device_option) {
std::vector<ANeuralNetworksDevice*> nnapi_target_devices;

39
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -3,6 +3,7 @@
#include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h"
#include "core/common/common.h"
#include "core/common/logging/logging.h"
#include "core/common/string_utils.h"
#include "core/framework/allocatormgr.h"
@ -66,9 +67,26 @@ NnapiExecutionProvider::NnapiExecutionProvider(uint32_t nnapi_flags,
});
InsertAllocator(CreateAllocator(cpu_memory_info));
// it should works for both Android and x86 wrapper
// it should works for both Android and x86 wrapper
nnapi_handle_ = nnapi::NnApiImplementation();
ORT_ENFORCE(nnapi_handle_ != nullptr, "Failed to get NnApiImplementation");
bool cpu_disabled = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED;
bool cpu_only = nnapi_flags_ & NNAPI_FLAG_CPU_ONLY;
ORT_ENFORCE((cpu_disabled && cpu_only) == false, "Both NNAPI_FLAG_CPU_DISABLED and NNAPI_FLAG_CPU_ONLY are set");
target_device_option_ = nnapi::TargetDeviceOption::ALL_DEVICES;
if (cpu_disabled) {
target_device_option_ = (nnapi::TargetDeviceOption::CPU_DISABLED);
} else if (cpu_only) {
target_device_option_ = (nnapi::TargetDeviceOption::CPU_ONLY);
}
// May we could just mark it as unavailable instead of throwing an error
ORT_THROW_IF_ERROR(GetTargetDevices(*nnapi_handle_, target_device_option_, nnapi_target_devices_, nnapi_target_devices_detail_));
LOGS_DEFAULT(VERBOSE) << "finding devices [" << nnapi_target_devices_detail_ << "] in NNAPI";
}
NnapiExecutionProvider::~NnapiExecutionProvider() {}
@ -90,15 +108,14 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view
// since we cannot get the runtime system API level, we have to specify it using compile definition.
const int32_t android_feature_level = [this]() {
#ifdef __ANDROID__
auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED
: nnapi::TargetDeviceOption::ALL_DEVICES;
return nnapi::GetNNAPIEffectiveFeatureLevelFromTargetDeviceOption(*nnapi_handle_, flag);
return GetNNAPIEffectiveFeatureLevel(*nnapi_handle_, nnapi_target_devices_);
#else
ORT_UNUSED_PARAMETER(nnapi_flags_);
ORT_UNUSED_PARAMETER(nnapi_handle_);
return ORT_NNAPI_MAX_SUPPORTED_API_LEVEL;
#endif
}();
LOGS_DEFAULT(VERBOSE) << "Finding Android API level: " << android_feature_level;
const nnapi::OpSupportCheckParams params{
android_feature_level,
!!(nnapi_flags_ & NNAPI_FLAG_USE_NCHW),
@ -271,20 +288,10 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
Node& fused_node = fused_node_and_graph.fused_node;
const onnxruntime::GraphViewer& graph_viewer(fused_node_and_graph.filtered_graph);
nnapi::ModelBuilder builder(graph_viewer, *nnapi_handle_);
nnapi::ModelBuilder builder(graph_viewer, *nnapi_handle_, nnapi_target_devices_, nnapi_target_devices_detail_);
builder.SetUseNCHW(nnapi_flags_ & NNAPI_FLAG_USE_NCHW);
builder.SetUseFp16(nnapi_flags_ & NNAPI_FLAG_USE_FP16);
bool cpu_disabled = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED;
bool cpu_only = nnapi_flags_ & NNAPI_FLAG_CPU_ONLY;
if (cpu_disabled && cpu_only) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Both NNAPI_FLAG_CPU_DISABLED and NNAPI_FLAG_CPU_ONLY are set");
} else if (cpu_disabled) {
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_DISABLED);
} else if (cpu_only) {
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_ONLY);
}
std::unique_ptr<nnapi::Model> nnapi_model;
ORT_RETURN_IF_ERROR(builder.Compile(nnapi_model));

3
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h
View file

@ -45,5 +45,8 @@ class NnapiExecutionProvider : public IExecutionProvider {
// nnapi handle for either Android NNAPI or x86 hooker.
const nnapi::NnApi* nnapi_handle_ = nullptr;
std::vector<ANeuralNetworksDevice*> nnapi_target_devices_;
std::string nnapi_target_devices_detail_; // Debug info for target devices
nnapi::TargetDeviceOption target_device_option_;
};
} // namespace onnxruntime