From adfd38edb93dde5464747427d6cc6347b6fc1904 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Mon, 6 Mar 2023 14:45:20 +0800 Subject: [PATCH] Fail early when getting device --- .../nnapi_builtin/builders/model_builder.cc | 15 ++++--- .../nnapi_builtin/builders/model_builder.h | 11 +++--- .../nnapi/nnapi_builtin/nnapi_api_helper.cc | 7 ++-- .../nnapi_builtin/nnapi_execution_provider.cc | 39 +++++++++++-------- .../nnapi_builtin/nnapi_execution_provider.h | 3 ++ 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 71175ae442..df9db4c5f9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -26,10 +26,12 @@ using namespace android::nn::wrapper; namespace onnxruntime { namespace nnapi { -ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle) - : nnapi_(nnapi_handle), graph_viewer_(graph_viewer), nnapi_model_{std::make_unique(nnapi_handle)}, shaper_{graph_viewer} { - ORT_THROW_IF_ERROR( - GetTargetDevices(nnapi_, target_device_option_, nnapi_target_devices_, nnapi_target_devices_detail_)); +ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle, + const std::vector& nnapi_target_devices, + const std::string& nnapi_target_devices_detail) + : nnapi_(nnapi_handle), graph_viewer_(graph_viewer), nnapi_model_{std::make_unique(nnapi_handle)}, shaper_{graph_viewer}, + nnapi_target_devices_(nnapi_target_devices), nnapi_target_devices_detail_(nnapi_target_devices_detail) { + nnapi_reference_device_ = nnapi_target_devices_detail_.find(nnapi_cpu) != std::string::npos ? nnapi_target_devices_.back() : nullptr; @@ -44,7 +46,7 @@ ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_h OperandType operandType(Type::op_type, InlinedVector{}); \ ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \ RETURN_STATUS_ON_ERROR_WITH_NOTE( \ - nnapi_.ANeuralNetworksModel_setOperandValue( \ + nnapi_.ANeuralNetworksModel_setOperandValue( \ nnapi_model_->model_, index, &value, sizeof(value)), \ "value: " + std::to_string(value)); \ return Status::OK(); \ @@ -589,7 +591,8 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { << nm_op_alloc_detail << "] are running in accelerators."; } #endif - + // When calling ANeuralNetworksCompilation_createForDevices, + // the CPU implementation is not used to handle the failure cases for model compilation and execution. if (use_create_for_devices) { RETURN_STATUS_ON_ERROR_WITH_NOTE( nnapi_.ANeuralNetworksCompilation_createForDevices( diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index aca09c9f67..29136aaec1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -30,7 +30,9 @@ class ModelBuilder { public: using Shape = Shaper::Shape; - ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle); + ModelBuilder(const GraphViewer& graph_viewer, const NnApi& nnapi_handle, + const std::vector& nnapi_target_devices, + const std::string& nnapi_target_devices_detail); common::Status Compile(std::unique_ptr& model); @@ -74,8 +76,6 @@ class ModelBuilder { // It is off by default void SetUseFp16(bool use_fp16) { use_fp16_ = use_fp16; } - void SetTargetDeviceOption(TargetDeviceOption option) { target_device_option_ = option; } - // Set NNAPI execution preference // Default preference is PREFER_FAST_SINGLE_ANSWER void SetExecutePreference( @@ -145,10 +145,9 @@ class ModelBuilder { std::unordered_set unique_names_; - TargetDeviceOption target_device_option_{TargetDeviceOption::ALL_DEVICES}; - std::vector nnapi_target_devices_; + const std::vector& nnapi_target_devices_; ANeuralNetworksDevice* nnapi_reference_device_{nullptr}; - std::string nnapi_target_devices_detail_; // Debug info for target devices + const std::string& nnapi_target_devices_detail_; // Debug info for target devices // feature_level, to decide if we can run this node on NNAPI int32_t nnapi_target_device_feature_level_ = 0; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc index a7f036dd46..f49ff2e13c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc @@ -102,7 +102,8 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi_handle.ANeuralNetworksDevice_getType(device, &device_type), "Getting " + std::to_string(i) + "th device's type"); - bool device_is_cpu = nnapi_cpu == device_name; + // https://developer.android.com/ndk/reference/group/neural-networks#aneuralnetworksdevice_gettype + bool device_is_cpu = device_type == ANEURALNETWORKS_DEVICE_CPU; if ((target_device_option == TargetDeviceOption::CPU_DISABLED && device_is_cpu) || (target_device_option == TargetDeviceOption::CPU_ONLY && !device_is_cpu)) { continue; @@ -119,7 +120,7 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev // put CPU device at the end // 1) it's helpful to accelerate nnapi compile, just assuming nnapi-reference has the lowest priority // and nnapi internally skip the last device if it has already found one. - // 2) we can easily exclude nnapi-reference for mode nnapi_disable_cpu_soft. + // 2) we can easily exclude nnapi-reference when not strict excluding CPU. // 3) we can easily log the detail of how op was assigned on NNAPI devices which is helpful for debugging. if (cpu_index != -1 && cpu_index != static_cast(nnapi_target_devices.size()) - 1) { std::swap(nnapi_target_devices[nnapi_target_devices.size() - 1], nnapi_target_devices[cpu_index]); @@ -128,7 +129,7 @@ Status GetTargetDevices(const NnApi& nnapi_handle, TargetDeviceOption target_dev return Status::OK(); } -// we Will get devices set first and then get the max feature level supported by all target devices +// Get devices-set first and then get the max feature level supported by all target devices // return -1 if failed. It's not necessary to handle the error here, because level=-1 will refuse all ops int32_t GetNNAPIEffectiveFeatureLevelFromTargetDeviceOption(const NnApi& nnapi_handle, TargetDeviceOption target_device_option) { std::vector nnapi_target_devices; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 96d3f95067..ed3b63516b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -3,6 +3,7 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" +#include "core/common/common.h" #include "core/common/logging/logging.h" #include "core/common/string_utils.h" #include "core/framework/allocatormgr.h" @@ -66,9 +67,26 @@ NnapiExecutionProvider::NnapiExecutionProvider(uint32_t nnapi_flags, }); InsertAllocator(CreateAllocator(cpu_memory_info)); -// it should works for both Android and x86 wrapper + // it should works for both Android and x86 wrapper nnapi_handle_ = nnapi::NnApiImplementation(); ORT_ENFORCE(nnapi_handle_ != nullptr, "Failed to get NnApiImplementation"); + + bool cpu_disabled = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED; + bool cpu_only = nnapi_flags_ & NNAPI_FLAG_CPU_ONLY; + + ORT_ENFORCE((cpu_disabled && cpu_only) == false, "Both NNAPI_FLAG_CPU_DISABLED and NNAPI_FLAG_CPU_ONLY are set"); + + target_device_option_ = nnapi::TargetDeviceOption::ALL_DEVICES; + if (cpu_disabled) { + target_device_option_ = (nnapi::TargetDeviceOption::CPU_DISABLED); + } else if (cpu_only) { + target_device_option_ = (nnapi::TargetDeviceOption::CPU_ONLY); + } + + // May we could just mark it as unavailable instead of throwing an error + ORT_THROW_IF_ERROR(GetTargetDevices(*nnapi_handle_, target_device_option_, nnapi_target_devices_, nnapi_target_devices_detail_)); + + LOGS_DEFAULT(VERBOSE) << "finding devices [" << nnapi_target_devices_detail_ << "] in NNAPI"; } NnapiExecutionProvider::~NnapiExecutionProvider() {} @@ -90,15 +108,14 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view // since we cannot get the runtime system API level, we have to specify it using compile definition. const int32_t android_feature_level = [this]() { #ifdef __ANDROID__ - auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED - : nnapi::TargetDeviceOption::ALL_DEVICES; - return nnapi::GetNNAPIEffectiveFeatureLevelFromTargetDeviceOption(*nnapi_handle_, flag); + return GetNNAPIEffectiveFeatureLevel(*nnapi_handle_, nnapi_target_devices_); #else - ORT_UNUSED_PARAMETER(nnapi_flags_); + ORT_UNUSED_PARAMETER(nnapi_handle_); return ORT_NNAPI_MAX_SUPPORTED_API_LEVEL; #endif }(); LOGS_DEFAULT(VERBOSE) << "Finding Android API level: " << android_feature_level; + const nnapi::OpSupportCheckParams params{ android_feature_level, !!(nnapi_flags_ & NNAPI_FLAG_USE_NCHW), @@ -271,20 +288,10 @@ common::Status NnapiExecutionProvider::Compile(const std::vector nnapi_model; ORT_RETURN_IF_ERROR(builder.Compile(nnapi_model)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h index f54cd3412d..24264ba991 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h @@ -45,5 +45,8 @@ class NnapiExecutionProvider : public IExecutionProvider { // nnapi handle for either Android NNAPI or x86 hooker. const nnapi::NnApi* nnapi_handle_ = nullptr; + std::vector nnapi_target_devices_; + std::string nnapi_target_devices_detail_; // Debug info for target devices + nnapi::TargetDeviceOption target_device_option_; }; } // namespace onnxruntime