saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove disable_cpu_soft temporarily

Browse source
This commit is contained in:
wejoncy 2023-03-03 11:44:43 +08:00 committed by JiCheng
parent 30151323da
commit 028c2372fa
7 changed files with 23 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h
View file

@ -45,12 +45,9 @@ enum NNAPIFlags {
// and NNAPI_FLAG_CPU_ONLY flags are set
NNAPI_FLAG_CPU_ONLY = 0x008,
// DISABLE_CPU may bring perf improvement, but may also make it initialize failed.
// A soft disabling flag is used to try disable CPU first, and if it fails, fallback to CPU again.
NNAPI_FLAG_CPU_DISABLED_SOFT = 0x010,
// Keep NNAPI_FLAG_LAST at the end of the enum definition
// And assign the last NNAPIFlag to it
NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_DISABLED_SOFT,
NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_ONLY,
};
#ifdef __cplusplus

37
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -538,34 +538,29 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
[](bool is_supported) { return is_supported; });
// allowing fall back to cpu if it's not strict CPU_DISABLED mode
if (!all_ops_supported) {
if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) {
// There are some ops not supported by the list of the target devices
// Fail the Compile
//
// TODO, add some logic to not fail for some cases
// Such as, if there are some acceptable fall back to cpu (nnapi-reference)
// and cpu is not in the target devices list
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"The model cannot run using current set of target devices, ",
nnapi_target_devices_detail_);
}
// There are some ops not supported by the list of the target devices
// Fail the Compile
//
// TODO, add some logic to not fail for some cases
// Such as, if there are some acceptable fall back to cpu (nnapi-reference)
// and cpu is not in the target devices list
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"The model cannot run using current set of target devices, ",
nnapi_target_devices_detail_);
} else {
use_create_for_devices = true;
}
}
#ifndef NDEBUG
if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1) ||
(target_device_option_ == TargetDeviceOption::CPU_DISABLED_SOFT)) {
if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1)) {
auto* supported_ops = supported_ops_holder.get();
if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) {
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices(
nnapi_model_->model_, nnapi_target_devices_.data(),
static_cast<uint32_t>(nnapi_target_devices_.size() - 1), supported_ops),
"on getSupportedOperationsForDevices");
}
RETURN_STATUS_ON_ERROR_WITH_NOTE(
nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices(
nnapi_model_->model_, nnapi_target_devices_.data(),
static_cast<uint32_t>(nnapi_target_devices_.size() - 1), supported_ops),
"on getSupportedOperationsForDevices");
std::unordered_map<std::string, int32_t> optype_support;
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();

3
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc
View file

@ -74,8 +74,7 @@ Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_o
RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getType(device, &device_type),
"Getting " + std::to_string(i) + "th device's type");
bool device_is_cpu = nnapi_cpu == device_name;
if ((target_device_option == TargetDeviceOption::CPU_DISABLED ||
target_device_option == TargetDeviceOption::CPU_DISABLED_SOFT) && device_is_cpu) {
if ((target_device_option == TargetDeviceOption::CPU_DISABLED) && device_is_cpu) {
continue;
}

1
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h
View file

@ -21,7 +21,6 @@ enum class TargetDeviceOption : int8_t {
*/
CPU_DISABLED, // use all available target devices except CPU
CPU_DISABLED_SOFT, // try best to use all target devices except CPU or fallback to CPU if can't
CPU_ONLY, // use CPU only
};

4
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -90,7 +90,6 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view
const auto* nnapi_handle = NnApiImplementation();
auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED
: nnapi::TargetDeviceOption::ALL_DEVICES;
flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT) ? nnapi::TargetDeviceOption::CPU_DISABLED_SOFT : flag;
return nnapi::GetNNAPIFeatureLevelWithDeviceTag(nnapi_handle, flag);
#else
ORT_UNUSED_PARAMETER(nnapi_flags_);
@ -275,7 +274,6 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
builder.SetUseFp16(nnapi_flags_ & NNAPI_FLAG_USE_FP16);
bool cpu_disabled = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED;
bool cpu_disabled_soft = nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT;
bool cpu_only = nnapi_flags_ & NNAPI_FLAG_CPU_ONLY;
if (cpu_disabled && cpu_only) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Both NNAPI_FLAG_CPU_DISABLED and NNAPI_FLAG_CPU_ONLY are set");
@ -283,8 +281,6 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_DISABLED);
} else if (cpu_only) {
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_ONLY);
} else if (cpu_disabled_soft) {
builder.SetTargetDeviceOption(nnapi::TargetDeviceOption::CPU_DISABLED_SOFT);
}
std::unique_ptr<nnapi::Model> nnapi_model;

9
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -84,15 +84,14 @@ namespace perftest {
"\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"
"\t [TensorRT only] [trt_context_memory_sharing_enable]: Enable TensorRT context memory sharing between subgraphs.\n"
"\t [TensorRT only] [trt_layer_norm_fp32_fallback]: Force Pow + Reduce ops in layer norm to run in FP32 to avoid overflow.\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n"
"\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n"
"\t [NNAPI only] [NNAPI_FLAG_USE_NCHW]: Use the NCHW layout in NNAPI EP.\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED]: Prevent NNAPI from using CPU devices.\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED_SOFT]: Try to prevent NNAPI from using CPU devices, but allow falling-back .\n"
"\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n"
"\t [Usage]: -e <provider_name> -i '<key1> <key2>'\n"
"\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n\n"
"\t [Usage]: -e <provider_name> -i '<key1> <key2>'\n\n"
"\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n"
"\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n"
"\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n"
"\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n"

8
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -546,13 +546,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
nnapi_flags |= NNAPI_FLAG_USE_NCHW;
} else if (key == "NNAPI_FLAG_CPU_DISABLED") {
nnapi_flags |= NNAPI_FLAG_CPU_DISABLED;
} else if (key == "NNAPI_FLAG_CPU_DISABLED_SOFT") {
nnapi_flags |= NNAPI_FLAG_CPU_DISABLED_SOFT;
} else if (key == "NNAPI_FLAG_CPU_ONLY") {
nnapi_flags |= NNAPI_FLAG_CPU_ONLY;
} else if (key.empty()) {
} else {
ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_DISABLED_SOFT', 'NNAPI_FLAG_CPU_ONLY'] \n");
ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_ONLY'] \n");
}
}
@ -797,10 +795,6 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) {
auto allocator = Ort::AllocatorWithDefaultOptions();
Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(),
input_node_dim.size(), tensor_info.GetElementType());
//static std::string const qq{"Who was Jim Henson?"};
//static std::string const aa{"Jim Henson was a nice puppet"};
//const char* const input_strings[] = {qq.c_str(), aa.c_str()};
//input_tensor.FillStringTensor(input_strings, 1U);
InitializeTensorWithSeed(seed, input_tensor);
PreLoadTestData(0, i, std::move(input_tensor));
}