From 028c2372fac9e270aff6f7e1f49345d067e92c71 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Fri, 3 Mar 2023 11:44:43 +0800 Subject: [PATCH] remove disable_cpu_soft temporarily --- .../providers/nnapi/nnapi_provider_factory.h | 5 +-- .../nnapi_builtin/builders/model_builder.cc | 37 ++++++++----------- .../nnapi/nnapi_builtin/nnapi_api_helper.cc | 3 +- .../nnapi/nnapi_builtin/nnapi_api_helper.h | 1 - .../nnapi_builtin/nnapi_execution_provider.cc | 4 -- .../test/perftest/command_args_parser.cc | 9 ++--- onnxruntime/test/perftest/ort_test_session.cc | 8 +--- 7 files changed, 23 insertions(+), 44 deletions(-) diff --git a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h index 9e42761918..a0aeb5ac26 100644 --- a/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h +++ b/include/onnxruntime/core/providers/nnapi/nnapi_provider_factory.h @@ -45,12 +45,9 @@ enum NNAPIFlags { // and NNAPI_FLAG_CPU_ONLY flags are set NNAPI_FLAG_CPU_ONLY = 0x008, - // DISABLE_CPU may bring perf improvement, but may also make it initialize failed. - // A soft disabling flag is used to try disable CPU first, and if it fails, fallback to CPU again. - NNAPI_FLAG_CPU_DISABLED_SOFT = 0x010, // Keep NNAPI_FLAG_LAST at the end of the enum definition // And assign the last NNAPIFlag to it - NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_DISABLED_SOFT, + NNAPI_FLAG_LAST = NNAPI_FLAG_CPU_ONLY, }; #ifdef __cplusplus diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 6e0e0ab3b9..703d75c87e 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -538,34 +538,29 @@ Status ModelBuilder::Compile(std::unique_ptr& model) { [](bool is_supported) { return is_supported; }); // allowing fall back to cpu if it's not strict CPU_DISABLED mode if (!all_ops_supported) { - if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) { - // There are some ops not supported by the list of the target devices - // Fail the Compile - // - // TODO, add some logic to not fail for some cases - // Such as, if there are some acceptable fall back to cpu (nnapi-reference) - // and cpu is not in the target devices list - return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "The model cannot run using current set of target devices, ", - nnapi_target_devices_detail_); - } + // There are some ops not supported by the list of the target devices + // Fail the Compile + // + // TODO, add some logic to not fail for some cases + // Such as, if there are some acceptable fall back to cpu (nnapi-reference) + // and cpu is not in the target devices list + return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, + "The model cannot run using current set of target devices, ", + nnapi_target_devices_detail_); + } else { use_create_for_devices = true; } } #ifndef NDEBUG - if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1) || - (target_device_option_ == TargetDeviceOption::CPU_DISABLED_SOFT)) { + if ((nnapi_reference_device_ && nnapi_target_devices_.size() > 1)) { auto* supported_ops = supported_ops_holder.get(); - - if (target_device_option_ != TargetDeviceOption::CPU_DISABLED_SOFT) { - RETURN_STATUS_ON_ERROR_WITH_NOTE( - nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices( - nnapi_model_->model_, nnapi_target_devices_.data(), - static_cast(nnapi_target_devices_.size() - 1), supported_ops), - "on getSupportedOperationsForDevices"); - } + RETURN_STATUS_ON_ERROR_WITH_NOTE( + nnapi_->ANeuralNetworksModel_getSupportedOperationsForDevices( + nnapi_model_->model_, nnapi_target_devices_.data(), + static_cast(nnapi_target_devices_.size() - 1), supported_ops), + "on getSupportedOperationsForDevices"); std::unordered_map optype_support; const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc index 49264cca26..141ab38ba8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.cc @@ -74,8 +74,7 @@ Status GetTargetDevices(const ::NnApi* nnapi, TargetDeviceOption target_device_o RETURN_STATUS_ON_ERROR_WITH_NOTE(nnapi->ANeuralNetworksDevice_getType(device, &device_type), "Getting " + std::to_string(i) + "th device's type"); bool device_is_cpu = nnapi_cpu == device_name; - if ((target_device_option == TargetDeviceOption::CPU_DISABLED || - target_device_option == TargetDeviceOption::CPU_DISABLED_SOFT) && device_is_cpu) { + if ((target_device_option == TargetDeviceOption::CPU_DISABLED) && device_is_cpu) { continue; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h index a48b63de6e..16941eb6b7 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_api_helper.h @@ -21,7 +21,6 @@ enum class TargetDeviceOption : int8_t { */ CPU_DISABLED, // use all available target devices except CPU - CPU_DISABLED_SOFT, // try best to use all target devices except CPU or fallback to CPU if can't CPU_ONLY, // use CPU only }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index f9a2a73473..616f69bef1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -90,7 +90,6 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const auto* nnapi_handle = NnApiImplementation(); auto flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED) ? nnapi::TargetDeviceOption::CPU_DISABLED : nnapi::TargetDeviceOption::ALL_DEVICES; - flag = (nnapi_flags_ & NNAPI_FLAG_CPU_DISABLED_SOFT) ? nnapi::TargetDeviceOption::CPU_DISABLED_SOFT : flag; return nnapi::GetNNAPIFeatureLevelWithDeviceTag(nnapi_handle, flag); #else ORT_UNUSED_PARAMETER(nnapi_flags_); @@ -275,7 +274,6 @@ common::Status NnapiExecutionProvider::Compile(const std::vector nnapi_model; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 6dd7b83907..4806240e3a 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -84,15 +84,14 @@ namespace perftest { "\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n" "\t [TensorRT only] [trt_context_memory_sharing_enable]: Enable TensorRT context memory sharing between subgraphs.\n" "\t [TensorRT only] [trt_layer_norm_fp32_fallback]: Force Pow + Reduce ops in layer norm to run in FP32 to avoid overflow.\n" - "\t [Usage]: -e -i '| |'\n" - "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n\n" + "\t [Usage]: -e -i '| |'\n\n" + "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" "\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n" "\t [NNAPI only] [NNAPI_FLAG_USE_NCHW]: Use the NCHW layout in NNAPI EP.\n" "\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED]: Prevent NNAPI from using CPU devices.\n" - "\t [NNAPI only] [NNAPI_FLAG_CPU_DISABLED_SOFT]: Try to prevent NNAPI from using CPU devices, but allow falling-back .\n" "\t [NNAPI only] [NNAPI_FLAG_CPU_ONLY]: Using CPU only in NNAPI EP.\n" - "\t [Usage]: -e -i ' '\n" - "\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n\n" + "\t [Usage]: -e -i ' '\n\n" + "\t [Example] [For NNAPI EP] -e nnapi -i \" NNAPI_FLAG_USE_FP16 NNAPI_FLAG_USE_NCHW NNAPI_FLAG_CPU_DISABLED \"\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" "\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 22e9ec2eb8..870a02fc9d 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -546,13 +546,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); nnapi_flags |= NNAPI_FLAG_USE_NCHW; } else if (key == "NNAPI_FLAG_CPU_DISABLED") { nnapi_flags |= NNAPI_FLAG_CPU_DISABLED; - } else if (key == "NNAPI_FLAG_CPU_DISABLED_SOFT") { - nnapi_flags |= NNAPI_FLAG_CPU_DISABLED_SOFT; } else if (key == "NNAPI_FLAG_CPU_ONLY") { nnapi_flags |= NNAPI_FLAG_CPU_ONLY; } else if (key.empty()) { } else { - ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_DISABLED_SOFT', 'NNAPI_FLAG_CPU_ONLY'] \n"); + ORT_THROW("[ERROR] [NNAPI] wrong key type entered. Choose from the following runtime key options that are available for NNAPI. ['NNAPI_FLAG_USE_FP16', 'NNAPI_FLAG_USE_NCHW', 'NNAPI_FLAG_CPU_DISABLED', 'NNAPI_FLAG_CPU_ONLY'] \n"); } } @@ -797,10 +795,6 @@ bool OnnxRuntimeTestSession::PopulateGeneratedInputTestData(int32_t seed) { auto allocator = Ort::AllocatorWithDefaultOptions(); Ort::Value input_tensor = Ort::Value::CreateTensor(allocator, (const int64_t*)input_node_dim.data(), input_node_dim.size(), tensor_info.GetElementType()); - //static std::string const qq{"Who was Jim Henson?"}; - //static std::string const aa{"Jim Henson was a nice puppet"}; - //const char* const input_strings[] = {qq.c_str(), aa.c_str()}; - //input_tensor.FillStringTensor(input_strings, 1U); InitializeTensorWithSeed(seed, input_tensor); PreLoadTestData(0, i, std::move(input_tensor)); }