From 1bebc88069ad51fc8540967afa39724c3f73c938 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 12 May 2023 07:57:11 -0700 Subject: [PATCH] [SNPE EP] Add option to enable SNPE init caching feature (#15917) ### Description [SNPE EP] Add option to enable SNPE init caching feature ### Motivation and Context To save model initialization time --- .../onnxruntime/core/session/onnxruntime_c_api.h | 1 + onnxruntime/core/providers/snpe/snpe_lib.cc | 3 ++- .../core/providers/snpe/snpe_runtime_options.cc | 15 ++++++++++++--- .../core/providers/snpe/snpe_runtime_options.h | 16 +++++++++++++--- onnxruntime/test/onnx/main.cc | 7 ++++++- onnxruntime/test/perftest/command_args_parser.cc | 1 + onnxruntime/test/perftest/ort_test_session.cc | 6 +++++- 7 files changed, 40 insertions(+), 9 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index fa373f5d7e..203a975b2b 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3558,6 +3558,7 @@ struct OrtApi { * "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT". * "ITENSOR" -- default, ITensor which is float only. * "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model + * "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. * If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail. * * XNNPACK supported keys: diff --git a/onnxruntime/core/providers/snpe/snpe_lib.cc b/onnxruntime/core/providers/snpe/snpe_lib.cc index 1d5065a906..e526ece1b2 100644 --- a/onnxruntime/core/providers/snpe/snpe_lib.cc +++ b/onnxruntime/core/providers/snpe/snpe_lib.cc @@ -134,7 +134,8 @@ Status SnpeLib::InitializeSnpe(zdl::DlContainer::IDlContainer* container, snpe_builder.setOutputTensors(dl_output_tensor_names) .setRuntimeProcessor(snpe_settings.GetRuntimeTarget().Get()) .setExecutionPriorityHint(snpe_settings.GetExecutionPriority()) - .setUseUserSuppliedBuffers(use_user_buffer); + .setUseUserSuppliedBuffers(use_user_buffer) + .setInitCacheMode(snpe_settings.GetInitCacheMode()); #ifdef __ANDROID__ // use sustained performance mode on android variants. LOGS_DEFAULT(INFO) << "setPerformanceProfile to SUSTAINED_HIGH_PERFORMANCE for Android environment!"; diff --git a/onnxruntime/core/providers/snpe/snpe_runtime_options.cc b/onnxruntime/core/providers/snpe/snpe_runtime_options.cc index 18494a9bf9..08659858b7 100644 --- a/onnxruntime/core/providers/snpe/snpe_runtime_options.cc +++ b/onnxruntime/core/providers/snpe/snpe_runtime_options.cc @@ -11,13 +11,14 @@ namespace snpe { constexpr const char* OPT_RUNTIME = "runtime"; constexpr const char* OPT_PRIORITY = "priority"; constexpr const char* BUFFER_TYPE = "buffer_type"; +constexpr const char* ENABLE_INIT_CACHE = "enable_init_cache"; void SnpeRuntimeOptions::ParseOptions() { if (const auto runtime_opt_it = runtime_options_.find(OPT_RUNTIME); runtime_opt_it != runtime_options_.end()) { runtime_target_ = SnpeRuntimeWrapper(runtime_opt_it->second); - LOGS_DEFAULT(INFO) << "Located user specified runtime target: " << runtime_opt_it->second; + LOGS_DEFAULT(VERBOSE) << "Located user specified runtime target: " << runtime_opt_it->second; } - LOGS_DEFAULT(INFO) << "Runtime target: " << runtime_target_.ToString(); + LOGS_DEFAULT(VERBOSE) << "Runtime target: " << runtime_target_.ToString(); // Option Priority if (const auto priority_opt_it = runtime_options_.find(OPT_PRIORITY); priority_opt_it != runtime_options_.end()) { @@ -30,7 +31,7 @@ void SnpeRuntimeOptions::ParseOptions() { execution_priority_ = zdl::DlSystem::ExecutionPriorityHint_t::LOW; } - LOGS_DEFAULT(INFO) << "Located user specified execution priority " << priority_opt_it->second; + LOGS_DEFAULT(VERBOSE) << "Located user specified execution priority " << priority_opt_it->second; } // buffer type @@ -49,6 +50,14 @@ void SnpeRuntimeOptions::ParseOptions() { LOGS_DEFAULT(ERROR) << "Invalid buffer type: " << buffer_type_it->second; buffer_type_ = BufferType::UNKNOWN; } + LOGS_DEFAULT(VERBOSE) << "Buffer type: " << buffer_type_it->second; + } + + if (const auto enable_init_cache_pos = runtime_options_.find(ENABLE_INIT_CACHE); enable_init_cache_pos != runtime_options_.end()) { + if (enable_init_cache_pos->second == "1") { + enable_init_cache_ = true; + LOGS_DEFAULT(VERBOSE) << "enable_init_cache enabled."; + } } } diff --git a/onnxruntime/core/providers/snpe/snpe_runtime_options.h b/onnxruntime/core/providers/snpe/snpe_runtime_options.h index e4789257d1..32571e7b99 100644 --- a/onnxruntime/core/providers/snpe/snpe_runtime_options.h +++ b/onnxruntime/core/providers/snpe/snpe_runtime_options.h @@ -24,12 +24,17 @@ enum class BufferType : int { class SnpeRuntimeOptions { public: - SnpeRuntimeOptions() - : runtime_target_(), execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), runtime_options_(), buffer_type_(BufferType::ITENSOR) { + SnpeRuntimeOptions() : runtime_target_(), + execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), + runtime_options_(), + buffer_type_(BufferType::ITENSOR) { } explicit SnpeRuntimeOptions(const std::unordered_map& options) - : runtime_target_(), execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), runtime_options_(options), buffer_type_(BufferType::ITENSOR) { + : runtime_target_(), + execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), + runtime_options_(options), + buffer_type_(BufferType::ITENSOR) { ParseOptions(); } @@ -45,6 +50,10 @@ class SnpeRuntimeOptions { return buffer_type_; } + bool GetInitCacheMode() const { + return enable_init_cache_; + } + private: void ParseOptions(); @@ -53,6 +62,7 @@ class SnpeRuntimeOptions { zdl::DlSystem::ExecutionPriorityHint_t execution_priority_; std::unordered_map runtime_options_; BufferType buffer_type_; + bool enable_init_cache_ = false; }; } // namespace snpe diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index a00e3bc4f2..63df472816 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -57,6 +57,7 @@ void usage() { "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" "\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n" + "\t [SNPE only] [enable_init_cache]: enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. \n" "\t [Usage]: -e -i '| |' \n\n" "\t [Example] [For SNPE EP] -e snpe -i \"runtime|CPU priority|low\" \n\n" "\t-o [optimization level]: Default is 99. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n" @@ -519,8 +520,12 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)" ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'. select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } + } else if (key == "enable_init_cache") { + if (value != "1") { + ORT_THROW("Set to 1 to enable_init_cache."); + } } else { - ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n"); + ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n"); } snpe_options[key] = value; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 9d5813c5b3..2cf4f15dad 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -95,6 +95,7 @@ namespace perftest { "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" "\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n" "\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n" + "\t [SNPE only] [enable_init_cache]: enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. \n" "\t [Usage]: -e -i '| |' \n\n" "\t [Example] [For SNPE EP] -e snpe -i \"runtime|CPU priority|low\" \n\n" "\t-T [Set intra op thread affinities]: Specify intra op thread affinity string\n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 77d8b8c31c..d3a547d244 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -597,8 +597,12 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)" ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'. select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } + } else if (key == "enable_init_cache") { + if (value != "1") { + ORT_THROW("Set to 1 to enable_init_cache."); + } } else { - ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n"); + ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n"); } snpe_options[key] = value;