saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[SNPE EP] Add option to enable SNPE init caching feature (#15917)

Browse source 
### Description
[SNPE EP] Add option to enable SNPE init caching feature

### Motivation and Context
To save model initialization time
This commit is contained in:
Hector Li 2023-05-12 07:57:11 -07:00 committed by GitHub
parent 7c4e8267e7
commit 1bebc88069
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 40 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -3558,6 +3558,7 @@ struct OrtApi {
* "buffer_type": ITensor or user buffers, options: "ITENSOR", user buffer with different types - "TF8", "TF16", "UINT8", "FLOAT".
* "ITENSOR" -- default, ITensor which is float only.
* "TF8" -- quantized model required, "FLOAT" -- for both quantized or non-quantized model
* "enable_init_cache": enable SNPE init caching feature, set to 1 to enabled it. Disabled by default.
* If SNPE is not available (due to a non Snpe enabled build or its dependencies not being installed), this function will fail.
*
* XNNPACK supported keys:

3
onnxruntime/core/providers/snpe/snpe_lib.cc
View file

@ -134,7 +134,8 @@ Status SnpeLib::InitializeSnpe(zdl::DlContainer::IDlContainer* container,
snpe_builder.setOutputTensors(dl_output_tensor_names)
.setRuntimeProcessor(snpe_settings.GetRuntimeTarget().Get())
.setExecutionPriorityHint(snpe_settings.GetExecutionPriority())
.setUseUserSuppliedBuffers(use_user_buffer);
.setUseUserSuppliedBuffers(use_user_buffer)
.setInitCacheMode(snpe_settings.GetInitCacheMode());
#ifdef __ANDROID__
// use sustained performance mode on android variants.
LOGS_DEFAULT(INFO) << "setPerformanceProfile to SUSTAINED_HIGH_PERFORMANCE for Android environment!";

15
onnxruntime/core/providers/snpe/snpe_runtime_options.cc
View file

@ -11,13 +11,14 @@ namespace snpe {
constexpr const char* OPT_RUNTIME = "runtime";
constexpr const char* OPT_PRIORITY = "priority";
constexpr const char* BUFFER_TYPE = "buffer_type";
constexpr const char* ENABLE_INIT_CACHE = "enable_init_cache";
void SnpeRuntimeOptions::ParseOptions() {
if (const auto runtime_opt_it = runtime_options_.find(OPT_RUNTIME); runtime_opt_it != runtime_options_.end()) {
runtime_target_ = SnpeRuntimeWrapper(runtime_opt_it->second);
LOGS_DEFAULT(INFO) << "Located user specified runtime target: " << runtime_opt_it->second;
LOGS_DEFAULT(VERBOSE) << "Located user specified runtime target: " << runtime_opt_it->second;
}
LOGS_DEFAULT(INFO) << "Runtime target: " << runtime_target_.ToString();
LOGS_DEFAULT(VERBOSE) << "Runtime target: " << runtime_target_.ToString();
// Option Priority
if (const auto priority_opt_it = runtime_options_.find(OPT_PRIORITY); priority_opt_it != runtime_options_.end()) {
@ -30,7 +31,7 @@ void SnpeRuntimeOptions::ParseOptions() {
execution_priority_ = zdl::DlSystem::ExecutionPriorityHint_t::LOW;
}
LOGS_DEFAULT(INFO) << "Located user specified execution priority " << priority_opt_it->second;
LOGS_DEFAULT(VERBOSE) << "Located user specified execution priority " << priority_opt_it->second;
}
// buffer type
@ -49,6 +50,14 @@ void SnpeRuntimeOptions::ParseOptions() {
LOGS_DEFAULT(ERROR) << "Invalid buffer type: " << buffer_type_it->second;
buffer_type_ = BufferType::UNKNOWN;
}
LOGS_DEFAULT(VERBOSE) << "Buffer type: " << buffer_type_it->second;
}
if (const auto enable_init_cache_pos = runtime_options_.find(ENABLE_INIT_CACHE); enable_init_cache_pos != runtime_options_.end()) {
if (enable_init_cache_pos->second == "1") {
enable_init_cache_ = true;
LOGS_DEFAULT(VERBOSE) << "enable_init_cache enabled.";
}
}
}

16
onnxruntime/core/providers/snpe/snpe_runtime_options.h
View file

@ -24,12 +24,17 @@ enum class BufferType : int {
class SnpeRuntimeOptions {
public:
SnpeRuntimeOptions()
: runtime_target_(), execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), runtime_options_(), buffer_type_(BufferType::ITENSOR) {
SnpeRuntimeOptions() : runtime_target_(),
execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL),
runtime_options_(),
buffer_type_(BufferType::ITENSOR) {
}
explicit SnpeRuntimeOptions(const std::unordered_map<std::string, std::string>& options)
: runtime_target_(), execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL), runtime_options_(options), buffer_type_(BufferType::ITENSOR) {
: runtime_target_(),
execution_priority_(zdl::DlSystem::ExecutionPriorityHint_t::NORMAL),
runtime_options_(options),
buffer_type_(BufferType::ITENSOR) {
ParseOptions();
}
@ -45,6 +50,10 @@ class SnpeRuntimeOptions {
return buffer_type_;
}
bool GetInitCacheMode() const {
return enable_init_cache_;
}
private:
void ParseOptions();
@ -53,6 +62,7 @@ class SnpeRuntimeOptions {
zdl::DlSystem::ExecutionPriorityHint_t execution_priority_;
std::unordered_map<std::string, std::string> runtime_options_;
BufferType buffer_type_;
bool enable_init_cache_ = false;
};
} // namespace snpe

7
onnxruntime/test/onnx/main.cc
View file

@ -57,6 +57,7 @@ void usage() {
"\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n"
"\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n"
"\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n"
"\t [SNPE only] [enable_init_cache]: enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. \n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>' \n\n"
"\t [Example] [For SNPE EP] -e snpe -i \"runtime|CPU priority|low\" \n\n"
"\t-o [optimization level]: Default is 99. Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).\n"
@ -519,8 +520,12 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)"
ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'.
select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
}
} else if (key == "enable_init_cache") {
if (value != "1") {
ORT_THROW("Set to 1 to enable_init_cache.");
}
} else {
ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n");
ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n");
}
snpe_options[key] = value;

1
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -95,6 +95,7 @@ namespace perftest {
"\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n"
"\t [SNPE only] [priority]: execution priority, options: 'low', 'normal'. \n"
"\t [SNPE only] [buffer_type]: options: 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. default: ITENSOR'. \n"
"\t [SNPE only] [enable_init_cache]: enable SNPE init caching feature, set to 1 to enabled it. Disabled by default. \n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>' \n\n"
"\t [Example] [For SNPE EP] -e snpe -i \"runtime|CPU priority|low\" \n\n"
"\t-T [Set intra op thread affinities]: Specify intra op thread affinity string\n"

6
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -597,8 +597,12 @@ select from 'CPU', 'GPU_FP32', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n)"
ORT_THROW(R"(Wrong configuration value for the key 'buffer_type'.
select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
}
} else if (key == "enable_init_cache") {
if (value != "1") {
ORT_THROW("Set to 1 to enable_init_cache.");
}
} else {
ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type'] \n");
ORT_THROW("Wrong key type entered. Choose from options: ['runtime', 'priority', 'buffer_type', 'enable_init_cache'] \n");
}
snpe_options[key] = value;