mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-07 04:39:07 +00:00
[QNN EP] Expose device-level session options (#19212)
### Description - Adds the following session options to configure the device: - `soc_model`: The SoC model number. Refer to the QNN SDK documentation for valid values. Defaults to "0" (unknown). - `htp_arch`: The minimum HTP architecture the driver will use to select compatible QNN operators. - `device_id`: The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). ### Motivation and Context Allow more configuration.
This commit is contained in:
Adrian Lizarraga
GitHub
parent
373ebac167
commit
8d9d751179
12 changed files with 292 additions and 118 deletions
|
|
@ -3608,6 +3608,14 @@ struct OrtApi {
|
|||
* - "1": Faster preparation time, less optimal graph.
|
||||
* - "2": Longer preparation time, more optimal graph.
|
||||
* - "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific details.
|
||||
* "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. Defaults to "0" (unknown).
|
||||
* "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. Available options:
|
||||
* - "0": Default (none).
|
||||
* - "68"
|
||||
* - "69"
|
||||
* - "73"
|
||||
* - "75"
|
||||
* "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device).
|
||||
*
|
||||
* SNPE supported keys:
|
||||
* "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16",
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "core/framework/endian_utils.h"
|
||||
#include "core/common/logging/capture.h"
|
||||
#include "core/providers/qnn/builder/onnx_ctx_model_helper.h"
|
||||
#include "core/providers/qnn/builder/qnn_configs_helper.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <winmeta.h>
|
||||
|
|
@ -329,9 +330,37 @@ Status QnnBackendManager::CreateDevice() {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
qnn::QnnConfigsBuilder<QnnDevice_Config_t, QnnHtpDevice_CustomConfig_t> device_configs_builder(QNN_DEVICE_CONFIG_INIT,
|
||||
{});
|
||||
if (qnn_backend_type_ == QnnBackendType::HTP) {
|
||||
// Set SoC Model. The *enum* Qnn_SocModel_t is deprecated and will not be updated in the future. Therefore,
|
||||
// must use the latest SDK documentation to get the SoC model of the latest HW.
|
||||
if (soc_model_ != QNN_SOC_MODEL_UNKNOWN) {
|
||||
QnnHtpDevice_CustomConfig_t& custom_config = device_configs_builder.PushCustomConfig();
|
||||
custom_config.option = QNN_HTP_DEVICE_CONFIG_OPTION_SOC;
|
||||
custom_config.socModel = soc_model_;
|
||||
|
||||
QnnDevice_Config_t& device_config = device_configs_builder.PushConfig();
|
||||
device_config.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM;
|
||||
device_config.customConfig = &custom_config;
|
||||
}
|
||||
|
||||
// Set the minimum HTP architecture. The driver will use ops that are compatible with this minimum architecture.
|
||||
if (htp_arch_ != QNN_HTP_DEVICE_ARCH_NONE) {
|
||||
QnnHtpDevice_CustomConfig_t& custom_config = device_configs_builder.PushCustomConfig();
|
||||
custom_config.option = QNN_HTP_DEVICE_CONFIG_OPTION_ARCH;
|
||||
custom_config.arch.arch = htp_arch_;
|
||||
custom_config.arch.deviceId = device_id_;
|
||||
|
||||
QnnDevice_Config_t& device_config = device_configs_builder.PushConfig();
|
||||
device_config.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM;
|
||||
device_config.customConfig = &custom_config;
|
||||
}
|
||||
}
|
||||
|
||||
LOGS_DEFAULT(INFO) << "Create device.";
|
||||
if (nullptr != qnn_interface_.deviceCreate) {
|
||||
auto result = qnn_interface_.deviceCreate(log_handle_, nullptr, &device_handle_);
|
||||
auto result = qnn_interface_.deviceCreate(log_handle_, device_configs_builder.GetQnnConfigs(), &device_handle_);
|
||||
if (QNN_SUCCESS != result) {
|
||||
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to create device. Error: ", result);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include <vector>
|
||||
#include "HTP/QnnHtpDevice.h"
|
||||
#include "QnnLog.h"
|
||||
#include "QnnTypes.h"
|
||||
#include "System/QnnSystemInterface.h"
|
||||
#include "core/common/status.h"
|
||||
#include "core/common/logging/logging.h"
|
||||
|
|
@ -35,13 +36,19 @@ class QnnBackendManager {
|
|||
uint32_t rpc_control_latency,
|
||||
HtpPerformanceMode htp_performance_mode,
|
||||
ContextPriority context_priority,
|
||||
std::string&& qnn_saver_path)
|
||||
std::string&& qnn_saver_path,
|
||||
uint32_t device_id,
|
||||
QnnHtpDevice_Arch_t htp_arch,
|
||||
uint32_t soc_model)
|
||||
: backend_path_(backend_path),
|
||||
profiling_level_(profiling_level),
|
||||
rpc_control_latency_(rpc_control_latency),
|
||||
htp_performance_mode_(htp_performance_mode),
|
||||
context_priority_(context_priority),
|
||||
qnn_saver_path_(qnn_saver_path) {
|
||||
qnn_saver_path_(qnn_saver_path),
|
||||
device_id_(device_id),
|
||||
htp_arch_(htp_arch),
|
||||
soc_model_(soc_model) {
|
||||
}
|
||||
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QnnBackendManager);
|
||||
|
||||
|
|
@ -233,6 +240,9 @@ class QnnBackendManager {
|
|||
#endif
|
||||
const std::string qnn_saver_path_;
|
||||
uint32_t htp_power_config_client_id_ = 0;
|
||||
uint32_t device_id_ = 0;
|
||||
QnnHtpDevice_Arch_t htp_arch_ = QNN_HTP_DEVICE_ARCH_NONE;
|
||||
uint32_t soc_model_ = QNN_SOC_MODEL_UNKNOWN;
|
||||
};
|
||||
|
||||
} // namespace qnn
|
||||
|
|
|
|||
90
onnxruntime/core/providers/qnn/builder/qnn_configs_helper.h
Normal file
90
onnxruntime/core/providers/qnn/builder/qnn_configs_helper.h
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <core/common/inlined_containers_fwd.h>
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace qnn {
|
||||
|
||||
/**
|
||||
* Helper class for building a null-terminated list of QNN configurations.
|
||||
* A QNN configuration consists of multiple objects with references to each other. This
|
||||
* class ensures that all configuration objects have the same lifetime, so that they remain valid
|
||||
* across calls to qnn_interface.xxxCreate().
|
||||
*/
|
||||
template <typename BaseConfigType, typename CustomConfigType>
|
||||
class QnnConfigsBuilder {
|
||||
public:
|
||||
/**
|
||||
* Initializes the config build. Provide the initial/default value for each config struct type.
|
||||
* \param base_config_init The initial/default value for objects of type BaseConfigType.
|
||||
* \param custom_config_init The initial/default value for objects of type CustomConfigType.
|
||||
*/
|
||||
QnnConfigsBuilder(BaseConfigType base_config_init, CustomConfigType custom_config_init)
|
||||
: base_config_init_(std::move(base_config_init)), custom_config_init_(std::move(custom_config_init)) {}
|
||||
|
||||
/**
|
||||
* Returns a pointer to the beginning of a null-terminated array of QNN base configurations.
|
||||
* This result is typically passed to QNN's xxxCreate() APIs.
|
||||
*
|
||||
* \return Pointer to null-terminated BaseConfigType* array.
|
||||
*/
|
||||
const BaseConfigType** GetQnnConfigs() {
|
||||
if (config_ptrs_.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!IsNullTerminated()) {
|
||||
config_ptrs_.push_back(nullptr);
|
||||
}
|
||||
|
||||
return config_ptrs_.data();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and returns a reference to a new custom QNN configuration object. The object is initialized to
|
||||
* the QNN recommended default value. The caller is meant to override fields in this object.
|
||||
*
|
||||
* \return A reference to a default CustomConfigType object.
|
||||
*/
|
||||
CustomConfigType& PushCustomConfig() {
|
||||
custom_configs_.push_back(custom_config_init_);
|
||||
return custom_configs_.back();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and returns a reference to a new QNN configuration object. The object is initialized to
|
||||
* the QNN recommended default value. The caller is meant to override fields in this object.
|
||||
*
|
||||
* \return A reference to a default BaseConfigType object.
|
||||
*/
|
||||
BaseConfigType& PushConfig() {
|
||||
configs_.push_back(base_config_init_);
|
||||
BaseConfigType& config = configs_.back();
|
||||
|
||||
// Add pointer to this new config to the list of config pointers.
|
||||
if (IsNullTerminated()) {
|
||||
config_ptrs_.back() = &config; // Replace last nullptr entry.
|
||||
} else {
|
||||
config_ptrs_.push_back(&config);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsNullTerminated() const {
|
||||
return !config_ptrs_.empty() && config_ptrs_.back() == nullptr;
|
||||
}
|
||||
|
||||
BaseConfigType base_config_init_;
|
||||
CustomConfigType custom_config_init_;
|
||||
InlinedVector<CustomConfigType> custom_configs_;
|
||||
InlinedVector<BaseConfigType> configs_;
|
||||
InlinedVector<const BaseConfigType*> config_ptrs_;
|
||||
};
|
||||
|
||||
} // namespace qnn
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#include "core/providers/qnn/builder/qnn_graph_configs_helper.h"
|
||||
|
||||
#include "HTP/QnnHtpGraph.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace qnn {
|
||||
|
||||
const QnnGraph_Config_t** QnnGraphConfigsBuilder::GetQnnGraphConfigs() {
|
||||
if (graph_config_ptrs_.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!IsNullTerminated()) {
|
||||
graph_config_ptrs_.push_back(nullptr);
|
||||
}
|
||||
|
||||
return graph_config_ptrs_.data();
|
||||
}
|
||||
|
||||
QnnHtpGraph_CustomConfig_t& QnnGraphConfigsBuilder::PushHtpGraphCustomConfig() {
|
||||
htp_custom_graph_configs_.push_back(QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT);
|
||||
return htp_custom_graph_configs_.back();
|
||||
}
|
||||
|
||||
QnnGraph_Config_t& QnnGraphConfigsBuilder::PushGraphConfig() {
|
||||
graph_configs_.push_back(QNN_GRAPH_CONFIG_INIT);
|
||||
QnnGraph_Config_t& config = graph_configs_.back();
|
||||
|
||||
// Add pointer to this new graph config to the list of graph config pointers.
|
||||
if (IsNullTerminated()) {
|
||||
graph_config_ptrs_.back() = &config; // Replace last nullptr entry.
|
||||
} else {
|
||||
graph_config_ptrs_.push_back(&config);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
} // namespace qnn
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <core/common/inlined_containers_fwd.h>
|
||||
|
||||
#include "HTP/QnnHtpGraph.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
namespace qnn {
|
||||
|
||||
/**
|
||||
* Helper class for building a null-terminated list of QNN Graph configurations.
|
||||
* A QNN configuration consists of multiple objects with references to each other. This
|
||||
* class ensures that all configuration objects have the same lifetime, so that they remain valid
|
||||
* across the call to graphCreate().
|
||||
*/
|
||||
class QnnGraphConfigsBuilder {
|
||||
public:
|
||||
/**
|
||||
* Returns a pointer to the beginning of a null-terminated array of QNN Graph configurations.
|
||||
* This result is passed QNN's graphCreate() API.
|
||||
*
|
||||
* \return Pointer to null-terminated QnnGraph_Config_t* array.
|
||||
*/
|
||||
const QnnGraph_Config_t** GetQnnGraphConfigs();
|
||||
|
||||
/**
|
||||
* Creates and returns a reference to a new HTP graph configuration object. The object is initialized to
|
||||
* the QNN recommended default value. The caller is meant to override fields in this object.
|
||||
*
|
||||
* \return A reference to a default QnnHtpGraph_CustomConfig_t object.
|
||||
*/
|
||||
QnnHtpGraph_CustomConfig_t& PushHtpGraphCustomConfig();
|
||||
|
||||
/**
|
||||
* Creates and returns a reference to a new graph configuration object. The object is initialized to
|
||||
* the QNN recommended default value. The caller is meant to override fields in this object.
|
||||
*
|
||||
* \return A reference to a default QnnGraph_Config_t object.
|
||||
*/
|
||||
QnnGraph_Config_t& PushGraphConfig();
|
||||
|
||||
private:
|
||||
bool IsNullTerminated() const {
|
||||
return !graph_config_ptrs_.empty() && graph_config_ptrs_.back() == nullptr;
|
||||
}
|
||||
|
||||
InlinedVector<QnnHtpGraph_CustomConfig_t> htp_custom_graph_configs_;
|
||||
InlinedVector<QnnGraph_Config_t> graph_configs_;
|
||||
InlinedVector<const QnnGraph_Config_t*> graph_config_ptrs_;
|
||||
};
|
||||
|
||||
} // namespace qnn
|
||||
} // namespace onnxruntime
|
||||
|
|
@ -111,6 +111,22 @@ void QNNExecutionProvider::ParseHtpGraphFinalizationOptimizationMode(const std::
|
|||
}
|
||||
}
|
||||
|
||||
static void ParseHtpArchitecture(const std::string& htp_arch_string, QnnHtpDevice_Arch_t& qnn_htp_arch) {
|
||||
if (htp_arch_string.empty() || htp_arch_string == "0") {
|
||||
qnn_htp_arch = QNN_HTP_DEVICE_ARCH_NONE;
|
||||
} else if (htp_arch_string == "68") {
|
||||
qnn_htp_arch = QNN_HTP_DEVICE_ARCH_V68;
|
||||
} else if (htp_arch_string == "69") {
|
||||
qnn_htp_arch = QNN_HTP_DEVICE_ARCH_V69;
|
||||
} else if (htp_arch_string == "73") {
|
||||
qnn_htp_arch = QNN_HTP_DEVICE_ARCH_V73;
|
||||
} else if (htp_arch_string == "75") {
|
||||
qnn_htp_arch = QNN_HTP_DEVICE_ARCH_V75;
|
||||
} else {
|
||||
LOGS_DEFAULT(WARNING) << "Invalid HTP architecture: " << htp_arch_string;
|
||||
}
|
||||
}
|
||||
|
||||
QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_options_map,
|
||||
const SessionOptions* session_options)
|
||||
: IExecutionProvider{onnxruntime::kQnnExecutionProvider, true} {
|
||||
|
|
@ -223,13 +239,49 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio
|
|||
}
|
||||
}
|
||||
|
||||
static const std::string QNN_DEVICE_ID = "device_id";
|
||||
uint32_t device_id = 0;
|
||||
auto dev_id_pos = provider_options_map.find(QNN_DEVICE_ID);
|
||||
if (dev_id_pos != provider_options_map.end()) {
|
||||
int value = std::stoi(dev_id_pos->second);
|
||||
if (value < 0) {
|
||||
LOGS_DEFAULT(WARNING) << "Invalid device ID '" << value
|
||||
<< "', only >= 0 allowed. Set to " << device_id << ".";
|
||||
} else {
|
||||
device_id = static_cast<uint32_t>(value);
|
||||
}
|
||||
}
|
||||
|
||||
static const std::string QNN_HTP_ARCH = "htp_arch";
|
||||
QnnHtpDevice_Arch_t htp_arch = QNN_HTP_DEVICE_ARCH_NONE;
|
||||
auto htp_arch_pos = provider_options_map.find(QNN_HTP_ARCH);
|
||||
if (htp_arch_pos != provider_options_map.end()) {
|
||||
ParseHtpArchitecture(htp_arch_pos->second, htp_arch);
|
||||
}
|
||||
|
||||
static const std::string QNN_SOC_MODEL = "soc_model";
|
||||
uint32_t soc_model = QNN_SOC_MODEL_UNKNOWN;
|
||||
auto soc_model_pos = provider_options_map.find(QNN_SOC_MODEL);
|
||||
if (soc_model_pos != provider_options_map.end()) {
|
||||
int value = std::stoi(soc_model_pos->second);
|
||||
if (value < 0) {
|
||||
LOGS_DEFAULT(WARNING) << "Invalid SoC Model '" << value
|
||||
<< "', only >= 0 allowed. Set to " << soc_model << ".";
|
||||
} else {
|
||||
soc_model = static_cast<uint32_t>(value);
|
||||
}
|
||||
}
|
||||
|
||||
qnn_backend_manager_ = std::make_unique<qnn::QnnBackendManager>(
|
||||
std::move(backend_path),
|
||||
profiling_level,
|
||||
rpc_control_latency,
|
||||
htp_performance_mode,
|
||||
context_priority,
|
||||
std::move(qnn_saver_path));
|
||||
std::move(qnn_saver_path),
|
||||
device_id,
|
||||
htp_arch,
|
||||
soc_model);
|
||||
}
|
||||
|
||||
bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit,
|
||||
|
|
@ -512,25 +564,25 @@ Status QNNExecutionProvider::CreateComputeFunc(std::vector<NodeComputeInfo>& nod
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
void QNNExecutionProvider::InitQnnGraphConfigs(qnn::QnnGraphConfigsBuilder& configs_builder) const {
|
||||
void QNNExecutionProvider::InitQnnGraphConfigs(qnn::QnnConfigsBuilder<QnnGraph_Config_t, QnnHtpGraph_CustomConfig_t>& configs_builder) const {
|
||||
if (qnn_backend_manager_->GetQnnBackendType() == qnn::QnnBackendType::HTP) {
|
||||
if (htp_graph_finalization_opt_mode_ != qnn::HtpGraphFinalizationOptimizationMode::kDefault) {
|
||||
QnnHtpGraph_CustomConfig_t& htp_graph_opt_config = configs_builder.PushHtpGraphCustomConfig();
|
||||
QnnHtpGraph_CustomConfig_t& htp_graph_opt_config = configs_builder.PushCustomConfig();
|
||||
htp_graph_opt_config.option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION;
|
||||
htp_graph_opt_config.optimizationOption.type = QNN_HTP_GRAPH_OPTIMIZATION_TYPE_FINALIZE_OPTIMIZATION_FLAG;
|
||||
htp_graph_opt_config.optimizationOption.floatValue = static_cast<float>(htp_graph_finalization_opt_mode_);
|
||||
|
||||
QnnGraph_Config_t& graph_opt_config = configs_builder.PushGraphConfig();
|
||||
QnnGraph_Config_t& graph_opt_config = configs_builder.PushConfig();
|
||||
graph_opt_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
|
||||
graph_opt_config.customConfig = &htp_graph_opt_config;
|
||||
}
|
||||
|
||||
if (vtcm_size_in_mb_ > 0) {
|
||||
QnnHtpGraph_CustomConfig_t& htp_graph_opt_config_vtcm = configs_builder.PushHtpGraphCustomConfig();
|
||||
QnnHtpGraph_CustomConfig_t& htp_graph_opt_config_vtcm = configs_builder.PushCustomConfig();
|
||||
htp_graph_opt_config_vtcm.option = QNN_HTP_GRAPH_CONFIG_OPTION_VTCM_SIZE;
|
||||
htp_graph_opt_config_vtcm.vtcmSizeInMB = static_cast<uint32_t>(vtcm_size_in_mb_);
|
||||
|
||||
QnnGraph_Config_t& graph_opt_config_vtcm = configs_builder.PushGraphConfig();
|
||||
QnnGraph_Config_t& graph_opt_config_vtcm = configs_builder.PushConfig();
|
||||
graph_opt_config_vtcm.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
|
||||
graph_opt_config_vtcm.customConfig = &htp_graph_opt_config_vtcm;
|
||||
}
|
||||
|
|
@ -547,10 +599,11 @@ Status QNNExecutionProvider::CompileFromOrtGraph(const std::vector<FusedNodeAndG
|
|||
std::unique_ptr<qnn::QnnModel> qnn_model = std::make_unique<qnn::QnnModel>(logger,
|
||||
qnn_backend_manager_.get());
|
||||
|
||||
qnn::QnnGraphConfigsBuilder graph_configs_builder;
|
||||
qnn::QnnConfigsBuilder<QnnGraph_Config_t, QnnHtpGraph_CustomConfig_t> graph_configs_builder(QNN_GRAPH_CONFIG_INIT,
|
||||
QNN_HTP_GRAPH_CUSTOM_CONFIG_INIT);
|
||||
InitQnnGraphConfigs(graph_configs_builder);
|
||||
|
||||
ORT_RETURN_IF_ERROR(qnn_model->ComposeGraph(graph_viewer, fused_node, graph_configs_builder.GetQnnGraphConfigs()));
|
||||
ORT_RETURN_IF_ERROR(qnn_model->ComposeGraph(graph_viewer, fused_node, graph_configs_builder.GetQnnConfigs()));
|
||||
ORT_RETURN_IF_ERROR(qnn_model->FinalizeGraphs());
|
||||
ORT_RETURN_IF_ERROR(qnn_model->SetupQnnInputOutput());
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@
|
|||
|
||||
#include "core/framework/execution_provider.h"
|
||||
#include "core/framework/session_options.h"
|
||||
#include "core/graph/model.h"
|
||||
#include <string>
|
||||
#include "core/providers/qnn/builder/qnn_backend_manager.h"
|
||||
#include "core/providers/qnn/builder/qnn_model.h"
|
||||
#include "core/providers/qnn/builder/qnn_graph_configs_helper.h"
|
||||
#include "core/graph/model.h"
|
||||
#include "core/providers/qnn/builder/qnn_configs_helper.h"
|
||||
#include "HTP/QnnHtpGraph.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
|
||||
|
|
@ -58,7 +59,7 @@ class QNNExecutionProvider : public IExecutionProvider {
|
|||
|
||||
void ParseHtpGraphFinalizationOptimizationMode(const std::string& htp_graph_finalization_opt_mode_string);
|
||||
|
||||
void InitQnnGraphConfigs(qnn::QnnGraphConfigsBuilder& configs_holder) const;
|
||||
void InitQnnGraphConfigs(qnn::QnnConfigsBuilder<QnnGraph_Config_t, QnnHtpGraph_CustomConfig_t>& configs_builder) const;
|
||||
|
||||
private:
|
||||
qnn::HtpGraphFinalizationOptimizationMode htp_graph_finalization_opt_mode_ = qnn::HtpGraphFinalizationOptimizationMode::kDefault;
|
||||
|
|
|
|||
|
|
@ -60,6 +60,10 @@ void usage() {
|
|||
"\t [QNN only] [qnn_saver_path]: QNN Saver backend path. e.g '/folderpath/libQnnSaver.so'.\n"
|
||||
"\t [QNN only] [htp_graph_finalization_optimization_mode]: QNN graph finalization optimization mode, options: \n"
|
||||
"\t '0', '1', '2', '3', default is '0'.\n"
|
||||
"\t [QNN only] [soc_model]: The SoC Model number. Refer to QNN SDK documentation for specific values. Defaults to '0' (unknown). \n"
|
||||
"\t [QNN only] [htp_arch]: The minimum HTP architecture. The driver will use ops compatible with this architecture. \n"
|
||||
"\t Options are '0', '68', '69', '73', '75'. Defaults to '0' (none). \n"
|
||||
"\t [QNN only] [device_id]: The ID of the device to use when setting 'htp_arch'. Defaults to '0' (for single device). \n"
|
||||
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>' \n\n"
|
||||
"\t [Example] [For QNN EP] -e qnn -i \"profiling_level|detailed backend_path|/folderpath/libQnnCpu.so\" \n\n"
|
||||
"\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n"
|
||||
|
|
@ -483,7 +487,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
if (supported_profiling_level.find(value) == supported_profiling_level.end()) {
|
||||
ORT_THROW("Supported profiling_level: off, basic, detailed");
|
||||
}
|
||||
} else if (key == "rpc_control_latency" || key == "vtcm_mb") {
|
||||
} else if (key == "rpc_control_latency" || key == "vtcm_mb" || key == "soc_model" || key == "device_id") {
|
||||
// no validation
|
||||
} else if (key == "htp_performance_mode") {
|
||||
std::set<std::string> supported_htp_perf_mode = {"burst", "balanced", "default", "high_performance",
|
||||
|
|
@ -512,10 +516,20 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
|
|||
std::string str = str_stream.str();
|
||||
ORT_THROW("Wrong value for htp_graph_finalization_optimization_mode. select from: " + str);
|
||||
}
|
||||
} else if (key == "htp_arch") {
|
||||
std::unordered_set<std::string> supported_htp_archs = {"0", "68", "69", "73", "75"};
|
||||
if (supported_htp_archs.find(value) == supported_htp_archs.end()) {
|
||||
std::ostringstream str_stream;
|
||||
std::copy(supported_htp_archs.begin(), supported_htp_archs.end(),
|
||||
std::ostream_iterator<std::string>(str_stream, ","));
|
||||
std::string str = str_stream.str();
|
||||
ORT_THROW("Wrong value for htp_arch. select from: " + str);
|
||||
}
|
||||
} else {
|
||||
ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path',
|
||||
'profiling_level', 'rpc_control_latency', 'vtcm_mb', 'htp_performance_mode',
|
||||
'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority'])");
|
||||
'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority',
|
||||
'soc_model', 'htp_arch', 'device_id'])");
|
||||
}
|
||||
|
||||
qnn_options[key] = value;
|
||||
|
|
|
|||
|
|
@ -78,6 +78,10 @@ namespace perftest {
|
|||
"\t [QNN only] [qnn_saver_path]: QNN Saver backend path. e.g '/folderpath/libQnnSaver.so'.\n"
|
||||
"\t [QNN only] [htp_graph_finalization_optimization_mode]: QNN graph finalization optimization mode, options: \n"
|
||||
"\t '0', '1', '2', '3', default is '0'.\n"
|
||||
"\t [QNN only] [soc_model]: The SoC Model number. Refer to QNN SDK documentation for specific values. Defaults to '0' (unknown). \n"
|
||||
"\t [QNN only] [htp_arch]: The minimum HTP architecture. The driver will use ops compatible with this architecture. \n"
|
||||
"\t Options are '0', '68', '69', '73', '75'. Defaults to '0' (none). \n"
|
||||
"\t [QNN only] [device_id]: The ID of the device to use when setting 'htp_arch'. Defaults to '0' (for single device). \n"
|
||||
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
|
||||
"\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU_FP32 enable_npu_fast_compile|true num_of_threads|5 enable_opencl_throttling|true cache_dir|\"<path>\"\"\n"
|
||||
"\t [Example] [For QNN EP] -e qnn -i \"backend_path|/folderpath/libQnnCpu.so\" \n\n"
|
||||
|
|
|
|||
|
|
@ -343,7 +343,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
if (supported_profiling_level.find(value) == supported_profiling_level.end()) {
|
||||
ORT_THROW("Supported profiling_level: off, basic, detailed");
|
||||
}
|
||||
} else if (key == "rpc_control_latency" || key == "vtcm_mb") {
|
||||
} else if (key == "rpc_control_latency" || key == "vtcm_mb" || key == "soc_model" || key == "device_id") {
|
||||
// no validation
|
||||
} else if (key == "htp_performance_mode") {
|
||||
std::set<std::string> supported_htp_perf_mode = {"burst", "balanced", "default", "high_performance",
|
||||
|
|
@ -372,10 +372,20 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
|
|||
if (supported_qnn_context_priority.find(value) == supported_qnn_context_priority.end()) {
|
||||
ORT_THROW("Supported qnn_context_priority: low, normal, normal_high, high");
|
||||
}
|
||||
} else if (key == "htp_arch") {
|
||||
std::unordered_set<std::string> supported_htp_archs = {"0", "68", "69", "73", "75"};
|
||||
if (supported_htp_archs.find(value) == supported_htp_archs.end()) {
|
||||
std::ostringstream str_stream;
|
||||
std::copy(supported_htp_archs.begin(), supported_htp_archs.end(),
|
||||
std::ostream_iterator<std::string>(str_stream, ","));
|
||||
std::string str = str_stream.str();
|
||||
ORT_THROW("Wrong value for htp_arch. select from: " + str);
|
||||
}
|
||||
} else {
|
||||
ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path',
|
||||
'profiling_level', 'rpc_control_latency', 'vtcm_mb', 'htp_performance_mode',
|
||||
'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority'])");
|
||||
'qnn_saver_path', 'htp_graph_finalization_optimization_mode', 'qnn_context_priority', 'soc_model',
|
||||
'htp_arch', 'device_id'])");
|
||||
}
|
||||
|
||||
qnn_options[key] = value;
|
||||
|
|
|
|||
|
|
@ -176,7 +176,10 @@ TEST(QnnEP, TestDisableCPUFallback_ConflictingConfig) {
|
|||
// types and shapes.
|
||||
static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp, bool enable_qnn_saver = false,
|
||||
std::string htp_graph_finalization_opt_mode = "",
|
||||
std::string qnn_context_priority = "") {
|
||||
std::string qnn_context_priority = "",
|
||||
std::string soc_model = "",
|
||||
std::string htp_arch = "",
|
||||
std::string device_id = "") {
|
||||
Ort::SessionOptions so;
|
||||
|
||||
// Ensure all type/shape inference warnings result in errors!
|
||||
|
|
@ -205,6 +208,18 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp, bo
|
|||
options["qnn_context_priority"] = std::move(qnn_context_priority);
|
||||
}
|
||||
|
||||
if (!soc_model.empty()) {
|
||||
options["soc_model"] = std::move(soc_model);
|
||||
}
|
||||
|
||||
if (!htp_arch.empty()) {
|
||||
options["htp_arch"] = std::move(htp_arch);
|
||||
}
|
||||
|
||||
if (!device_id.empty()) {
|
||||
options["device_id"] = std::move(device_id);
|
||||
}
|
||||
|
||||
so.AppendExecutionProvider("QNN", options);
|
||||
|
||||
Ort::Session session(*ort_env, ort_model_path, so);
|
||||
|
|
@ -519,6 +534,45 @@ TEST_F(QnnHTPBackendTests, HTPGraphFinalizationOptimizationModes) {
|
|||
}
|
||||
}
|
||||
|
||||
// Test that models run with various SoC model values
|
||||
TEST_F(QnnHTPBackendTests, HTPSocModels) {
|
||||
constexpr std::array<const char*, 3> soc_models = { "", // No explicit SoC model specified
|
||||
"0", // "Unknown"
|
||||
#if defined(_M_ARM64)
|
||||
"37" }; // SC8280X
|
||||
#elif defined(__linux__)
|
||||
"30" }; // SM8350
|
||||
#else
|
||||
"" };
|
||||
#endif
|
||||
|
||||
for (auto soc_model : soc_models) {
|
||||
RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.quant.onnx",
|
||||
true, // use_htp
|
||||
false, // enable_qnn_saver
|
||||
"", // htp_graph_finalization_opt_mode
|
||||
"", // qnn_context_priority
|
||||
soc_model);
|
||||
}
|
||||
}
|
||||
|
||||
// Test that models run with various HTP architecture values (and set device_id)
|
||||
TEST_F(QnnHTPBackendTests, HTPArchValues) {
|
||||
constexpr std::array<const char*, 3> htp_archs = {"", // No explicit arch specified
|
||||
"0", // "None"
|
||||
"68"}; // v68
|
||||
for (auto htp_arch : htp_archs) {
|
||||
RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.quant.onnx",
|
||||
true, // use_htp
|
||||
false, // enable_qnn_saver
|
||||
"", // htp_graph_finalization_opt_mode
|
||||
"", // qnn_context_priority
|
||||
"", // soc_model
|
||||
htp_arch, // htp_arch
|
||||
"0"); // device_id
|
||||
}
|
||||
}
|
||||
|
||||
// Test that models run with high QNN context priority.
|
||||
TEST_F(QnnHTPBackendTests, QnnContextPriorityHigh) {
|
||||
RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.quant.onnx",
|
||||
|
|
|
|||
Loading…
Reference in a new issue