saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-03 03:58:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable user to set QNN HTP performance mode for every session run (#19521)

Browse source 
### Description
Currently, the QNN HTP performance mode is set during session creation, there's no way to change it afterwards. There's requirement to set it high performance mode for high priority request and set it back to low performance mode later to save the power when the incoming request is idle for example.

Now, still keeps the performance mode at the session level in QNN EP options which is used at the default one. Ort QNN EP will set it once if user set it.
And there are setting (qnn.htp_perf_mode and qnn.htp_perf_mode_post_run) in run option to change the performance mode before and after session run. There's recommended scenario that user set the mode to high performance mode before the the inference sun so that user can get the result back ASAP. And set the mode to low performance mode after the inference to save the power.
This commit is contained in:
Hector Li 2024-02-22 17:04:59 -08:00 committed by GitHub
parent 5e5c36f6df
commit 4ab497603e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 575 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
include/onnxruntime/core/framework/execution_provider.h
View file

@ -33,6 +33,8 @@ class Node;
#include "core/framework/stream_handles.h"
#include "core/framework/tuning_context.h"
struct OrtRunOptions;
namespace onnxruntime {
/**
@ -51,6 +53,8 @@ struct NodeComputeInfo {
DestroyFunctionStateFunc release_state_func;
};
using RunOptions = OrtRunOptions;
enum class DataLayout {
NCHW,
NHWC,
@ -184,7 +188,7 @@ class IExecutionProvider {
Run may not be finished on device This function should be regarded as the
point after which a new Run would start to submit commands from CPU
*/
virtual common::Status OnRunStart() { return Status::OK(); }
virtual common::Status OnRunStart(const onnxruntime::RunOptions& /*run_options*/) { return Status::OK(); }
/**
Called when InferenceSession::Run ended
@ -192,7 +196,9 @@ class IExecutionProvider {
may not be finished on device This function should be regarded as the point
that all commands of current Run has been submmited by CPU
*/
virtual common::Status OnRunEnd(bool /*sync_stream*/) { return Status::OK(); }
virtual common::Status OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& /*run_options*/) {
return Status::OK();
}
/**
Indicate whether the graph capturing mode (e.g., cuda graph) is enabled for

12
include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h
View file

@ -30,3 +30,15 @@ static const char* const kOrtRunOptionsConfigEnableMemoryArenaShrinkage = "memor
// Per default it will be set to '0'
// Taking CUDA EP as an example, it omit triggering cudaStreamSynchronize on the compute stream.
static const char* const kOrtRunOptionsConfigDisableSynchronizeExecutionProviders = "disable_synchronize_execution_providers";
// Set HTP performance mode for QNN HTP backend before session run.
// options for HTP performance mode: "burst", "balanced", "default", "high_performance",
// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver",
// "sustained_high_performance". Default to "default".
static const char* const kOrtRunOptionsConfigQnnPerfMode = "qnn.htp_perf_mode";
// Set HTP performance mode for QNN HTP backend post session run.
static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_mode_post_run";
// Set RPC control latency for QNN HTP backend
static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency";

4
onnxruntime/core/framework/stream_execution_context.cc
View file

@ -181,11 +181,13 @@ void RunSince(size_t stream_idx, StreamExecutionContext& ctx, SessionScope& sess
}
#ifdef USE_CANN
// Leave it to CANN EP to fill the gap if they want to use run_options
static onnxruntime::RunOptions run_options;
// For CANN EP, it is necessary to explicitly create a corresponding Context for each thread in the thread pool,
// which is different from CUDA Runtime API, but similar to CUDA Driver API.
auto& execution_providers = ctx.GetSessionState().GetExecutionProviders();
for (auto& xp : execution_providers) {
auto status = xp->OnRunStart();
auto status = xp->OnRunStart(run_options);
if (!status.IsOK()) {
ctx.SetStatus(status);
return;

2
onnxruntime/core/providers/cann/cann_execution_provider.cc
View file

@ -1045,7 +1045,7 @@ CANNExecutionProvider::~CANNExecutionProvider() {
}
// All threads share the same context and stream
Status CANNExecutionProvider::OnRunStart() {
Status CANNExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
CANN_RETURN_IF_ERROR(aclrtSetDevice(info_.device_id));
return Status::OK();

2
onnxruntime/core/providers/cann/cann_execution_provider.h
View file

@ -33,7 +33,7 @@ class CANNExecutionProvider : public IExecutionProvider {
explicit CANNExecutionProvider(const CANNExecutionProviderInfo& info);
virtual ~CANNExecutionProvider();
Status OnRunStart() override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
template <typename T>
Status Fill(Tensor* y, void* addr, aclrtStream stream) const {

4
onnxruntime/core/providers/cuda/cuda_execution_provider.cc
View file

@ -386,7 +386,7 @@ Status CUDAExecutionProvider::Sync() const {
return Status::OK();
}
Status CUDAExecutionProvider::OnRunStart() {
Status CUDAExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
// always set CUDA device when session::Run() in case it runs in a worker thread
CUDA_RETURN_IF_ERROR(cudaSetDevice(GetDeviceId()));
if (IsGraphCaptureEnabled() && GetPerThreadContext().IsGraphCaptureAllowed() && !GetPerThreadContext().IsGraphCaptured()) {
@ -396,7 +396,7 @@ Status CUDAExecutionProvider::OnRunStart() {
return Status::OK();
}
Status CUDAExecutionProvider::OnRunEnd(bool sync_stream) {
Status CUDAExecutionProvider::OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& /*run_options*/) {
if (IsGraphCaptureEnabled() && !GetPerThreadContext().IsGraphCaptured()) {
if (GetPerThreadContext().IsGraphCaptureAllowed()) {
GetPerThreadContext().CaptureEnd();

5
onnxruntime/core/providers/cuda/cuda_execution_provider.h
View file

@ -29,9 +29,9 @@ class CUDAExecutionProvider : public IExecutionProvider {
Status Sync() const override;
Status OnRunStart() override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
DataLayout GetPreferredLayout() const override;
@ -115,6 +115,7 @@ class CUDAExecutionProvider : public IExecutionProvider {
PerThreadContext(OrtDevice::DeviceId device_id, cudaStream_t stream, size_t cuda_mem_limit, ArenaExtendStrategy arena_extend_strategy,
CUDAExecutionProviderExternalAllocatorInfo external_alloc_info, OrtArenaCfg* arena_cfg);
~PerThreadContext();
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerThreadContext);
cublasHandle_t CublasHandle() const {
return cublas_handle_;

4
onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionProvider.h
View file

@ -270,7 +270,7 @@ namespace Dml
return m_impl->OnSessionInitializationEnd();
}
virtual onnxruntime::Status Sync() const final override
onnxruntime::Status Sync() const final override
{
// Completely wait until the device has completed all preceding tasks.
// The application could have called SynchronizeBoundOutputs().
@ -278,7 +278,7 @@ namespace Dml
return Status::OK();
}
virtual onnxruntime::Status OnRunEnd(bool /*sync_stream*/) final override
onnxruntime::Status OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& /*run_options*/) final override
{
// Flush any pending work to the GPU, but don't block for completion, permitting it
// to overlap other work.

4
onnxruntime/core/providers/js/js_execution_provider.cc
View file

@ -756,7 +756,7 @@ std::unique_ptr<onnxruntime::IDataTransfer> JsExecutionProvider::GetDataTransfer
JsExecutionProvider::~JsExecutionProvider() {
}
Status JsExecutionProvider::OnRunStart() {
Status JsExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
if (IsGraphCaptureEnabled() && IsGraphCaptureAllowed() && !IsGraphCaptured()) {
LOGS(*GetLogger(), INFO) << "Capturing the webgpu graph for this model";
EM_ASM({ Module.jsepCaptureBegin(); });
@ -764,7 +764,7 @@ Status JsExecutionProvider::OnRunStart() {
return Status::OK();
}
Status JsExecutionProvider::OnRunEnd(bool sync_stream) {
Status JsExecutionProvider::OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& /*run_options*/) {
if (IsGraphCaptureEnabled() && !IsGraphCaptured()) {
if (IsGraphCaptureAllowed()) {
EM_ASM({ Module.jsepCaptureEnd(); });

4
onnxruntime/core/providers/js/js_execution_provider.h
View file

@ -59,8 +59,8 @@ class JsExecutionProvider : public IExecutionProvider {
std::vector<AllocatorPtr> CreatePreferredAllocators() override;
Status OnRunStart() override;
Status OnRunEnd(bool sync_stream) override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
bool IsGraphCaptureEnabled() const override;
bool IsGraphCaptured() const override;

4
onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc
View file

@ -1383,11 +1383,11 @@ Status MIGraphXExecutionProvider::Sync() const {
return Status::OK();
}
Status MIGraphXExecutionProvider::OnRunStart() {
Status MIGraphXExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
return Status::OK();
}
Status MIGraphXExecutionProvider::OnRunEnd(bool) {
Status MIGraphXExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& /*run_options*/) {
auto status = hipStreamQuery(stream_);
if (status != hipSuccess) {

4
onnxruntime/core/providers/migraphx/migraphx_execution_provider.h
View file

@ -56,9 +56,9 @@ class MIGraphXExecutionProvider : public IExecutionProvider {
#ifdef MIGRAPHX_STREAM_SYNC
Status Sync() const override;
Status OnRunStart() override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
#endif
std::vector<std::unique_ptr<ComputeCapability>>

75
onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc
View file

@ -634,11 +634,6 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool load_
LOGS(logger, VERBOSE) << "CreateContext succeed.";
}
if (htp_performance_mode_ != HtpPerformanceMode::kHtpDefault) {
ORT_RETURN_IF_ERROR(SetHtpPowerConfig());
LOGS(logger, VERBOSE) << "SetHtpPowerConfig succeed.";
}
LOGS(logger, VERBOSE) << "QNN SetupBackend succeed";
backend_setup_completed_ = true;
@ -646,7 +641,7 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool load_
return Status::OK();
}
Status QnnBackendManager::SetHtpPowerConfig() {
Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id, uint32_t core_id, uint32_t& htp_power_config_id) {
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
@ -656,23 +651,37 @@ Status QnnBackendManager::SetHtpPowerConfig() {
"HTP infra type = ", htp_infra->infraType, ", which is not perf infra type.");
QnnHtpDevice_PerfInfrastructure_t& htp_perf_infra = htp_infra->perfInfra;
// Get power client id
status = htp_perf_infra.createPowerConfigId(/*device_id=*/0, /*core_id=*/0, &htp_power_config_client_id_);
status = htp_perf_infra.createPowerConfigId(device_id, core_id, &htp_power_config_id);
ORT_RETURN_IF(QNN_SUCCESS != status, "createPowerConfigId failed.");
return Status::OK();
}
Status QnnBackendManager::SetHtpPowerConfig(uint32_t htp_power_config_client_id,
HtpPerformanceMode htp_performance_mode) {
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
auto* htp_infra = static_cast<QnnHtpDevice_Infrastructure_t*>(qnn_device_infra);
ORT_RETURN_IF(QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF != htp_infra->infraType,
"HTP infra type = ", htp_infra->infraType, ", which is not perf infra type.");
QnnHtpDevice_PerfInfrastructure_t& htp_perf_infra = htp_infra->perfInfra;
constexpr const int kNumConfigs = 1;
std::vector<QnnHtpPerfInfrastructure_PowerConfig_t> power_configs(
kNumConfigs);
QnnHtpPerfInfrastructure_PowerConfig_t& dcvs_config = power_configs[0];
dcvs_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3;
QnnHtpPerfInfrastructure_DcvsV3_t& dcvs_v3 = dcvs_config.dcvsV3Config;
dcvs_v3.contextId = htp_power_config_client_id_;
dcvs_v3.contextId = htp_power_config_client_id;
dcvs_v3.setSleepDisable = 0;
dcvs_v3.sleepDisable = 0;
dcvs_v3.setDcvsEnable = 1;
dcvs_v3.dcvsEnable = kDcvsDisable;
dcvs_v3.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE;
// choose performance mode
switch (htp_performance_mode_) {
switch (htp_performance_mode) {
case HtpPerformanceMode::kHtpBurst:
dcvs_v3.setSleepLatency = 1; // true
dcvs_v3.sleepLatency = kSleepMinLatency;
@ -771,25 +780,40 @@ Status QnnBackendManager::SetHtpPowerConfig() {
dcvs_v3.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_NOM_PLUS;
break;
default:
ORT_THROW("Invalid performance profile %d", static_cast<int>(htp_performance_mode_));
ORT_THROW("Invalid performance profile %d", static_cast<int>(htp_performance_mode));
break;
}
std::vector<const QnnHtpPerfInfrastructure_PowerConfig_t*> perf_power_configs_ptr = ObtainNullTermPtrVector(power_configs);
status = htp_perf_infra.setPowerConfig(htp_power_config_client_id_, perf_power_configs_ptr.data());
status = htp_perf_infra.setPowerConfig(htp_power_config_client_id, perf_power_configs_ptr.data());
ORT_RETURN_IF(QNN_SUCCESS != status, "setPowerConfig failed for HTP performance mode.");
// Set rpc control latency here, but note that v68 doesn't support rpc polling mode.
if (rpc_control_latency_ != 0) {
return Status::OK();
}
Status QnnBackendManager::SetRpcControlLatency(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency) {
if (rpc_control_latency != 0) {
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
auto* htp_infra = static_cast<QnnHtpDevice_Infrastructure_t*>(qnn_device_infra);
ORT_RETURN_IF(QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF != htp_infra->infraType,
"HTP infra type = ", htp_infra->infraType, ", which is not perf infra type.");
QnnHtpDevice_PerfInfrastructure_t& htp_perf_infra = htp_infra->perfInfra;
// Set rpc control latency here, but note that v68 doesn't support rpc polling mode.
constexpr int kNumRpcPollingPowerConfigs = 2;
std::vector<QnnHtpPerfInfrastructure_PowerConfig_t> rpc_power_configs(kNumRpcPollingPowerConfigs);
QnnHtpPerfInfrastructure_PowerConfig_t& rpc_control_latency = rpc_power_configs[0];
QnnHtpPerfInfrastructure_PowerConfig_t& rpc_control_latency_cfg = rpc_power_configs[0];
// v68 doesn't support this.
QnnHtpPerfInfrastructure_PowerConfig_t& rpc_polling_time = rpc_power_configs[1];
rpc_control_latency.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY;
rpc_control_latency_cfg.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY;
rpc_polling_time.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME;
rpc_control_latency.rpcControlLatencyConfig = rpc_control_latency_;
perf_power_configs_ptr = ObtainNullTermPtrVector(rpc_power_configs);
status = htp_perf_infra.setPowerConfig(htp_power_config_client_id_, perf_power_configs_ptr.data());
rpc_control_latency_cfg.rpcControlLatencyConfig = rpc_control_latency;
std::vector<const QnnHtpPerfInfrastructure_PowerConfig_t*> perf_power_configs_ptr =
ObtainNullTermPtrVector(rpc_power_configs);
status = htp_perf_infra.setPowerConfig(htp_power_config_client_id, perf_power_configs_ptr.data());
ORT_RETURN_IF(QNN_SUCCESS != status, "setPowerConfig failed for RPC control latency.");
}
@ -810,11 +834,7 @@ void QnnBackendManager::Split(std::vector<std::string>& split_string,
}
}
Status QnnBackendManager::DestroyHTPPowerConfigID() {
if (htp_performance_mode_ == HtpPerformanceMode::kHtpDefault) {
return Status::OK();
}
Status QnnBackendManager::DestroyHTPPowerConfigID(uint32_t htp_power_config_id) {
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
@ -824,7 +844,7 @@ Status QnnBackendManager::DestroyHTPPowerConfigID() {
"HTP infra type = ", htp_infra->infraType, ", which is not perf infra type.");
QnnHtpDevice_PerfInfrastructure_t& htp_perf_infra = htp_infra->perfInfra;
Qnn_ErrorHandle_t destroy_ret = htp_perf_infra.destroyPowerConfigId(htp_power_config_client_id_);
Qnn_ErrorHandle_t destroy_ret = htp_perf_infra.destroyPowerConfigId(htp_power_config_id);
ORT_RETURN_IF(QNN_SUCCESS != destroy_ret, "destroyPowerConfigId failed.");
return Status::OK();
}
@ -834,12 +854,7 @@ void QnnBackendManager::ReleaseResources() {
return;
}
auto result = DestroyHTPPowerConfigID();
if (Status::OK() != result) {
ORT_THROW("Failed to DestroyHTPPowerConfigID.");
}
result = ReleaseContext();
auto result = ReleaseContext();
if (Status::OK() != result) {
ORT_THROW("Failed to ReleaseContext.");
}

19
onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h
View file

@ -33,8 +33,6 @@ class QnnBackendManager {
public:
QnnBackendManager(std::string&& backend_path,
ProfilingLevel profiling_level,
uint32_t rpc_control_latency,
HtpPerformanceMode htp_performance_mode,
ContextPriority context_priority,
std::string&& qnn_saver_path,
uint32_t device_id,
@ -42,8 +40,6 @@ class QnnBackendManager {
uint32_t soc_model)
: backend_path_(backend_path),
profiling_level_(profiling_level),
rpc_control_latency_(rpc_control_latency),
htp_performance_mode_(htp_performance_mode),
context_priority_(context_priority),
qnn_saver_path_(qnn_saver_path),
device_id_(device_id),
@ -92,7 +88,13 @@ class QnnBackendManager {
Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context);
Status SetHtpPowerConfig();
Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
Status SetHtpPowerConfig(uint32_t htp_power_config_client_id,
HtpPerformanceMode htp_performance_mode);
Status SetRpcControlLatency(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency);
const QNN_INTERFACE_VER_TYPE& GetQnnInterface() { return qnn_interface_; }
@ -141,6 +143,8 @@ class QnnBackendManager {
const std::string& GetSdkVersion() { return sdk_build_version_; }
Status DestroyHTPPowerConfigID(uint32_t htp_power_config_id);
private:
void* LoadLib(const char* file_name, int flags, std::string& error_msg);
@ -150,8 +154,6 @@ class QnnBackendManager {
Status UnloadLib(void* handle);
Status DestroyHTPPowerConfigID();
void* LibFunction(void* handle, const char* symbol, std::string& error_msg);
template <class T>
@ -232,15 +234,12 @@ class QnnBackendManager {
QnnBackendType qnn_backend_type_ = QnnBackendType::CPU;
Qnn_ProfileHandle_t profile_backend_handle_ = nullptr;
std::vector<std::string> op_package_paths_;
uint32_t rpc_control_latency_ = 0;
HtpPerformanceMode htp_performance_mode_;
ContextPriority context_priority_;
std::string sdk_build_version_ = "";
#ifdef _WIN32
std::set<HMODULE> mod_handles_;
#endif
const std::string qnn_saver_path_;
uint32_t htp_power_config_client_id_ = 0;
uint32_t device_id_ = 0;
QnnHtpDevice_Arch_t htp_arch_ = QNN_HTP_DEVICE_ARCH_NONE;
uint32_t soc_model_ = QNN_SOC_MODEL_UNKNOWN;

198
onnxruntime/core/providers/qnn/qnn_execution_provider.cc
View file

@ -7,6 +7,7 @@
#include "core/framework/compute_capability.h"
#include "core/graph/graph_viewer.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/session/onnxruntime_run_options_config_keys.h"
#include "core/session/onnxruntime_cxx_api.h"
#include "core/framework/kernel_registry.h"
#include "core/platform/env.h"
@ -18,11 +19,36 @@
#include "core/providers/qnn/builder/op_builder_factory.h"
#include "core/providers/qnn/builder/qnn_def.h"
#include "core/providers/qnn/builder/onnx_ctx_model_helper.h"
#include "core/framework/run_options.h"
namespace onnxruntime {
constexpr const char* QNN = "QNN";
static std::unique_ptr<std::vector<std::function<void()>>> s_run_on_unload_;
void RunOnUnload(std::function<void()> function) {
OrtMutex mutex;
std::lock_guard<OrtMutex> guard(mutex);
if (!s_run_on_unload_) {
s_run_on_unload_ = std::make_unique<std::vector<std::function<void()>>>();
}
s_run_on_unload_->push_back(std::move(function));
}
struct OnUnload {
~OnUnload() {
if (!s_run_on_unload_)
return;
for (auto& function : *s_run_on_unload_)
function();
s_run_on_unload_.reset();
}
} g_on_unload;
static void ParseProfilingLevel(std::string profiling_level_string,
qnn::ProfilingLevel& profiling_level) {
std::transform(profiling_level_string.begin(),
@ -193,18 +219,18 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio
}
static const std::string RPC_CONTROL_LANTENCY = "rpc_control_latency";
uint32_t rpc_control_latency = 0;
auto latency_pos = provider_options_map.find(RPC_CONTROL_LANTENCY);
if (latency_pos != provider_options_map.end()) {
rpc_control_latency = static_cast<uint32_t>(std::stoul(latency_pos->second));
LOGS_DEFAULT(VERBOSE) << "rpc_control_latency: " << rpc_control_latency;
default_rpc_control_latency_ = static_cast<uint32_t>(std::stoul(latency_pos->second));
LOGS_DEFAULT(VERBOSE) << "rpc_control_latency: " << default_rpc_control_latency_;
}
qnn::HtpPerformanceMode htp_performance_mode = qnn::HtpPerformanceMode::kHtpDefault;
// default_htp_performance_mode from QNN EP option.
// set it once only for each thread as default so user don't need to set it for every session run
static const std::string HTP_PERFORMANCE_MODE = "htp_performance_mode";
auto htp_performance_mode_pos = provider_options_map.find(HTP_PERFORMANCE_MODE);
if (htp_performance_mode_pos != provider_options_map.end()) {
ParseHtpPerformanceMode(htp_performance_mode_pos->second, htp_performance_mode);
ParseHtpPerformanceMode(htp_performance_mode_pos->second, default_htp_performance_mode_);
}
htp_graph_finalization_opt_mode_ = qnn::HtpGraphFinalizationOptimizationMode::kDefault;
@ -241,15 +267,14 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio
}
static const std::string QNN_DEVICE_ID = "device_id";
uint32_t device_id = 0;
auto dev_id_pos = provider_options_map.find(QNN_DEVICE_ID);
if (dev_id_pos != provider_options_map.end()) {
int value = std::stoi(dev_id_pos->second);
if (value < 0) {
LOGS_DEFAULT(WARNING) << "Invalid device ID '" << value
<< "', only >= 0 allowed. Set to " << device_id << ".";
<< "', only >= 0 allowed. Set to " << device_id_ << ".";
} else {
device_id = static_cast<uint32_t>(value);
device_id_ = static_cast<uint32_t>(value);
}
}
@ -276,15 +301,23 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio
qnn_backend_manager_ = std::make_unique<qnn::QnnBackendManager>(
std::move(backend_path),
profiling_level,
rpc_control_latency,
htp_performance_mode,
context_priority,
std::move(qnn_saver_path),
device_id,
device_id_,
htp_arch,
soc_model);
}
QNNExecutionProvider::~QNNExecutionProvider() {
// clean up thread local context caches
std::lock_guard<OrtMutex> lock(context_state_.mutex);
for (const auto& cache_weak : context_state_.caches_to_update_on_destruction) {
const auto cache = cache_weak.lock();
if (!cache) continue;
ORT_IGNORE_RETURN_VALUE(cache->erase(this));
}
}
bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit,
const logging::Logger& logger) const {
const std::string& op_type = node_unit.OpType();
@ -725,4 +758,147 @@ const InlinedVector<const Node*> QNNExecutionProvider::GetEpContextNodes() const
return ep_context_nodes;
}
QNNExecutionProvider::PerThreadContext::PerThreadContext(qnn::QnnBackendManager* qnn_backend_manager,
uint32_t device_id,
uint32_t core_id,
qnn::HtpPerformanceMode default_htp_performance_mode,
uint32_t default_rpc_control_latency)
: qnn_backend_manager_(qnn_backend_manager) {
Status rt = qnn_backend_manager_->CreateHtpPowerCfgId(device_id, core_id, htp_power_config_id_);
is_htp_power_config_id_valid_ = rt.IsOK();
// default_htp_performance_mode and default_rpc_control_latency are from QNN EP option.
// set it once only for each thread as default so user don't need to set it for every session run
if (is_htp_power_config_id_valid_) {
if (qnn::HtpPerformanceMode::kHtpDefault != default_htp_performance_mode) {
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->SetHtpPowerConfig(htp_power_config_id_,
default_htp_performance_mode));
}
if (default_rpc_control_latency > 0) {
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->SetRpcControlLatency(htp_power_config_id_,
default_rpc_control_latency));
}
}
}
QNNExecutionProvider::PerThreadContext::~PerThreadContext() {
if (is_htp_power_config_id_valid_) {
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->DestroyHTPPowerConfigID(htp_power_config_id_));
}
}
QNNExecutionProvider::PerThreadContext& QNNExecutionProvider::GetPerThreadContext() const {
const auto& per_thread_context_cache = PerThreadContextCache();
// try to use cached context
auto cached_context_it = per_thread_context_cache->find(this);
if (cached_context_it != per_thread_context_cache->end()) {
auto cached_context = cached_context_it->second.lock();
ORT_ENFORCE(cached_context);
return *cached_context;
}
// get context and update cache
std::shared_ptr<PerThreadContext> context;
{
std::lock_guard<OrtMutex> lock(context_state_.mutex);
// get or create a context
if (context_state_.retired_context_pool.empty()) {
uint32_t core_id = 0;
context = std::make_shared<PerThreadContext>(qnn_backend_manager_.get(), device_id_, core_id,
default_htp_performance_mode_, default_rpc_control_latency_);
} else {
context = context_state_.retired_context_pool.back();
context_state_.retired_context_pool.pop_back();
}
// insert into active_contexts, should not already be present
const auto active_contexts_insert_result = context_state_.active_contexts.insert(context);
ORT_ENFORCE(active_contexts_insert_result.second);
// insert into caches_to_update_on_destruction, may already be present
ORT_IGNORE_RETURN_VALUE(context_state_.caches_to_update_on_destruction.insert(per_thread_context_cache));
}
per_thread_context_cache->insert(std::make_pair(this, context));
return *context;
}
void QNNExecutionProvider::ReleasePerThreadContext() const {
const auto& per_thread_context_cache = PerThreadContextCache();
auto cached_context_it = per_thread_context_cache->find(this);
ORT_ENFORCE(cached_context_it != per_thread_context_cache->end());
auto cached_context = cached_context_it->second.lock();
ORT_ENFORCE(cached_context);
{
std::lock_guard<OrtMutex> lock(context_state_.mutex);
context_state_.active_contexts.erase(cached_context);
context_state_.retired_context_pool.push_back(cached_context);
}
per_thread_context_cache->erase(cached_context_it);
}
Status QNNExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_options) {
auto backend_type = qnn_backend_manager_->GetQnnBackendType();
if (qnn::QnnBackendType::HTP != backend_type && qnn::QnnBackendType::DSP != backend_type) {
return Status::OK();
}
std::string htp_perf_mode = "";
qnn::HtpPerformanceMode htp_performance_mode = qnn::HtpPerformanceMode::kHtpDefault;
if (run_options.config_options.TryGetConfigEntry(kOrtRunOptionsConfigQnnPerfMode, htp_perf_mode)) {
// set power mode
ParseHtpPerformanceMode(htp_perf_mode, htp_performance_mode);
}
std::string rpc_latency = "";
uint32_t rpc_control_latency = 0;
if (run_options.config_options.TryGetConfigEntry(kOrtRunOptionsConfigQnnRpcControlLatency, rpc_latency)) {
rpc_control_latency = static_cast<uint32_t>(std::stoul(rpc_latency));
LOGS_DEFAULT(VERBOSE) << "rpc_control_latency: " << rpc_control_latency;
}
if (GetPerThreadContext().IsHtpPowerConfigIdValid()) {
if (qnn::HtpPerformanceMode::kHtpDefault != htp_performance_mode) {
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetHtpPowerConfig(GetPerThreadContext().GetHtpPowerConfigId(),
htp_performance_mode));
}
if (rpc_control_latency > 0) {
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetRpcControlLatency(GetPerThreadContext().GetHtpPowerConfigId(),
rpc_control_latency));
}
}
return Status::OK();
}
Status QNNExecutionProvider::OnRunEnd(bool /*sync_stream*/, const onnxruntime::RunOptions& run_options) {
auto backend_type = qnn_backend_manager_->GetQnnBackendType();
if (qnn::QnnBackendType::HTP != backend_type && qnn::QnnBackendType::DSP != backend_type) {
return Status::OK();
}
std::string htp_perf_mode = "";
qnn::HtpPerformanceMode htp_performance_mode = qnn::HtpPerformanceMode::kHtpDefault;
if (run_options.config_options.TryGetConfigEntry(kOrtRunOptionsConfigQnnPerfModePostRun, htp_perf_mode)) {
// set power mode
ParseHtpPerformanceMode(htp_perf_mode, htp_performance_mode);
}
if (qnn::HtpPerformanceMode::kHtpDefault != htp_performance_mode) {
if (!GetPerThreadContext().IsHtpPowerConfigIdValid()) {
return Status::OK();
}
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetHtpPowerConfig(GetPerThreadContext().GetHtpPowerConfigId(),
htp_performance_mode));
}
return Status::OK();
}
} // namespace onnxruntime

73
onnxruntime/core/providers/qnn/qnn_execution_provider.h
View file

@ -12,14 +12,19 @@
#include "core/providers/qnn/builder/qnn_model.h"
#include "core/providers/qnn/builder/qnn_configs_helper.h"
#include "HTP/QnnHtpGraph.h"
#include <vector>
#include <set>
#include <unordered_map>
namespace onnxruntime {
void RunOnUnload(std::function<void()> function);
// Logical device representation.
class QNNExecutionProvider : public IExecutionProvider {
public:
explicit QNNExecutionProvider(const ProviderOptions& provider_options_map, const SessionOptions* session_options);
virtual ~QNNExecutionProvider() = default;
virtual ~QNNExecutionProvider();
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QNNExecutionProvider);
// we implement the Compile that takes FusedNodeAndGraph instances
@ -40,6 +45,10 @@ class QNNExecutionProvider : public IExecutionProvider {
const InlinedVector<const Node*> GetEpContextNodes() const override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
private:
bool IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit,
const logging::Logger& logger) const;
@ -72,6 +81,68 @@ class QNNExecutionProvider : public IExecutionProvider {
int32_t vtcm_size_in_mb_ = 0;
std::unique_ptr<onnxruntime::Model> qnn_ep_context_model_;
ModelMetadefIdGenerator metadef_id_generator_;
uint32_t device_id_ = 0;
qnn::HtpPerformanceMode default_htp_performance_mode_ = qnn::HtpPerformanceMode::kHtpDefault;
uint32_t default_rpc_control_latency_ = 0;
class PerThreadContext final {
public:
PerThreadContext(qnn::QnnBackendManager* qnn_backend_manager,
uint32_t device_id, uint32_t core_id,
qnn::HtpPerformanceMode default_htp_performance_mode,
uint32_t default_rpc_control_latency);
~PerThreadContext();
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerThreadContext);
bool IsHtpPowerConfigIdValid() { return is_htp_power_config_id_valid_; }
uint32_t GetHtpPowerConfigId() { return htp_power_config_id_; }
private:
bool is_htp_power_config_id_valid_ = false;
uint32_t htp_power_config_id_ = 0;
qnn::QnnBackendManager* qnn_backend_manager_;
};
using PerThreadContextMap = std::unordered_map<const QNNExecutionProvider*, std::weak_ptr<PerThreadContext>>;
struct ContextCacheHolder {
ContextCacheHolder() {
RunOnUnload([&, weak_p_ = std::weak_ptr<PerThreadContextMap>(p)] {
if (auto lock = weak_p_.lock())
p.reset();
});
}
std::shared_ptr<PerThreadContextMap> p = std::make_shared<PerThreadContextMap>();
};
static const std::shared_ptr<PerThreadContextMap>& PerThreadContextCache() {
thread_local const ContextCacheHolder per_thread_context_cache;
return per_thread_context_cache.p;
}
struct PerThreadContextState {
// contexts that are currently active
std::set<std::shared_ptr<PerThreadContext>, std::owner_less<std::shared_ptr<PerThreadContext>>> active_contexts;
// contexts available for reuse
std::vector<std::shared_ptr<PerThreadContext>> retired_context_pool;
// weak references to thread local caches from which this QNNExecutionProvider instance's entry should be removed
// upon destruction
std::set<std::weak_ptr<PerThreadContextMap>, std::owner_less<std::weak_ptr<PerThreadContextMap>>>
caches_to_update_on_destruction;
// synchronizes access to PerThreadContextState members
OrtMutex mutex;
};
// The execution provider maintains the PerThreadContexts in this structure.
// Synchronization is required to update the contained structures.
// On the other hand, access to an individual PerThreadContext is assumed to be from a single thread at a time,
// so synchronization is not required for that.
mutable PerThreadContextState context_state_;
PerThreadContext& GetPerThreadContext() const;
void ReleasePerThreadContext() const;
};
} // namespace onnxruntime

4
onnxruntime/core/providers/rocm/rocm_execution_provider.cc
View file

@ -353,7 +353,7 @@ Status ROCMExecutionProvider::Sync() const {
return Status::OK();
}
Status ROCMExecutionProvider::OnRunStart() {
Status ROCMExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
// always set ROCM device when session::Run() in case it runs in a worker thread
HIP_RETURN_IF_ERROR(hipSetDevice(GetDeviceId()));
if (IsGraphCaptureEnabled() && GetPerThreadContext().IsGraphCaptureAllowed() && !GetPerThreadContext().IsGraphCaptured()) {
@ -363,7 +363,7 @@ Status ROCMExecutionProvider::OnRunStart() {
return Status::OK();
}
Status ROCMExecutionProvider::OnRunEnd(bool sync_stream) {
Status ROCMExecutionProvider::OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& /*run_options*/) {
if (IsGraphCaptureEnabled() && !GetPerThreadContext().IsGraphCaptured()) {
if (GetPerThreadContext().IsGraphCaptureAllowed()) {
GetPerThreadContext().CaptureEnd();

4
onnxruntime/core/providers/rocm/rocm_execution_provider.h
View file

@ -28,9 +28,9 @@ class ROCMExecutionProvider : public IExecutionProvider {
Status Sync() const override;
Status OnRunStart() override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
const void* GetExecutionHandle() const noexcept override {
// The ROCM interface does not return anything interesting.

4
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -1818,11 +1818,11 @@ std::unique_ptr<IDataTransfer> TensorrtExecutionProvider::GetDataTransfer() cons
return onnxruntime::CreateGPUDataTransfer();
}
Status TensorrtExecutionProvider::OnRunStart() {
Status TensorrtExecutionProvider::OnRunStart(const onnxruntime::RunOptions& /*run_options*/) {
return Status::OK();
}
Status TensorrtExecutionProvider::OnRunEnd(bool sync_stream) {
Status TensorrtExecutionProvider::OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& /*run_options*/) {
if (sync_stream && external_stream_) {
CUDA_RETURN_IF_ERROR(cudaStreamSynchronize(stream_));
}

4
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -233,8 +233,8 @@ class TensorrtExecutionProvider : public IExecutionProvider {
common::Status Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,
std::vector<NodeComputeInfo>& node_compute_funcs) override;
Status OnRunStart() override;
Status OnRunEnd(bool sync_stream) override;
Status OnRunStart(const onnxruntime::RunOptions& run_options) override;
Status OnRunEnd(bool sync_stream, const onnxruntime::RunOptions& run_options) override;
ProviderOptions GetProviderOptions() const override {
return TensorrtExecutionProviderInfo::ToProviderOptions(info_);

12
onnxruntime/core/session/inference_session.cc
View file

@ -2289,8 +2289,8 @@ Status InferenceSession::PartialRun(onnxruntime::RunOptions& run_options,
// TODO: only call OnRunStart for all providers in-use
for (auto& xp : execution_providers_) {
// call OnRunStart and add to exec_providers_to_stop if successful
auto start_func = [&xp, &exec_providers_to_stop]() {
auto status = xp->OnRunStart();
auto start_func = [&xp, &exec_providers_to_stop, run_options]() {
auto status = xp->OnRunStart(run_options);
if (status.IsOK())
exec_providers_to_stop.push_back(xp.get());
@ -2326,7 +2326,7 @@ Status InferenceSession::PartialRun(onnxruntime::RunOptions& run_options,
// info all execution providers InferenceSession:Run ended
for (auto* xp : exec_providers_to_stop) {
auto status = xp->OnRunEnd(/*sync_stream*/ false);
auto status = xp->OnRunEnd(/*sync_stream*/ false, run_options);
ORT_CHECK_AND_SET_RETVAL(status);
}
@ -2448,8 +2448,8 @@ Status InferenceSession::Run(const RunOptions& run_options,
// TODO: only call OnRunStart for all providers in-use
for (auto& xp : execution_providers_) {
// call OnRunStart and add to exec_providers_to_stop if successful
auto start_func = [&xp, &exec_providers_to_stop]() {
auto status = xp->OnRunStart();
auto start_func = [&xp, &exec_providers_to_stop, &run_options]() {
auto status = xp->OnRunStart(run_options);
if (status.IsOK())
exec_providers_to_stop.push_back(xp.get());
@ -2490,7 +2490,7 @@ Status InferenceSession::Run(const RunOptions& run_options,
// info all execution providers InferenceSession:Run ended
for (auto* xp : exec_providers_to_stop) {
bool synchronize_execution_providers = run_options.config_options.GetConfigOrDefault(kOrtRunOptionsConfigDisableSynchronizeExecutionProviders, "0") == "0";
auto status = xp->OnRunEnd(synchronize_execution_providers);
auto status = xp->OnRunEnd(synchronize_execution_providers, run_options);
ORT_CHECK_AND_SET_RETVAL(status);
}

13
onnxruntime/test/providers/cuda/test_cases/cuda_execution_provider_test.cc
View file

@ -22,6 +22,8 @@ TEST(TestDeferredRelease, WithArena) {
CUDAExecutionProvider ep(info);
AllocatorPtr gpu_alloctor = ep.CreatePreferredAllocators()[0];
RunOptions run_opts;
run_opts.run_tag = "log1";
// Allocator for call cudaMallocHost and cudaFreeHost
// For details, see CUDAPinnedAllocator in cuda_allocator.cc.
AllocatorPtr cpu_pinned_alloc = ep.CreatePreferredAllocators()[1];
@ -31,7 +33,7 @@ TEST(TestDeferredRelease, WithArena) {
// 10 MB
const size_t n_bytes = 10 * 1000000;
const int64_t n_allocs = 64;
ORT_THROW_IF_ERROR(ep.OnRunStart());
ORT_THROW_IF_ERROR(ep.OnRunStart(run_opts));
for (size_t i = 0; i < n_allocs; ++i) {
// Allocate 10MB CUDA pinned memory.
auto pinned_buffer = IAllocator::MakeUniquePtr<void>(cpu_pinned_alloc, n_bytes);
@ -44,7 +46,7 @@ TEST(TestDeferredRelease, WithArena) {
cpu_pinned_alloc->GetStats(&stats);
ASSERT_EQ(stats.num_allocs, n_allocs);
ORT_THROW_IF_ERROR(stream.CleanUpOnRunEnd());
ORT_THROW_IF_ERROR(ep.OnRunEnd(true));
ORT_THROW_IF_ERROR(ep.OnRunEnd(true, run_opts));
}
TEST(TestDeferredRelease, WithoutArena) {
@ -52,6 +54,9 @@ TEST(TestDeferredRelease, WithoutArena) {
CUDAExecutionProviderInfo info;
CUDAExecutionProvider ep(info);
RunOptions run_opts;
run_opts.run_tag = "log1";
OrtDevice pinned_device{OrtDevice::CPU, OrtDevice::MemType::CUDA_PINNED, DEFAULT_CPU_ALLOCATOR_DEVICE_ID};
// Create allocator without BFCArena
AllocatorCreationInfo pinned_memory_info(
@ -70,7 +75,7 @@ TEST(TestDeferredRelease, WithoutArena) {
// 10 MB
const size_t n_bytes = 10 * 1000000;
const int64_t n_allocs = 64;
ORT_THROW_IF_ERROR(ep.OnRunStart());
ORT_THROW_IF_ERROR(ep.OnRunStart(run_opts));
for (size_t i = 0; i < n_allocs; ++i) {
// Allocate 10MB CUDA pinned memory.
auto pinned_buffer = IAllocator::MakeUniquePtr<void>(cuda_pinned_alloc, n_bytes);
@ -79,7 +84,7 @@ TEST(TestDeferredRelease, WithoutArena) {
}
ORT_THROW_IF_ERROR(stream.CleanUpOnRunEnd());
ORT_THROW_IF_ERROR(ep.OnRunEnd(true));
ORT_THROW_IF_ERROR(ep.OnRunEnd(true, run_opts));
}
} // namespace test

213
onnxruntime/test/providers/qnn/qnn_basic_test.cc
View file

@ -7,6 +7,7 @@
#include "core/session/onnxruntime_cxx_api.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/session/onnxruntime_run_options_config_keys.h"
#include "core/providers/cpu/cpu_provider_factory.h" // For OrtSessionOptionsAppendExecutionProvider_CPU
#include "core/session/inference_session.h"
@ -332,19 +333,23 @@ static void CreateModelInMemory(std::unique_ptr<ModelAndBuilder>& result,
static void RunSessionAndVerify(InferenceSession& session, const RunOptions& run_options, const NameMLValMap& feeds,
const std::vector<std::string>& output_names,
const std::vector<std::vector<int64_t>>& output_shapes,
const std::vector<std::vector<float>>& expected_values) {
std::vector<OrtValue> fetches;
auto status = session.Run(run_options, feeds, output_names, &fetches);
ASSERT_TRUE(status.IsOK());
const std::vector<std::vector<float>>& expected_values,
int loop_count = 10) {
// Let it run for a while
for (int it = 0; it < loop_count; ++it) {
std::vector<OrtValue> fetches;
auto status = session.Run(run_options, feeds, output_names, &fetches);
ASSERT_TRUE(status.IsOK());
for (size_t i = 0; i < fetches.size(); i++) {
auto& tensor = fetches[i].Get<Tensor>();
TensorShape expected_shape(output_shapes[i]);
ASSERT_EQ(expected_shape, tensor.Shape());
for (size_t i = 0; i < fetches.size(); i++) {
auto& tensor = fetches[i].Get<Tensor>();
TensorShape expected_shape(output_shapes[i]);
ASSERT_EQ(expected_shape, tensor.Shape());
gsl::span<const float> actual = tensor.DataAsSpan<float>();
gsl::span<const float> expected(expected_values[i].data(), expected_values[i].size());
ASSERT_EQ(expected, actual);
gsl::span<const float> actual = tensor.DataAsSpan<float>();
gsl::span<const float> expected(expected_values[i].data(), expected_values[i].size());
ASSERT_EQ(expected, actual);
}
}
}
@ -404,11 +409,11 @@ TEST_F(QnnCPUBackendTests, MultithreadSessionRun) {
std::vector<std::thread> threads;
constexpr int num_threads = 5;
constexpr int loop_count = 10;
for (int i = 0; i < num_threads; i++) {
threads.push_back(std::thread(RunSessionAndVerify, std::ref(session_obj), run_opts,
model->builder.feeds_, model->builder.output_names_,
output_shapes, output_values));
output_shapes, output_values, loop_count));
}
for (auto& th : threads) {
@ -484,11 +489,191 @@ TEST_F(QnnHTPBackendTests, MultithreadSessionRun) {
std::vector<std::thread> threads;
constexpr int num_threads = 5;
constexpr int loop_count = 10;
for (int i = 0; i < num_threads; i++) {
threads.push_back(std::thread(RunSessionAndVerify, std::ref(session_obj), run_opts,
model->builder.feeds_, model->builder.output_names_,
output_shapes, output_values));
output_shapes, output_values, loop_count));
}
for (auto& th : threads) {
th.join();
}
}
// Tests running a single session in multiple threads on the HTP backend with run option to set power config
TEST_F(QnnHTPBackendTests, MultithreadHtpPowerCfgSessionRunOption) {
std::unique_ptr<ModelAndBuilder> model;
std::vector<float> input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
std::vector<int64_t> shape = {1, 3, 2};
std::vector<std::vector<int64_t>> output_shapes = {shape};
std::vector<std::vector<float>> output_values = {{3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}};
CreateModelInMemory(model,
QDQBuildAdd3Tensors<uint8_t>(TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data)),
"add3.qdq");
SessionOptions session_opts;
session_opts.session_logid = "logger0";
InferenceSession session_obj{session_opts, GetEnvironment()};
onnxruntime::ProviderOptions options;
#if defined(_WIN32)
options["backend_path"] = "QnnHtp.dll";
#else
options["backend_path"] = "libQnnHtp.so";
#endif
auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts);
EXPECT_TRUE(session_obj.RegisterExecutionProvider(std::move(qnn_ep)).IsOK());
auto status = session_obj.Load(model->model_data.data(), static_cast<int>(model->model_data.size()));
ASSERT_TRUE(status.IsOK());
status = session_obj.Initialize();
ASSERT_TRUE(status.IsOK());
std::vector<std::thread> threads;
constexpr int num_threads = 5;
constexpr int loop_count = 10;
std::vector<std::string> perf_modes{
"burst", "balanced", "default", "high_performance", "high_power_saver",
"low_balanced", "extreme_power_saver", "low_power_saver", "power_saver"};
size_t post_i = perf_modes.size() - 1;
ASSERT_TRUE(post_i > num_threads);
for (int i = 0; i < num_threads; ++i, --post_i) {
RunOptions run_opts;
run_opts.run_tag = session_opts.session_logid;
auto rt = run_opts.config_options.AddConfigEntry(kOrtRunOptionsConfigQnnPerfMode, perf_modes[i].c_str());
ASSERT_TRUE(rt.IsOK());
rt = run_opts.config_options.AddConfigEntry(kOrtRunOptionsConfigQnnPerfModePostRun, perf_modes[post_i].c_str());
ASSERT_TRUE(rt.IsOK());
threads.push_back(std::thread(RunSessionAndVerify, std::ref(session_obj), run_opts,
model->builder.feeds_, model->builder.output_names_,
output_shapes, output_values, loop_count));
}
for (auto& th : threads) {
th.join();
}
}
// Tests running a single session in multiple threads on the HTP backend with EP option to set default power config
TEST_F(QnnHTPBackendTests, MultithreadDefaultHtpPowerCfgFromEpOption) {
std::unique_ptr<ModelAndBuilder> model;
std::vector<float> input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
std::vector<int64_t> shape = {1, 3, 2};
std::vector<std::vector<int64_t>> output_shapes = {shape};
std::vector<std::vector<float>> output_values = {{3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}};
CreateModelInMemory(model,
QDQBuildAdd3Tensors<uint8_t>(TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data)),
"add3.qdq");
SessionOptions session_opts;
session_opts.session_logid = "logger0";
RunOptions run_opts;
run_opts.run_tag = session_opts.session_logid;
InferenceSession session_obj{session_opts, GetEnvironment()};
onnxruntime::ProviderOptions options;
#if defined(_WIN32)
options["backend_path"] = "QnnHtp.dll";
#else
options["backend_path"] = "libQnnHtp.so";
#endif
options["htp_performance_mode"] = "burst";
auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts);
EXPECT_TRUE(session_obj.RegisterExecutionProvider(std::move(qnn_ep)).IsOK());
auto status = session_obj.Load(model->model_data.data(), static_cast<int>(model->model_data.size()));
ASSERT_TRUE(status.IsOK());
status = session_obj.Initialize();
ASSERT_TRUE(status.IsOK());
std::vector<std::thread> threads;
constexpr int num_threads = 5;
constexpr int loop_count = 10;
for (int i = 0; i < num_threads; i++) {
threads.push_back(std::thread(RunSessionAndVerify, std::ref(session_obj), run_opts,
model->builder.feeds_, model->builder.output_names_,
output_shapes, output_values, loop_count));
}
for (auto& th : threads) {
th.join();
}
}
// Tests running a single session in multiple threads on the HTP backend with
// EP option to set default power config + run option to set power config for each run
TEST_F(QnnHTPBackendTests, MultithreadHtpPowerCfgDefaultAndRunOption) {
std::unique_ptr<ModelAndBuilder> model;
std::vector<float> input_data = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
std::vector<int64_t> shape = {1, 3, 2};
std::vector<std::vector<int64_t>> output_shapes = {shape};
std::vector<std::vector<float>> output_values = {{3.0f, 6.0f, 9.0f, 12.0f, 15.0f, 18.0f}};
CreateModelInMemory(model,
QDQBuildAdd3Tensors<uint8_t>(TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data),
TestInputDef<float>(shape, false, input_data)),
"add3.qdq");
SessionOptions session_opts;
session_opts.session_logid = "logger0";
InferenceSession session_obj{session_opts, GetEnvironment()};
onnxruntime::ProviderOptions options;
#if defined(_WIN32)
options["backend_path"] = "QnnHtp.dll";
#else
options["backend_path"] = "libQnnHtp.so";
#endif
options["htp_performance_mode"] = "burst";
auto qnn_ep = QnnExecutionProviderWithOptions(options, &session_opts);
EXPECT_TRUE(session_obj.RegisterExecutionProvider(std::move(qnn_ep)).IsOK());
auto status = session_obj.Load(model->model_data.data(), static_cast<int>(model->model_data.size()));
ASSERT_TRUE(status.IsOK());
status = session_obj.Initialize();
ASSERT_TRUE(status.IsOK());
std::vector<std::thread> threads;
constexpr int num_threads = 5;
constexpr int loop_count = 10;
std::vector<std::string> perf_modes{
"burst", "balanced", "default", "high_performance", "high_power_saver",
"low_balanced", "extreme_power_saver", "low_power_saver", "power_saver"};
size_t post_i = perf_modes.size() - 1;
ASSERT_TRUE(post_i > num_threads);
for (int i = 0; i < num_threads; ++i, --post_i) {
RunOptions run_opts;
run_opts.run_tag = session_opts.session_logid;
auto rt = run_opts.config_options.AddConfigEntry(kOrtRunOptionsConfigQnnPerfMode, perf_modes[i].c_str());
ASSERT_TRUE(rt.IsOK());
rt = run_opts.config_options.AddConfigEntry(kOrtRunOptionsConfigQnnPerfModePostRun, perf_modes[post_i].c_str());
ASSERT_TRUE(rt.IsOK());
threads.push_back(std::thread(RunSessionAndVerify, std::ref(session_obj), run_opts,
model->builder.feeds_, model->builder.output_names_,
output_shapes, output_values, loop_count));
}
for (auto& th : threads) {