diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4b911e3482..486e2ff2b9 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3597,6 +3597,9 @@ struct OrtApi { * "rpc_control_latency": QNN RPC control latency. * "htp_performance_mode": QNN performance mode, options: "burst", "balanced", "default", "high_performance", * "high_power_saver", "low_balanced", "low_power_saver", "power_saver", "sustained_high_performance". Default to "default". + * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will + * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and + * may alter model/EP partitioning. Use only for debugging. * * SNPE supported keys: * "runtime": SNPE runtime engine, options: "CPU", "CPU_FLOAT32", "GPU", "GPU_FLOAT32_16_HYBRID", "GPU_FLOAT16", diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 8e31124ce4..e2083371ac 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -30,12 +30,20 @@ typedef Qnn_ErrorHandle_t (*QnnSystemInterfaceGetProvidersFn_t)(const QnnSystemI constexpr const char* QNN_PROVIDER = "ORTQNNEP"; +static Qnn_Version_t GetQnnInterfaceApiVersion(const QnnInterface_t* qnn_interface) { + return qnn_interface->apiVersion.coreApiVersion; +} + +static Qnn_Version_t GetQnnInterfaceApiVersion(const QnnSystemInterface_t* qnn_interface) { + return qnn_interface->systemApiVersion; +} + template -Status QnnBackendManager::GetQnnInterfaceProviders(const char* lib_path, - const char* interface_provider_name, - void** backend_lib_handle, - T*** interface_providers, - uint32_t& num_providers) { +Status QnnBackendManager::GetQnnInterfaceProvider(const char* lib_path, + const char* interface_provider_name, + void** backend_lib_handle, + Qnn_Version_t req_version, + T** interface_provider) { std::string error_msg; *backend_lib_handle = LoadLib(lib_path, static_cast(DlOpenFlag::DL_NOW) | static_cast(DlOpenFlag::DL_LOCAL), @@ -47,10 +55,36 @@ Status QnnBackendManager::GetQnnInterfaceProviders(const char* lib_path, GetInterfaceProviders = ResolveSymbol(*backend_lib_handle, interface_provider_name, *logger_); ORT_RETURN_IF(nullptr == GetInterfaceProviders, "Failed to get QNN providers!"); - auto result = GetInterfaceProviders((const T***)interface_providers, &num_providers); + T** interface_providers{nullptr}; + uint32_t num_providers{0}; + + auto result = GetInterfaceProviders((const T***)&interface_providers, &num_providers); ORT_RETURN_IF((QNN_SUCCESS != result || nullptr == *interface_providers || 0 == num_providers), "Failed to get QNN providers."); + bool found_valid_interface{false}; + for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { + Qnn_Version_t interface_version = GetQnnInterfaceApiVersion(interface_providers[pIdx]); + + LOGS_DEFAULT(VERBOSE) << lib_path << " interface version: " << interface_version.major << "." + << interface_version.minor << "." << interface_version.patch; + + // Check the interface's API version against the required version. + // Major versions must match. The interface's minor version must be greater OR equal with a suitable patch version. + if (interface_version.major == req_version.major) { + bool minor_and_patch_version_ok = (interface_version.minor > req_version.minor) || + (interface_version.minor == req_version.minor && + interface_version.patch >= req_version.patch); + if (minor_and_patch_version_ok) { + found_valid_interface = true; + *interface_provider = interface_providers[pIdx]; + break; + } + } + } + + ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid interface for ", lib_path); + return Status::OK(); } @@ -76,38 +110,89 @@ void QnnBackendManager::SetQnnBackendType(uint32_t backend_id) { } Status QnnBackendManager::LoadBackend() { - QnnInterface_t** interface_providers{nullptr}; - uint32_t num_providers{0}; - auto rt = GetQnnInterfaceProviders(backend_path_.c_str(), - "QnnInterface_getProviders", - &backend_lib_handle_, - &interface_providers, - num_providers); + QnnInterface_t* backend_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(backend_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle_, + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &backend_interface_provider); + ORT_RETURN_IF_ERROR(rt); + qnn_interface_ = backend_interface_provider->QNN_INTERFACE_VER_NAME; + auto backend_id = backend_interface_provider->backendId; + SetQnnBackendType(backend_id); + + Qnn_Version_t backend_interface_version = GetQnnInterfaceApiVersion(backend_interface_provider); + LOGS_DEFAULT(INFO) << "Found valid interface, version: " << backend_interface_version.major + << "." << backend_interface_version.minor << "." << backend_interface_version.patch + << " backend provider name: " << backend_interface_provider->providerName + << " backend id: " << backend_id; + + return Status::OK(); +} + +// Loads the intended backend (e.g., HTP, CPU, etc) to get its type, and then +// sets QNN Saver as the active backend. QNN op builders will still see the intended backend (e.g., HTP) +// as the backend type to ensure they emit the expected QNN API calls. +// +// QNN Saver is a "debugging" backend that serializes all QNN API calls (and weights) into local files. +// This information can be used to debug issues by replaying QNN API calls with another backend. +Status QnnBackendManager::LoadQnnSaverBackend() { + void* backend_lib_handle = nullptr; + + // Helper that unloads the intended backend library handle when the `unload_backend_lib` variable + // goes out of scope. Similar to `defer` in other languages. + auto unload_backend_lib = gsl::finally([&] { + if (backend_lib_handle != nullptr) { + auto result = UnloadLib(backend_lib_handle); + if (Status::OK() != result) { + ORT_THROW("Failed to unload backend library."); + } + } + }); + + // Load the intended backend (e.g., HTP, CPU) to ensure it is valid and to get its type. + QnnInterface_t* backend_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(backend_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle, + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &backend_interface_provider); ORT_RETURN_IF_ERROR(rt); - bool found_valid_interface{false}; - LOGS_DEFAULT(VERBOSE) << "QNN_API_VERSION_MAJOR: " << QNN_API_VERSION_MAJOR - << " QNN_API_VERSION_MINOR: " << QNN_API_VERSION_MINOR; - for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { - LOGS_DEFAULT(VERBOSE) << "interface_providers major: " << interface_providers[pIdx]->apiVersion.coreApiVersion.major - << " interface_providers minor: " << interface_providers[pIdx]->apiVersion.coreApiVersion.minor; - if (QNN_API_VERSION_MAJOR == interface_providers[pIdx]->apiVersion.coreApiVersion.major && - QNN_API_VERSION_MINOR <= interface_providers[pIdx]->apiVersion.coreApiVersion.minor) { - found_valid_interface = true; - qnn_interface_ = interface_providers[pIdx]->QNN_INTERFACE_VER_NAME; - auto backend_id = interface_providers[pIdx]->backendId; - SetQnnBackendType(backend_id); + // Set the "intended" backend type so that QNN builders still make the expected QNN API calls. + auto backend_id = backend_interface_provider->backendId; + SetQnnBackendType(backend_id); - LOGS_DEFAULT(INFO) << "Found valid interface, version: " << QNN_API_VERSION_MAJOR - << "." << QNN_API_VERSION_MINOR - << " backend provider name: " << interface_providers[pIdx]->providerName - << " backend id: " << backend_id; - break; - } - } + // Load the QNN Saver backend and set it as the activate backend. + QnnInterface_t* saver_interface_provider{nullptr}; + auto saver_rt = GetQnnInterfaceProvider(qnn_saver_path_.c_str(), + "QnnInterface_getProviders", + &backend_lib_handle_, // NOTE: QNN Saver library handle is set + {QNN_API_VERSION_MAJOR, + QNN_API_VERSION_MINOR, + QNN_API_VERSION_PATCH}, + &saver_interface_provider); + ORT_RETURN_IF_ERROR(saver_rt); + qnn_interface_ = saver_interface_provider->QNN_INTERFACE_VER_NAME; // NOTE: QNN Saver will provide the interfaces - ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid interface."); + Qnn_Version_t backend_interface_version = GetQnnInterfaceApiVersion(backend_interface_provider); + Qnn_Version_t saver_interface_version = GetQnnInterfaceApiVersion(saver_interface_provider); + + LOGS_DEFAULT(INFO) << "Using QNN Saver version: " << saver_interface_version.major << "." + << saver_interface_version.minor << "." << saver_interface_version.patch + << " provider name : " << saver_interface_provider->providerName; + + LOGS_DEFAULT(INFO) << "Intended backend provider name: " << backend_interface_provider->providerName + << " backend id: " << backend_id + << " interface version: " << backend_interface_version.major + << "." << backend_interface_version.minor << "." << backend_interface_version.patch; return Status::OK(); } @@ -120,34 +205,22 @@ Status QnnBackendManager::LoadQnnSystemLib() { #endif // #ifdef _WIN32 std::filesystem::path lib_file_path(backend_path_.c_str()); std::string sys_file_path(lib_file_path.remove_filename().string() + system_lib_file); - QnnSystemInterface_t** system_interface_providers{nullptr}; - uint32_t num_providers = 0; - auto rt = GetQnnInterfaceProviders(sys_file_path.c_str(), - "QnnSystemInterface_getProviders", - &system_lib_handle_, - &system_interface_providers, - num_providers); + QnnSystemInterface_t* system_interface_provider{nullptr}; + auto rt = GetQnnInterfaceProvider(sys_file_path.c_str(), + "QnnSystemInterface_getProviders", + &system_lib_handle_, + {QNN_SYSTEM_API_VERSION_MAJOR, + QNN_SYSTEM_API_VERSION_MINOR, + QNN_SYSTEM_API_VERSION_PATCH}, + &system_interface_provider); ORT_RETURN_IF_ERROR(rt); + Qnn_Version_t system_interface_version = GetQnnInterfaceApiVersion(system_interface_provider); + qnn_sys_interface_ = system_interface_provider->QNN_SYSTEM_INTERFACE_VER_NAME; - bool found_valid_interface{false}; - for (size_t pIdx = 0; pIdx < num_providers; pIdx++) { - LOGS_DEFAULT(VERBOSE) << "system_interface_providers major: " << system_interface_providers[pIdx]->systemApiVersion.major - << " system_interface_providers minor: " << system_interface_providers[pIdx]->systemApiVersion.minor; - int64_t systems_version_major = static_cast(system_interface_providers[pIdx]->systemApiVersion.major); - int64_t systems_version_minor = static_cast(system_interface_providers[pIdx]->systemApiVersion.minor); - if (systems_version_major == QNN_SYSTEM_API_VERSION_MAJOR && - systems_version_minor >= QNN_SYSTEM_API_VERSION_MINOR) { - found_valid_interface = true; - qnn_sys_interface_ = system_interface_providers[pIdx]->QNN_SYSTEM_INTERFACE_VER_NAME; - LOGS_DEFAULT(INFO) << "Found valid system interface, version: " << QNN_API_VERSION_MAJOR - << "." << QNN_API_VERSION_MINOR - << " backend provider name: " << system_interface_providers[pIdx]->providerName; - break; - } - } - - ORT_RETURN_IF_NOT(found_valid_interface, "Unable to find a valid system interface."); + LOGS_DEFAULT(INFO) << "Found valid system interface, version: " << system_interface_version.major + << "." << system_interface_version.minor + << " backend provider name: " << system_interface_provider->providerName; return Status::OK(); } @@ -643,7 +716,12 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool load_ return Status::OK(); } - ORT_RETURN_IF_ERROR(LoadBackend()); + if (qnn_saver_path_.empty()) { + ORT_RETURN_IF_ERROR(LoadBackend()); + } else { + ORT_RETURN_IF_ERROR(LoadQnnSaverBackend()); + } + LOGS(logger, VERBOSE) << "LoadBackend succeed."; if (load_from_cached_context) { diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 4ca63a042c..402f842c7a 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -25,14 +25,16 @@ class QnnModel; class QnnBackendManager { public: - QnnBackendManager(std::string backend_path, + QnnBackendManager(std::string&& backend_path, ProfilingLevel profiling_level, uint32_t rpc_control_latency, - HtpPerformanceMode htp_performance_mode) + HtpPerformanceMode htp_performance_mode, + std::string&& qnn_saver_path) : backend_path_(backend_path), profiling_level_(profiling_level), rpc_control_latency_(rpc_control_latency), - htp_performance_mode_(htp_performance_mode) { + htp_performance_mode_(htp_performance_mode), + qnn_saver_path_(qnn_saver_path) { } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QnnBackendManager); @@ -140,6 +142,8 @@ class QnnBackendManager { Status LoadQnnSystemLib(); + Status LoadQnnSaverBackend(); + Status UnloadLib(void* handle); void* LibFunction(void* handle, const char* symbol, std::string& error_msg); @@ -155,11 +159,11 @@ class QnnBackendManager { } template - Status GetQnnInterfaceProviders(const char* lib_path, - const char* interface_provider_name, - void** backend_lib_handle, - T*** interface_providers, - uint32_t& num_providers); + Status GetQnnInterfaceProvider(const char* lib_path, + const char* interface_provider_name, + void** backend_lib_handle, + Qnn_Version_t req_version, + T** interface_provider); bool IsDevicePropertySupported(); @@ -210,6 +214,7 @@ class QnnBackendManager { #ifdef _WIN32 std::set mod_handles_; #endif + const std::string qnn_saver_path_; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 7bbfe807da..ec5316eb13 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -104,9 +104,10 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio static const std::string BACKEND_PATH = "backend_path"; auto backend_path_pos = runtime_options_.find(BACKEND_PATH); + std::string backend_path; if (backend_path_pos != runtime_options_.end()) { - backend_path_ = backend_path_pos->second; - LOGS_DEFAULT(VERBOSE) << "Backend path: " << backend_path_; + backend_path = backend_path_pos->second; + LOGS_DEFAULT(VERBOSE) << "Backend path: " << backend_path; } else { LOGS_DEFAULT(ERROR) << "No backend path provided."; } @@ -131,10 +132,21 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio ParseHtpPerformanceMode(htp_performance_mode_pos->second); } - qnn_backend_manager_ = std::make_unique(backend_path_, - profiling_level_, - rpc_control_latency_, - htp_performance_mode_); + // Enable use of QNN Saver if the user provides a path the QNN Saver backend library. + static const std::string QNN_SAVER_PATH_KEY = "qnn_saver_path"; + std::string qnn_saver_path; + auto qnn_saver_path_pos = runtime_options_.find(QNN_SAVER_PATH_KEY); + if (qnn_saver_path_pos != runtime_options_.end()) { + qnn_saver_path = qnn_saver_path_pos->second; + LOGS_DEFAULT(VERBOSE) << "User specified QNN Saver path: " << qnn_saver_path; + } + + qnn_backend_manager_ = std::make_unique( + std::move(backend_path), + profiling_level_, + rpc_control_latency_, + htp_performance_mode_, + std::move(qnn_saver_path)); } bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit, diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index 2fe507b70a..3827e2044e 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -58,7 +58,6 @@ class QNNExecutionProvider : public IExecutionProvider { private: ProviderOptions runtime_options_; - std::string backend_path_; qnn::ProfilingLevel profiling_level_ = qnn::ProfilingLevel::OFF; qnn::HtpPerformanceMode htp_performance_mode_ = qnn::HtpPerformanceMode::kHtpDefault; std::unique_ptr qnn_backend_manager_; diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 062ca4ece8..287d657a2c 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -56,6 +56,7 @@ void usage() { "\t [QNN only] [rpc_control_latency]: QNN rpc control latency. default to 10.\n" "\t [QNN only] [htp_performance_mode]: QNN performance mode, options: 'burst', 'balanced', 'default', 'high_performance', \n" "\t 'high_power_saver', 'low_balanced', 'low_power_saver', 'power_saver', 'sustained_high_performance'. Default to 'default'. \n" + "\t [QNN only] [qnn_saver_path]: QNN Saver backend path. e.g '/folderpath/libQnnSaver.so'.\n" "\t [Usage]: -e -i '| |' \n\n" "\t [Example] [For QNN EP] -e qnn -i \"profiling_level|detailed backend_path|/folderpath/libQnnCpu.so\" \n\n" "\t [SNPE only] [runtime]: SNPE runtime, options: 'CPU', 'GPU', 'GPU_FLOAT16', 'DSP', 'AIP_FIXED_TF'. \n" @@ -477,6 +478,8 @@ int real_main(int argc, char* argv[], Ort::Env& env) { std::string str = str_stream.str(); ORT_THROW("Wrong value for htp_performance_mode. select from: " + str); } + } else if (key == "qnn_saver_path") { + // no validation } else { ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path', 'qnn_context_cache_enable', 'qnn_context_cache_path', 'profiling_level', 'rpc_control_latency', 'htp_performance_mode'])"); diff --git a/onnxruntime/test/providers/qnn/qnn_basic_test.cc b/onnxruntime/test/providers/qnn/qnn_basic_test.cc index a441e828c0..5f63813d8d 100644 --- a/onnxruntime/test/providers/qnn/qnn_basic_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_basic_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "core/session/onnxruntime_cxx_api.h" #include "core/session/onnxruntime_session_options_config_keys.h" @@ -172,7 +173,7 @@ TEST(QnnEP, TestDisableCPUFallback_ConflictingConfig) { // The models passed to this function are subgraphs extracted from a larger model that exhibited // shape inferencing issues on QNN. Thus, the models are expected to have a specific input/output // types and shapes. -static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { +static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp, bool enable_qnn_saver = false) { Ort::SessionOptions so; // Ensure all type/shape inference warnings result in errors! @@ -183,8 +184,14 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { #if defined(_WIN32) options["backend_path"] = use_htp ? "QnnHtp.dll" : "QnnCpu.dll"; + if (enable_qnn_saver) { + options["qnn_saver_path"] = "QnnSaver.dll"; + } #else options["backend_path"] = use_htp ? "libQnnHtp.so" : "libQnnCpu.so"; + if (enable_qnn_saver) { + options["qnn_saver_path"] = "libQnnSaver.so"; + } #endif so.AppendExecutionProvider("QNN", options); @@ -226,7 +233,7 @@ static void RunNHWCResizeModel(const ORTCHAR_T* ort_model_path, bool use_htp) { auto typeshape = ort_output.GetTensorTypeAndShapeInfo(); std::vector output_shape = typeshape.GetShape(); - ASSERT_THAT(output_shape, ::testing::ElementsAre(1, 6, 7, 10)); + EXPECT_THAT(output_shape, ::testing::ElementsAre(1, 6, 7, 10)); } // Test shape inference of NHWC Resize operator (opset 11) that uses @@ -253,6 +260,23 @@ TEST_F(QnnCPUBackendTests, TestNHWCResizeShapeInference_sizes_opset18) { RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", false); } +// Test that QNN Saver generates the expected files for a model meant to run on the QNN CPU backend. +TEST_F(QnnCPUBackendTests, QnnSaver_OutputFiles) { + const std::filesystem::path qnn_saver_output_dir = "saver_output"; + + // Remove pre-existing QNN Saver output files. Note that fs::remove_all() can handle non-existing paths. + std::filesystem::remove_all(qnn_saver_output_dir); + ASSERT_FALSE(std::filesystem::exists(qnn_saver_output_dir)); + + RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", + false, // use_htp + true); // enable_qnn_saver + + // Check that QNN Saver output files exist. + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "saver_output.c")); + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "params.bin")); +} + #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) // Test shape inference of QDQ NHWC Resize operator (opset 18) that uses @@ -261,6 +285,23 @@ TEST_F(QnnHTPBackendTests, TestNHWCResizeShapeInference_qdq_sizes_opset18) { RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.quant.onnx", true); } +// Test that QNN Saver generates the expected files for a model meant to run on the QNN HTP backend. +TEST_F(QnnHTPBackendTests, QnnSaver_OutputFiles) { + const std::filesystem::path qnn_saver_output_dir = "saver_output"; + + // Remove pre-existing QNN Saver output files. Note that fs::remove_all() can handle non-existing paths. + std::filesystem::remove_all(qnn_saver_output_dir); + ASSERT_FALSE(std::filesystem::exists(qnn_saver_output_dir)); + + RunNHWCResizeModel(ORT_MODEL_FOLDER "nhwc_resize_sizes_opset18.onnx", + true, // use_htp + true); // enable_qnn_saver + + // Check that QNN Saver output files exist. + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "saver_output.c")); + EXPECT_TRUE(std::filesystem::exists(qnn_saver_output_dir / "params.bin")); +} + #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) #endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index 51df93f885..a067c9c53e 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -9,6 +9,7 @@ #include "test/util/include/default_providers.h" #include "test/util/include/test/test_environment.h" +#include "core/platform/env_var_utils.h" #include "core/common/span_utils.h" #include "core/framework/compute_capability.h" #include "core/graph/graph.h" @@ -41,7 +42,22 @@ std::vector GetFloatDataInRange(float min_val, float max_val, size_t num_ return data; } -void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOptions& provider_options, +void TryEnableQNNSaver(ProviderOptions& qnn_options) { + // Allow dumping QNN API calls to file by setting an environment variable that enables the QNN Saver backend. + constexpr auto kEnableQNNSaverEnvironmentVariableName = "ORT_UNIT_TEST_ENABLE_QNN_SAVER"; + static std::optional enable_qnn_saver = onnxruntime::ParseEnvironmentVariable( + kEnableQNNSaverEnvironmentVariableName); + + if (enable_qnn_saver.has_value() && *enable_qnn_saver != 0) { +#if defined(_WIN32) + qnn_options["qnn_saver_path"] = "QnnSaver.dll"; +#else + qnn_options["qnn_saver_path"] = "libQnnSaver.so"; +#endif // defined(_WIN32) + } +} + +void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions provider_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err, logging::Severity log_severity) { EPVerificationParams verification_params; @@ -65,6 +81,7 @@ void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOption // Serialize the model to a string. std::string model_data; model.ToProto().SerializeToString(&model_data); + TryEnableQNNSaver(provider_options); RunAndVerifyOutputsWithEP(AsByteSpan(model_data.data(), model_data.size()), "QNN_EP_TestLogID", QnnExecutionProviderWithOptions(provider_options), helper.feeds_, verification_params); diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.h b/onnxruntime/test/providers/qnn/qnn_test_utils.h index 14c62f98f6..b4c84d893c 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.h +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.h @@ -220,6 +220,25 @@ void InferenceModel(const std::string& model_data, const char* log_id, ExpectedEPNodeAssignment expected_ep_assignment, const NameMLValMap& feeds, std::vector& output_vals); +/** + * If the ORT_UNIT_TEST_ENABLE_QNN_SAVER environment variable is enabled (set to 1), this function modifies + * the QNN EP provider options to enable the QNN Saver backend, which dumps QNN API calls (and weights) to disk. + * + * - saver_output/saver_output.c: C file containing all QNN API calls. + * - saver_output/params.bin: binary file containing all input/output/parameter tensor data provided during tensor + * creation, op config validation, and graph execution. + * + * Enabling the QNN Saver backend has 2 note-worthy effects: + * 1. All QNN API calls will succeed. + * 2. Inference output returns dummy data. + * + * Because output files from QNN Saver are always overwritten, it is recommended to run individual unit tests via the + * --gtest_filter command-line option. Ex: --gtest_filter=QnnHTPBackendTests.Resize_DownSample_Linear_AlignCorners + * + * \param qnn_options QNN EP provider options that may be modified to enable QNN Saver. + */ +void TryEnableQNNSaver(ProviderOptions& qnn_options); + /** * Tests the accuracy of a QDQ model on QNN EP by runnning 3 inferences: * @@ -240,7 +259,7 @@ void InferenceModel(const std::string& model_data, const char* log_id, */ template inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTestQDQModelFn& qdq_model_fn, - const ProviderOptions& qnn_options, int opset_version, + ProviderOptions qnn_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err = 1e-4f, logging::Severity log_severity = logging::Severity::kERROR) { // Add kMSDomain to cover contrib op like Gelu @@ -300,6 +319,7 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTe qdq_model.ToProto().SerializeToString(&qdq_model_data); // Run QDQ model on QNN EP and collect outputs. + TryEnableQNNSaver(qnn_options); std::vector qnn_qdq_outputs; InferenceModel(qdq_model_data, "qdq_model_logger", QnnExecutionProviderWithOptions(qnn_options), expected_ep_assignment, qdq_helper.feeds_, qnn_qdq_outputs); @@ -538,7 +558,7 @@ inline GetTestQDQModelFn BuildQDQOpTestCase(const std::string& op_typ * \param fp32_abs_err The acceptable error between CPU EP and QNN EP. * \param log_severity The logger's minimum severity level. */ -void RunQnnModelTest(const GetTestModelFn& build_test_case, const ProviderOptions& provider_options, +void RunQnnModelTest(const GetTestModelFn& build_test_case, ProviderOptions provider_options, int opset_version, ExpectedEPNodeAssignment expected_ep_assignment, float fp32_abs_err = 1e-5f, logging::Severity log_severity = logging::Severity::kERROR);