From a2fd8a6fc083f43d6535f5acd24219c140812c87 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 24 Nov 2023 20:41:27 -0800 Subject: [PATCH] [QNN EP] Return INVALID_GRAPH if failed to load from context binary (#18485) ### Description [QNN EP] Return INVALID_GRAPH if failed to load from context binary ### Motivation and Context Make sure QNN EP return INVALID_GRAPH if error encountered with the context binary file --- .../qnn/builder/onnx_ctx_model_helper.cc | 194 +++++++++--------- .../qnn/builder/onnx_ctx_model_helper.h | 99 ++++----- .../qnn/builder/qnn_backend_manager.h | 1 - .../providers/qnn/qnn_execution_provider.cc | 87 ++++---- .../providers/qnn/qnn_execution_provider.h | 5 +- .../test/providers/qnn/simple_op_htp_test.cc | 58 +++++- 6 files changed, 238 insertions(+), 206 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc index bd9986e661..234b957816 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc @@ -60,10 +60,10 @@ Status CreateNodeArgs(const std::vector& names, return Status::OK(); } -Status QnnCacheModelHandler::GetEpContextFromModel(const std::string& ctx_onnx_model_path, - QnnBackendManager* qnn_backend_manager, - QnnModel& qnn_model, - const logging::Logger& logger) { +Status GetEpContextFromModel(const onnxruntime::PathString& ctx_onnx_model_path, + QnnBackendManager* qnn_backend_manager, + QnnModel& qnn_model, + const logging::Logger& logger) { using namespace onnxruntime; std::shared_ptr model; ORT_RETURN_IF_ERROR(Model::Load(ToPathString(ctx_onnx_model_path), model, {}, logger)); @@ -74,10 +74,10 @@ Status QnnCacheModelHandler::GetEpContextFromModel(const std::string& ctx_onnx_m qnn_model); } -Status QnnCacheModelHandler::GetEpContextFromGraph(const onnxruntime::GraphViewer& graph_viewer, - const std::string& ctx_onnx_model_path, - QnnBackendManager* qnn_backend_manager, - QnnModel& qnn_model) { +Status GetEpContextFromGraph(const onnxruntime::GraphViewer& graph_viewer, + const onnxruntime::PathString& ctx_onnx_model_path, + QnnBackendManager* qnn_backend_manager, + QnnModel& qnn_model) { const auto& node = graph_viewer.Nodes().begin(); NodeAttrHelper node_helper(*node); bool is_embed_mode = node_helper.Get(EMBED_MODE, true); @@ -89,11 +89,11 @@ Status QnnCacheModelHandler::GetEpContextFromGraph(const onnxruntime::GraphViewe } std::string external_qnn_context_binary_file_name = node_helper.Get(EP_CACHE_CONTEXT, ""); + std::filesystem::path folder_path = std::filesystem::path(ctx_onnx_model_path).parent_path(); + std::filesystem::path context_binary_path = folder_path.append(external_qnn_context_binary_file_name); - std::string context_binary_path(std::filesystem::path(ctx_onnx_model_path).parent_path().string() + - "/" + external_qnn_context_binary_file_name); size_t buffer_size{0}; - std::ifstream cache_file(context_binary_path.c_str(), std::ifstream::binary); + std::ifstream cache_file(context_binary_path.string().c_str(), std::ifstream::binary); ORT_RETURN_IF(!cache_file || !cache_file.good(), "Failed to open cache file."); cache_file.seekg(0, cache_file.end); @@ -112,114 +112,122 @@ Status QnnCacheModelHandler::GetEpContextFromGraph(const onnxruntime::GraphViewe qnn_model); } -Status QnnCacheModelHandler::GetMetadataFromEpContextModel(const std::string& ctx_onnx_model_path, - std::string& model_name, - std::string& model_description, - std::string& graph_partition_name, - std::string& cache_source, - const logging::Logger& logger) { - if (!is_metadata_ready_) { - using namespace onnxruntime; - std::shared_ptr model; - ORT_RETURN_IF_ERROR(Model::Load(ToPathString(ctx_onnx_model_path), model, {}, logger)); - const auto& graph = GraphViewer(model->MainGraph()); - const auto& node = graph.Nodes().begin(); - NodeAttrHelper node_helper(*node); - model_name_ = graph.Name(); - model_description_ = graph.Description(); - graph_partition_name_ = node_helper.Get(PARTITION_NAME, ""); - cache_source_ = node_helper.Get(SOURCE, ""); - is_metadata_ready_ = true; +Status LoadQnnCtxFromOnnxModel(const onnxruntime::GraphViewer& graph_viewer, + const onnxruntime::PathString& ctx_onnx_model_path, + bool is_qnn_ctx_model, + bool is_ctx_cache_file_exist, + QnnBackendManager* qnn_backend_manager, + QnnModel& qnn_model, + const logging::Logger& logger) { + Status status; + if (is_qnn_ctx_model) { + status = GetEpContextFromGraph(graph_viewer, ctx_onnx_model_path, qnn_backend_manager, qnn_model); + } else if (is_ctx_cache_file_exist) { + status = GetEpContextFromModel(ctx_onnx_model_path, qnn_backend_manager, qnn_model, logger); + } + + if (!status.IsOK()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "Failed to load from EpContextModel. ", status.ErrorMessage()); } - model_name = model_name_; - model_description = model_description_; - graph_partition_name = graph_partition_name_; - cache_source = cache_source_; return Status::OK(); } -bool QnnCacheModelHandler::IsContextCacheFileExists(const std::string& customer_context_cache_path, - const std::string& model_description, - const onnxruntime::PathString& model_pathstring) { - // Avoid duplicate work - if (ctx_file_exists_) { - return ctx_file_exists_; - } - model_description_ = model_description; - // Use user provided context cache file path if exist, otherwise try model_file.onnx_ctx.onnx by default - if (customer_context_cache_path.empty()) { - context_cache_path_ = PathToUTF8String(model_pathstring) + "_qnn_ctx.onnx"; - } else { - context_cache_path_ = customer_context_cache_path; - } +Status GetMetadataFromEpContextModel(const onnxruntime::PathString& ctx_onnx_model_path, + std::string& model_name, + std::string& model_description, + std::string& graph_partition_name, + std::string& cache_source, + const logging::Logger& logger) { + using namespace onnxruntime; + std::shared_ptr model; + ORT_RETURN_IF_ERROR(Model::Load(ctx_onnx_model_path, model, {}, logger)); + const auto& graph = GraphViewer(model->MainGraph()); + const auto& node = graph.Nodes().begin(); + NodeAttrHelper node_helper(*node); + model_name = graph.Name(); + model_description = graph.Description(); + graph_partition_name = node_helper.Get(PARTITION_NAME, ""); + cache_source = node_helper.Get(SOURCE, ""); - ctx_file_exists_ = std::filesystem::is_regular_file(context_cache_path_) && std::filesystem::exists(context_cache_path_); - - return ctx_file_exists_; + return Status::OK(); } -Status QnnCacheModelHandler::ValidateWithContextFile(const std::string& model_name, - const std::string& graph_partition_name, - const logging::Logger& logger) { - ORT_RETURN_IF(!ctx_file_exists_, "Qnn context binary file not exist for some reason!"); +bool IsContextCacheFileExists(const std::string& customer_context_cache_path, + const onnxruntime::PathString& model_pathstring, + onnxruntime::PathString& context_cache_path) { + // Use user provided context cache file path if exist, otherwise try model_file.onnx_ctx.onnx by default + if (!customer_context_cache_path.empty()) { + context_cache_path = ToPathString(customer_context_cache_path); + } else if (!model_pathstring.empty()) { + context_cache_path = model_pathstring + ToPathString("_qnn_ctx.onnx"); + } + return std::filesystem::is_regular_file(context_cache_path) && std::filesystem::exists(context_cache_path); +} + +Status ValidateWithContextFile(const onnxruntime::PathString& context_cache_path, + const std::string& model_name, + const std::string& model_description, + const std::string& graph_partition_name, + const logging::Logger& logger) { std::string model_name_from_ctx_cache; std::string model_description_from_ctx_cache; std::string graph_partition_name_from_ctx_cache; std::string cache_source; - ORT_RETURN_IF_ERROR(GetMetadataFromEpContextModel(context_cache_path_, - model_name_from_ctx_cache, - model_description_from_ctx_cache, - graph_partition_name_from_ctx_cache, - cache_source, - logger)); + auto status = GetMetadataFromEpContextModel(context_cache_path, + model_name_from_ctx_cache, + model_description_from_ctx_cache, + graph_partition_name_from_ctx_cache, + cache_source, + logger); + if (!status.IsOK()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, "Failed to get metadata from EpContextModel."); + } // The source attribute from the skeleton onnx file indicate whether it's generated from QNN toolchain or ORT if (cache_source != kQnnExecutionProvider) { + LOGS(logger, VERBOSE) << "Context binary cache is not generated by Ort."; return Status::OK(); } - ORT_RETURN_IF(model_name != model_name_from_ctx_cache, - "Model file name from context cache metadata: " + model_name_from_ctx_cache + - " is different with target: " + model_name + - ". Please make sure the context binary file matches the model."); + if (model_name != model_name_from_ctx_cache || + model_description != model_description_from_ctx_cache || + graph_partition_name != graph_partition_name_from_ctx_cache) { + std::string message = onnxruntime::MakeString("Metadata mismatch. onnx: ", + model_name, " ", model_description, " ", graph_partition_name, + " vs epcontext: ", + model_name_from_ctx_cache, " ", + model_description_from_ctx_cache, " ", + graph_partition_name_from_ctx_cache); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_GRAPH, message); + } - ORT_RETURN_IF(model_description_ != model_description_from_ctx_cache, - "Model description from context cache metadata: " + model_description_from_ctx_cache + - " is different with target: " + model_description_ + - ". Please make sure the context binary file matches the model."); - - ORT_RETURN_IF(graph_partition_name != graph_partition_name_from_ctx_cache && get_capability_round_2_, - "Graph name from context cache metadata: " + graph_partition_name_from_ctx_cache + - " is different with target: " + graph_partition_name + - ". You may need to re-generate the context binary file."); - - get_capability_round_2_ = true; return Status::OK(); } -Status QnnCacheModelHandler::GenerateCtxCacheOnnxModel(unsigned char* buffer, - uint64_t buffer_size, - const std::string& sdk_build_version, - const std::vector& fused_nodes_and_graphs, - const std::unordered_map>& qnn_models, - const logging::Logger& logger) { +Status GenerateCtxCacheOnnxModel(const std::string model_name, + const std::string model_description, + unsigned char* buffer, + uint64_t buffer_size, + const std::string& sdk_build_version, + const std::vector& fused_nodes_and_graphs, + const std::unordered_map>& qnn_models, + const onnxruntime::PathString& context_cache_path, + bool qnn_context_embed_mode, + const logging::Logger& logger) { std::unordered_map domain_to_version = {{kOnnxDomain, 11}, {kMSDomain, 1}}; - Model model(model_name_, false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), + Model model(model_name, false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), domain_to_version, {}, logger); auto& graph = model.MainGraph(); - graph.SetDescription(model_description_); + graph.SetDescription(model_description); using namespace ONNX_NAMESPACE; int index = 0; // Still need more work to support multiple partition, it's out of EP's scope. // Already have code to make sure it's single partition before this method get invoked. for (const auto& fused_node_graph : fused_nodes_and_graphs) { - const onnxruntime::GraphViewer& graph_viewer(fused_node_graph.filtered_graph); Node& fused_node = fused_node_graph.fused_node; - // graph_viewer.Name() is generated in GetCapability, e.g QNN_[hash_id]_[id] - // dump graph_viewer.Name() as metadata in context cache binary file, so that we can validate it in GetCapability auto qnn_model_kv = qnn_models.find(fused_node.Name()); ORT_RETURN_IF(qnn_model_kv == qnn_models.end(), fused_node.Name(), " not exist in QnnModel table."); @@ -229,7 +237,7 @@ Status QnnCacheModelHandler::GenerateCtxCacheOnnxModel(unsigned char* buffer, ORT_RETURN_IF_ERROR(CreateNodeArgs(qnn_model->GetInputNames(), qnn_model->GetInputsInfo(), inputs, graph)); ORT_RETURN_IF_ERROR(CreateNodeArgs(qnn_model->GetOutputNames(), qnn_model->GetOutputsInfo(), outputs, graph)); - const std::string& graph_name = graph_viewer.Name(); + const std::string& graph_name = fused_node.Name(); auto& ep_node = graph.AddNode(graph_name, EPCONTEXT_OP, "Onnx Qnn context binary cache for graph partition: " + graph_name, @@ -240,13 +248,13 @@ Status QnnCacheModelHandler::GenerateCtxCacheOnnxModel(unsigned char* buffer, // Only dump the context buffer once since all QNN graph are in one single context if (0 == index) { - if (qnn_context_embed_mode_) { + if (qnn_context_embed_mode) { std::string cache_payload(buffer, buffer + buffer_size); ep_node.AddAttribute(EP_CACHE_CONTEXT, cache_payload); } else { - std::string context_cache_path(context_cache_path_ + "_" + graph_name + ".bin"); - std::string context_cache_name(std::filesystem::path(context_cache_path).filename().string()); - std::ofstream of_stream(context_cache_path.c_str(), std::ofstream::binary); + onnxruntime::PathString context_bin_path = context_cache_path + ToPathString("_" + graph_name + ".bin"); + std::string context_cache_name(std::filesystem::path(context_bin_path).filename().string()); + std::ofstream of_stream(context_bin_path.c_str(), std::ofstream::binary); if (!of_stream) { LOGS(logger, ERROR) << "Failed to open create context file."; return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to open context cache file."); @@ -257,7 +265,7 @@ Status QnnCacheModelHandler::GenerateCtxCacheOnnxModel(unsigned char* buffer, } else { ep_node.AddAttribute(MAIN_CONTEXT, static_cast(0)); } - int64_t embed_mode = qnn_context_embed_mode_ ? static_cast(1) : static_cast(0); + int64_t embed_mode = qnn_context_embed_mode ? static_cast(1) : static_cast(0); ep_node.AddAttribute(EMBED_MODE, embed_mode); ep_node.AddAttribute(EP_SDK_VER, sdk_build_version); ep_node.AddAttribute(PARTITION_NAME, graph_name); @@ -265,7 +273,7 @@ Status QnnCacheModelHandler::GenerateCtxCacheOnnxModel(unsigned char* buffer, ++index; } ORT_RETURN_IF_ERROR(graph.Resolve()); - ORT_RETURN_IF_ERROR(Model::Save(model, context_cache_path_)); + ORT_RETURN_IF_ERROR(Model::Save(model, context_cache_path)); return Status::OK(); } diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h index e9ca87a679..0011d0f43f 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h @@ -38,77 +38,50 @@ Status CreateNodeArgs(const std::vector& names, std::vector& node_args, onnxruntime::Graph& graph); -class QnnCacheModelHandler { - public: - QnnCacheModelHandler(bool qnn_context_embed_mode) : qnn_context_embed_mode_(qnn_context_embed_mode) { - } - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QnnCacheModelHandler); +bool IsContextCacheFileExists(const std::string& customer_context_cache_path, + const onnxruntime::PathString& model_pathstring, + onnxruntime::PathString& context_cache_path); - Status LoadQnnCtxFromOnnxModel(const onnxruntime::GraphViewer& graph_viewer, - const std::string& ctx_onnx_model_path, - bool is_qnn_ctx_model, - bool is_ctx_cache_file_exist, - QnnBackendManager* qnn_backend_manager, - QnnModel& qnn_model, - const logging::Logger& logger) { - if (is_qnn_ctx_model) { - return GetEpContextFromGraph(graph_viewer, ctx_onnx_model_path, qnn_backend_manager, qnn_model); - } else if (is_ctx_cache_file_exist) { - return GetEpContextFromModel(ctx_onnx_model_path, qnn_backend_manager, qnn_model, logger); - } - return Status::OK(); - } +Status GetEpContextFromModel(const onnxruntime::PathString& ctx_onnx_model_path, + QnnBackendManager* qnn_backend_manager, + QnnModel& qnn_model, + const logging::Logger& logger); - bool IsContextCacheFileExists(const std::string& customer_context_cache_path, - const std::string& model_description, - const onnxruntime::PathString& model_pathstring); +Status GetEpContextFromGraph(const onnxruntime::GraphViewer& graph_viewer, + const onnxruntime::PathString& ctx_onnx_model_path, + QnnBackendManager* qnn_backend_manager, + QnnModel& qnn_model); - bool GetIsContextCacheFileExists() const { - return ctx_file_exists_; - } - - Status ValidateWithContextFile(const std::string& model_name, - const std::string& graph_name, - const logging::Logger& logger); - - Status GetMetadataFromEpContextModel(const std::string& ctx_onnx_model_path, - std::string& model_name, - std::string& model_description, - std::string& graph_partition_name, - std::string& cache_source, - const logging::Logger& logger); - - Status GenerateCtxCacheOnnxModel(unsigned char* buffer, - uint64_t buffer_size, - const std::string& sdk_build_version, - const std::vector& fused_nodes_and_graphs, - const std::unordered_map>& qnn_models, - const logging::Logger& logger); - - private: - Status GetEpContextFromModel(const std::string& ctx_onnx_model_path, +Status LoadQnnCtxFromOnnxModel(const onnxruntime::GraphViewer& graph_viewer, + const onnxruntime::PathString& ctx_onnx_model_path, + bool is_qnn_ctx_model, + bool is_ctx_cache_file_exist, QnnBackendManager* qnn_backend_manager, QnnModel& qnn_model, const logging::Logger& logger); - Status GetEpContextFromGraph(const onnxruntime::GraphViewer& graph_viewer, - const std::string& ctx_onnx_model_path, - QnnBackendManager* qnn_backend_manager, - QnnModel& qnn_model); +Status ValidateWithContextFile(const onnxruntime::PathString& context_cache_path, + const std::string& model_name, + const std::string& model_description, + const std::string& graph_partition_name, + const logging::Logger& logger); - private: - bool is_metadata_ready_ = false; - // model_name_ to cache_source_ -- metadata get from generated Qnn context binary Onnx model - std::string model_name_ = ""; - std::string model_description_ = ""; - std::string graph_partition_name_ = ""; - std::string cache_source_ = ""; - - std::string context_cache_path_ = ""; - bool ctx_file_exists_ = false; - bool get_capability_round_2_ = false; - bool qnn_context_embed_mode_ = true; -}; // QnnCacheModelHandler +Status GetMetadataFromEpContextModel(const onnxruntime::PathString& ctx_onnx_model_path, + std::string& model_name, + std::string& model_description, + std::string& graph_partition_name, + std::string& cache_source, + const logging::Logger& logger); +Status GenerateCtxCacheOnnxModel(const std::string model_name, + const std::string model_description, + unsigned char* buffer, + uint64_t buffer_size, + const std::string& sdk_build_version, + const std::vector& fused_nodes_and_graphs, + const std::unordered_map>& qnn_models, + const onnxruntime::PathString& context_cache_path, + bool qnn_context_embed_mode, + const logging::Logger& logger); } // namespace qnn } // namespace onnxruntime diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index aac82c89d6..4edccea661 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -22,7 +22,6 @@ namespace onnxruntime { namespace qnn { class QnnModel; -class QnnCacheModelHandler; class QnnBackendManager { public: diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index 8acd0d68b7..c7b309ae47 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -16,20 +16,12 @@ #include "core/providers/qnn/builder/qnn_model_wrapper.h" #include "core/providers/qnn/builder/op_builder_factory.h" #include "core/providers/qnn/builder/qnn_def.h" +#include "core/providers/qnn/builder/onnx_ctx_model_helper.h" namespace onnxruntime { constexpr const char* QNN = "QNN"; -std::string GetFileNameFromModelPath(onnxruntime::Path model_path) { - auto model_path_components = model_path.GetComponents(); - // There's no model path if model loaded from buffer stead of file - if (model_path_components.empty()) { - return ""; - } - return PathToUTF8String(model_path_components.back()); -} - void QNNExecutionProvider::ParseProfilingLevel(std::string profiling_level_string) { std::transform(profiling_level_string.begin(), profiling_level_string.end(), @@ -134,16 +126,15 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio static const std::string CONTEXT_CACHE_PATH = "qnn_context_cache_path"; auto context_cache_path_pos = provider_options_map.find(CONTEXT_CACHE_PATH); if (context_cache_path_pos != provider_options_map.end()) { - context_cache_path_ = context_cache_path_pos->second; - LOGS_DEFAULT(VERBOSE) << "User specified context cache path: " << context_cache_path_; + context_cache_path_cfg_ = context_cache_path_pos->second; + LOGS_DEFAULT(VERBOSE) << "User specified context cache path: " << context_cache_path_cfg_; } - bool qnn_context_embed_mode = true; static const std::string CONTEXT_CACHE_EMBED_MODE = "qnn_context_embed_mode"; auto context_cache_embed_mode_pos = provider_options_map.find(CONTEXT_CACHE_EMBED_MODE); if (context_cache_embed_mode_pos != provider_options_map.end()) { - qnn_context_embed_mode = context_cache_embed_mode_pos->second == "1"; - LOGS_DEFAULT(VERBOSE) << "User specified context cache embed mode: " << qnn_context_embed_mode; + qnn_context_embed_mode_ = context_cache_embed_mode_pos->second == "1"; + LOGS_DEFAULT(VERBOSE) << "User specified context cache embed mode: " << qnn_context_embed_mode_; } static const std::string BACKEND_PATH = "backend_path"; @@ -206,7 +197,6 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio htp_performance_mode_, context_priority_, std::move(qnn_saver_path)); - qnn_cache_model_handler_ = std::make_unique(qnn_context_embed_mode); } bool QNNExecutionProvider::IsNodeSupported(qnn::QnnModelWrapper& qnn_model_wrapper, const NodeUnit& node_unit, @@ -343,9 +333,10 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer // This is for case: QDQ model + Onnx Qnn context cache model if (context_cache_enabled_ && !is_qnn_ctx_model) { - load_from_cached_context = qnn_cache_model_handler_->IsContextCacheFileExists(context_cache_path_, - graph_viewer.Description(), - graph_viewer.ModelPath().ToPathString()); + onnxruntime::PathString context_cache_path; + load_from_cached_context = qnn::IsContextCacheFileExists(context_cache_path_cfg_, + graph_viewer.ModelPath().ToPathString(), + context_cache_path); } // Load from cached context will load the QnnSystem lib and skip the Qnn context creation @@ -444,17 +435,6 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer } const size_t num_of_partitions = result.size(); - - if (!is_qnn_ctx_model && load_from_cached_context && 1 == num_of_partitions) { - rt = qnn_cache_model_handler_->ValidateWithContextFile(GetFileNameFromModelPath(graph_viewer.ModelPath()), - result[0]->sub_graph->GetMetaDef()->name, - logger); - if (Status::OK() != rt) { - LOGS(logger, ERROR) << "QNN failed to validate context cache metadata: " << rt.ErrorMessage(); - return result; - } - } - const auto summary_msg = MakeString("Number of partitions supported by QNN EP: ", num_of_partitions, ", number of nodes in the graph: ", num_nodes_in_graph, ", number of nodes supported by QNN: ", num_of_supported_nodes); @@ -547,25 +527,38 @@ Status QNNExecutionProvider::Compile(const std::vector& fused bool is_qnn_ctx_model = false; ORT_RETURN_IF_ERROR(qnn::IsFusedGraphHasCtxNode(fused_nodes_and_graphs, is_qnn_ctx_model)); - bool is_ctx_file_exist = qnn_cache_model_handler_->GetIsContextCacheFileExists(); + onnxruntime::PathString context_cache_path; + bool is_ctx_file_exist = qnn::IsContextCacheFileExists(context_cache_path_cfg_, + graph_viewer.ModelPath().ToPathString(), + context_cache_path); + const std::string& model_name = graph_viewer.GetGraph().Name(); + const std::string& model_description = graph_viewer.GetGraph().Description(); + const std::string& graph_meta_id = fused_node.Name(); + if (fused_nodes_and_graphs.size() == 1 && !is_qnn_ctx_model && is_ctx_file_exist) { + ORT_RETURN_IF_ERROR(qnn::ValidateWithContextFile(context_cache_path, + model_name, + model_description, + graph_meta_id, + logger)); + } + if (is_qnn_ctx_model || (context_cache_enabled_ && is_ctx_file_exist)) { ORT_RETURN_IF(fused_nodes_and_graphs.size() != 1, "Only support single partition for context cache feature."); std::unique_ptr qnn_model = std::make_unique(logger, qnn_backend_manager_.get()); // Load and execute from cached context if exist - ORT_RETURN_IF_ERROR(qnn_cache_model_handler_->LoadQnnCtxFromOnnxModel(graph_viewer, - context_cache_path_, - is_qnn_ctx_model, - is_ctx_file_exist, - qnn_backend_manager_.get(), - *(qnn_model.get()), - logger)); + ORT_RETURN_IF_ERROR(qnn::LoadQnnCtxFromOnnxModel(graph_viewer, + context_cache_path, + is_qnn_ctx_model, + is_ctx_file_exist, + qnn_backend_manager_.get(), + *(qnn_model.get()), + logger)); ORT_RETURN_IF_ERROR(qnn_model->SetGraphInputOutputInfo(graph_viewer, fused_node)); ORT_RETURN_IF_ERROR(qnn_model->SetupQnnInputOutput()); // fused node name is QNNExecutionProvider_QNN_[hash_id]_[id] // the name here should be same with context->node_name in compute_info - LOGS(logger, VERBOSE) << "fused node name: " << fused_node.Name(); - qnn_models_.emplace(fused_node.Name(), std::move(qnn_model)); + qnn_models_.emplace(graph_meta_id, std::move(qnn_model)); ORT_RETURN_IF_ERROR(CreateComputeFunc(node_compute_funcs, logger)); return Status::OK(); @@ -576,12 +569,16 @@ Status QNNExecutionProvider::Compile(const std::vector& fused ORT_RETURN_IF(fused_nodes_and_graphs.size() != 1, "Only support single partition for context cache feature."); uint64_t buffer_size(0); auto context_buffer = qnn_backend_manager_->GetContextBinaryBuffer(buffer_size); - ORT_RETURN_IF_ERROR(qnn_cache_model_handler_->GenerateCtxCacheOnnxModel(context_buffer.get(), - buffer_size, - qnn_backend_manager_->GetSdkVersion(), - fused_nodes_and_graphs, - qnn_models_, - logger)); + ORT_RETURN_IF_ERROR(qnn::GenerateCtxCacheOnnxModel(model_name, + model_description, + context_buffer.get(), + buffer_size, + qnn_backend_manager_->GetSdkVersion(), + fused_nodes_and_graphs, + qnn_models_, + context_cache_path, + qnn_context_embed_mode_, + logger)); } return Status::OK(); } diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.h b/onnxruntime/core/providers/qnn/qnn_execution_provider.h index cf0bff8890..8c99a916a6 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.h +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.h @@ -8,7 +8,6 @@ #include #include "core/providers/qnn/builder/qnn_backend_manager.h" #include "core/providers/qnn/builder/qnn_model.h" -#include "core/providers/qnn/builder/onnx_ctx_model_helper.h" #include "core/providers/qnn/builder/qnn_graph_configs_helper.h" namespace onnxruntime { @@ -71,10 +70,10 @@ class QNNExecutionProvider : public IExecutionProvider { std::unordered_map> qnn_models_; uint32_t rpc_control_latency_ = 0; bool context_cache_enabled_ = false; - std::string context_cache_path_ = ""; + std::string context_cache_path_cfg_ = ""; bool disable_cpu_ep_fallback_ = false; // True if CPU EP fallback has been disabled for this session. - std::unique_ptr qnn_cache_model_handler_; qnn::ContextPriority context_priority_ = qnn::ContextPriority::NORMAL; + bool qnn_context_embed_mode_ = true; }; } // namespace onnxruntime diff --git a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc index 9fcb5744ad..3435bd71aa 100644 --- a/onnxruntime/test/providers/qnn/simple_op_htp_test.cc +++ b/onnxruntime/test/providers/qnn/simple_op_htp_test.cc @@ -786,7 +786,7 @@ TEST_F(QnnHTPBackendTests, ContextBinaryCacheNonEmbedModeTest) { // Check the Onnx skeleton file is generated EXPECT_TRUE(std::filesystem::exists(context_binary_file.c_str())); // Check the Qnn context cache binary file is generated - EXPECT_TRUE(std::filesystem::exists("qnn_context_cache_non_embed.onnx_QNN_8283143575221199085_1.bin")); + EXPECT_TRUE(std::filesystem::exists("qnn_context_cache_non_embed.onnx_QNNExecutionProvider_QNN_8283143575221199085_1_0.bin")); // 2nd run loads and run from QDQ model + Onnx skeleton file + Qnn context cache binary file TestQDQModelAccuracy(BuildOpTestCase(op_type, {input_def}, {}, {}), @@ -806,6 +806,62 @@ TEST_F(QnnHTPBackendTests, ContextBinaryCacheNonEmbedModeTest) { context_binary_file); } +// Run QDQ model on HTP 2 times +// 1st run will generate the Onnx skeleton file + Qnn context cache binary file +// Then delete the context bin file to make the 2nd sesssion.Initialize() return the status with code INVALID_GRAPH +TEST_F(QnnHTPBackendTests, ContextBinaryCache_InvalidGraph) { + ProviderOptions provider_options; +#if defined(_WIN32) + provider_options["backend_path"] = "QnnHtp.dll"; +#else + provider_options["backend_path"] = "libQnnHtp.so"; +#endif + provider_options["qnn_context_cache_enable"] = "1"; + const std::string context_binary_file = "./qnn_context_cache_non_embed.onnx"; + provider_options["qnn_context_cache_path"] = context_binary_file; + provider_options["qnn_context_embed_mode"] = "0"; + + const TestInputDef input_def({1, 2, 3}, false, -10.0f, 10.0f); + const std::string op_type = "Atan"; + + // Runs model with DQ-> Atan-> Q and compares the outputs of the CPU and QNN EPs. + // 1st run will generate the Onnx skeleton file + Qnn context cache binary file + TestQDQModelAccuracy(BuildOpTestCase(op_type, {input_def}, {}, {}), + BuildQDQOpTestCase(op_type, {input_def}, {}, {}), + provider_options, + 14, + ExpectedEPNodeAssignment::All); + + // Check the Onnx skeleton file is generated + EXPECT_TRUE(std::filesystem::exists(context_binary_file.c_str())); + // Check the Qnn context cache binary file is generated + std::filesystem::path context_bin = "qnn_context_cache_non_embed.onnx_QNNExecutionProvider_QNN_8283143575221199085_1_0.bin"; + EXPECT_TRUE(std::filesystem::exists(context_bin)); + // Delete the Qnn context cache binary file + EXPECT_TRUE(std::filesystem::remove(context_bin)); + + // loads and run from Onnx skeleton file + Qnn context cache binary file + onnx::ModelProto model_proto; + onnxruntime::Model qnn_ctx_model; + // Load the QNN context cache model from path specified + ASSERT_STATUS_OK(qnn_ctx_model.Load(ToPathString(context_binary_file), model_proto)); + std::string qnn_ctx_model_data; + model_proto.SerializeToString(&qnn_ctx_model_data); + + SessionOptions so; + so.session_logid = "qnn_ctx_model_logger"; + RunOptions run_options; + run_options.run_tag = so.session_logid; + + InferenceSessionWrapper session_object{so, GetEnvironment()}; + + std::string provider_type = kCpuExecutionProvider; + ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(QnnExecutionProviderWithOptions(provider_options))); + ASSERT_STATUS_OK(session_object.Load(qnn_ctx_model_data.data(), static_cast(qnn_ctx_model_data.size()))); + // Verify the return status with code INVALID_GRAPH + ASSERT_TRUE(session_object.Initialize().Code() == common::StatusCode::INVALID_GRAPH); +} + // Run QDQ model on HTP with 2 inputs // 1st run will generate the Qnn context cache onnx file // 2nd run will load and run from QDQ model + Qnn context cache model