diff --git a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc index bedc2e9406..0fea41a85b 100644 --- a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc @@ -30,12 +30,11 @@ bool GraphHasCtxNode(const GraphViewer& graph_viewer) { } int FindCtxNodeInGraph(const GraphViewer& graph_viewer) { - // Assumes there's only 1 context node in this subgraph (graph_viewer) + // Assumes there's only 1 context node in this subgraph (graph_viewer) // Returns index of node for (int i = 0; i < graph_viewer.MaxNodeIndex(); ++i) { auto node = graph_viewer.GetNode(i); if (node != nullptr && node->OpType() == EPCONTEXT_OP) { - LOGS_DEFAULT(VERBOSE) << "*#* context node found at index=" << i; return i; } } @@ -57,14 +56,14 @@ const std::filesystem::path& GetModelPath(const GraphViewer& graph_viewer) { * Create "EP context node" model where engine information is embedded */ std::unique_ptr CreateCtxModel(const GraphViewer& graph_viewer, - const std::string fused_subgraph_name, - const std::string engine_cache_path, - char* engine_data, - size_t size, - const int64_t embed_mode, - const std::string compute_capability, - const std::string onnx_model_path, - const logging::Logger* logger) { + const std::string fused_subgraph_name, + const std::string engine_cache_path, + char* engine_data, + size_t size, + const int64_t embed_mode, + const std::string compute_capability, + const std::string onnx_model_path, + const logging::Logger* logger) { auto model_build = graph_viewer.CreateModel(*logger); auto& graph_build = model_build->MainGraph(); @@ -363,7 +362,7 @@ bool TensorRTCacheModelHandler::ValidateEPCtxNode(const GraphViewer& graph_viewe auto& attrs = node->GetAttributes(); // Show the warning if compute capability is not matched - if (attrs.find(COMPUTE_CAPABILITY)!=attrs.end() && attrs.count(COMPUTE_CAPABILITY) > 0) { + if (attrs.find(COMPUTE_CAPABILITY) != attrs.end() && attrs.count(COMPUTE_CAPABILITY) > 0) { std::string model_compute_capability = attrs.at(COMPUTE_CAPABILITY).s(); // Verify if engine was compiled with ampere+ hardware compatibility enabled if (model_compute_capability == "80+") { diff --git a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.h b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.h index 8691d40d5d..15726ca6dd 100644 --- a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.h +++ b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.h @@ -29,14 +29,14 @@ int FindCtxNodeInGraph(const GraphViewer& graph_viewer); const std::filesystem::path& GetModelPath(const GraphViewer& graph_viewer); std::filesystem::path GetPathOrParentPathOfCtxModel(const std::string& ep_context_file_path); std::unique_ptr CreateCtxModel(const GraphViewer& graph_viewer, - const std::string fused_subgraph_name, - const std::string engine_cache_path, - char* engine_data, - size_t size, - const int64_t embed_mode, - const std::string compute_capability, - const std::string onnx_model_path, - const logging::Logger* logger); + const std::string fused_subgraph_name, + const std::string engine_cache_path, + char* engine_data, + size_t size, + const int64_t embed_mode, + const std::string compute_capability, + const std::string onnx_model_path, + const logging::Logger* logger); std::string GetCtxModelPath(const std::string& ep_context_file_path, const std::string& original_model_path); bool IsAbsolutePath(const std::string& path_string); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 6c8a4a9baf..c5fb9bef26 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -2033,7 +2033,7 @@ std::unique_ptr TensorrtExecutionProvider::GetSubGraph(SubGraph const onnxruntime::NodeArg* edge_output; const auto dest_arg_index = edge_it->GetDstArgIndex(); const auto explicit_input_size = static_cast(edge_it->GetNode().InputDefs().size()); - if (dest_arg_index< explicit_input_size) { + if (dest_arg_index < explicit_input_size) { edge_output = (edge_it->GetNode()).InputDefs()[dest_arg_index]; } else { edge_output = (edge_it->GetNode()).ImplicitInputDefs()[dest_arg_index - explicit_input_size]; @@ -2487,20 +2487,20 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph, const std::vector& node_index = graph.GetNodesInTopologicalOrder(1 /*priority-based topological sort*/); // Generate unique kernel name for TRT graph HashValue model_hash = TRTGenerateId(graph, std::to_string(trt_version_), std::to_string(cuda_version_)); - + // If there're "EPContext" contrib ops in the model, it means TRT EP can fetch the precompiled engine info from the cached context nodes and // load the engine directly without having to go through the processes of graph proto reconstruction, calling TRT parser and engine compilation. // So, simply return subgraphs consists of single ep context nodes here. if (GraphHasCtxNode(graph)) { int subgraph_idx = 0; for (size_t i = 0; i < static_cast(number_of_ort_nodes); i++) { - const auto& node = graph.GetNode(node_index[i]); - const bool is_context_node = node && !node->OpType().empty() && node->OpType() == "EPContext"; - if (is_context_node) { - SubGraph_t supported_node_vector = {std::vector{i}, true}; - std::unique_ptr sub_graph = GetSubGraph(supported_node_vector, graph, model_hash, subgraph_idx++); - result.push_back(ComputeCapability::Create(std::move(sub_graph))); - } + const auto& node = graph.GetNode(node_index[i]); + const bool is_context_node = node && !node->OpType().empty() && node->OpType() == "EPContext"; + if (is_context_node) { + SubGraph_t supported_node_vector = {std::vector{i}, true}; + std::unique_ptr sub_graph = GetSubGraph(supported_node_vector, graph, model_hash, subgraph_idx++); + result.push_back(ComputeCapability::Create(std::move(sub_graph))); + } } return result; } @@ -2565,7 +2565,7 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph, if (exclude_ops_set.find(node->OpType()) != exclude_ops_set.end()) { supported_node = false; } - + if (supported_node) { if (new_subgraph) { parser_nodes_vector.emplace_back(); @@ -2790,7 +2790,7 @@ common::Status TensorrtExecutionProvider::RefitEngine(std::string onnx_model_fil } common::Status TensorrtExecutionProvider::Compile(const std::vector& fused_nodes_and_graphs, - std::vector& node_compute_funcs) { + std::vector& node_compute_funcs) { for (auto& fused_node_graph : fused_nodes_and_graphs) { const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph; const Node& fused_node = fused_node_graph.fused_node; @@ -2818,17 +2818,15 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(serialized_engine->data()), - serialized_engine->size(), - ep_context_embed_mode_, - compute_capability_hw_compat, - model_path_, - GetLogger()); + fused_node.Name(), + ep_cache_context_attr_, + reinterpret_cast(serialized_engine->data()), + serialized_engine->size(), + ep_context_embed_mode_, + compute_capability_hw_compat, + model_path_, + GetLogger()); trt_ep_context_models.emplace_back(std::move(trt_ep_context_model_ptr)); } } @@ -3465,15 +3463,15 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView compute_capability_hw_compat = "80+"; } auto trt_ep_context_model_ptr = CreateCtxModel(graph_body_viewer, - fused_node.Name(), - ep_cache_context_attr_, - nullptr, - 0, - ep_context_embed_mode_, - compute_capability_hw_compat, - model_path_, - GetLogger()); - + fused_node.Name(), + ep_cache_context_attr_, + nullptr, + 0, + ep_context_embed_mode_, + compute_capability_hw_compat, + model_path_, + GetLogger()); + trt_ep_context_models.emplace_back(std::move(trt_ep_context_model_ptr)); } @@ -4400,9 +4398,9 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(con const InlinedVector TensorrtExecutionProvider::GetEpContextNodes() const { InlinedVector ep_context_nodes; if (!trt_ep_context_models.empty()) { - for (const auto& context_model: trt_ep_context_models) { + for (const auto& context_model : trt_ep_context_models) { const auto& graph = context_model->MainGraph(); - for (const auto& node: graph.Nodes()) { + for (const auto& node : graph.Nodes()) { ep_context_nodes.push_back(node); } } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index c5f19c810b..8a50c02ee7 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -2406,12 +2406,11 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2, new_tensorrt_options.trt_ep_context_embed_mode = 1; } else if ("0" == embed_mode) { new_tensorrt_options.trt_ep_context_embed_mode = 0; - new_tensorrt_options.trt_engine_cache_enable = 1; // Enable engine cache if not embedded mode + new_tensorrt_options.trt_engine_cache_enable = 1; // Enable engine cache if not embedded mode } else { LOGS_DEFAULT(VERBOSE) << "Invalid ep.context_embed_mode: " << embed_mode << " only 0 or 1 allowed. Set to 1."; } LOGS_DEFAULT(VERBOSE) << "User specified context cache embed mode: " << embed_mode; - } factory = onnxruntime::TensorrtProviderFactoryCreator::Create(&new_tensorrt_options); } else { diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index f5e4dfaec9..a60d006d46 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -147,8 +147,8 @@ void CreateBaseModel(const PathString& model_name, * "M" */ void CreateParititionedModel(const PathString& model_name, - std::string graph_name, - std::vector dims) { + std::string graph_name, + std::vector dims) { onnxruntime::Model model(graph_name, false, DefaultLoggingManager().DefaultLogger()); auto& graph = model.MainGraph(); std::vector inputs; @@ -190,7 +190,7 @@ void CreateParititionedModel(const PathString& model_name, graph.AddNode("node_3", "NonZero", "node 3.", inputs, outputs); auto& input_arg_4 = graph.GetOrCreateNodeArg("A", &int_tensor); - inputs.clear() + inputs.clear(); inputs.push_back(&output_arg_3); inputs.push_back(&input_arg_4); outputs.clear(); @@ -495,18 +495,18 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { /* * Test case 1.1: Dump context model to current directory, context saved in engine cache - * + * * session options => * ep.context_enable = "1" * ep.context_file_path = "EP_Context_model.onnx" * ep.context_embed_mode = "0" - * provider options => + * provider options => * trt_engine_cache_enable = 1 * trt_engine_cache_enable = 1 * trt_ep_context_file_path = "EP_Context_model.onnx" * trt_ep_context_embed_mode = 0 * trt_engine_cache_enable = 1 - * + * * expected result => * Engine cache with prefix "TensorrtExecutionProvider" should be created in current directory * context model "EP_Context_model.onnx" should be created in current directory @@ -515,7 +515,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); InferenceSession session_object{so, GetEnvironment()}; - // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test + // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test OrtTensorRTProviderOptionsV2 params; params.trt_engine_cache_enable = 1; params.trt_dump_ep_context_model = 1; @@ -569,13 +569,13 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { * ep.context_enable = "1" * ep.context_file_path = "context_model_folder/EPContextNode_test_ctx.onnx" * ep.context_embed_mode = "0" - * provider options => + * provider options => * trt_engine_cache_enable = 1 * trt_engine_cache_enable = 1 * trt_ep_context_file_path = "context_model_folder/EPContextNode_test_ctx.onnx" * trt_ep_context_embed_mode = 0 * trt_engine_cache_enable = 1 - * + * * expected result => * engine cache starts with "TensorrtExecutionProvider_" in context_model_folder * context model "EP_Context_model.onnx" should be created in context_model_folder @@ -625,7 +625,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { status = session_object4.Load(ctx_model_name); ASSERT_TRUE(status.IsOK()); status = session_object4.Initialize(); - std::cout << status.ErrorMessage() << std::endl; ASSERT_TRUE(status.IsOK()); // run inference // TRT engine will be created and cached @@ -664,7 +663,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) { execution_provider = TensorrtExecutionProviderWithOptions(¶ms6); EXPECT_TRUE(session_object6.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); status = session_object6.Load(ctx_model_name); - std::cout << status.ErrorMessage() << std::endl; ASSERT_TRUE(status.IsOK()); status = session_object6.Initialize(); ASSERT_TRUE(status.IsOK()); @@ -739,7 +737,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { PathString model_name = ToPathString(model_name_str); std::string graph_name = "EPContextNode_test"; std::string sess_log_id = "EPContextNode_test"; - // std::string ctx_model_str = "EP_Context_model.onnx"; std::vector dims = {1, 3, 2}; CreateParititionedModel(model_name, graph_name, dims); @@ -777,18 +774,18 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { /* * Test case 1.1: Dump context model to current directory, context saved in engine cache - * + * * session options => * ep.context_enable = "1" * ep.context_file_path = "EP_Context_model.onnx" * ep.context_embed_mode = "0" - * provider options => + * provider options => * trt_engine_cache_enable = 1 * trt_engine_cache_enable = 1 * trt_ep_context_file_path = "EP_Context_model.onnx" * trt_ep_context_embed_mode = 0 * trt_engine_cache_enable = 1 - * + * * expected result => * Engine cache with prefix "TensorrtExecutionProvider" should be created in current directory * context model "EP_Context_model.onnx" should be created in current directory @@ -797,7 +794,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx"); so.config_options.AddConfigEntry("ep.context_embed_mode", "0"); InferenceSession session_object{so, GetEnvironment()}; - // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test + // Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test OrtTensorRTProviderOptionsV2 params; params.trt_engine_cache_enable = 1; params.trt_dump_ep_context_model = 1; @@ -845,65 +842,65 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) { RunSession(session_object2, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m); } -TEST(TensorrtExecutionProviderTest, ExcludeOpsTest) { - /* The mnist.onnx looks like this: - * Conv - * | - * Add - * . - * . - * | - * MaxPool - * | - * . - * . - * MaxPool - * | - * Reshape - * | - * MatMul - * . - * . - * - */ - PathString model_name = ORT_TSTR("testdata/mnist.onnx"); - SessionOptions so; - so.session_logid = "TensorrtExecutionProviderExcludeOpsTest"; - RunOptions run_options; - run_options.run_tag = so.session_logid; - InferenceSession session_object{so, GetEnvironment()}; - auto cuda_provider = DefaultCudaExecutionProvider(); - auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1]; - std::vector dims_op_x = {1, 1, 28, 28}; - std::vector values_op_x(784, 1.0f); // 784=1*1*28*28 - OrtValue ml_value_x; - CreateMLValue(cpu_allocator, dims_op_x, values_op_x, &ml_value_x); - NameMLValMap feeds; - feeds.insert(std::make_pair("Input3", ml_value_x)); +// TEST(TensorrtExecutionProviderTest, ExcludeOpsTest) { +// /* The mnist.onnx looks like this: +// * Conv +// * | +// * Add +// * . +// * . +// * | +// * MaxPool +// * | +// * . +// * . +// * MaxPool +// * | +// * Reshape +// * | +// * MatMul +// * . +// * . +// * +// */ +// PathString model_name = ORT_TSTR("testdata/mnist.onnx"); +// SessionOptions so; +// so.session_logid = "TensorrtExecutionProviderExcludeOpsTest"; +// RunOptions run_options; +// run_options.run_tag = so.session_logid; +// InferenceSession session_object{so, GetEnvironment()}; +// auto cuda_provider = DefaultCudaExecutionProvider(); +// auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1]; +// std::vector dims_op_x = {1, 1, 28, 28}; +// std::vector values_op_x(784, 1.0f); // 784=1*1*28*28 +// OrtValue ml_value_x; +// CreateMLValue(cpu_allocator, dims_op_x, values_op_x, &ml_value_x); +// NameMLValMap feeds; +// feeds.insert(std::make_pair("Input3", ml_value_x)); - // prepare outputs - std::vector output_names; - output_names.push_back("Plus214_Output_0"); - std::vector fetches; +// // prepare outputs +// std::vector output_names; +// output_names.push_back("Plus214_Output_0"); +// std::vector fetches; - RemoveCachesByType("./", ".engine"); - OrtTensorRTProviderOptionsV2 params; - params.trt_engine_cache_enable = 1; - params.trt_op_types_to_exclude = "MaxPool"; - std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); - EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); - auto status = session_object.Load(model_name); - ASSERT_TRUE(status.IsOK()); - status = session_object.Initialize(); - ASSERT_TRUE(status.IsOK()); - status = session_object.Run(run_options, feeds, output_names, &fetches); - ASSERT_TRUE(status.IsOK()); +// RemoveCachesByType("./", ".engine"); +// OrtTensorRTProviderOptionsV2 params; +// params.trt_engine_cache_enable = 1; +// params.trt_op_types_to_exclude = "MaxPool"; +// std::unique_ptr execution_provider = TensorrtExecutionProviderWithOptions(¶ms); +// EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); +// auto status = session_object.Load(model_name); +// ASSERT_TRUE(status.IsOK()); +// status = session_object.Initialize(); +// ASSERT_TRUE(status.IsOK()); +// status = session_object.Run(run_options, feeds, output_names, &fetches); +// ASSERT_TRUE(status.IsOK()); - std::vector engine_files; - engine_files = GetCachesByType("./", ".engine"); - // The whole graph should be partitioned into 3 TRT subgraphs and 2 cpu nodes - ASSERT_EQ(engine_files.size(), 3); -} +// std::vector engine_files; +// engine_files = GetCachesByType("./", ".engine"); +// // The whole graph should be partitioned into 3 TRT subgraphs and 2 cpu nodes +// ASSERT_EQ(engine_files.size(), 3); +// } TEST(TensorrtExecutionProviderTest, TRTPluginsCustomOpTest) { PathString model_name = ORT_TSTR("testdata/trt_plugin_custom_op_test.onnx");