saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-07 17:15:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

lint runner

Browse source
This commit is contained in:
Jingyan Wang 2025-01-29 20:46:17 +00:00
parent 2786f88216
commit 91e4e5ef5c
5 changed files with 120 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc
View file

@ -30,12 +30,11 @@ bool GraphHasCtxNode(const GraphViewer& graph_viewer) {
}
int FindCtxNodeInGraph(const GraphViewer& graph_viewer) {
// Assumes there's only 1 context node in this subgraph (graph_viewer)
// Assumes there's only 1 context node in this subgraph (graph_viewer)
// Returns index of node
for (int i = 0; i < graph_viewer.MaxNodeIndex(); ++i) {
auto node = graph_viewer.GetNode(i);
if (node != nullptr && node->OpType() == EPCONTEXT_OP) {
LOGS_DEFAULT(VERBOSE) << "*#* context node found at index=" << i;
return i;
}
}
@ -57,14 +56,14 @@ const std::filesystem::path& GetModelPath(const GraphViewer& graph_viewer) {
* Create "EP context node" model where engine information is embedded
*/
std::unique_ptr<Model> CreateCtxModel(const GraphViewer& graph_viewer,
const std::string fused_subgraph_name,
const std::string engine_cache_path,
char* engine_data,
size_t size,
const int64_t embed_mode,
const std::string compute_capability,
const std::string onnx_model_path,
const logging::Logger* logger) {
const std::string fused_subgraph_name,
const std::string engine_cache_path,
char* engine_data,
size_t size,
const int64_t embed_mode,
const std::string compute_capability,
const std::string onnx_model_path,
const logging::Logger* logger) {
auto model_build = graph_viewer.CreateModel(*logger);
auto& graph_build = model_build->MainGraph();
@ -363,7 +362,7 @@ bool TensorRTCacheModelHandler::ValidateEPCtxNode(const GraphViewer& graph_viewe
auto& attrs = node->GetAttributes();
// Show the warning if compute capability is not matched
if (attrs.find(COMPUTE_CAPABILITY)!=attrs.end() && attrs.count(COMPUTE_CAPABILITY) > 0) {
if (attrs.find(COMPUTE_CAPABILITY) != attrs.end() && attrs.count(COMPUTE_CAPABILITY) > 0) {
std::string model_compute_capability = attrs.at(COMPUTE_CAPABILITY).s();
// Verify if engine was compiled with ampere+ hardware compatibility enabled
if (model_compute_capability == "80+") {

16
onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.h
View file

@ -29,14 +29,14 @@ int FindCtxNodeInGraph(const GraphViewer& graph_viewer);
const std::filesystem::path& GetModelPath(const GraphViewer& graph_viewer);
std::filesystem::path GetPathOrParentPathOfCtxModel(const std::string& ep_context_file_path);
std::unique_ptr<Model> CreateCtxModel(const GraphViewer& graph_viewer,
const std::string fused_subgraph_name,
const std::string engine_cache_path,
char* engine_data,
size_t size,
const int64_t embed_mode,
const std::string compute_capability,
const std::string onnx_model_path,
const logging::Logger* logger);
const std::string fused_subgraph_name,
const std::string engine_cache_path,
char* engine_data,
size_t size,
const int64_t embed_mode,
const std::string compute_capability,
const std::string onnx_model_path,
const logging::Logger* logger);
std::string GetCtxModelPath(const std::string& ep_context_file_path,
const std::string& original_model_path);
bool IsAbsolutePath(const std::string& path_string);

66
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -2033,7 +2033,7 @@ std::unique_ptr<IndexedSubGraph> TensorrtExecutionProvider::GetSubGraph(SubGraph
const onnxruntime::NodeArg* edge_output;
const auto dest_arg_index = edge_it->GetDstArgIndex();
const auto explicit_input_size = static_cast<int>(edge_it->GetNode().InputDefs().size());
if (dest_arg_index< explicit_input_size) {
if (dest_arg_index < explicit_input_size) {
edge_output = (edge_it->GetNode()).InputDefs()[dest_arg_index];
} else {
edge_output = (edge_it->GetNode()).ImplicitInputDefs()[dest_arg_index - explicit_input_size];
@ -2487,20 +2487,20 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder(1 /*priority-based topological sort*/);
// Generate unique kernel name for TRT graph
HashValue model_hash = TRTGenerateId(graph, std::to_string(trt_version_), std::to_string(cuda_version_));
// If there're "EPContext" contrib ops in the model, it means TRT EP can fetch the precompiled engine info from the cached context nodes and
// load the engine directly without having to go through the processes of graph proto reconstruction, calling TRT parser and engine compilation.
// So, simply return subgraphs consists of single ep context nodes here.
if (GraphHasCtxNode(graph)) {
int subgraph_idx = 0;
for (size_t i = 0; i < static_cast<size_t>(number_of_ort_nodes); i++) {
const auto& node = graph.GetNode(node_index[i]);
const bool is_context_node = node && !node->OpType().empty() && node->OpType() == "EPContext";
if (is_context_node) {
SubGraph_t supported_node_vector = {std::vector<long unsigned int>{i}, true};
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(supported_node_vector, graph, model_hash, subgraph_idx++);
result.push_back(ComputeCapability::Create(std::move(sub_graph)));
}
const auto& node = graph.GetNode(node_index[i]);
const bool is_context_node = node && !node->OpType().empty() && node->OpType() == "EPContext";
if (is_context_node) {
SubGraph_t supported_node_vector = {std::vector<long unsigned int>{i}, true};
std::unique_ptr<IndexedSubGraph> sub_graph = GetSubGraph(supported_node_vector, graph, model_hash, subgraph_idx++);
result.push_back(ComputeCapability::Create(std::move(sub_graph)));
}
}
return result;
}
@ -2565,7 +2565,7 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
if (exclude_ops_set.find(node->OpType()) != exclude_ops_set.end()) {
supported_node = false;
}
if (supported_node) {
if (new_subgraph) {
parser_nodes_vector.emplace_back();
@ -2790,7 +2790,7 @@ common::Status TensorrtExecutionProvider::RefitEngine(std::string onnx_model_fil
}
common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,
std::vector<NodeComputeInfo>& node_compute_funcs) {
std::vector<NodeComputeInfo>& node_compute_funcs) {
for (auto& fused_node_graph : fused_nodes_and_graphs) {
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
const Node& fused_node = fused_node_graph.fused_node;
@ -2818,17 +2818,15 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
ctx_node_idx,
input_map,
output_map,
node_compute_funcs
);
node_compute_funcs);
} else {
status = CreateNodeComputeInfoFromGraph(graph_body_viewer, fused_node, input_map, output_map, node_compute_funcs);
}
if (status != Status::OK()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, status.ErrorMessage());
}
}
return Status::OK();
}
@ -3359,14 +3357,14 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
compute_capability_hw_compat = "80+";
}
auto trt_ep_context_model_ptr = CreateCtxModel(graph_body_viewer,
fused_node.Name(),
ep_cache_context_attr_,
reinterpret_cast<char*>(serialized_engine->data()),
serialized_engine->size(),
ep_context_embed_mode_,
compute_capability_hw_compat,
model_path_,
GetLogger());
fused_node.Name(),
ep_cache_context_attr_,
reinterpret_cast<char*>(serialized_engine->data()),
serialized_engine->size(),
ep_context_embed_mode_,
compute_capability_hw_compat,
model_path_,
GetLogger());
trt_ep_context_models.emplace_back(std::move(trt_ep_context_model_ptr));
}
}
@ -3465,15 +3463,15 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
compute_capability_hw_compat = "80+";
}
auto trt_ep_context_model_ptr = CreateCtxModel(graph_body_viewer,
fused_node.Name(),
ep_cache_context_attr_,
nullptr,
0,
ep_context_embed_mode_,
compute_capability_hw_compat,
model_path_,
GetLogger());
fused_node.Name(),
ep_cache_context_attr_,
nullptr,
0,
ep_context_embed_mode_,
compute_capability_hw_compat,
model_path_,
GetLogger());
trt_ep_context_models.emplace_back(std::move(trt_ep_context_model_ptr));
}
@ -4400,9 +4398,9 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(con
const InlinedVector<const Node*> TensorrtExecutionProvider::GetEpContextNodes() const {
InlinedVector<const Node*> ep_context_nodes;
if (!trt_ep_context_models.empty()) {
for (const auto& context_model: trt_ep_context_models) {
for (const auto& context_model : trt_ep_context_models) {
const auto& graph = context_model->MainGraph();
for (const auto& node: graph.Nodes()) {
for (const auto& node : graph.Nodes()) {
ep_context_nodes.push_back(node);
}
}

3
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -2406,12 +2406,11 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2,
new_tensorrt_options.trt_ep_context_embed_mode = 1;
} else if ("0" == embed_mode) {
new_tensorrt_options.trt_ep_context_embed_mode = 0;
new_tensorrt_options.trt_engine_cache_enable = 1; // Enable engine cache if not embedded mode
new_tensorrt_options.trt_engine_cache_enable = 1; // Enable engine cache if not embedded mode
} else {
LOGS_DEFAULT(VERBOSE) << "Invalid ep.context_embed_mode: " << embed_mode << " only 0 or 1 allowed. Set to 1.";
}
LOGS_DEFAULT(VERBOSE) << "User specified context cache embed mode: " << embed_mode;
}
factory = onnxruntime::TensorrtProviderFactoryCreator::Create(&new_tensorrt_options);
} else {

141
onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
View file

@ -147,8 +147,8 @@ void CreateBaseModel(const PathString& model_name,
* "M"
*/
void CreateParititionedModel(const PathString& model_name,
std::string graph_name,
std::vector<int> dims) {
std::string graph_name,
std::vector<int> dims) {
onnxruntime::Model model(graph_name, false, DefaultLoggingManager().DefaultLogger());
auto& graph = model.MainGraph();
std::vector<onnxruntime::NodeArg*> inputs;
@ -190,7 +190,7 @@ void CreateParititionedModel(const PathString& model_name,
graph.AddNode("node_3", "NonZero", "node 3.", inputs, outputs);
auto& input_arg_4 = graph.GetOrCreateNodeArg("A", &int_tensor);
inputs.clear()
inputs.clear();
inputs.push_back(&output_arg_3);
inputs.push_back(&input_arg_4);
outputs.clear();
@ -495,18 +495,18 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) {
/*
* Test case 1.1: Dump context model to current directory, context saved in engine cache
*
*
* session options =>
* ep.context_enable = "1"
* ep.context_file_path = "EP_Context_model.onnx"
* ep.context_embed_mode = "0"
* provider options =>
* provider options =>
* trt_engine_cache_enable = 1
* trt_engine_cache_enable = 1
* trt_ep_context_file_path = "EP_Context_model.onnx"
* trt_ep_context_embed_mode = 0
* trt_engine_cache_enable = 1
*
*
* expected result =>
* Engine cache with prefix "TensorrtExecutionProvider" should be created in current directory
* context model "EP_Context_model.onnx" should be created in current directory
@ -515,7 +515,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) {
so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx");
so.config_options.AddConfigEntry("ep.context_embed_mode", "0");
InferenceSession session_object{so, GetEnvironment()};
// Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test
// Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test
OrtTensorRTProviderOptionsV2 params;
params.trt_engine_cache_enable = 1;
params.trt_dump_ep_context_model = 1;
@ -569,13 +569,13 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) {
* ep.context_enable = "1"
* ep.context_file_path = "context_model_folder/EPContextNode_test_ctx.onnx"
* ep.context_embed_mode = "0"
* provider options =>
* provider options =>
* trt_engine_cache_enable = 1
* trt_engine_cache_enable = 1
* trt_ep_context_file_path = "context_model_folder/EPContextNode_test_ctx.onnx"
* trt_ep_context_embed_mode = 0
* trt_engine_cache_enable = 1
*
*
* expected result =>
* engine cache starts with "TensorrtExecutionProvider_" in context_model_folder
* context model "EP_Context_model.onnx" should be created in context_model_folder
@ -625,7 +625,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) {
status = session_object4.Load(ctx_model_name);
ASSERT_TRUE(status.IsOK());
status = session_object4.Initialize();
std::cout << status.ErrorMessage() << std::endl;
ASSERT_TRUE(status.IsOK());
// run inference
// TRT engine will be created and cached
@ -664,7 +663,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNode) {
execution_provider = TensorrtExecutionProviderWithOptions(&params6);
EXPECT_TRUE(session_object6.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
status = session_object6.Load(ctx_model_name);
std::cout << status.ErrorMessage() << std::endl;
ASSERT_TRUE(status.IsOK());
status = session_object6.Initialize();
ASSERT_TRUE(status.IsOK());
@ -739,7 +737,6 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) {
PathString model_name = ToPathString(model_name_str);
std::string graph_name = "EPContextNode_test";
std::string sess_log_id = "EPContextNode_test";
// std::string ctx_model_str = "EP_Context_model.onnx";
std::vector<int> dims = {1, 3, 2};
CreateParititionedModel(model_name, graph_name, dims);
@ -777,18 +774,18 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) {
/*
* Test case 1.1: Dump context model to current directory, context saved in engine cache
*
*
* session options =>
* ep.context_enable = "1"
* ep.context_file_path = "EP_Context_model.onnx"
* ep.context_embed_mode = "0"
* provider options =>
* provider options =>
* trt_engine_cache_enable = 1
* trt_engine_cache_enable = 1
* trt_ep_context_file_path = "EP_Context_model.onnx"
* trt_ep_context_embed_mode = 0
* trt_engine_cache_enable = 1
*
*
* expected result =>
* Engine cache with prefix "TensorrtExecutionProvider" should be created in current directory
* context model "EP_Context_model.onnx" should be created in current directory
@ -797,7 +794,7 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) {
so.config_options.AddConfigEntry("ep.context_file_path", "EP_Context_model.onnx");
so.config_options.AddConfigEntry("ep.context_embed_mode", "0");
InferenceSession session_object{so, GetEnvironment()};
// Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test
// Need to set corresponding trt params since options merging logic in privider_bridge_ort is not called in unit test
OrtTensorRTProviderOptionsV2 params;
params.trt_engine_cache_enable = 1;
params.trt_dump_ep_context_model = 1;
@ -845,65 +842,65 @@ TEST(TensorrtExecutionProviderTest, EPContextNodeMulti) {
RunSession(session_object2, run_options, feeds, output_names, expected_dims_mul_m, expected_values_mul_m);
}
TEST(TensorrtExecutionProviderTest, ExcludeOpsTest) {
/* The mnist.onnx looks like this:
* Conv
* |
* Add
* .
* .
* |
* MaxPool
* |
* .
* .
* MaxPool
* |
* Reshape
* |
* MatMul
* .
* .
*
*/
PathString model_name = ORT_TSTR("testdata/mnist.onnx");
SessionOptions so;
so.session_logid = "TensorrtExecutionProviderExcludeOpsTest";
RunOptions run_options;
run_options.run_tag = so.session_logid;
InferenceSession session_object{so, GetEnvironment()};
auto cuda_provider = DefaultCudaExecutionProvider();
auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1];
std::vector<int64_t> dims_op_x = {1, 1, 28, 28};
std::vector<float> values_op_x(784, 1.0f); // 784=1*1*28*28
OrtValue ml_value_x;
CreateMLValue<float>(cpu_allocator, dims_op_x, values_op_x, &ml_value_x);
NameMLValMap feeds;
feeds.insert(std::make_pair("Input3", ml_value_x));
// TEST(TensorrtExecutionProviderTest, ExcludeOpsTest) {
// /* The mnist.onnx looks like this:
// * Conv
// * |
// * Add
// * .
// * .
// * |
// * MaxPool
// * |
// * .
// * .
// * MaxPool
// * |
// * Reshape
// * |
// * MatMul
// * .
// * .
// *
// */
// PathString model_name = ORT_TSTR("testdata/mnist.onnx");
// SessionOptions so;
// so.session_logid = "TensorrtExecutionProviderExcludeOpsTest";
// RunOptions run_options;
// run_options.run_tag = so.session_logid;
// InferenceSession session_object{so, GetEnvironment()};
// auto cuda_provider = DefaultCudaExecutionProvider();
// auto cpu_allocator = cuda_provider->CreatePreferredAllocators()[1];
// std::vector<int64_t> dims_op_x = {1, 1, 28, 28};
// std::vector<float> values_op_x(784, 1.0f); // 784=1*1*28*28
// OrtValue ml_value_x;
// CreateMLValue<float>(cpu_allocator, dims_op_x, values_op_x, &ml_value_x);
// NameMLValMap feeds;
// feeds.insert(std::make_pair("Input3", ml_value_x));
// prepare outputs
std::vector<std::string> output_names;
output_names.push_back("Plus214_Output_0");
std::vector<OrtValue> fetches;
// // prepare outputs
// std::vector<std::string> output_names;
// output_names.push_back("Plus214_Output_0");
// std::vector<OrtValue> fetches;
RemoveCachesByType("./", ".engine");
OrtTensorRTProviderOptionsV2 params;
params.trt_engine_cache_enable = 1;
params.trt_op_types_to_exclude = "MaxPool";
std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
auto status = session_object.Load(model_name);
ASSERT_TRUE(status.IsOK());
status = session_object.Initialize();
ASSERT_TRUE(status.IsOK());
status = session_object.Run(run_options, feeds, output_names, &fetches);
ASSERT_TRUE(status.IsOK());
// RemoveCachesByType("./", ".engine");
// OrtTensorRTProviderOptionsV2 params;
// params.trt_engine_cache_enable = 1;
// params.trt_op_types_to_exclude = "MaxPool";
// std::unique_ptr<IExecutionProvider> execution_provider = TensorrtExecutionProviderWithOptions(&params);
// EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK());
// auto status = session_object.Load(model_name);
// ASSERT_TRUE(status.IsOK());
// status = session_object.Initialize();
// ASSERT_TRUE(status.IsOK());
// status = session_object.Run(run_options, feeds, output_names, &fetches);
// ASSERT_TRUE(status.IsOK());
std::vector<fs::path> engine_files;
engine_files = GetCachesByType("./", ".engine");
// The whole graph should be partitioned into 3 TRT subgraphs and 2 cpu nodes
ASSERT_EQ(engine_files.size(), 3);
}
// std::vector<fs::path> engine_files;
// engine_files = GetCachesByType("./", ".engine");
// // The whole graph should be partitioned into 3 TRT subgraphs and 2 cpu nodes
// ASSERT_EQ(engine_files.size(), 3);
// }
TEST(TensorrtExecutionProviderTest, TRTPluginsCustomOpTest) {
PathString model_name = ORT_TSTR("testdata/trt_plugin_custom_op_test.onnx");