saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Consolidate TensorRT subgraphs to reduce inference overhead (#11211)

Browse source 
* add trt node list consolidation

* add more log

* fix typo

* seperate cycle detection and removal

* update

* change function name

Co-authored-by: Ubuntu <azureuser@orttrtlinuxdev.bxgbzpva45kedp3rhbsbit4phb.jx.internal.cloudapp.net>
This commit is contained in:
stevenlix 2022-04-14 11:05:27 -07:00 committed by GitHub
parent a00d24066a
commit 5216a43c9d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -837,9 +837,9 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
}
// Detect and remove cycles from supported node list
void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const GraphViewer& graph) const {
bool TensorrtExecutionProvider::DetectTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const GraphViewer& graph, bool remove_cycles) const {
const std::vector<NodeIndex>& node_index = graph.GetNodesInTopologicalOrder();
bool trt_cycle = true;
bool trt_cycle = true, cycle_detected = false;
while (trt_cycle) {
trt_cycle = false;
std::unordered_map<std::string, size_t> node_to_index_map;
@ -923,12 +923,13 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
for (size_t i = 0; i < graph_size; ++i) {
if (FindCycleHelper(i, adjacency_map, visited, st, cycles)) {
has_cycle = true;
cycle_detected = true;
break;
}
}
// Remove TensorRT subgraph from the supported node list if it's part of the cycle
if (has_cycle) {
if (has_cycle && remove_cycles) {
for (size_t i = 0; i < cycles.size(); ++i) {
auto loc = index_to_node_map.find(cycles[i]);
if (loc != index_to_node_map.end() && loc->second.find("TRTKernel") != std::string::npos) {
@ -943,6 +944,7 @@ void TensorrtExecutionProvider::RemoveTensorRTGraphCycles(SubGraphCollection_t&
delete[] visited;
delete[] st;
}
return cycle_detected;
}
std::vector<std::unique_ptr<ComputeCapability>>
@ -976,7 +978,24 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
}
// Detect and remove cycles from supported node list
RemoveTensorRTGraphCycles(supported_nodes_vector, graph);
DetectTensorRTGraphCycles(supported_nodes_vector, graph);
// Consolidate supported node list
if (supported_nodes_vector.size() > 1) {
nodes_vector.clear();
for (const auto& group : supported_nodes_vector) {
if (!group.first.empty()) {
nodes_vector.insert(nodes_vector.end(), group.first.begin(), group.first.end());
}
}
SubGraphCollection_t consolidated_supported_nodes_vector = {{nodes_vector, true}};
if (DetectTensorRTGraphCycles(consolidated_supported_nodes_vector, graph, false)) {
LOGS_DEFAULT(INFO) << "[TensorRT EP] TensorRT nodes are not consolidated because graph will have cycles after consolidation";
} else {
LOGS_DEFAULT(INFO) << "[TensorRT EP] TensorRT nodes are consolidated into one subgraph";
supported_nodes_vector = consolidated_supported_nodes_vector;
}
}
// Construct subgraph capability from node list
std::vector<std::unique_ptr<ComputeCapability>> result;

2
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -191,7 +191,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
SubGraphCollection_t GetSupportedList(SubGraphCollection_t supported_nodes_list, int iterations, const int max_iterations,
const GraphViewer& graph, bool* early_termination) const;
void RemoveTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const GraphViewer& graph) const;
bool DetectTensorRTGraphCycles(SubGraphCollection_t& supported_nodes_vector, const GraphViewer& graph, bool remove_cycles = true) const;
/**
Get a unique_lock object to control the concurrency behavior.