saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

lintrunner -a

Browse source
This commit is contained in:
Chi Lo 2025-01-22 23:17:43 +00:00
parent 8785c3cce7
commit 08e22bf099
3 changed files with 9 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -2481,7 +2481,7 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
std::vector<size_t> nodes_vector(number_of_ort_nodes);
std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0);
std::set<std::string> exclude_ops_set; // currently not support to exclude ops
std::set<std::string> exclude_ops_set; // currently not support to exclude ops
/*
* There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) when running TRT EP with TRT 10.
@ -2668,10 +2668,10 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
}
}
#if NV_TENSORRT_MAJOR >= 10
#if NV_TENSORRT_MAJOR >= 10
// TRT EP will take appropriate actions later to prevent performance degradation if the graph has DDS op that run by TRT 10.
is_dds_op_in_graph_ = IsDDSOpInSubGraph(graph, result, dds_op_set_);
#endif
#endif
const size_t number_of_subgraphs = supported_nodes_vector.size();
if (number_of_trt_nodes == 0) {
@ -2773,8 +2773,7 @@ common::Status TensorrtExecutionProvider::RefitEngine(std::string onnx_model_fil
common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,
std::vector<NodeComputeInfo>& node_compute_funcs) {
#if NV_TENSORRT_MAJOR >= 10
#if NV_TENSORRT_MAJOR >= 10
// There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) when running TRT EP with TRT 10.
// The issue arises because when cudaStreamSynchronize is called after inference, GPU memory is released back to the OS.
// As a result, for the next inference run, TRT reallocates GPU memory from the OS, introducing overhead and leading to performance degradation.
@ -2784,7 +2783,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
runtime_->setGpuAllocator(trt_gpu_allocator_.get());
}
#endif
for (auto& fused_node_graph : fused_nodes_and_graphs) {
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
const Node& fused_node = fused_node_graph.fused_node;

2
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -641,7 +641,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
/**
* Check if DDS op is in the ComputeCapability/subgraph.
*/
*/
bool IsDDSOpInSubGraph(const GraphViewer& graph,
std::vector<std::unique_ptr<ComputeCapability>>& compute_capabilities,
std::unordered_set<std::string>& dds_op_set) const;

6
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_helper.cc
View file

@ -261,13 +261,13 @@ void TensorrtExecutionProvider::SetAllGraphInputs(Graph& graph) const {
// Check if DDS op is in the ComputeCapability/subgraph.
bool TensorrtExecutionProvider::IsDDSOpInSubGraph(const GraphViewer& graph,
std::vector<std::unique_ptr<ComputeCapability>>& compute_capabilities,
std::unordered_set<std::string>& dds_op_set) const {
std::vector<std::unique_ptr<ComputeCapability>>& compute_capabilities,
std::unordered_set<std::string>& dds_op_set) const {
auto is_dds_op = [&](const auto& node) {
if (dds_op_set.find(node->OpType()) != dds_op_set.end()) return true;
return false;
};
for (auto& compute_capability : compute_capabilities) {
auto& indexed_sub_graph = compute_capability->SubGraph();
for (auto i : indexed_sub_graph->Nodes()) {