saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-25 19:48:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update

Browse source
This commit is contained in:
Chi Lo 2025-01-16 23:52:10 +00:00
parent f3d58e8158
commit 74f81f0542
2 changed files with 36 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -1723,8 +1723,16 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
{
auto lock = GetApiLock();
runtime_ = std::unique_ptr<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger(detailed_build_log_)));
trt_gpu_allocator_ = std::unique_ptr<PoolAllocator>();
runtime_->setGpuAllocator(trt_gpu_allocator_.get());
#if NV_TENSORRT_MAJOR >= 10
// There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) when running TRT EP with TRT 10.
// The reason is when cudaStreamSynchronize being called after inference, the gpu memory will be released back to OS and for the next inference
// run, TRT will allocate gpu memory from OS again. This introduces overheads and end up having performance degradation.
// The fix is to increase mem pool threshold so TRT can hold the allocated memory to prevent overhead.
if (is_dds_op_in_graph_) {
trt_gpu_allocator_ = std::make_unique<onnxruntime::PoolAllocator>();
runtime_->setGpuAllocator(trt_gpu_allocator_.get());
}
#endif
}
trt_version_ = getInferLibVersion();
@ -2325,6 +2333,12 @@ SubGraphCollection_t TensorrtExecutionProvider::GetSupportedList(SubGraphCollect
* TODO: Remove the subgraph_node_index
*/
next_nodes_list[i].first[j] = group.first[subgraph_node_index[next_nodes_list[i].first[j]]];
// Check if it's DDS op. TRT EP will have corresponding action later to prevent performance degradation if the graph has DDS op that run by TRT 10.
if (!is_dds_op_in_graph_) {
const auto& node = graph.GetNode(next_nodes_list[i].first[j]);
if (dds_op_set_.find(node->OpType()) != dds_op_set_.end()) is_dds_op_in_graph_ = true;
}
}
nodes_list_output.push_back(next_nodes_list[i]);
}
@ -2483,17 +2497,16 @@ TensorrtExecutionProvider::GetCapability(const GraphViewer& graph,
std::vector<size_t> nodes_vector(number_of_ort_nodes);
std::iota(std::begin(nodes_vector), std::end(nodes_vector), 0);
std::set<std::string> exclude_ops_set;
std::set<std::string> exclude_ops_set; // currently not support to exclude ops
/*
* There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) in TRT 10.
* TRT EP automatically excludes DDS ops from running on TRT.
* There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) when running TRT EP with TRT 10.
* DDS op needs special handling here.
*/
if (trt_version_ >= 100000 && trt_version_ < 110000) {
exclude_ops_set.insert("NonMaxSuppression");
exclude_ops_set.insert("NonZero");
exclude_ops_set.insert("RoiAlign");
LOGS_DEFAULT(VERBOSE) << "There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) in TRT 10. TRT EP automatically excludes DDS ops from running on TRT, if applicable";
if (trt_version_ >= 100000) {
dds_op_set_.insert("NonMaxSuppression");
dds_op_set_.insert("NonZero");
dds_op_set_.insert("RoiAlign");
}
SubGraphCollection_t parser_nodes_vector, supported_nodes_vector;

13
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -114,6 +114,11 @@ template <typename T>
using unique_pointer = std::unique_ptr<T, TensorrtInferDeleter>;
}; // namespace tensorrt_ptr
#if NV_TENSORRT_MAJOR >= 10
/*
* Application-implemented class for controlling asynchronous (stream ordered) memory allocation on the GPU.
*
*/
class PoolAllocator : public nvinfer1::IGpuAsyncAllocator {
public:
PoolAllocator() {
@ -124,6 +129,10 @@ class PoolAllocator : public nvinfer1::IGpuAsyncAllocator {
poolProps.location.id = 0;
cudaMemPoolCreate(&mPool, &poolProps);
auto maxThreshold = std::numeric_limits<std::uint64_t>::max();
// cudaMemPoolAttrReleaseThreshold:
// Amount of reserved memory in bytes to hold onto before trying to release memory back to the OS.
// When more than the release threshold bytes of memory are held by the memory pool, the allocator
// will try to release memory back to the OS on the next call to stream, event or context synchronize
cudaMemPoolSetAttribute(mPool, cudaMemPoolAttrReleaseThreshold, &maxThreshold);
}
@ -147,6 +156,7 @@ class PoolAllocator : public nvinfer1::IGpuAsyncAllocator {
private:
cudaMemPool_t mPool{nullptr};
};
#endif
//
// Class to allocate memory for outputs with data-dependent shapes. The sizes of those are unknown so pre-allocation is
@ -386,6 +396,9 @@ class TensorrtExecutionProvider : public IExecutionProvider {
std::unordered_set<std::string> control_flow_op_set_ = {"If", "Loop", "Scan"};
mutable std::unordered_map<std::string, std::unique_ptr<SubGraphContext>> subgraph_context_map_;
mutable std::unordered_set<std::string> dds_op_set_;
mutable bool is_dds_op_in_graph_ = false;
mutable std::unique_ptr<nvinfer1::IBuilder> builder_;
// Following maps that hold TRT objects will be accessible by different threads if ORT is using multithreading.