saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[TensorRT EP] Support engine hardware compatibility (#20669)

Browse source 
### Description
<!-- Describe your changes. -->
- Introduce option `trt_engine_hw_compatible` to support engine hardware
compatibility for Ampere+ GPUs
- This enables `nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS` flag
when generating engines
- This option has been validated on sm80/86 GPUs, as engine can be
reused across different ampere+ arch:
- Client side need to enable this option as well to leverage existing
sm80+ engines
- If this option is enabled by users which TRT<8.6 or sm<80, there will
be a warning showing this option not supported

Engine naming:
| When | `trt_engine_hw_compat=false` | `trt_engine_hw_compat=true` |
| -------------- |
------------------------------------------------------------ |
------------------------------------------------------------ |
| A100 (sm80) |
TensorrtExecutionProvider_TRTKernel_graph_torch-jit-export_9454133937466702238_0_0_sm**80**.engine
|
TensorrtExecutionProvider_TRTKernel_graph_torch-jit-export_9454133937466702238_0_0_sm**80+**.engine
|
| RTX3080 (sm86) |
TensorrtExecutionProvider_TRTKernel_graph_torch-jit-export_9454133937466702238_0_0_sm**86**.engine
|
TensorrtExecutionProvider_TRTKernel_graph_torch-jit-export_9454133937466702238_0_0_sm**80+**.engine
|


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
Reference:
https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#hardware-compat

---------

Co-authored-by: Chi Lo <54722500+chilo-ms@users.noreply.github.com>
This commit is contained in:
Yifan Li 2024-05-28 18:12:56 -07:00 committed by GitHub
parent 535e9d7114
commit d44be41e1c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 84 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h
View file

@ -80,4 +80,5 @@ struct OrtTensorRTProviderOptionsV2 {
// the "trt_weight_stripped_engine_enable" option is enabled)
const char* trt_engine_cache_prefix{nullptr}; // specify engine cache prefix
int trt_engine_hw_compatible{0}; // Enable hardware compatibility. Default 0 = false, nonzero = true
};

8
onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc
View file

@ -367,7 +367,13 @@ bool TensorRTCacheModelHandler::ValidateEPCtxNode(const GraphViewer& graph_viewe
// Show the warning if compute capability is not matched
if (attrs.count(COMPUTE_CAPABILITY) > 0) {
std::string model_compute_capability = attrs.at(COMPUTE_CAPABILITY).s();
if (model_compute_capability != compute_capability_) {
// Verify if engine was compiled with ampere+ hardware compatibility enabled
if (model_compute_capability == "80+") {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Engine is compatible to all Ampere+ GPU (except Jetson)";
if (std::stoi(compute_capability_) < 80) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] However, this GPU doesn't match. The compute capability of the GPU: " << compute_capability_;
}
} else if (model_compute_capability != compute_capability_) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Engine was compiled for a different compatibility level and might not work or perform suboptimal";
LOGS_DEFAULT(WARNING) << "[TensorRT EP] The compute capability of the engine: " << model_compute_capability;
LOGS_DEFAULT(WARNING) << "[TensorRT EP] The compute capability of the GPU: " << compute_capability_;

63
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -1301,6 +1301,7 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
profile_max_shapes = info.profile_max_shapes;
profile_opt_shapes = info.profile_opt_shapes;
cuda_graph_enable_ = info.cuda_graph_enable;
engine_hw_compatible_ = info.engine_hw_compatible;
} else {
try {
const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations);
@ -1539,6 +1540,22 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
cache_path_ = GetPathOrParentPathOfCtxModel(ep_context_file_path_).append(cache_path_).string();
}
// Hardware compatibility: pre-check on environment
if (engine_cache_enable_ && engine_hw_compatible_) {
#if NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR > 5 || NV_TENSORRT_MAJOR > 8
if (std::stoi(compute_capability_) < 80) {
LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled as GPU arch < 80. ";
engine_hw_compatible_ = false;
} else if (std::stoi(compute_capability_) == 87) {
LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled on Jetson Orin. ";
engine_hw_compatible_ = false;
}
#else
LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled as TRT < 8.6. ";
engine_hw_compatible_ = false;
#endif
}
if (engine_cache_enable_ || int8_enable_ || timing_cache_enable_) {
if (!cache_path_.empty() && !fs::is_directory(cache_path_)) {
if (!fs::create_directory(cache_path_)) {
@ -1666,7 +1683,8 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
<< ", trt_dump_ep_context_model: " << dump_ep_context_model_
<< ", trt_ep_context_file_path: " << ep_context_file_path_
<< ", trt_ep_context_embed_mode: " << ep_context_embed_mode_
<< ", trt_cache_prefix: " << cache_prefix_;
<< ", trt_cache_prefix: " << cache_prefix_
<< ", trt_engine_hw_compatible: " << engine_hw_compatible_;
}
TensorrtExecutionProvider::~TensorrtExecutionProvider() {
@ -2916,9 +2934,17 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
cache_path = GetCachePath(cache_path_, trt_node_name_with_precision);
}
std::string cache_hw_compat = "_sm" + compute_capability_;
// Enable hardware compatility mode if assigned
if (engine_cache_enable_ && engine_hw_compatible_) {
trt_config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS);
cache_hw_compat = "_sm80+";
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Hardware compatibility is enabled when loading and capturing engine cache.";
}
// Name the engine cache based on GPU compute capacity and reduce the chance of loading an incompatible cache
// Note: Engine cache generated on a GPU with large memory might not be loadable on a GPU with smaller memory, even if they share the same compute capacity
const std::string cache_path_prefix = cache_path + "_sm" + compute_capability_;
const std::string cache_path_prefix = cache_path + cache_hw_compat;
std::string engine_cache_path = cache_path_prefix + ".engine";
const std::string encrypted_engine_cache_path = engine_cache_path + ".encrypted";
const std::string profile_cache_path = cache_path_prefix + ".profile";
@ -3078,13 +3104,16 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
auto cache_file_name = std::filesystem::path(engine_cache_path).filename();
ep_cache_context_attr_ = std::filesystem::path(engine_cache_relative_path_to_context_model_dir).append(cache_file_name.string()).string();
}
std::string compute_capability_hw_compat = compute_capability_;
if (engine_cache_enable_ && engine_hw_compatible_) {
compute_capability_hw_compat = "80+";
}
std::unique_ptr<ONNX_NAMESPACE::ModelProto> model_proto{CreateCtxModel(graph_body_viewer,
ep_cache_context_attr_,
reinterpret_cast<char*>(serialized_engine->data()),
serialized_engine->size(),
ep_context_embed_mode_,
compute_capability_,
compute_capability_hw_compat,
model_path_,
GetLogger())};
DumpCtxModel(model_proto.get(), ctx_model_path_);
@ -3165,12 +3194,16 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
auto cache_file_name = std::filesystem::path(engine_cache_path).filename();
ep_cache_context_attr_ = std::filesystem::path(engine_cache_relative_path_to_context_model_dir).append(cache_file_name.string()).string();
}
std::string compute_capability_hw_compat = compute_capability_;
if (engine_cache_enable_ && engine_hw_compatible_) {
compute_capability_hw_compat = "80+";
}
model_proto_.reset(CreateCtxModel(graph_body_viewer,
ep_cache_context_attr_,
nullptr,
0,
ep_context_embed_mode_,
compute_capability_,
compute_capability_hw_compat,
model_path_,
GetLogger()));
if (ep_context_embed_mode_ == 0) {
@ -3197,7 +3230,7 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
context_memory_sharing_enable_, &max_ctx_mem_size_, dynamic_range_map, engine_decryption_enable_,
engine_decryption_, engine_encryption_, timing_cache_enable_, global_cache_path_, force_timing_cache_match_,
detailed_build_log_, build_heuristics_enable_, sparsity_enable_, builder_optimization_level_,
auxiliary_streams_, !tactic_sources_.empty(), tactics, cuda_graph_enable_, cache_prefix_, cache_suffix};
auxiliary_streams_, !tactic_sources_.empty(), tactics, cuda_graph_enable_, cache_prefix_, cache_suffix, engine_hw_compatible_};
*state = p.release();
return 0;
};
@ -3260,7 +3293,17 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
} else {
cache_path = GetCachePath(trt_state->engine_cache_path, trt_state->trt_node_name_with_precision);
}
const std::string cache_path_prefix = cache_path + "_sm" + compute_capability_;
// Enable hardware compatility mode if assigned
std::string cache_hw_compat = "_sm" + compute_capability_;
if (engine_cache_enable_ && engine_hw_compatible_) {
cache_hw_compat = "_sm80+";
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Hardware compatibility is enabled when loading and capturing engine cache.";
}
// Name the engine cache based on GPU compute capacity and reduce the chance of loading an incompatible cache
// Note: Engine cache generated on a GPU with large memory might not be loadable on a GPU with smaller memory, even if they share the same compute capacity
const std::string cache_path_prefix = cache_path + cache_hw_compat;
std::string engine_cache_path = cache_path_prefix + ".engine";
const std::string encrypted_engine_cache_path = engine_cache_path + ".encrypted";
const std::string profile_cache_path = cache_path_prefix + ".profile";
@ -3451,6 +3494,12 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView
}
}
// Enable hardware compatility mode if assigned
if (trt_state->engine_hw_compatible) {
trt_config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS);
LOGS_DEFAULT(INFO) << "[TensorRT EP] Re-generate engine with hardware compatibility enabled.";
}
// Build engine
std::unique_ptr<nvinfer1::IHostMemory> serialized_engine;
{

2
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -193,6 +193,7 @@ struct TensorrtFuncState {
bool cuda_graph_enable = 0;
std::string cache_prefix;
std::string cache_suffix;
bool engine_hw_compatible = false;
};
// Minimum information to construct kernel function state for direct engine load code path
@ -320,6 +321,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
bool detailed_build_log_ = false;
bool cuda_graph_enable_ = false;
std::string cache_prefix_;
bool engine_hw_compatible_ = false;
// The OrtAllocator object will be get during ep compute time
// and should be kept for the lifetime of TRT EP object.

6
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc
View file

@ -53,6 +53,8 @@ constexpr const char* kCudaGraphEnable = "trt_cuda_graph_enable";
constexpr const char* kEpContextEmbedMode = "trt_ep_context_embed_mode";
constexpr const char* kEpContextFilePath = "trt_ep_context_file_path";
constexpr const char* kDumpEpContextModel = "trt_dump_ep_context_model";
constexpr const char* kEngineHwCompatible = "trt_engine_hw_compatible";
} // namespace provider_option_names
} // namespace tensorrt
@ -119,6 +121,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions
.AddAssignmentToReference(tensorrt::provider_option_names::kDumpEpContextModel, info.dump_ep_context_model)
.AddAssignmentToReference(tensorrt::provider_option_names::kEpContextFilePath, info.ep_context_file_path)
.AddAssignmentToReference(tensorrt::provider_option_names::kEpContextEmbedMode, info.ep_context_embed_mode)
.AddAssignmentToReference(tensorrt::provider_option_names::kEngineHwCompatible, info.engine_hw_compatible)
.Parse(options)); // add new provider option here.
info.user_compute_stream = user_compute_stream;
@ -169,6 +172,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE
{tensorrt::provider_option_names::kDumpEpContextModel, MakeStringWithClassicLocale(info.dump_ep_context_model)},
{tensorrt::provider_option_names::kEpContextFilePath, MakeStringWithClassicLocale(info.ep_context_file_path)},
{tensorrt::provider_option_names::kEpContextEmbedMode, MakeStringWithClassicLocale(info.ep_context_embed_mode)},
{tensorrt::provider_option_names::kEngineHwCompatible, MakeStringWithClassicLocale(info.engine_hw_compatible)},
};
return options;
}
@ -229,6 +233,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const OrtTensor
{tensorrt::provider_option_names::kEpContextFilePath, kEpContextFilePath_},
{tensorrt::provider_option_names::kDumpEpContextModel, MakeStringWithClassicLocale(info.trt_dump_ep_context_model)},
{tensorrt::provider_option_names::kEpContextEmbedMode, MakeStringWithClassicLocale(info.trt_ep_context_embed_mode)},
{tensorrt::provider_option_names::kEngineHwCompatible, MakeStringWithClassicLocale(info.trt_engine_hw_compatible)},
};
return options;
}
@ -330,5 +335,6 @@ void TensorrtExecutionProviderInfo::UpdateProviderOptions(void* provider_options
trt_provider_options_v2.trt_dump_ep_context_model = internal_options.dump_ep_context_model;
trt_provider_options_v2.trt_ep_context_embed_mode = internal_options.ep_context_embed_mode;
trt_provider_options_v2.trt_ep_context_file_path = copy_string_if_needed(internal_options.ep_context_file_path);
trt_provider_options_v2.trt_engine_hw_compatible = internal_options.engine_hw_compatible;
}
} // namespace onnxruntime

1
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h
View file

@ -57,6 +57,7 @@ struct TensorrtExecutionProviderInfo {
std::string ep_context_file_path{""};
int ep_context_embed_mode{0};
std::string engine_cache_prefix{""};
bool engine_hw_compatible{false};
static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);
static ProviderOptions ToProviderOptions(const TensorrtExecutionProviderInfo& info);

1
onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
View file

@ -115,6 +115,7 @@ struct Tensorrt_Provider : Provider {
info.ep_context_file_path = options.trt_ep_context_file_path == nullptr ? "" : options.trt_ep_context_file_path;
info.ep_context_embed_mode = options.trt_ep_context_embed_mode;
info.engine_cache_prefix = options.trt_engine_cache_prefix == nullptr ? "" : options.trt_engine_cache_prefix;
info.engine_hw_compatible = options.trt_engine_hw_compatible != 0;
return std::make_shared<TensorrtProviderFactory>(info);
}

1
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -1720,6 +1720,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti
trt_options_converted.trt_ep_context_file_path = "";
trt_options_converted.trt_ep_context_embed_mode = 0;
trt_options_converted.trt_engine_cache_prefix = "";
trt_options_converted.trt_engine_hw_compatible = 0;
return trt_options_converted;
}

8
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -793,6 +793,14 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_ep_context_embed_mode' should be a positive integer number i.e. '1'.\n");
}
} else if (option.first == "trt_engine_hw_compatible") {
if (option.second == "True" || option.second == "true") {
params.trt_engine_hw_compatible = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_engine_hw_compatible = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_hw_compatible' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else {
ORT_THROW("Invalid TensorRT EP option: ", option.first);
}

1
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -112,6 +112,7 @@ namespace perftest {
"\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n"
"\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n"
"\t [TensorRT only] [trt_engine_cache_prefix]: Customize engine cache prefix when trt_engine_cache_enable is true.\n"
"\t [TensorRT only] [trt_engine_hw_compatible]: Enable hardware compatibility. Engines ending with '_sm80+' can be re-used across all Ampere+ GPU (a hardware-compatible engine may have lower throughput and/or higher latency than its non-hardware-compatible counterpart).\n"
"\t [TensorRT only] [trt_weight_stripped_engine_enable]: Enable weight-stripped engine build.\n"
"\t [TensorRT only] [trt_onnx_model_folder_path]: Folder path for the ONNX model with weights.\n"
"\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"