diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h index f71b60a9f5..d008058821 100644 --- a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h +++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h @@ -80,4 +80,5 @@ struct OrtTensorRTProviderOptionsV2 { // the "trt_weight_stripped_engine_enable" option is enabled) const char* trt_engine_cache_prefix{nullptr}; // specify engine cache prefix + int trt_engine_hw_compatible{0}; // Enable hardware compatibility. Default 0 = false, nonzero = true }; diff --git a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc index 959da4944f..2171ce056e 100644 --- a/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/tensorrt/onnx_ctx_model_helper.cc @@ -367,7 +367,13 @@ bool TensorRTCacheModelHandler::ValidateEPCtxNode(const GraphViewer& graph_viewe // Show the warning if compute capability is not matched if (attrs.count(COMPUTE_CAPABILITY) > 0) { std::string model_compute_capability = attrs.at(COMPUTE_CAPABILITY).s(); - if (model_compute_capability != compute_capability_) { + // Verify if engine was compiled with ampere+ hardware compatibility enabled + if (model_compute_capability == "80+") { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] Engine is compatible to all Ampere+ GPU (except Jetson)"; + if (std::stoi(compute_capability_) < 80) { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] However, this GPU doesn't match. The compute capability of the GPU: " << compute_capability_; + } + } else if (model_compute_capability != compute_capability_) { LOGS_DEFAULT(WARNING) << "[TensorRT EP] Engine was compiled for a different compatibility level and might not work or perform suboptimal"; LOGS_DEFAULT(WARNING) << "[TensorRT EP] The compute capability of the engine: " << model_compute_capability; LOGS_DEFAULT(WARNING) << "[TensorRT EP] The compute capability of the GPU: " << compute_capability_; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 0c3886ccf6..dff74a404a 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -1301,6 +1301,7 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv profile_max_shapes = info.profile_max_shapes; profile_opt_shapes = info.profile_opt_shapes; cuda_graph_enable_ = info.cuda_graph_enable; + engine_hw_compatible_ = info.engine_hw_compatible; } else { try { const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations); @@ -1539,6 +1540,22 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv cache_path_ = GetPathOrParentPathOfCtxModel(ep_context_file_path_).append(cache_path_).string(); } + // Hardware compatibility: pre-check on environment + if (engine_cache_enable_ && engine_hw_compatible_) { +#if NV_TENSORRT_MAJOR == 8 && NV_TENSORRT_MINOR > 5 || NV_TENSORRT_MAJOR > 8 + if (std::stoi(compute_capability_) < 80) { + LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled as GPU arch < 80. "; + engine_hw_compatible_ = false; + } else if (std::stoi(compute_capability_) == 87) { + LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled on Jetson Orin. "; + engine_hw_compatible_ = false; + } +#else + LOGS_DEFAULT(WARNING) << "Engine hardware compatibility cannot be enabled as TRT < 8.6. "; + engine_hw_compatible_ = false; +#endif + } + if (engine_cache_enable_ || int8_enable_ || timing_cache_enable_) { if (!cache_path_.empty() && !fs::is_directory(cache_path_)) { if (!fs::create_directory(cache_path_)) { @@ -1666,7 +1683,8 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv << ", trt_dump_ep_context_model: " << dump_ep_context_model_ << ", trt_ep_context_file_path: " << ep_context_file_path_ << ", trt_ep_context_embed_mode: " << ep_context_embed_mode_ - << ", trt_cache_prefix: " << cache_prefix_; + << ", trt_cache_prefix: " << cache_prefix_ + << ", trt_engine_hw_compatible: " << engine_hw_compatible_; } TensorrtExecutionProvider::~TensorrtExecutionProvider() { @@ -2916,9 +2934,17 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView cache_path = GetCachePath(cache_path_, trt_node_name_with_precision); } + std::string cache_hw_compat = "_sm" + compute_capability_; + // Enable hardware compatility mode if assigned + if (engine_cache_enable_ && engine_hw_compatible_) { + trt_config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS); + cache_hw_compat = "_sm80+"; + LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Hardware compatibility is enabled when loading and capturing engine cache."; + } + // Name the engine cache based on GPU compute capacity and reduce the chance of loading an incompatible cache // Note: Engine cache generated on a GPU with large memory might not be loadable on a GPU with smaller memory, even if they share the same compute capacity - const std::string cache_path_prefix = cache_path + "_sm" + compute_capability_; + const std::string cache_path_prefix = cache_path + cache_hw_compat; std::string engine_cache_path = cache_path_prefix + ".engine"; const std::string encrypted_engine_cache_path = engine_cache_path + ".encrypted"; const std::string profile_cache_path = cache_path_prefix + ".profile"; @@ -3078,13 +3104,16 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView auto cache_file_name = std::filesystem::path(engine_cache_path).filename(); ep_cache_context_attr_ = std::filesystem::path(engine_cache_relative_path_to_context_model_dir).append(cache_file_name.string()).string(); } - + std::string compute_capability_hw_compat = compute_capability_; + if (engine_cache_enable_ && engine_hw_compatible_) { + compute_capability_hw_compat = "80+"; + } std::unique_ptr model_proto{CreateCtxModel(graph_body_viewer, ep_cache_context_attr_, reinterpret_cast(serialized_engine->data()), serialized_engine->size(), ep_context_embed_mode_, - compute_capability_, + compute_capability_hw_compat, model_path_, GetLogger())}; DumpCtxModel(model_proto.get(), ctx_model_path_); @@ -3165,12 +3194,16 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView auto cache_file_name = std::filesystem::path(engine_cache_path).filename(); ep_cache_context_attr_ = std::filesystem::path(engine_cache_relative_path_to_context_model_dir).append(cache_file_name.string()).string(); } + std::string compute_capability_hw_compat = compute_capability_; + if (engine_cache_enable_ && engine_hw_compatible_) { + compute_capability_hw_compat = "80+"; + } model_proto_.reset(CreateCtxModel(graph_body_viewer, ep_cache_context_attr_, nullptr, 0, ep_context_embed_mode_, - compute_capability_, + compute_capability_hw_compat, model_path_, GetLogger())); if (ep_context_embed_mode_ == 0) { @@ -3197,7 +3230,7 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView context_memory_sharing_enable_, &max_ctx_mem_size_, dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_, timing_cache_enable_, global_cache_path_, force_timing_cache_match_, detailed_build_log_, build_heuristics_enable_, sparsity_enable_, builder_optimization_level_, - auxiliary_streams_, !tactic_sources_.empty(), tactics, cuda_graph_enable_, cache_prefix_, cache_suffix}; + auxiliary_streams_, !tactic_sources_.empty(), tactics, cuda_graph_enable_, cache_prefix_, cache_suffix, engine_hw_compatible_}; *state = p.release(); return 0; }; @@ -3260,7 +3293,17 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView } else { cache_path = GetCachePath(trt_state->engine_cache_path, trt_state->trt_node_name_with_precision); } - const std::string cache_path_prefix = cache_path + "_sm" + compute_capability_; + + // Enable hardware compatility mode if assigned + std::string cache_hw_compat = "_sm" + compute_capability_; + if (engine_cache_enable_ && engine_hw_compatible_) { + cache_hw_compat = "_sm80+"; + LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Hardware compatibility is enabled when loading and capturing engine cache."; + } + + // Name the engine cache based on GPU compute capacity and reduce the chance of loading an incompatible cache + // Note: Engine cache generated on a GPU with large memory might not be loadable on a GPU with smaller memory, even if they share the same compute capacity + const std::string cache_path_prefix = cache_path + cache_hw_compat; std::string engine_cache_path = cache_path_prefix + ".engine"; const std::string encrypted_engine_cache_path = engine_cache_path + ".encrypted"; const std::string profile_cache_path = cache_path_prefix + ".profile"; @@ -3451,6 +3494,12 @@ Status TensorrtExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphView } } + // Enable hardware compatility mode if assigned + if (trt_state->engine_hw_compatible) { + trt_config->setHardwareCompatibilityLevel(nvinfer1::HardwareCompatibilityLevel::kAMPERE_PLUS); + LOGS_DEFAULT(INFO) << "[TensorRT EP] Re-generate engine with hardware compatibility enabled."; + } + // Build engine std::unique_ptr serialized_engine; { diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index 389cd471db..f4dae57487 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -193,6 +193,7 @@ struct TensorrtFuncState { bool cuda_graph_enable = 0; std::string cache_prefix; std::string cache_suffix; + bool engine_hw_compatible = false; }; // Minimum information to construct kernel function state for direct engine load code path @@ -320,6 +321,7 @@ class TensorrtExecutionProvider : public IExecutionProvider { bool detailed_build_log_ = false; bool cuda_graph_enable_ = false; std::string cache_prefix_; + bool engine_hw_compatible_ = false; // The OrtAllocator object will be get during ep compute time // and should be kept for the lifetime of TRT EP object. diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index 7f7587543d..9fe39f5921 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -53,6 +53,8 @@ constexpr const char* kCudaGraphEnable = "trt_cuda_graph_enable"; constexpr const char* kEpContextEmbedMode = "trt_ep_context_embed_mode"; constexpr const char* kEpContextFilePath = "trt_ep_context_file_path"; constexpr const char* kDumpEpContextModel = "trt_dump_ep_context_model"; +constexpr const char* kEngineHwCompatible = "trt_engine_hw_compatible"; + } // namespace provider_option_names } // namespace tensorrt @@ -119,6 +121,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions .AddAssignmentToReference(tensorrt::provider_option_names::kDumpEpContextModel, info.dump_ep_context_model) .AddAssignmentToReference(tensorrt::provider_option_names::kEpContextFilePath, info.ep_context_file_path) .AddAssignmentToReference(tensorrt::provider_option_names::kEpContextEmbedMode, info.ep_context_embed_mode) + .AddAssignmentToReference(tensorrt::provider_option_names::kEngineHwCompatible, info.engine_hw_compatible) .Parse(options)); // add new provider option here. info.user_compute_stream = user_compute_stream; @@ -169,6 +172,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE {tensorrt::provider_option_names::kDumpEpContextModel, MakeStringWithClassicLocale(info.dump_ep_context_model)}, {tensorrt::provider_option_names::kEpContextFilePath, MakeStringWithClassicLocale(info.ep_context_file_path)}, {tensorrt::provider_option_names::kEpContextEmbedMode, MakeStringWithClassicLocale(info.ep_context_embed_mode)}, + {tensorrt::provider_option_names::kEngineHwCompatible, MakeStringWithClassicLocale(info.engine_hw_compatible)}, }; return options; } @@ -229,6 +233,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const OrtTensor {tensorrt::provider_option_names::kEpContextFilePath, kEpContextFilePath_}, {tensorrt::provider_option_names::kDumpEpContextModel, MakeStringWithClassicLocale(info.trt_dump_ep_context_model)}, {tensorrt::provider_option_names::kEpContextEmbedMode, MakeStringWithClassicLocale(info.trt_ep_context_embed_mode)}, + {tensorrt::provider_option_names::kEngineHwCompatible, MakeStringWithClassicLocale(info.trt_engine_hw_compatible)}, }; return options; } @@ -330,5 +335,6 @@ void TensorrtExecutionProviderInfo::UpdateProviderOptions(void* provider_options trt_provider_options_v2.trt_dump_ep_context_model = internal_options.dump_ep_context_model; trt_provider_options_v2.trt_ep_context_embed_mode = internal_options.ep_context_embed_mode; trt_provider_options_v2.trt_ep_context_file_path = copy_string_if_needed(internal_options.ep_context_file_path); + trt_provider_options_v2.trt_engine_hw_compatible = internal_options.engine_hw_compatible; } } // namespace onnxruntime diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h index df9d845657..3b859ea2da 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h @@ -57,6 +57,7 @@ struct TensorrtExecutionProviderInfo { std::string ep_context_file_path{""}; int ep_context_embed_mode{0}; std::string engine_cache_prefix{""}; + bool engine_hw_compatible{false}; static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); static ProviderOptions ToProviderOptions(const TensorrtExecutionProviderInfo& info); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc index 7beeba336d..6430ffab09 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -115,6 +115,7 @@ struct Tensorrt_Provider : Provider { info.ep_context_file_path = options.trt_ep_context_file_path == nullptr ? "" : options.trt_ep_context_file_path; info.ep_context_embed_mode = options.trt_ep_context_embed_mode; info.engine_cache_prefix = options.trt_engine_cache_prefix == nullptr ? "" : options.trt_engine_cache_prefix; + info.engine_hw_compatible = options.trt_engine_hw_compatible != 0; return std::make_shared(info); } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index fdd2eadf7f..3e5ec3662f 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1720,6 +1720,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti trt_options_converted.trt_ep_context_file_path = ""; trt_options_converted.trt_ep_context_embed_mode = 0; trt_options_converted.trt_engine_cache_prefix = ""; + trt_options_converted.trt_engine_hw_compatible = 0; return trt_options_converted; } diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index f964cfbdae..7f9a6e13d7 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -793,6 +793,14 @@ std::unique_ptr CreateExecutionProviderInstance( } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_ep_context_embed_mode' should be a positive integer number i.e. '1'.\n"); } + } else if (option.first == "trt_engine_hw_compatible") { + if (option.second == "True" || option.second == "true") { + params.trt_engine_hw_compatible = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_engine_hw_compatible = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_hw_compatible' should be 'True' or 'False'. Default value is 'False'.\n"); + } } else { ORT_THROW("Invalid TensorRT EP option: ", option.first); } diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index e4b7e087c0..175079d819 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -112,6 +112,7 @@ namespace perftest { "\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n" "\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n" "\t [TensorRT only] [trt_engine_cache_prefix]: Customize engine cache prefix when trt_engine_cache_enable is true.\n" + "\t [TensorRT only] [trt_engine_hw_compatible]: Enable hardware compatibility. Engines ending with '_sm80+' can be re-used across all Ampere+ GPU (a hardware-compatible engine may have lower throughput and/or higher latency than its non-hardware-compatible counterpart).\n" "\t [TensorRT only] [trt_weight_stripped_engine_enable]: Enable weight-stripped engine build.\n" "\t [TensorRT only] [trt_onnx_model_folder_path]: Folder path for the ONNX model with weights.\n" "\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"