From 143551092fea84eac84a25aa7e5740a292df90b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20M=C3=BCller?= <44298237+gedoensmax@users.noreply.github.com> Date: Fri, 12 May 2023 22:29:30 +0200 Subject: [PATCH] fix: setting builder optimization level to TRT 8.6 default (#15897) The actual released default level is 3 and not the previously used 2. Just a small sample of the effects: ![Screenshot 2023-05-10 at 15 49 55](https://github.com/microsoft/onnxruntime/assets/44298237/5a694446-22c0-4943-9ddf-80670781878f) --- .../core/providers/tensorrt/tensorrt_provider_options.h | 2 +- .../providers/tensorrt/tensorrt_execution_provider.cc | 8 ++++---- .../core/providers/tensorrt/tensorrt_execution_provider.h | 4 ++-- .../providers/tensorrt/tensorrt_execution_provider_info.h | 4 +++- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 2 +- onnxruntime/test/providers/cpu/model_tests.cc | 2 +- .../test/providers/tensorrt/tensorrt_basic_test.cc | 8 ++++---- 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h index 3bd6dac54b..600e255bcd 100644 --- a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h +++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h @@ -36,7 +36,7 @@ struct OrtTensorRTProviderOptionsV2 { int trt_detailed_build_log; // Enable detailed build step logging on TensorRT EP with timing for each engine build. Default 0 = false, nonzero = true int trt_build_heuristics_enable; // Build engine using heuristics to reduce build time. Default 0 = false, nonzero = true int trt_sparsity_enable; // Control if sparsity can be used by TRT. Default 0 = false, 1 = true - int trt_builder_optimization_level; // Set the builder optimization level. WARNING: levels below 2 do not guarantee good engine performance, but greatly improve build time. Default 2, valid range [0-4] + int trt_builder_optimization_level; // Set the builder optimization level. WARNING: levels below 3 do not guarantee good engine performance, but greatly improve build time. Default 3, valid range [0-5] int trt_auxiliary_streams; // Set maximum number of auxiliary streams per inference stream. Setting this value to 0 will lead to optimal memory usage. Default -1 = heuristics const char* trt_tactic_sources; // pecify the tactics to be used by adding (+) or removing (-) tactics from the default // tactic sources (default = all available tactics) e.g. "-CUDNN,+CUBLAS" available keys: "CUBLAS"|"CUBLAS_LT"|"CUDNN"|"EDGE_MASK_CONVOLUTIONS" diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 5526ee1151..5b186a037e 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -1960,7 +1960,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector 5 && NV_TENSORRT_MAJOR >= 8 // switch optimizaion level - if (builder_optimization_level_ != 2) { + if (builder_optimization_level_ != 3) { trt_config->setBuilderOptimizationLevel(builder_optimization_level_); LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder optimization level is set to " << builder_optimization_level_; } @@ -1971,7 +1971,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector= 0) { @@ -2378,7 +2378,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector 5 && NV_TENSORRT_MAJOR >= 8 // switch optimizaion level - if (trt_state->builder_optimization_level != 2) { + if (trt_state->builder_optimization_level != 3) { trt_config->setBuilderOptimizationLevel(trt_state->builder_optimization_level); LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder optimization level is set to " << builder_optimization_level_; } @@ -2389,7 +2389,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorauxiliary_streams; } #else - if (trt_state->builder_optimization_level != 2) { + if (trt_state->builder_optimization_level != 3) { LOGS_DEFAULT(WARNING) << "[TensorRT EP] Builder optimization level can only be used on TRT 8.6 onwards!"; } if (trt_state->auxiliary_streams >= 0) { diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index e4d5261bf2..a5f94c1c23 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -131,7 +131,7 @@ struct TensorrtFuncState { bool detailed_build_log = false; bool build_heuristics_enable = false; bool sparsity_enable = false; - int builder_optimization_level = 2; + int builder_optimization_level = 3; int auxiliary_streams = -1; bool filter_tactic_sources = false; nvinfer1::TacticSources tactic_sources; @@ -190,7 +190,7 @@ class TensorrtExecutionProvider : public IExecutionProvider { bool engine_cache_enable_ = false; bool build_heuristics_enable_ = false; bool sparsity_enable_ = false; - int builder_optimization_level_ = 2; + int builder_optimization_level_ = 3; int auxiliary_streams_ = -1; std::string tactic_sources_; std::string cache_path_, engine_decryption_lib_path_; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h index f77717b976..7235bb6940 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h @@ -11,6 +11,8 @@ #include "core/session/onnxruntime_c_api.h" #include "core/framework/library_handles.h" +#define TRT_DEFAULT_OPTIMIZER_LEVEL 3 + namespace onnxruntime { // Information needed to construct trt execution providers. struct TensorrtExecutionProviderInfo { @@ -40,7 +42,7 @@ struct TensorrtExecutionProviderInfo { bool detailed_build_log{false}; bool build_heuristics_enable{false}; bool sparsity_enable{false}; - int builder_optimization_level{2}; + int builder_optimization_level{3}; int auxiliary_streams{-1}; std::string tactic_sources{""}; std::string extra_plugin_lib_paths{""}; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 9a4fe6900c..7d92cc3f7e 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1305,7 +1305,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti trt_options_converted.trt_layer_norm_fp32_fallback = 0; trt_options_converted.trt_build_heuristics_enable = 0; trt_options_converted.trt_sparsity_enable = 0; - trt_options_converted.trt_builder_optimization_level = 2; + trt_options_converted.trt_builder_optimization_level = 3; trt_options_converted.trt_auxiliary_streams = -1; trt_options_converted.trt_tactic_sources = ""; trt_options_converted.trt_extra_plugin_lib_paths = ""; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d3a547d244..ae3f4ca05a 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -126,7 +126,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device bool trt_detailed_build_log = false; bool trt_build_heuristics_enable = false; bool trt_sparsity_enable = false; - int trt_builder_optimization_level = 2; + int trt_builder_optimization_level = 3; int trt_auxiliary_streams = -1; std::string trt_tactic_sources = ""; std::string trt_extra_plugin_lib_paths = ""; diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 0d29dac792..46cae17c6b 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -702,7 +702,7 @@ TEST_P(ModelTest, Run) { OrtTensorRTProviderOptionsV2 params{0, 0, nullptr, 1000, 1, 1 << 30, 1, // enable fp16 0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0, 0, 0, 0, 0, 0, 0, 0, - 2, -1, nullptr, nullptr, nullptr, nullptr, nullptr}; + 3, -1, nullptr, nullptr, nullptr, nullptr, nullptr}; ortso.AppendExecutionProvider_TensorRT_V2(params); } else { diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index b90767b9ae..81cb844f68 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -159,7 +159,7 @@ void RunWithOneSessionSingleThreadInference(std::string model_name, std::string 0, 0, 0, - 2, + 3, -1, nullptr, nullptr, @@ -242,7 +242,7 @@ void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string 0, 0, 0, - 2, + 3, -1, nullptr, nullptr, @@ -396,7 +396,7 @@ TEST(TensorrtExecutionProviderTest, TRTPluginsCustomOpTest) { 0, 0, 0, - 2, + 3, -1, nullptr, nullptr, @@ -491,7 +491,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) { 0, 0, 0, - 2, + 3, -1, nullptr, nullptr,