fix: setting builder optimization level to TRT 8.6 default (#15897)

The actual released default level is 3 and not the previously used 2.

Just a small sample of the effects:
![Screenshot 2023-05-10 at 15 49
55](https://github.com/microsoft/onnxruntime/assets/44298237/5a694446-22c0-4943-9ddf-80670781878f)

include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h

@@ -36,7 +36,7 @@ struct OrtTensorRTProviderOptionsV2 {
   int trt_detailed_build_log;                   // Enable detailed build step logging on TensorRT EP with timing for each engine build. Default 0 = false, nonzero = true
   int trt_build_heuristics_enable;              // Build engine using heuristics to reduce build time. Default 0 = false, nonzero = true
   int trt_sparsity_enable;                      // Control if sparsity can be used by TRT. Default 0 = false, 1 = true
-  int trt_builder_optimization_level;           // Set the builder optimization level. WARNING: levels below 2 do not guarantee good engine performance, but greatly improve build time.  Default 2, valid range [0-4]
+  int trt_builder_optimization_level;           // Set the builder optimization level. WARNING: levels below 3 do not guarantee good engine performance, but greatly improve build time.  Default 3, valid range [0-5]
   int trt_auxiliary_streams;                    // Set maximum number of auxiliary streams per inference stream. Setting this value to 0 will lead to optimal memory usage. Default -1 = heuristics
   const char* trt_tactic_sources;               // pecify the tactics to be used by adding (+) or removing (-) tactics from the default
                                                 // tactic sources (default = all available tactics) e.g. "-CUDNN,+CUBLAS" available keys: "CUBLAS"|"CUBLAS_LT"|"CUDNN"|"EDGE_MASK_CONVOLUTIONS"

onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

@@ -1960,7 +1960,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
     }
 #if NV_TENSORRT_MINOR > 5 && NV_TENSORRT_MAJOR >= 8
     // switch optimizaion level
-    if (builder_optimization_level_ != 2) {
+    if (builder_optimization_level_ != 3) {
       trt_config->setBuilderOptimizationLevel(builder_optimization_level_);
       LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder optimization level is set to " << builder_optimization_level_;
     }
@@ -1971,7 +1971,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
       LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Auxiliary streams are se to " << auxiliary_streams_;
     }
 #else
-    if (builder_optimization_level_ != 2) {
+    if (builder_optimization_level_ != 3) {
       LOGS_DEFAULT(WARNING) << "[TensorRT EP] Builder optimization level can only be used on TRT 8.6 onwards!";
     }
     if (auxiliary_streams_ >= 0) {
@@ -2378,7 +2378,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
         }
 #if NV_TENSORRT_MINOR > 5 && NV_TENSORRT_MAJOR >= 8
         // switch optimizaion level
-        if (trt_state->builder_optimization_level != 2) {
+        if (trt_state->builder_optimization_level != 3) {
           trt_config->setBuilderOptimizationLevel(trt_state->builder_optimization_level);
           LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Builder optimization level is set to " << builder_optimization_level_;
         }
@@ -2389,7 +2389,7 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
           LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] Auxiliary streams are se to " << trt_state->auxiliary_streams;
         }
 #else
-        if (trt_state->builder_optimization_level != 2) {
+        if (trt_state->builder_optimization_level != 3) {
           LOGS_DEFAULT(WARNING) << "[TensorRT EP] Builder optimization level can only be used on TRT 8.6 onwards!";
         }
         if (trt_state->auxiliary_streams >= 0) {

onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h

@@ -131,7 +131,7 @@ struct TensorrtFuncState {
   bool detailed_build_log = false;
   bool build_heuristics_enable = false;
   bool sparsity_enable = false;
-  int builder_optimization_level = 2;
+  int builder_optimization_level = 3;
   int auxiliary_streams = -1;
   bool filter_tactic_sources = false;
   nvinfer1::TacticSources tactic_sources;
@@ -190,7 +190,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
   bool engine_cache_enable_ = false;
   bool build_heuristics_enable_ = false;
   bool sparsity_enable_ = false;
-  int builder_optimization_level_ = 2;
+  int builder_optimization_level_ = 3;
   int auxiliary_streams_ = -1;
   std::string tactic_sources_;
   std::string cache_path_, engine_decryption_lib_path_;

onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h

@@ -11,6 +11,8 @@
 #include "core/session/onnxruntime_c_api.h"
 #include "core/framework/library_handles.h"
 
+#define TRT_DEFAULT_OPTIMIZER_LEVEL 3
+
 namespace onnxruntime {
 // Information needed to construct trt execution providers.
 struct TensorrtExecutionProviderInfo {
@@ -40,7 +42,7 @@ struct TensorrtExecutionProviderInfo {
   bool detailed_build_log{false};
   bool build_heuristics_enable{false};
   bool sparsity_enable{false};
-  int builder_optimization_level{2};
+  int builder_optimization_level{3};
   int auxiliary_streams{-1};
   std::string tactic_sources{""};
   std::string extra_plugin_lib_paths{""};

onnxruntime/core/session/provider_bridge_ort.cc

@@ -1305,7 +1305,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti
   trt_options_converted.trt_layer_norm_fp32_fallback = 0;
   trt_options_converted.trt_build_heuristics_enable = 0;
   trt_options_converted.trt_sparsity_enable = 0;
-  trt_options_converted.trt_builder_optimization_level = 2;
+  trt_options_converted.trt_builder_optimization_level = 3;
   trt_options_converted.trt_auxiliary_streams = -1;
   trt_options_converted.trt_tactic_sources = "";
   trt_options_converted.trt_extra_plugin_lib_paths = "";

onnxruntime/test/perftest/ort_test_session.cc

@@ -126,7 +126,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
     bool trt_detailed_build_log = false;
     bool trt_build_heuristics_enable = false;
     bool trt_sparsity_enable = false;
-    int trt_builder_optimization_level = 2;
+    int trt_builder_optimization_level = 3;
     int trt_auxiliary_streams = -1;
     std::string trt_tactic_sources = "";
     std::string trt_extra_plugin_lib_paths = "";

onnxruntime/test/providers/cpu/model_tests.cc

@@ -702,7 +702,7 @@ TEST_P(ModelTest, Run) {
           OrtTensorRTProviderOptionsV2 params{0, 0, nullptr, 1000, 1, 1 << 30,
                                               1,  // enable fp16
                                               0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0, 0, 0, 0, 0, 0, 0, 0,
-                                              2, -1, nullptr, nullptr, nullptr, nullptr, nullptr};
+                                              3, -1, nullptr, nullptr, nullptr, nullptr, nullptr};
 
           ortso.AppendExecutionProvider_TensorRT_V2(params);
         } else {

onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc

@@ -159,7 +159,7 @@ void RunWithOneSessionSingleThreadInference(std::string model_name, std::string
       0,
       0,
       0,
-      2,
+      3,
       -1,
       nullptr,
       nullptr,
@@ -242,7 +242,7 @@ void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string
       0,
       0,
       0,
-      2,
+      3,
       -1,
       nullptr,
       nullptr,
@@ -396,7 +396,7 @@ TEST(TensorrtExecutionProviderTest, TRTPluginsCustomOpTest) {
       0,
       0,
       0,
-      2,
+      3,
       -1,
       nullptr,
       nullptr,
@@ -491,7 +491,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
       0,
       0,
       0,
-      2,
+      3,
       -1,
       nullptr,
       nullptr,