diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 927ca70161..b57ed2f570 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -299,12 +299,20 @@ typedef struct OrtTensorRTProviderOptions { int device_id; // cuda device id. int has_user_compute_stream; // indicator of user specified CUDA compute stream. void* user_compute_stream; // user specified CUDA compute stream. - int has_trt_options; // override environment variables with following TensorRT settings at runtime. + int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability + int trt_min_subgraph_size; // minimum size of TensorRT subgraphs size_t trt_max_workspace_size; // maximum workspace size for TensorRT. int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true + int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true + int trt_dla_core; // DLA core number. Default 0 + int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true + int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true + const char* trt_engine_cache_path; // specify engine cache path + int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true + const char* trt_engine_decryption_lib_path; // specify engine decryption library path int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true } OrtTensorRTProviderOptions; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index 40d299bb82..ff05b6bea3 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -394,89 +394,135 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv } // Get environment variables - const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations); - if (!max_partition_iterations_env.empty()) { - max_partition_iterations_ = std::stoi(max_partition_iterations_env); - } - - const std::string min_subgraph_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMinSubgraphSize); - if (!min_subgraph_size_env.empty()) { - min_subgraph_size_ = std::stoi(min_subgraph_size_env); - } - if (info.has_trt_options) { + max_partition_iterations_ = info.max_partition_iterations; + min_subgraph_size_ = info.min_subgraph_size; max_workspace_size_ = info.max_workspace_size; + fp16_enable_ = info.fp16_enable; + int8_enable_ = info.int8_enable; + if (int8_enable_) { + int8_calibration_cache_name_ = info.int8_calibration_table_name; + int8_use_native_tensorrt_calibration_table_ = info.int8_use_native_calibration_table; + } + if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8 + dla_enable_ = info.dla_enable; + dla_core_ = info.dla_core; + } + dump_subgraphs_ = info.dump_subgraphs; + engine_cache_enable_ = info.engine_cache_enable; + if (engine_cache_enable_ || int8_enable_) { + cache_path_ = info.engine_cache_path; + } + engine_decryption_enable_ = info.engine_decryption_enable; + if (engine_decryption_enable_) { + engine_decryption_lib_path_ = info.engine_decryption_lib_path; + } + force_sequential_engine_build_ = info.force_sequential_engine_build; } else { + const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations); + if (!max_partition_iterations_env.empty()) { + max_partition_iterations_ = std::stoi(max_partition_iterations_env); + } + + const std::string min_subgraph_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMinSubgraphSize); + if (!min_subgraph_size_env.empty()) { + min_subgraph_size_ = std::stoi(min_subgraph_size_env); + } + const std::string max_workspace_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxWorkspaceSize); if (!max_workspace_size_env.empty()) { max_workspace_size_ = std::stoull(max_workspace_size_env); } - } - if (info.has_trt_options) { - fp16_enable_ = info.fp16_enable; - } else { const std::string fp16_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kFP16Enable); if (!fp16_enable_env.empty()) { fp16_enable_ = (std::stoi(fp16_enable_env) == 0 ? false : true); } - } - if (info.has_trt_options) { - int8_enable_ = info.int8_enable; - } else { const std::string int8_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8Enable); if (!int8_enable_env.empty()) { int8_enable_ = (std::stoi(int8_enable_env) == 0 ? false : true); } - } - if (int8_enable_) { - if (info.has_trt_options) { - int8_calibration_cache_name_ = info.int8_calibration_table_name; - } else { + if (int8_enable_) { const std::string int8_calibration_cache_name_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8CalibrationTableName); if (!int8_calibration_cache_name_env.empty()) { int8_calibration_cache_name_ = int8_calibration_cache_name_env; } - } - if (info.has_trt_options) { - int8_use_native_tensorrt_calibration_table_ = info.int8_use_native_calibration_table; - } else { const std::string int8_use_native_tensorrt_calibration_table_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8UseNativeTensorrtCalibrationTable); if (!int8_use_native_tensorrt_calibration_table_env.empty()) { int8_use_native_tensorrt_calibration_table_ = (std::stoi(int8_use_native_tensorrt_calibration_table_env) == 0 ? false : true); } } - } - if (info.has_trt_options) { - force_sequential_engine_build_ = info.force_sequential_engine_build; - } else { + if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8 + const std::string dla_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLAEnable); + if (!dla_enable_env.empty()) { + dla_enable_ = (std::stoi(dla_enable_env) == 0 ? false : true); + } + + if (dla_enable_) { + const std::string dla_core_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLACore); + if (!dla_core_env.empty()) { + dla_core_ = std::stoi(dla_core_env); + } + } + } + + const std::string dump_subgraphs_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDumpSubgraphs); + if (!dump_subgraphs_env.empty()) { + dump_subgraphs_ = (std::stoi(dump_subgraphs_env) == 0 ? false : true); + } + + const std::string engine_cache_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCacheEnable); + if (!engine_cache_enable_env.empty()) { + engine_cache_enable_ = (std::stoi(engine_cache_enable_env) == 0 ? false : true); + } + + if (engine_cache_enable_ || int8_enable_) { + const std::string engine_cache_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCachePath); + cache_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kCachePath); + if (!engine_cache_path.empty() && cache_path_.empty()) { + cache_path_ = engine_cache_path; + LOGS_DEFAULT(WARNING) << "[TensorRT EP] ORT_TENSORRT_ENGINE_CACHE_PATH is deprecated! Please use ORT_TENSORRT_CACHE_PATH to specify engine cache path"; + } + } + + const std::string engine_decryption_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionEnable); + if (!engine_decryption_enable_env.empty()) { + engine_decryption_enable_ = (std::stoi(engine_decryption_enable_env) == 0 ? false : true); + } + + if (engine_decryption_enable_) { + engine_decryption_lib_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionLibPath); + } + const std::string force_sequential_engine_build_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kForceSequentialEngineBuild); if (!force_sequential_engine_build_env.empty()) { force_sequential_engine_build_ = (std::stoi(force_sequential_engine_build_env) == 0 ? false : true); } } - const std::string dump_subgraphs_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDumpSubgraphs); - if (!dump_subgraphs_env.empty()) { - dump_subgraphs_ = (std::stoi(dump_subgraphs_env) == 0 ? false : true); + // Validate setting + if (max_partition_iterations_ <= 0) { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_max_partition_iterations must be a positive integer value. Set it to 1000"; + max_partition_iterations_ = 1000; } - - const std::string engine_cache_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCacheEnable); - if (!engine_cache_enable_env.empty()) { - engine_cache_enable_ = (std::stoi(engine_cache_enable_env) == 0 ? false : true); + if (min_subgraph_size_ <= 0) { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_min_subgraph_size must be a positive integer value. Set it to 1"; + min_subgraph_size_ = 1; + } + if (max_workspace_size_ <= 0) { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_max_workspace_size must be a positive integer value. Set it to 1073741824 (1GB)"; + max_workspace_size_ = 1 << 30; + } + if (dla_core_ < 0) { + LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_dla_core must be a non-negative integer value. Set it to 0"; + dla_core_ = 0; } if (engine_cache_enable_ || int8_enable_) { - const std::string engine_cache_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCachePath); - cache_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kCachePath); - if (!engine_cache_path.empty() && cache_path_.empty()) { - cache_path_ = engine_cache_path; - LOGS_DEFAULT(WARNING) << "[TensorRT EP] ORT_TENSORRT_ENGINE_CACHE_PATH is deprecated! Please use ORT_TENSORRT_CACHE_PATH to specify engine cache path"; - } if (!cache_path_.empty() && !fs::is_directory(cache_path_)) { if (!fs::create_directory(cache_path_)) { throw std::runtime_error("Failed to create directory " + cache_path_); @@ -485,34 +531,14 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv runtime_ = tensorrt_ptr::unique_pointer(nvinfer1::createInferRuntime(GetTensorrtLogger())); } - const std::string engine_decryption_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionEnable); - if (!engine_decryption_enable_env.empty()) { - engine_decryption_enable_ = (std::stoi(engine_decryption_enable_env) == 0 ? false : true); - } - if (engine_decryption_enable_) { - std::string engine_decryption_lib_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionLibPath); - LIBTYPE handle = OPENLIB(engine_decryption_lib_path.c_str()); + LIBTYPE handle = OPENLIB(engine_decryption_lib_path_.c_str()); if (handle == nullptr) { ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "TensorRT EP could not open shared library from " + engine_decryption_lib_path); + "TensorRT EP could not open shared library from " + engine_decryption_lib_path_); } engine_decryption_ = (int (*)(const char*, char*, size_t*))LIBFUNC(handle, "decrypt"); } - - if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8 - const std::string dla_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLAEnable); - if (!dla_enable_env.empty()) { - dla_enable_ = (std::stoi(dla_enable_env) == 0 ? false : true); - } - - if (dla_enable_) { - const std::string dla_core_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLACore); - if (!dla_core_env.empty()) { - dla_core_ = std::stoi(dla_core_env); - } - } - } } TensorrtExecutionProvider::~TensorrtExecutionProvider() { diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index 858303430e..82b347768c 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -18,16 +18,16 @@ static const std::string kFP16Enable = "ORT_TENSORRT_FP16_ENABLE"; static const std::string kINT8Enable = "ORT_TENSORRT_INT8_ENABLE"; static const std::string kINT8CalibrationTableName = "ORT_TENSORRT_INT8_CALIBRATION_TABLE_NAME"; static const std::string kINT8UseNativeTensorrtCalibrationTable = "ORT_TENSORRT_INT8_USE_NATIVE_CALIBRATION_TABLE"; +static const std::string kDLAEnable = "ORT_TENSORRT_DLA_ENABLE"; +static const std::string kDLACore = "ORT_TENSORRT_DLA_CORE"; static const std::string kDumpSubgraphs = "ORT_TENSORRT_DUMP_SUBGRAPHS"; static const std::string kEngineCacheEnable = "ORT_TENSORRT_ENGINE_CACHE_ENABLE"; static const std::string kCachePath = "ORT_TENSORRT_CACHE_PATH"; +static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE"; +static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH"; static const std::string kForceSequentialEngineBuild= "ORT_TENSORRT_FORCE_SEQUENTIAL_ENGINE_BUILD"; // Old env variable for backward compatibility static const std::string kEngineCachePath = "ORT_TENSORRT_ENGINE_CACHE_PATH"; -static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE"; -static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH"; -static const std::string kDLAEnable = "ORT_TENSORRT_DLA_ENABLE"; -static const std::string kDLACore = "ORT_TENSORRT_DLA_CORE"; } // namespace tensorrt_env_vars class TensorrtLogger : public nvinfer1::ILogger { @@ -148,7 +148,7 @@ class TensorrtExecutionProvider : public IExecutionProvider { bool int8_use_native_tensorrt_calibration_table_ = false; bool dump_subgraphs_ = false; bool engine_cache_enable_ = false; - std::string cache_path_; + std::string cache_path_, engine_decryption_lib_path_; tensorrt_ptr::unique_pointer runtime_ = nullptr; OrtMutex tensorrt_mu_; int device_id_; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index 529f5a4e7c..3eb91671d4 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -12,13 +12,21 @@ namespace onnxruntime { namespace tensorrt { namespace provider_option_names { constexpr const char* kDeviceId = "device_id"; -constexpr const char* kHasTrtOptions = "has_trt_options"; +constexpr const char* kMaxPartitionIterations = "trt_max_partition_iterations"; +constexpr const char* kMinSubgraphSize = "trt_min_subgraph_size"; constexpr const char* kMaxWorkspaceSize = "trt_max_workspace_size"; constexpr const char* kFp16Enable = "trt_fp16_enable"; constexpr const char* kInt8Enable = "trt_int8_enable"; constexpr const char* kInt8CalibTable = "trt_int8_calibration_table_name"; constexpr const char* kInt8UseNativeCalibTable = "trt_int8_use_native_calibration_table"; -//constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build"; +constexpr const char* kDLAEnable = "trt_dla_enable"; +constexpr const char* kDLACore = "trt_dla_core"; +constexpr const char* kDumpSubgraphs = "trt_dump_subgraphs"; +constexpr const char* kEngineCacheEnable = "trt_engine_cache_enable"; +constexpr const char* kCachePath = "trt_engine_cache_path"; +constexpr const char* kDecryptionEnable = "trt_engine_decryption_enable"; +constexpr const char* kDecryptionLibPath = "trt_engine_decryption_lib_path"; +constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build"; } // namespace provider_option_names } // namespace tensorrt @@ -40,13 +48,21 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions ", must be between 0 (inclusive) and ", num_devices, " (exclusive)."); return Status::OK(); }) - .AddAssignmentToReference(tensorrt::provider_option_names::kHasTrtOptions, info.has_trt_options) + .AddAssignmentToReference(tensorrt::provider_option_names::kMaxPartitionIterations, info.max_partition_iterations) + .AddAssignmentToReference(tensorrt::provider_option_names::kMinSubgraphSize, info.min_subgraph_size) .AddAssignmentToReference(tensorrt::provider_option_names::kMaxWorkspaceSize, info.max_workspace_size) .AddAssignmentToReference(tensorrt::provider_option_names::kFp16Enable, info.fp16_enable) .AddAssignmentToReference(tensorrt::provider_option_names::kInt8Enable, info.int8_enable) .AddAssignmentToReference(tensorrt::provider_option_names::kInt8CalibTable, info.int8_calibration_table_name) .AddAssignmentToReference(tensorrt::provider_option_names::kInt8UseNativeCalibTable, info.int8_use_native_calibration_table) - //.AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build) + .AddAssignmentToReference(tensorrt::provider_option_names::kDLAEnable, info.dla_enable) + .AddAssignmentToReference(tensorrt::provider_option_names::kDLACore, info.dla_core) + .AddAssignmentToReference(tensorrt::provider_option_names::kDumpSubgraphs, info.dump_subgraphs) + .AddAssignmentToReference(tensorrt::provider_option_names::kEngineCacheEnable, info.engine_cache_enable) + .AddAssignmentToReference(tensorrt::provider_option_names::kCachePath, info.engine_cache_path) + .AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionEnable, info.engine_decryption_enable) + .AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionLibPath, info.engine_decryption_lib_path) + .AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build) .Parse(options)); return info; @@ -55,13 +71,21 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtExecutionProviderInfo& info) { const ProviderOptions options{ {tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, - {tensorrt::provider_option_names::kHasTrtOptions, MakeStringWithClassicLocale(info.has_trt_options)}, + {tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.max_partition_iterations)}, + {tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.min_subgraph_size)}, {tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.max_workspace_size)}, {tensorrt::provider_option_names::kFp16Enable, MakeStringWithClassicLocale(info.fp16_enable)}, {tensorrt::provider_option_names::kInt8Enable, MakeStringWithClassicLocale(info.int8_enable)}, {tensorrt::provider_option_names::kInt8CalibTable, MakeStringWithClassicLocale(info.int8_calibration_table_name)}, {tensorrt::provider_option_names::kInt8UseNativeCalibTable, MakeStringWithClassicLocale(info.int8_use_native_calibration_table)}, - //{tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)}, + {tensorrt::provider_option_names::kDLAEnable, MakeStringWithClassicLocale(info.dla_enable)}, + {tensorrt::provider_option_names::kDLACore, MakeStringWithClassicLocale(info.dla_core)}, + {tensorrt::provider_option_names::kDumpSubgraphs, MakeStringWithClassicLocale(info.dump_subgraphs)}, + {tensorrt::provider_option_names::kEngineCacheEnable, MakeStringWithClassicLocale(info.engine_cache_enable)}, + {tensorrt::provider_option_names::kCachePath, MakeStringWithClassicLocale(info.engine_cache_path)}, + {tensorrt::provider_option_names::kDecryptionEnable, MakeStringWithClassicLocale(info.engine_decryption_enable)}, + {tensorrt::provider_option_names::kDecryptionLibPath, MakeStringWithClassicLocale(info.engine_decryption_lib_path)}, + {tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)}, }; return options; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h index 7395f52a25..2b82bcb1fd 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h @@ -16,11 +16,20 @@ struct TensorrtExecutionProviderInfo { bool has_user_compute_stream{false}; void* user_compute_stream{nullptr}; bool has_trt_options{false}; + int max_partition_iterations{1000}; + int min_subgraph_size{1}; size_t max_workspace_size{1 << 30}; bool fp16_enable{false}; bool int8_enable{false}; std::string int8_calibration_table_name{""}; bool int8_use_native_calibration_table{false}; + bool dla_enable{false}; + int dla_core{0}; + bool dump_subgraphs{false}; + bool engine_cache_enable{false}; + std::string engine_cache_path{""}; + bool engine_decryption_enable{false}; + std::string engine_decryption_lib_path{""}; bool force_sequential_engine_build{false}; static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc index 617fb615d9..86fb41d8cd 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -29,6 +29,7 @@ std::unique_ptr TensorrtProviderFactory::CreateProvider() { std::shared_ptr CreateExecutionProviderFactory_Tensorrt(int device_id) { TensorrtExecutionProviderInfo info; info.device_id = device_id; + info.has_trt_options = false; return std::make_shared(info); } @@ -40,6 +41,7 @@ struct Tensorrt_Provider : Provider { std::shared_ptr CreateExecutionProviderFactory(int device_id) override { TensorrtExecutionProviderInfo info; info.device_id = device_id; + info.has_trt_options = false; return std::make_shared(info); } @@ -49,12 +51,21 @@ struct Tensorrt_Provider : Provider { info.device_id = options.device_id; info.has_user_compute_stream = options.has_user_compute_stream; info.user_compute_stream = options.user_compute_stream; - info.has_trt_options = options.has_trt_options; + info.has_trt_options = true; + info.max_partition_iterations = options.trt_max_partition_iterations; + info.min_subgraph_size = options.trt_min_subgraph_size; info.max_workspace_size = options.trt_max_workspace_size; info.fp16_enable = options.trt_fp16_enable; info.int8_enable = options.trt_int8_enable; info.int8_calibration_table_name = options.trt_int8_calibration_table_name == nullptr ? "" : options.trt_int8_calibration_table_name; info.int8_use_native_calibration_table = options.trt_int8_use_native_calibration_table; + info.dla_enable = options.trt_dla_enable; + info.dla_core = options.trt_dla_core; + info.dump_subgraphs = options.trt_dump_subgraphs; + info.engine_cache_enable = options.trt_engine_cache_enable; + info.engine_cache_path = options.trt_engine_cache_path == nullptr ? "" : options.trt_engine_cache_path; + info.engine_decryption_enable = options.trt_engine_decryption_enable; + info.engine_decryption_lib_path = options.trt_engine_decryption_lib_path == nullptr ? "" : options.trt_engine_decryption_lib_path; info.force_sequential_engine_build = options.trt_force_sequential_engine_build; return std::make_shared(info); } diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 39fbec01ab..a9c422bd78 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -575,8 +575,27 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector sess->GetSessionOptions().enable_cpu_mem_arena)); } else if (type == kTensorrtExecutionProvider) { #ifdef USE_TENSORRT - OrtTensorRTProviderOptions params{0, 0, nullptr, 0, 1 << 30, 0, 0, nullptr, 0, 0}; - std::string trt_int8_calibration_table_name; + OrtTensorRTProviderOptions params{ + 0, + 0, + nullptr, + 1000, + 1, + 1 << 30, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + nullptr, + 0, + nullptr, + 0}; + + std::string calibration_table, cache_path, lib_path; auto it = provider_options_map.find(type); if (it != provider_options_map.end()) { for (auto option : it->second) { @@ -586,13 +605,17 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'device_id' should be a number i.e. '0'.\n"); } - } else if (option.first == "has_trt_options") { - if (option.second == "True" || option.second == "true") { - params.has_trt_options = true; - } else if (option.second == "False" || option.second == "false") { - params.has_trt_options = false; + } else if (option.first == "trt_max_partition_iterations") { + if (!option.second.empty()) { + params.trt_max_partition_iterations = std::stoi(option.second); } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'has_trt_options' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_max_partition_iterations' should be a positive integer number i.e. '1000'.\n"); + } + } else if (option.first == "trt_min_subgraph_size") { + if (!option.second.empty()) { + params.trt_min_subgraph_size = std::stoi(option.second); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_min_subgraph_size' should be a positive integer number i.e. '1'.\n"); } } else if (option.first == "trt_max_workspace_size") { if (!option.second.empty()) { @@ -618,8 +641,8 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } } else if (option.first == "trt_int8_calibration_table_name") { if (!option.second.empty()) { - trt_int8_calibration_table_name = option.second; - params.trt_int8_calibration_table_name = trt_int8_calibration_table_name.c_str(); + calibration_table = option.second; + params.trt_int8_calibration_table_name = calibration_table.c_str(); } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_calibration_table_name' should be a file name i.e. 'cal_table'.\n"); } @@ -631,6 +654,66 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); } + } else if (option.first == "trt_dla_enable") { + if (option.second == "True" || option.second == "true") { + params.trt_dla_enable = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_dla_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + } + } else if (option.first == "trt_dla_core") { + if (!option.second.empty()) { + params.trt_dla_core = std::stoi(option.second); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_core' should be a positive integer number i.e. '0'.\n"); + } + } else if (option.first == "trt_dump_subgraphs") { + if (option.second == "True" || option.second == "true") { + params.trt_dump_subgraphs = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_dump_subgraphs = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + } + } else if (option.first == "trt_engine_cache_enable") { + if (option.second == "True" || option.second == "true") { + params.trt_engine_cache_enable = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_engine_cache_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + } + } else if (option.first == "trt_engine_cache_path") { + if (!option.second.empty()) { + cache_path = option.second; + params.trt_engine_cache_path = cache_path.c_str(); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_path' should be a path string i.e. 'engine_cache'.\n"); + } + } else if (option.first == "trt_engine_decryption_enable") { + if (option.second == "True" || option.second == "true") { + params.trt_engine_decryption_enable = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_engine_decryption_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + } + } else if (option.first == "trt_engine_decryption_lib_path") { + if (!option.second.empty()) { + lib_path = option.second; + params.trt_engine_decryption_lib_path = lib_path.c_str(); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_lib_path' should be a path string i.e. 'decryption_lib'.\n"); + } + } else if (option.first == "trt_force_sequential_engine_build") { + if (option.second == "True" || option.second == "true") { + params.trt_force_sequential_engine_build = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_force_sequential_engine_build = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + } } else { ORT_THROW("Invalid TensorRT EP option: ", option.first); } diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 2c9ea95cd0..975d6957c8 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -313,12 +313,20 @@ int real_main(int argc, char* argv[], Ort::Env& env) { 0, 0, nullptr, - 0, + 1000, + 1, 1 << 30, 0, 0, nullptr, 0, + 0, + 0, + 0, + 0, + nullptr, + 0, + nullptr, 0}; OrtCUDAProviderOptions cuda_options{ diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index c201cf62f9..18835ca2d9 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -64,15 +64,23 @@ namespace perftest { "\t [OpenVINO only] [blob_dump_path]: Explicitly specify the path where you would like to dump and load the blobs for the use_compiled_network(save/load blob) feature. This overrides the default path.\n" "\t [Usage]: -e -i '| |'\n\n" "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU_FP32 enable_vpu_fast_compile|true num_of_threads|5 use_compiled_network|true blob_dump_path|\"\"\"\n" - "\t [TensorRT only] [use_trt_options]: Overrides TensorRT environment variables (if any) with following settings at runtime.\n" + "\t [TensorRT only] [trt_max_partition_iterations]: Maximum iterations for TensorRT parser to get capability.\n" + "\t [TensorRT only] [trt_min_subgraph_size]: Minimum size of TensorRT subgraphs.\n" "\t [TensorRT only] [trt_max_workspace_size]: Set TensorRT maximum workspace size in byte.\n" "\t [TensorRT only] [trt_fp16_enable]: Enable TensorRT FP16 precision.\n" "\t [TensorRT only] [trt_int8_enable]: Enable TensorRT INT8 precision.\n" "\t [TensorRT only] [trt_int8_calibration_table_name]: Specify INT8 calibration table name.\n" "\t [TensorRT only] [trt_int8_use_native_calibration_table]: Use Native TensorRT calibration table.\n" + "\t [TensorRT only] [trt_dla_enable]: Enable DLA in Jetson device.\n" + "\t [TensorRT only] [trt_dla_core]: DLA core number.\n" + "\t [TensorRT only] [trt_dump_subgraphs]: Dump TRT subgraph to onnx model.\n" + "\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n" + "\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n" + "\t [TensorRT only] [trt_engine_decryption_enable]: (experimental feature) Enable engine decryption.\n" + "\t [TensorRT only] [trt_engine_decryption_lib_path]: (experimental feature) Specify engine decryption library path.\n" "\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n" "\t [Usage]: -e -i '| |'\n\n" - "\t [Example] [For TensorRT EP] -e tensorrt -i 'use_trt_options|true trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" + "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" "\t-h: help\n"); } #ifdef _WIN32 diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 665dcb9170..64c58d9972 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -63,19 +63,28 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device #endif } else if (provider_name == onnxruntime::kTensorrtExecutionProvider) { #ifdef USE_TENSORRT - bool has_trt_options = false; + int device_id = 0; + int trt_max_partition_iterations = 1000; + int trt_min_subgraph_size = 1; size_t trt_max_workspace_size = 1 << 30; bool trt_fp16_enable = false; bool trt_int8_enable = false; std::string trt_int8_calibration_table_name = ""; bool trt_int8_use_native_calibration_table = false; + bool trt_dla_enable = false; + int trt_dla_core = 0; + bool trt_dump_subgraphs = false; + bool trt_engine_cache_enable = false; + std::string trt_engine_cache_path = ""; + bool trt_engine_decryption_enable = false; + std::string trt_engine_decryption_lib_path = ""; bool trt_force_sequential_engine_build = false; -#ifdef _MSC_VER + #ifdef _MSC_VER std::string ov_string = ToMBString(performance_test_config.run_config.ep_runtime_config_string); -#else + #else std::string ov_string = performance_test_config.run_config.ep_runtime_config_string; -#endif + #endif std::istringstream ss(ov_string); std::string token; while (ss >> token) { @@ -89,13 +98,23 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device auto key = token.substr(0, pos); auto value = token.substr(pos + 1); - if (key == "has_trt_options") { - if (value == "true" || value == "True") { - has_trt_options = true; - } else if (value == "false" || value == "False") { - has_trt_options = false; + if (key == "device_id") { + if (!value.empty()) { + device_id = std::stoi(value); } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'has_trt_options' should be a boolean i.e. true or false. Default value is false.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'device_id' should be a number.\n"); + } + } else if (key == "trt_max_partition_iterations") { + if (!value.empty()) { + trt_max_partition_iterations = std::stoi(value); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_max_partition_iterations' should be a number.\n"); + } + } else if (key == "trt_min_subgraph_size") { + if (!value.empty()) { + trt_min_subgraph_size = std::stoi(value); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_min_subgraph_size' should be a number.\n"); } } else if (key == "trt_max_workspace_size") { if (!value.empty()) { @@ -133,6 +152,56 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. true or false. Default value is false.\n"); } + } else if (key == "trt_dla_enable") { + if (value == "true" || value == "True") { + trt_dla_enable = true; + } else if (value == "false" || value == "False") { + trt_dla_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "trt_dla_core") { + if (!value.empty()) { + trt_dla_core = std::stoi(value); + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_core' should be a number.\n"); + } + } else if (key == "trt_dump_subgraphs") { + if (value == "true" || value == "True") { + trt_dump_subgraphs = true; + } else if (value == "false" || value == "False") { + trt_dump_subgraphs = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "trt_engine_cache_enable") { + if (value == "true" || value == "True") { + trt_engine_cache_enable = true; + } else if (value == "false" || value == "False") { + trt_engine_cache_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "trt_engine_cache_path") { + if (!value.empty()) { + trt_engine_cache_path = value; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_path' should be a non-emtpy string.\n"); + } + } else if (key == "trt_engine_decryption_enable") { + if (value == "true" || value == "True") { + trt_engine_decryption_enable = true; + } else if (value == "false" || value == "False") { + trt_engine_decryption_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. true or false. Default value is false.\n"); + } + } else if (key == "trt_engine_decryption_lib_path") { + if (!value.empty()) { + trt_engine_decryption_lib_path = value; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_lib_path' should be a non-emtpy string.\n"); + } } else if (key == "trt_force_sequential_engine_build") { if (value == "true" || value == "True") { trt_force_sequential_engine_build = true; @@ -142,19 +211,27 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. true or false. Default value is false.\n"); } } else { - ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['use_trt_options', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_force_sequential_engine_build'] \n"); + ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build'] \n"); } } OrtTensorRTProviderOptions tensorrt_options; - tensorrt_options.device_id = 0; + tensorrt_options.device_id = device_id; tensorrt_options.has_user_compute_stream = 0; tensorrt_options.user_compute_stream = nullptr; - tensorrt_options.has_trt_options = has_trt_options; + tensorrt_options.trt_max_partition_iterations = trt_max_partition_iterations; + tensorrt_options.trt_min_subgraph_size = trt_min_subgraph_size; tensorrt_options.trt_max_workspace_size = trt_max_workspace_size; tensorrt_options.trt_fp16_enable = trt_fp16_enable; tensorrt_options.trt_int8_enable = trt_int8_enable; tensorrt_options.trt_int8_calibration_table_name = trt_int8_calibration_table_name.c_str(); tensorrt_options.trt_int8_use_native_calibration_table = trt_int8_use_native_calibration_table; + tensorrt_options.trt_dla_enable = trt_dla_enable; + tensorrt_options.trt_dla_core = trt_dla_core; + tensorrt_options.trt_dump_subgraphs = trt_dump_subgraphs; + tensorrt_options.trt_engine_cache_enable = trt_engine_cache_enable; + tensorrt_options.trt_engine_cache_path = trt_engine_cache_path.c_str(); + tensorrt_options.trt_engine_decryption_enable = trt_engine_decryption_enable; + tensorrt_options.trt_engine_decryption_lib_path = trt_engine_decryption_lib_path.c_str(); tensorrt_options.trt_force_sequential_engine_build = trt_force_sequential_engine_build; session_options.AppendExecutionProvider_TensorRT(tensorrt_options); diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 53c320efe5..cdb543ef84 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -69,20 +69,35 @@ class TestInferenceSession(unittest.TestCase): options = sess.get_provider_options() option = options['TensorrtExecutionProvider'] self.assertIn('device_id', option) - self.assertIn('has_trt_options', option) + self.assertIn('trt_max_partition_iterations', option) + self.assertIn('trt_min_subgraph_size', option) self.assertIn('trt_max_workspace_size', option) self.assertIn('trt_fp16_enable', option) self.assertIn('trt_int8_enable', option) self.assertIn('trt_int8_calibration_table_name', option) self.assertIn('trt_int8_use_native_calibration_table', option) + self.assertIn('trt_dla_enable', option) + self.assertIn('trt_dla_core', option) + self.assertIn('trt_dump_subgraphs', option) + self.assertIn('trt_engine_cache_enable', option) + self.assertIn('trt_engine_cache_path', option) + self.assertIn('trt_engine_decryption_enable', option) + self.assertIn('trt_engine_decryption_lib_path', option) + self.assertIn('trt_force_sequential_engine_build', option) + max_partition_iterations = option['trt_max_partition_iterations'] + new_max_partition_iterations = int(max_partition_iterations) + 1 + min_subgraph_size = option['trt_min_subgraph_size'] + new_min_subgraph_size = int(max_partition_iterations) + 1 ori_max_workspace_size = option['trt_max_workspace_size'] new_max_workspace_size = int(ori_max_workspace_size) // 2 + dla_core = option['trt_dla_core'] + new_dla_core = int(dla_core) + 1 option = {} + option['trt_max_partition_iterations'] = new_max_partition_iterations + option['trt_min_subgraph_size'] = new_min_subgraph_size option['trt_max_workspace_size'] = new_max_workspace_size - trt_options = "true" - option['has_trt_options'] = trt_options fp16_enable = "true" option['trt_fp16_enable'] = fp16_enable int8_enable = "false" @@ -91,17 +106,40 @@ class TestInferenceSession(unittest.TestCase): option['trt_int8_calibration_table_name'] = calib_table_name int8_use_native_calibration_table = "true" option['trt_int8_use_native_calibration_table'] = int8_use_native_calibration_table + dla_enable = "true" + option['trt_dla_enable'] = dla_enable + option['trt_dla_core'] = new_dla_core + dump_subgraphs = "true" + option['trt_dump_subgraphs'] = dump_subgraphs + engine_cache_enable = "true" + option['trt_engine_cache_enable'] = engine_cache_enable + engine_cache_path = '/home/onnxruntime/engine_cache' + option['trt_engine_cache_path'] = engine_cache_path + engine_decryption_enable = "true" + option['trt_engine_decryption_enable'] = engine_decryption_enable + engine_decryption_lib_path = '/home/onnxruntime/decryption_lib' + option['trt_engine_decryption_lib_path'] = engine_decryption_lib_path + force_sequential_engine_build = "true" + option['trt_force_sequential_engine_build'] = force_sequential_engine_build sess.set_providers(['TensorrtExecutionProvider'], [option]) options = sess.get_provider_options() option = options['TensorrtExecutionProvider'] + self.assertEqual(option['trt_max_partition_iterations'], str(new_max_partition_iterations)) + self.assertEqual(option['trt_min_subgraph_size'], str(new_min_subgraph_size)) self.assertEqual(option['trt_max_workspace_size'], str(new_max_workspace_size)) self.assertEqual(option['trt_int8_calibration_table_name'], str(calib_table_name)) - self.assertEqual(option['has_trt_options'], '1') self.assertEqual(option['trt_fp16_enable'], '1') self.assertEqual(option['trt_int8_enable'], '0') self.assertEqual(option['trt_int8_use_native_calibration_table'], '1') - + self.assertEqual(option['trt_dla_enable'], '1') + self.assertEqual(option['trt_dla_core'], str(new_dla_core)) + self.assertEqual(option['trt_dump_subgraphs'], '1') + self.assertEqual(option['trt_engine_cache_enable'], '1') + self.assertEqual(option['trt_engine_cache_path'], str(engine_cache_path)) + self.assertEqual(option['trt_engine_decryption_enable'], '1') + self.assertEqual(option['engine_decryption_lib_path'], str(engine_decryption_lib_path)) + self.assertEqual(option['trt_force_sequential_engine_build'], '1') # We currently disable following test code since that not all test machines/GPUs have nvidia int8 capability diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index a5c9539124..72c70f9728 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -43,7 +43,25 @@ std::unique_ptr DefaultCpuExecutionProvider(bool enable_aren std::unique_ptr DefaultTensorrtExecutionProvider() { #ifdef USE_TENSORRT - OrtTensorRTProviderOptions params{0, 0, nullptr, 0, 1 << 30, 0, 0, nullptr, 0, 0}; + OrtTensorRTProviderOptions params{ + 0, + 0, + nullptr, + 1000, + 1, + 1 << 30, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + nullptr, + 0, + nullptr, + 0}; if (auto factory = CreateExecutionProviderFactory_Tensorrt(¶ms)) return factory->CreateProvider(); #endif