saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add more TensorRT env variables to provider options (#7698)

Browse source 
* add all trt env variables to provider options

* add python test

* Update onnxruntime_c_api.h

* fix issues

* validate values for options
This commit is contained in:
stevenlix 2021-05-16 22:09:52 -07:00 committed by GitHub
parent 943ab9dcef
commit 557b94637d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 421 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -299,12 +299,20 @@ typedef struct OrtTensorRTProviderOptions {
int device_id; // cuda device id.
int has_user_compute_stream; // indicator of user specified CUDA compute stream.
void* user_compute_stream; // user specified CUDA compute stream.
int has_trt_options; // override environment variables with following TensorRT settings at runtime.
int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability
int trt_min_subgraph_size; // minimum size of TensorRT subgraphs
size_t trt_max_workspace_size; // maximum workspace size for TensorRT.
int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true
int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true
const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name.
int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true
int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true
int trt_dla_core; // DLA core number. Default 0
int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true
int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true
const char* trt_engine_cache_path; // specify engine cache path
int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true
const char* trt_engine_decryption_lib_path; // specify engine decryption library path
int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true
} OrtTensorRTProviderOptions;

158
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -394,89 +394,135 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
}
// Get environment variables
const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations);
if (!max_partition_iterations_env.empty()) {
max_partition_iterations_ = std::stoi(max_partition_iterations_env);
}
const std::string min_subgraph_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMinSubgraphSize);
if (!min_subgraph_size_env.empty()) {
min_subgraph_size_ = std::stoi(min_subgraph_size_env);
}
if (info.has_trt_options) {
max_partition_iterations_ = info.max_partition_iterations;
min_subgraph_size_ = info.min_subgraph_size;
max_workspace_size_ = info.max_workspace_size;
fp16_enable_ = info.fp16_enable;
int8_enable_ = info.int8_enable;
if (int8_enable_) {
int8_calibration_cache_name_ = info.int8_calibration_table_name;
int8_use_native_tensorrt_calibration_table_ = info.int8_use_native_calibration_table;
}
if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8
dla_enable_ = info.dla_enable;
dla_core_ = info.dla_core;
}
dump_subgraphs_ = info.dump_subgraphs;
engine_cache_enable_ = info.engine_cache_enable;
if (engine_cache_enable_ || int8_enable_) {
cache_path_ = info.engine_cache_path;
}
engine_decryption_enable_ = info.engine_decryption_enable;
if (engine_decryption_enable_) {
engine_decryption_lib_path_ = info.engine_decryption_lib_path;
}
force_sequential_engine_build_ = info.force_sequential_engine_build;
} else {
const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations);
if (!max_partition_iterations_env.empty()) {
max_partition_iterations_ = std::stoi(max_partition_iterations_env);
}
const std::string min_subgraph_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMinSubgraphSize);
if (!min_subgraph_size_env.empty()) {
min_subgraph_size_ = std::stoi(min_subgraph_size_env);
}
const std::string max_workspace_size_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxWorkspaceSize);
if (!max_workspace_size_env.empty()) {
max_workspace_size_ = std::stoull(max_workspace_size_env);
}
}
if (info.has_trt_options) {
fp16_enable_ = info.fp16_enable;
} else {
const std::string fp16_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kFP16Enable);
if (!fp16_enable_env.empty()) {
fp16_enable_ = (std::stoi(fp16_enable_env) == 0 ? false : true);
}
}
if (info.has_trt_options) {
int8_enable_ = info.int8_enable;
} else {
const std::string int8_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8Enable);
if (!int8_enable_env.empty()) {
int8_enable_ = (std::stoi(int8_enable_env) == 0 ? false : true);
}
}
if (int8_enable_) {
if (info.has_trt_options) {
int8_calibration_cache_name_ = info.int8_calibration_table_name;
} else {
if (int8_enable_) {
const std::string int8_calibration_cache_name_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8CalibrationTableName);
if (!int8_calibration_cache_name_env.empty()) {
int8_calibration_cache_name_ = int8_calibration_cache_name_env;
}
}
if (info.has_trt_options) {
int8_use_native_tensorrt_calibration_table_ = info.int8_use_native_calibration_table;
} else {
const std::string int8_use_native_tensorrt_calibration_table_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kINT8UseNativeTensorrtCalibrationTable);
if (!int8_use_native_tensorrt_calibration_table_env.empty()) {
int8_use_native_tensorrt_calibration_table_ = (std::stoi(int8_use_native_tensorrt_calibration_table_env) == 0 ? false : true);
}
}
}
if (info.has_trt_options) {
force_sequential_engine_build_ = info.force_sequential_engine_build;
} else {
if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8
const std::string dla_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLAEnable);
if (!dla_enable_env.empty()) {
dla_enable_ = (std::stoi(dla_enable_env) == 0 ? false : true);
}
if (dla_enable_) {
const std::string dla_core_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLACore);
if (!dla_core_env.empty()) {
dla_core_ = std::stoi(dla_core_env);
}
}
}
const std::string dump_subgraphs_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDumpSubgraphs);
if (!dump_subgraphs_env.empty()) {
dump_subgraphs_ = (std::stoi(dump_subgraphs_env) == 0 ? false : true);
}
const std::string engine_cache_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCacheEnable);
if (!engine_cache_enable_env.empty()) {
engine_cache_enable_ = (std::stoi(engine_cache_enable_env) == 0 ? false : true);
}
if (engine_cache_enable_ || int8_enable_) {
const std::string engine_cache_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCachePath);
cache_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kCachePath);
if (!engine_cache_path.empty() && cache_path_.empty()) {
cache_path_ = engine_cache_path;
LOGS_DEFAULT(WARNING) << "[TensorRT EP] ORT_TENSORRT_ENGINE_CACHE_PATH is deprecated! Please use ORT_TENSORRT_CACHE_PATH to specify engine cache path";
}
}
const std::string engine_decryption_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionEnable);
if (!engine_decryption_enable_env.empty()) {
engine_decryption_enable_ = (std::stoi(engine_decryption_enable_env) == 0 ? false : true);
}
if (engine_decryption_enable_) {
engine_decryption_lib_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionLibPath);
}
const std::string force_sequential_engine_build_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kForceSequentialEngineBuild);
if (!force_sequential_engine_build_env.empty()) {
force_sequential_engine_build_ = (std::stoi(force_sequential_engine_build_env) == 0 ? false : true);
}
}
const std::string dump_subgraphs_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDumpSubgraphs);
if (!dump_subgraphs_env.empty()) {
dump_subgraphs_ = (std::stoi(dump_subgraphs_env) == 0 ? false : true);
// Validate setting
if (max_partition_iterations_ <= 0) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_max_partition_iterations must be a positive integer value. Set it to 1000";
max_partition_iterations_ = 1000;
}
const std::string engine_cache_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCacheEnable);
if (!engine_cache_enable_env.empty()) {
engine_cache_enable_ = (std::stoi(engine_cache_enable_env) == 0 ? false : true);
if (min_subgraph_size_ <= 0) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_min_subgraph_size must be a positive integer value. Set it to 1";
min_subgraph_size_ = 1;
}
if (max_workspace_size_ <= 0) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_max_workspace_size must be a positive integer value. Set it to 1073741824 (1GB)";
max_workspace_size_ = 1 << 30;
}
if (dla_core_ < 0) {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] TensorRT option trt_dla_core must be a non-negative integer value. Set it to 0";
dla_core_ = 0;
}
if (engine_cache_enable_ || int8_enable_) {
const std::string engine_cache_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kEngineCachePath);
cache_path_ = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kCachePath);
if (!engine_cache_path.empty() && cache_path_.empty()) {
cache_path_ = engine_cache_path;
LOGS_DEFAULT(WARNING) << "[TensorRT EP] ORT_TENSORRT_ENGINE_CACHE_PATH is deprecated! Please use ORT_TENSORRT_CACHE_PATH to specify engine cache path";
}
if (!cache_path_.empty() && !fs::is_directory(cache_path_)) {
if (!fs::create_directory(cache_path_)) {
throw std::runtime_error("Failed to create directory " + cache_path_);
@ -485,34 +531,14 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
runtime_ = tensorrt_ptr::unique_pointer<nvinfer1::IRuntime>(nvinfer1::createInferRuntime(GetTensorrtLogger()));
}
const std::string engine_decryption_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionEnable);
if (!engine_decryption_enable_env.empty()) {
engine_decryption_enable_ = (std::stoi(engine_decryption_enable_env) == 0 ? false : true);
}
if (engine_decryption_enable_) {
std::string engine_decryption_lib_path = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDecryptionLibPath);
LIBTYPE handle = OPENLIB(engine_decryption_lib_path.c_str());
LIBTYPE handle = OPENLIB(engine_decryption_lib_path_.c_str());
if (handle == nullptr) {
ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not open shared library from " + engine_decryption_lib_path);
"TensorRT EP could not open shared library from " + engine_decryption_lib_path_);
}
engine_decryption_ = (int (*)(const char*, char*, size_t*))LIBFUNC(handle, "decrypt");
}
if (fp16_enable_ || int8_enable_) { // DLA can only be enabled with FP16 or INT8
const std::string dla_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLAEnable);
if (!dla_enable_env.empty()) {
dla_enable_ = (std::stoi(dla_enable_env) == 0 ? false : true);
}
if (dla_enable_) {
const std::string dla_core_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kDLACore);
if (!dla_core_env.empty()) {
dla_core_ = std::stoi(dla_core_env);
}
}
}
}
TensorrtExecutionProvider::~TensorrtExecutionProvider() {

10
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -18,16 +18,16 @@ static const std::string kFP16Enable = "ORT_TENSORRT_FP16_ENABLE";
static const std::string kINT8Enable = "ORT_TENSORRT_INT8_ENABLE";
static const std::string kINT8CalibrationTableName = "ORT_TENSORRT_INT8_CALIBRATION_TABLE_NAME";
static const std::string kINT8UseNativeTensorrtCalibrationTable = "ORT_TENSORRT_INT8_USE_NATIVE_CALIBRATION_TABLE";
static const std::string kDLAEnable = "ORT_TENSORRT_DLA_ENABLE";
static const std::string kDLACore = "ORT_TENSORRT_DLA_CORE";
static const std::string kDumpSubgraphs = "ORT_TENSORRT_DUMP_SUBGRAPHS";
static const std::string kEngineCacheEnable = "ORT_TENSORRT_ENGINE_CACHE_ENABLE";
static const std::string kCachePath = "ORT_TENSORRT_CACHE_PATH";
static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE";
static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH";
static const std::string kForceSequentialEngineBuild= "ORT_TENSORRT_FORCE_SEQUENTIAL_ENGINE_BUILD";
// Old env variable for backward compatibility
static const std::string kEngineCachePath = "ORT_TENSORRT_ENGINE_CACHE_PATH";
static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE";
static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH";
static const std::string kDLAEnable = "ORT_TENSORRT_DLA_ENABLE";
static const std::string kDLACore = "ORT_TENSORRT_DLA_CORE";
} // namespace tensorrt_env_vars
class TensorrtLogger : public nvinfer1::ILogger {
@ -148,7 +148,7 @@ class TensorrtExecutionProvider : public IExecutionProvider {
bool int8_use_native_tensorrt_calibration_table_ = false;
bool dump_subgraphs_ = false;
bool engine_cache_enable_ = false;
std::string cache_path_;
std::string cache_path_, engine_decryption_lib_path_;
tensorrt_ptr::unique_pointer<nvinfer1::IRuntime> runtime_ = nullptr;
OrtMutex tensorrt_mu_;
int device_id_;

36
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc
View file

@ -12,13 +12,21 @@ namespace onnxruntime {
namespace tensorrt {
namespace provider_option_names {
constexpr const char* kDeviceId = "device_id";
constexpr const char* kHasTrtOptions = "has_trt_options";
constexpr const char* kMaxPartitionIterations = "trt_max_partition_iterations";
constexpr const char* kMinSubgraphSize = "trt_min_subgraph_size";
constexpr const char* kMaxWorkspaceSize = "trt_max_workspace_size";
constexpr const char* kFp16Enable = "trt_fp16_enable";
constexpr const char* kInt8Enable = "trt_int8_enable";
constexpr const char* kInt8CalibTable = "trt_int8_calibration_table_name";
constexpr const char* kInt8UseNativeCalibTable = "trt_int8_use_native_calibration_table";
//constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build";
constexpr const char* kDLAEnable = "trt_dla_enable";
constexpr const char* kDLACore = "trt_dla_core";
constexpr const char* kDumpSubgraphs = "trt_dump_subgraphs";
constexpr const char* kEngineCacheEnable = "trt_engine_cache_enable";
constexpr const char* kCachePath = "trt_engine_cache_path";
constexpr const char* kDecryptionEnable = "trt_engine_decryption_enable";
constexpr const char* kDecryptionLibPath = "trt_engine_decryption_lib_path";
constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build";
} // namespace provider_option_names
} // namespace tensorrt
@ -40,13 +48,21 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions
", must be between 0 (inclusive) and ", num_devices, " (exclusive).");
return Status::OK();
})
.AddAssignmentToReference(tensorrt::provider_option_names::kHasTrtOptions, info.has_trt_options)
.AddAssignmentToReference(tensorrt::provider_option_names::kMaxPartitionIterations, info.max_partition_iterations)
.AddAssignmentToReference(tensorrt::provider_option_names::kMinSubgraphSize, info.min_subgraph_size)
.AddAssignmentToReference(tensorrt::provider_option_names::kMaxWorkspaceSize, info.max_workspace_size)
.AddAssignmentToReference(tensorrt::provider_option_names::kFp16Enable, info.fp16_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kInt8Enable, info.int8_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kInt8CalibTable, info.int8_calibration_table_name)
.AddAssignmentToReference(tensorrt::provider_option_names::kInt8UseNativeCalibTable, info.int8_use_native_calibration_table)
//.AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build)
.AddAssignmentToReference(tensorrt::provider_option_names::kDLAEnable, info.dla_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kDLACore, info.dla_core)
.AddAssignmentToReference(tensorrt::provider_option_names::kDumpSubgraphs, info.dump_subgraphs)
.AddAssignmentToReference(tensorrt::provider_option_names::kEngineCacheEnable, info.engine_cache_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kCachePath, info.engine_cache_path)
.AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionEnable, info.engine_decryption_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionLibPath, info.engine_decryption_lib_path)
.AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build)
.Parse(options));
return info;
@ -55,13 +71,21 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions
ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtExecutionProviderInfo& info) {
const ProviderOptions options{
{tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)},
{tensorrt::provider_option_names::kHasTrtOptions, MakeStringWithClassicLocale(info.has_trt_options)},
{tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.max_partition_iterations)},
{tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.min_subgraph_size)},
{tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.max_workspace_size)},
{tensorrt::provider_option_names::kFp16Enable, MakeStringWithClassicLocale(info.fp16_enable)},
{tensorrt::provider_option_names::kInt8Enable, MakeStringWithClassicLocale(info.int8_enable)},
{tensorrt::provider_option_names::kInt8CalibTable, MakeStringWithClassicLocale(info.int8_calibration_table_name)},
{tensorrt::provider_option_names::kInt8UseNativeCalibTable, MakeStringWithClassicLocale(info.int8_use_native_calibration_table)},
//{tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)},
{tensorrt::provider_option_names::kDLAEnable, MakeStringWithClassicLocale(info.dla_enable)},
{tensorrt::provider_option_names::kDLACore, MakeStringWithClassicLocale(info.dla_core)},
{tensorrt::provider_option_names::kDumpSubgraphs, MakeStringWithClassicLocale(info.dump_subgraphs)},
{tensorrt::provider_option_names::kEngineCacheEnable, MakeStringWithClassicLocale(info.engine_cache_enable)},
{tensorrt::provider_option_names::kCachePath, MakeStringWithClassicLocale(info.engine_cache_path)},
{tensorrt::provider_option_names::kDecryptionEnable, MakeStringWithClassicLocale(info.engine_decryption_enable)},
{tensorrt::provider_option_names::kDecryptionLibPath, MakeStringWithClassicLocale(info.engine_decryption_lib_path)},
{tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)},
};
return options;

9
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h
View file

@ -16,11 +16,20 @@ struct TensorrtExecutionProviderInfo {
bool has_user_compute_stream{false};
void* user_compute_stream{nullptr};
bool has_trt_options{false};
int max_partition_iterations{1000};
int min_subgraph_size{1};
size_t max_workspace_size{1 << 30};
bool fp16_enable{false};
bool int8_enable{false};
std::string int8_calibration_table_name{""};
bool int8_use_native_calibration_table{false};
bool dla_enable{false};
int dla_core{0};
bool dump_subgraphs{false};
bool engine_cache_enable{false};
std::string engine_cache_path{""};
bool engine_decryption_enable{false};
std::string engine_decryption_lib_path{""};
bool force_sequential_engine_build{false};
static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);

13
onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
View file

@ -29,6 +29,7 @@ std::unique_ptr<IExecutionProvider> TensorrtProviderFactory::CreateProvider() {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Tensorrt(int device_id) {
TensorrtExecutionProviderInfo info;
info.device_id = device_id;
info.has_trt_options = false;
return std::make_shared<onnxruntime::TensorrtProviderFactory>(info);
}
@ -40,6 +41,7 @@ struct Tensorrt_Provider : Provider {
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory(int device_id) override {
TensorrtExecutionProviderInfo info;
info.device_id = device_id;
info.has_trt_options = false;
return std::make_shared<TensorrtProviderFactory>(info);
}
@ -49,12 +51,21 @@ struct Tensorrt_Provider : Provider {
info.device_id = options.device_id;
info.has_user_compute_stream = options.has_user_compute_stream;
info.user_compute_stream = options.user_compute_stream;
info.has_trt_options = options.has_trt_options;
info.has_trt_options = true;
info.max_partition_iterations = options.trt_max_partition_iterations;
info.min_subgraph_size = options.trt_min_subgraph_size;
info.max_workspace_size = options.trt_max_workspace_size;
info.fp16_enable = options.trt_fp16_enable;
info.int8_enable = options.trt_int8_enable;
info.int8_calibration_table_name = options.trt_int8_calibration_table_name == nullptr ? "" : options.trt_int8_calibration_table_name;
info.int8_use_native_calibration_table = options.trt_int8_use_native_calibration_table;
info.dla_enable = options.trt_dla_enable;
info.dla_core = options.trt_dla_core;
info.dump_subgraphs = options.trt_dump_subgraphs;
info.engine_cache_enable = options.trt_engine_cache_enable;
info.engine_cache_path = options.trt_engine_cache_path == nullptr ? "" : options.trt_engine_cache_path;
info.engine_decryption_enable = options.trt_engine_decryption_enable;
info.engine_decryption_lib_path = options.trt_engine_decryption_lib_path == nullptr ? "" : options.trt_engine_decryption_lib_path;
info.force_sequential_engine_build = options.trt_force_sequential_engine_build;
return std::make_shared<TensorrtProviderFactory>(info);
}

103
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -575,8 +575,27 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
sess->GetSessionOptions().enable_cpu_mem_arena));
} else if (type == kTensorrtExecutionProvider) {
#ifdef USE_TENSORRT
OrtTensorRTProviderOptions params{0, 0, nullptr, 0, 1 << 30, 0, 0, nullptr, 0, 0};
std::string trt_int8_calibration_table_name;
OrtTensorRTProviderOptions params{
0,
0,
nullptr,
1000,
1,
1 << 30,
0,
0,
nullptr,
0,
0,
0,
0,
0,
nullptr,
0,
nullptr,
0};
std::string calibration_table, cache_path, lib_path;
auto it = provider_options_map.find(type);
if (it != provider_options_map.end()) {
for (auto option : it->second) {
@ -586,13 +605,17 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'device_id' should be a number i.e. '0'.\n");
}
} else if (option.first == "has_trt_options") {
if (option.second == "True" || option.second == "true") {
params.has_trt_options = true;
} else if (option.second == "False" || option.second == "false") {
params.has_trt_options = false;
} else if (option.first == "trt_max_partition_iterations") {
if (!option.second.empty()) {
params.trt_max_partition_iterations = std::stoi(option.second);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'has_trt_options' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_max_partition_iterations' should be a positive integer number i.e. '1000'.\n");
}
} else if (option.first == "trt_min_subgraph_size") {
if (!option.second.empty()) {
params.trt_min_subgraph_size = std::stoi(option.second);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_min_subgraph_size' should be a positive integer number i.e. '1'.\n");
}
} else if (option.first == "trt_max_workspace_size") {
if (!option.second.empty()) {
@ -618,8 +641,8 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
}
} else if (option.first == "trt_int8_calibration_table_name") {
if (!option.second.empty()) {
trt_int8_calibration_table_name = option.second;
params.trt_int8_calibration_table_name = trt_int8_calibration_table_name.c_str();
calibration_table = option.second;
params.trt_int8_calibration_table_name = calibration_table.c_str();
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_calibration_table_name' should be a file name i.e. 'cal_table'.\n");
}
@ -631,6 +654,66 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else if (option.first == "trt_dla_enable") {
if (option.second == "True" || option.second == "true") {
params.trt_dla_enable = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_dla_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else if (option.first == "trt_dla_core") {
if (!option.second.empty()) {
params.trt_dla_core = std::stoi(option.second);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_core' should be a positive integer number i.e. '0'.\n");
}
} else if (option.first == "trt_dump_subgraphs") {
if (option.second == "True" || option.second == "true") {
params.trt_dump_subgraphs = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_dump_subgraphs = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else if (option.first == "trt_engine_cache_enable") {
if (option.second == "True" || option.second == "true") {
params.trt_engine_cache_enable = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_engine_cache_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else if (option.first == "trt_engine_cache_path") {
if (!option.second.empty()) {
cache_path = option.second;
params.trt_engine_cache_path = cache_path.c_str();
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_path' should be a path string i.e. 'engine_cache'.\n");
}
} else if (option.first == "trt_engine_decryption_enable") {
if (option.second == "True" || option.second == "true") {
params.trt_engine_decryption_enable = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_engine_decryption_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else if (option.first == "trt_engine_decryption_lib_path") {
if (!option.second.empty()) {
lib_path = option.second;
params.trt_engine_decryption_lib_path = lib_path.c_str();
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_lib_path' should be a path string i.e. 'decryption_lib'.\n");
}
} else if (option.first == "trt_force_sequential_engine_build") {
if (option.second == "True" || option.second == "true") {
params.trt_force_sequential_engine_build = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_force_sequential_engine_build = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
}
} else {
ORT_THROW("Invalid TensorRT EP option: ", option.first);
}

10
onnxruntime/test/onnx/main.cc
View file

@ -313,12 +313,20 @@ int real_main(int argc, char* argv[], Ort::Env& env) {
0,
0,
nullptr,
0,
1000,
1,
1 << 30,
0,
0,
nullptr,
0,
0,
0,
0,
0,
nullptr,
0,
nullptr,
0};
OrtCUDAProviderOptions cuda_options{

12
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -64,15 +64,23 @@ namespace perftest {
"\t [OpenVINO only] [blob_dump_path]: Explicitly specify the path where you would like to dump and load the blobs for the use_compiled_network(save/load blob) feature. This overrides the default path.\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
"\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU_FP32 enable_vpu_fast_compile|true num_of_threads|5 use_compiled_network|true blob_dump_path|\"<path>\"\"\n"
"\t [TensorRT only] [use_trt_options]: Overrides TensorRT environment variables (if any) with following settings at runtime.\n"
"\t [TensorRT only] [trt_max_partition_iterations]: Maximum iterations for TensorRT parser to get capability.\n"
"\t [TensorRT only] [trt_min_subgraph_size]: Minimum size of TensorRT subgraphs.\n"
"\t [TensorRT only] [trt_max_workspace_size]: Set TensorRT maximum workspace size in byte.\n"
"\t [TensorRT only] [trt_fp16_enable]: Enable TensorRT FP16 precision.\n"
"\t [TensorRT only] [trt_int8_enable]: Enable TensorRT INT8 precision.\n"
"\t [TensorRT only] [trt_int8_calibration_table_name]: Specify INT8 calibration table name.\n"
"\t [TensorRT only] [trt_int8_use_native_calibration_table]: Use Native TensorRT calibration table.\n"
"\t [TensorRT only] [trt_dla_enable]: Enable DLA in Jetson device.\n"
"\t [TensorRT only] [trt_dla_core]: DLA core number.\n"
"\t [TensorRT only] [trt_dump_subgraphs]: Dump TRT subgraph to onnx model.\n"
"\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n"
"\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n"
"\t [TensorRT only] [trt_engine_decryption_enable]: (experimental feature) Enable engine decryption.\n"
"\t [TensorRT only] [trt_engine_decryption_lib_path]: (experimental feature) Specify engine decryption library path.\n"
"\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'use_trt_options|true trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n"
"\t-h: help\n");
}
#ifdef _WIN32

103
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -63,19 +63,28 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
#endif
} else if (provider_name == onnxruntime::kTensorrtExecutionProvider) {
#ifdef USE_TENSORRT
bool has_trt_options = false;
int device_id = 0;
int trt_max_partition_iterations = 1000;
int trt_min_subgraph_size = 1;
size_t trt_max_workspace_size = 1 << 30;
bool trt_fp16_enable = false;
bool trt_int8_enable = false;
std::string trt_int8_calibration_table_name = "";
bool trt_int8_use_native_calibration_table = false;
bool trt_dla_enable = false;
int trt_dla_core = 0;
bool trt_dump_subgraphs = false;
bool trt_engine_cache_enable = false;
std::string trt_engine_cache_path = "";
bool trt_engine_decryption_enable = false;
std::string trt_engine_decryption_lib_path = "";
bool trt_force_sequential_engine_build = false;
#ifdef _MSC_VER
#ifdef _MSC_VER
std::string ov_string = ToMBString(performance_test_config.run_config.ep_runtime_config_string);
#else
#else
std::string ov_string = performance_test_config.run_config.ep_runtime_config_string;
#endif
#endif
std::istringstream ss(ov_string);
std::string token;
while (ss >> token) {
@ -89,13 +98,23 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
auto key = token.substr(0, pos);
auto value = token.substr(pos + 1);
if (key == "has_trt_options") {
if (value == "true" || value == "True") {
has_trt_options = true;
} else if (value == "false" || value == "False") {
has_trt_options = false;
if (key == "device_id") {
if (!value.empty()) {
device_id = std::stoi(value);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'has_trt_options' should be a boolean i.e. true or false. Default value is false.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'device_id' should be a number.\n");
}
} else if (key == "trt_max_partition_iterations") {
if (!value.empty()) {
trt_max_partition_iterations = std::stoi(value);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_max_partition_iterations' should be a number.\n");
}
} else if (key == "trt_min_subgraph_size") {
if (!value.empty()) {
trt_min_subgraph_size = std::stoi(value);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_min_subgraph_size' should be a number.\n");
}
} else if (key == "trt_max_workspace_size") {
if (!value.empty()) {
@ -133,6 +152,56 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_dla_enable") {
if (value == "true" || value == "True") {
trt_dla_enable = true;
} else if (value == "false" || value == "False") {
trt_dla_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_dla_core") {
if (!value.empty()) {
trt_dla_core = std::stoi(value);
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_core' should be a number.\n");
}
} else if (key == "trt_dump_subgraphs") {
if (value == "true" || value == "True") {
trt_dump_subgraphs = true;
} else if (value == "false" || value == "False") {
trt_dump_subgraphs = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_engine_cache_enable") {
if (value == "true" || value == "True") {
trt_engine_cache_enable = true;
} else if (value == "false" || value == "False") {
trt_engine_cache_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_engine_cache_path") {
if (!value.empty()) {
trt_engine_cache_path = value;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_path' should be a non-emtpy string.\n");
}
} else if (key == "trt_engine_decryption_enable") {
if (value == "true" || value == "True") {
trt_engine_decryption_enable = true;
} else if (value == "false" || value == "False") {
trt_engine_decryption_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_engine_decryption_lib_path") {
if (!value.empty()) {
trt_engine_decryption_lib_path = value;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_lib_path' should be a non-emtpy string.\n");
}
} else if (key == "trt_force_sequential_engine_build") {
if (value == "true" || value == "True") {
trt_force_sequential_engine_build = true;
@ -142,19 +211,27 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['use_trt_options', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_force_sequential_engine_build'] \n");
ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build'] \n");
}
}
OrtTensorRTProviderOptions tensorrt_options;
tensorrt_options.device_id = 0;
tensorrt_options.device_id = device_id;
tensorrt_options.has_user_compute_stream = 0;
tensorrt_options.user_compute_stream = nullptr;
tensorrt_options.has_trt_options = has_trt_options;
tensorrt_options.trt_max_partition_iterations = trt_max_partition_iterations;
tensorrt_options.trt_min_subgraph_size = trt_min_subgraph_size;
tensorrt_options.trt_max_workspace_size = trt_max_workspace_size;
tensorrt_options.trt_fp16_enable = trt_fp16_enable;
tensorrt_options.trt_int8_enable = trt_int8_enable;
tensorrt_options.trt_int8_calibration_table_name = trt_int8_calibration_table_name.c_str();
tensorrt_options.trt_int8_use_native_calibration_table = trt_int8_use_native_calibration_table;
tensorrt_options.trt_dla_enable = trt_dla_enable;
tensorrt_options.trt_dla_core = trt_dla_core;
tensorrt_options.trt_dump_subgraphs = trt_dump_subgraphs;
tensorrt_options.trt_engine_cache_enable = trt_engine_cache_enable;
tensorrt_options.trt_engine_cache_path = trt_engine_cache_path.c_str();
tensorrt_options.trt_engine_decryption_enable = trt_engine_decryption_enable;
tensorrt_options.trt_engine_decryption_lib_path = trt_engine_decryption_lib_path.c_str();
tensorrt_options.trt_force_sequential_engine_build = trt_force_sequential_engine_build;
session_options.AppendExecutionProvider_TensorRT(tensorrt_options);

48
onnxruntime/test/python/onnxruntime_test_python.py
View file

@ -69,20 +69,35 @@ class TestInferenceSession(unittest.TestCase):
options = sess.get_provider_options()
option = options['TensorrtExecutionProvider']
self.assertIn('device_id', option)
self.assertIn('has_trt_options', option)
self.assertIn('trt_max_partition_iterations', option)
self.assertIn('trt_min_subgraph_size', option)
self.assertIn('trt_max_workspace_size', option)
self.assertIn('trt_fp16_enable', option)
self.assertIn('trt_int8_enable', option)
self.assertIn('trt_int8_calibration_table_name', option)
self.assertIn('trt_int8_use_native_calibration_table', option)
self.assertIn('trt_dla_enable', option)
self.assertIn('trt_dla_core', option)
self.assertIn('trt_dump_subgraphs', option)
self.assertIn('trt_engine_cache_enable', option)
self.assertIn('trt_engine_cache_path', option)
self.assertIn('trt_engine_decryption_enable', option)
self.assertIn('trt_engine_decryption_lib_path', option)
self.assertIn('trt_force_sequential_engine_build', option)
max_partition_iterations = option['trt_max_partition_iterations']
new_max_partition_iterations = int(max_partition_iterations) + 1
min_subgraph_size = option['trt_min_subgraph_size']
new_min_subgraph_size = int(max_partition_iterations) + 1
ori_max_workspace_size = option['trt_max_workspace_size']
new_max_workspace_size = int(ori_max_workspace_size) // 2
dla_core = option['trt_dla_core']
new_dla_core = int(dla_core) + 1
option = {}
option['trt_max_partition_iterations'] = new_max_partition_iterations
option['trt_min_subgraph_size'] = new_min_subgraph_size
option['trt_max_workspace_size'] = new_max_workspace_size
trt_options = "true"
option['has_trt_options'] = trt_options
fp16_enable = "true"
option['trt_fp16_enable'] = fp16_enable
int8_enable = "false"
@ -91,17 +106,40 @@ class TestInferenceSession(unittest.TestCase):
option['trt_int8_calibration_table_name'] = calib_table_name
int8_use_native_calibration_table = "true"
option['trt_int8_use_native_calibration_table'] = int8_use_native_calibration_table
dla_enable = "true"
option['trt_dla_enable'] = dla_enable
option['trt_dla_core'] = new_dla_core
dump_subgraphs = "true"
option['trt_dump_subgraphs'] = dump_subgraphs
engine_cache_enable = "true"
option['trt_engine_cache_enable'] = engine_cache_enable
engine_cache_path = '/home/onnxruntime/engine_cache'
option['trt_engine_cache_path'] = engine_cache_path
engine_decryption_enable = "true"
option['trt_engine_decryption_enable'] = engine_decryption_enable
engine_decryption_lib_path = '/home/onnxruntime/decryption_lib'
option['trt_engine_decryption_lib_path'] = engine_decryption_lib_path
force_sequential_engine_build = "true"
option['trt_force_sequential_engine_build'] = force_sequential_engine_build
sess.set_providers(['TensorrtExecutionProvider'], [option])
options = sess.get_provider_options()
option = options['TensorrtExecutionProvider']
self.assertEqual(option['trt_max_partition_iterations'], str(new_max_partition_iterations))
self.assertEqual(option['trt_min_subgraph_size'], str(new_min_subgraph_size))
self.assertEqual(option['trt_max_workspace_size'], str(new_max_workspace_size))
self.assertEqual(option['trt_int8_calibration_table_name'], str(calib_table_name))
self.assertEqual(option['has_trt_options'], '1')
self.assertEqual(option['trt_fp16_enable'], '1')
self.assertEqual(option['trt_int8_enable'], '0')
self.assertEqual(option['trt_int8_use_native_calibration_table'], '1')
self.assertEqual(option['trt_dla_enable'], '1')
self.assertEqual(option['trt_dla_core'], str(new_dla_core))
self.assertEqual(option['trt_dump_subgraphs'], '1')
self.assertEqual(option['trt_engine_cache_enable'], '1')
self.assertEqual(option['trt_engine_cache_path'], str(engine_cache_path))
self.assertEqual(option['trt_engine_decryption_enable'], '1')
self.assertEqual(option['engine_decryption_lib_path'], str(engine_decryption_lib_path))
self.assertEqual(option['trt_force_sequential_engine_build'], '1')
# We currently disable following test code since that not all test machines/GPUs have nvidia int8 capability

20
onnxruntime/test/util/default_providers.cc
View file

@ -43,7 +43,25 @@ std::unique_ptr<IExecutionProvider> DefaultCpuExecutionProvider(bool enable_aren
std::unique_ptr<IExecutionProvider> DefaultTensorrtExecutionProvider() {
#ifdef USE_TENSORRT
OrtTensorRTProviderOptions params{0, 0, nullptr, 0, 1 << 30, 0, 0, nullptr, 0, 0};
OrtTensorRTProviderOptions params{
0,
0,
nullptr,
1000,
1,
1 << 30,
0,
0,
nullptr,
0,
0,
0,
0,
0,
nullptr,
0,
nullptr,
0};
if (auto factory = CreateExecutionProviderFactory_Tensorrt(&params))
return factory->CreateProvider();
#endif