saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-07 17:15:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Share execution context memory between TensorRT subgraphs (#11859)

Browse source 
* share trt context memory

* update parser to 8.4-EA

* update parser to 8.4-GA

* add context memory sharing enable option

* update parser to 8.2-GA

* fix format issue

* reverse orders

* fix format

* fix format

* fix issues
This commit is contained in:
stevenlix 2022-06-16 22:42:40 -07:00 committed by GitHub
parent 10478a09ca
commit bd65acd08d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 100 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h
View file

@ -29,4 +29,5 @@ struct OrtTensorRTProviderOptionsV2 {
int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true
const char* trt_engine_decryption_lib_path; // specify engine decryption library path
int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true
int trt_context_memory_sharing_enable; // enable context memory sharing between subgraphs. Default 0 = false, nonzero = true
};

56
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file

@ -289,6 +289,7 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
engine_decryption_lib_path_ = info.engine_decryption_lib_path;
}
force_sequential_engine_build_ = info.force_sequential_engine_build;
context_memory_sharing_enable_ = info.context_memory_sharing_enable;
} else {
const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations);
if (!max_partition_iterations_env.empty()) {
@ -373,6 +374,11 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
if (!force_sequential_engine_build_env.empty()) {
force_sequential_engine_build_ = (std::stoi(force_sequential_engine_build_env) == 0 ? false : true);
}
const std::string context_memory_sharing_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kContextMemorySharingEnable);
if (!context_memory_sharing_enable_env.empty()) {
context_memory_sharing_enable_ = (std::stoi(context_memory_sharing_enable_env) == 0 ? false : true);
}
}
// Validate setting
@ -436,7 +442,8 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv
<< ", trt_cache_path: " << cache_path_
<< ", trt_engine_decryption_enable: " << engine_decryption_enable_
<< ", trt_engine_decryption_lib_path: " << engine_decryption_lib_path_
<< ", trt_force_sequential_engine_build: " << force_sequential_engine_build_;
<< ", trt_force_sequential_engine_build: " << force_sequential_engine_build_
<< ", trt_context_memory_sharing_enable: " << context_memory_sharing_enable_;
}
TensorrtExecutionProvider::~TensorrtExecutionProvider() {
@ -1199,7 +1206,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
if (trt_engine != nullptr) {
// Build context
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
if (context_memory_sharing_enable_) {
size_t mem_size = trt_engine->getDeviceMemorySize();
if (mem_size > max_ctx_mem_size_) {
max_ctx_mem_size_ = mem_size;
context_memory_ = IAllocator::MakeUniquePtr<void>(allocator_, max_ctx_mem_size_);
}
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
} else {
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
}
if (trt_context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build execution context for fused node: " + fused_node.Name());
@ -1245,7 +1261,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
update_engine_cache = true;
// Build context
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
if (context_memory_sharing_enable_) {
size_t mem_size = trt_engine->getDeviceMemorySize();
if (mem_size > max_ctx_mem_size_) {
max_ctx_mem_size_ = mem_size;
context_memory_ = IAllocator::MakeUniquePtr<void>(allocator_, max_ctx_mem_size_);
}
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
} else {
trt_context = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
}
if (trt_context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL,
"TensorRT EP could not build execution context for fused node: " + fused_node.Name());
@ -1296,8 +1321,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
&networks_[context->node_name], input_info_[context->node_name], output_info_[context->node_name],
input_shape_ranges_[context->node_name], &tensorrt_mu_, fp16_enable_, int8_enable_, int8_calibration_cache_available_,
dla_enable_, dla_core_, &max_workspace_size_, trt_node_name_with_precision, engine_cache_enable_, cache_path_,
runtime_.get(), nullptr, allocator_, dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_,
update_engine_cache};
runtime_.get(), nullptr, allocator_, context_memory_sharing_enable_, &max_ctx_mem_size_, &context_memory_,
dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_, update_engine_cache};
*state = p.release();
return 0;
};
@ -1356,6 +1381,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
auto trt_context = trt_state->context->get();
auto trt_profile = &(trt_state->trt_profile);
auto alloc = trt_state->scratch_allocator;
auto context_memory = trt_state->context_memory;
auto max_context_mem_size_ptr = trt_state->max_context_mem_size_ptr;
int num_inputs = static_cast<int>(input_indexes.size());
int num_outputs = static_cast<int>(output_indexes.size());
bool engine_update = false;
@ -1542,8 +1569,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
trt_engine = trt_state->engine->get();
// Build context
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
if (trt_state->context_memory_sharing_enable) {
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContextWithoutDeviceMemory());
} else {
*(trt_state->context) = tensorrt_ptr::unique_pointer<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
}
if (trt_state->context == nullptr) {
return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context.");
}
@ -1835,6 +1867,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector<FusedNodeAnd
}
}
// Set execution context memory
if (trt_state->context_memory_sharing_enable) {
size_t mem_size = trt_engine->getDeviceMemorySize();
if (mem_size > *max_context_mem_size_ptr) {
*max_context_mem_size_ptr = mem_size;
*context_memory = IAllocator::MakeUniquePtr<void>(alloc, *max_context_mem_size_ptr);
}
trt_context->setDeviceMemory((*context_memory).get());
}
// Run TRT inference
if (!trt_context->enqueueV2(&buffers[0], stream, nullptr)) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "TensorRT EP execution context enqueue failed.");

7
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file

@ -26,6 +26,7 @@ static const std::string kCachePath = "ORT_TENSORRT_CACHE_PATH";
static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE";
static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH";
static const std::string kForceSequentialEngineBuild= "ORT_TENSORRT_FORCE_SEQUENTIAL_ENGINE_BUILD";
static const std::string kContextMemorySharingEnable= "ORT_TENSORRT_CONTEXT_MEMORY_SHARING_ENABLE";
// Old env variable for backward compatibility
static const std::string kEngineCachePath = "ORT_TENSORRT_ENGINE_CACHE_PATH";
} // namespace tensorrt_env_vars
@ -103,6 +104,9 @@ struct TensorrtFuncState {
nvinfer1::IRuntime* runtime = nullptr;
nvinfer1::IOptimizationProfile* trt_profile = nullptr;
AllocatorPtr scratch_allocator;
bool context_memory_sharing_enable;
size_t* max_context_mem_size_ptr = nullptr;
IAllocatorUniquePtr<void>* context_memory = nullptr;
std::unordered_map<std::string, float> dynamic_range_map;
bool engine_decryption_enable;
int (*engine_decryption)(const char*, char*, size_t*);
@ -166,6 +170,9 @@ class TensorrtExecutionProvider : public IExecutionProvider {
OrtMutex tensorrt_mu_;
int device_id_;
AllocatorPtr allocator_;
bool context_memory_sharing_enable_ = false;
size_t max_ctx_mem_size_ = 0;
IAllocatorUniquePtr<void> context_memory_ = nullptr;
mutable char model_path_[4096]; // Reserved for max path length
bool engine_decryption_enable_ = false;
int (*engine_decryption_)(const char*, char*, size_t*);

3
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc
View file

@ -28,6 +28,7 @@ constexpr const char* kDecryptionEnable = "trt_engine_decryption_enable";
constexpr const char* kDecryptionLibPath = "trt_engine_decryption_lib_path";
constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build";
// add new provider option name here.
constexpr const char* kContextMemorySharingEnable = "trt_context_memory_sharing_enable";
} // namespace provider_option_names
} // namespace tensorrt
@ -64,6 +65,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions
.AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionEnable, info.engine_decryption_enable)
.AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionLibPath, info.engine_decryption_lib_path)
.AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build)
.AddAssignmentToReference(tensorrt::provider_option_names::kContextMemorySharingEnable, info.context_memory_sharing_enable)
.Parse(options)); // add new provider option here.
return info;
@ -89,6 +91,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE
{tensorrt::provider_option_names::kDecryptionLibPath, MakeStringWithClassicLocale(info.engine_decryption_lib_path)},
{tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)},
// add new provider option here.
{tensorrt::provider_option_names::kContextMemorySharingEnable, MakeStringWithClassicLocale(info.context_memory_sharing_enable)},
};
return options;
}

1
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h
View file

@ -31,6 +31,7 @@ struct TensorrtExecutionProviderInfo {
bool engine_decryption_enable{false};
std::string engine_decryption_lib_path{""};
bool force_sequential_engine_build{false};
bool context_memory_sharing_enable{false};
static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);
static ProviderOptions ToProviderOptions(const TensorrtExecutionProviderInfo& info);

2
onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
View file

@ -68,6 +68,7 @@ struct Tensorrt_Provider : Provider {
info.engine_decryption_enable = options.trt_engine_decryption_enable != 0;
info.engine_decryption_lib_path = options.trt_engine_decryption_lib_path == nullptr ? "" : options.trt_engine_decryption_lib_path;
info.force_sequential_engine_build = options.trt_force_sequential_engine_build != 0;
info.context_memory_sharing_enable = options.trt_context_memory_sharing_enable != 0;
return std::make_shared<TensorrtProviderFactory>(info);
}
@ -133,6 +134,7 @@ struct Tensorrt_Provider : Provider {
}
trt_options.trt_force_sequential_engine_build = internal_options.force_sequential_engine_build;
trt_options.trt_context_memory_sharing_enable = internal_options.context_memory_sharing_enable;
}
ProviderOptions GetProviderOptions(const void* provider_options) override {

3
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -1179,7 +1179,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti
trt_options_converted.trt_force_sequential_engine_build = legacy_trt_options->trt_force_sequential_engine_build;
// Add new provider option below
// Use default value as this field is not available in OrtTensorRTProviderOptionsV
trt_options_converted.trt_context_memory_sharing_enable = 0;
return trt_options_converted;
}
@ -1470,6 +1470,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateTensorRTProviderOptions, _Outptr_ OrtTensorRT
(*out)->trt_engine_decryption_enable = false;
(*out)->trt_engine_decryption_lib_path = nullptr;
(*out)->trt_force_sequential_engine_build = false;
(*out)->trt_context_memory_sharing_enable = false;
return nullptr;
#else
ORT_UNUSED_PARAMETER(out);

25
onnxruntime/python/onnxruntime_pybind_state.cc
View file

@ -398,6 +398,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
nullptr,
0,
nullptr,
0,
0};
for (auto option : it->second) {
if (option.first == "device_id") {
@ -430,7 +431,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_fp16_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_fp16_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_fp16_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_int8_enable") {
if (option.second == "True" || option.second == "true") {
@ -438,7 +439,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_int8_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_int8_calibration_table_name") {
if (!option.second.empty()) {
@ -453,7 +454,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_int8_use_native_calibration_table = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_dla_enable") {
if (option.second == "True" || option.second == "true") {
@ -461,7 +462,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_dla_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_dla_core") {
if (!option.second.empty()) {
@ -475,7 +476,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_dump_subgraphs = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_engine_cache_enable") {
if (option.second == "True" || option.second == "true") {
@ -483,7 +484,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_engine_cache_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_engine_cache_path") {
if (!option.second.empty()) {
@ -498,7 +499,7 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_engine_decryption_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_engine_decryption_lib_path") {
if (!option.second.empty()) {
@ -513,7 +514,15 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
} else if (option.second == "False" || option.second == "false") {
params.trt_force_sequential_engine_build = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. 'True' or 'False'. Default value is False.\n");
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else if (option.first == "trt_context_memory_sharing_enable") {
if (option.second == "True" || option.second == "true") {
params.trt_context_memory_sharing_enable = true;
} else if (option.second == "False" || option.second == "false") {
params.trt_context_memory_sharing_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_context_memory_sharing_enable' should be 'True' or 'False'. Default value is 'False'.\n");
}
} else {
ORT_THROW("Invalid TensorRT EP option: ", option.first);

1
onnxruntime/python/tools/tensorrt/perf/mem_test/main.cpp
View file

@ -27,6 +27,7 @@ std::unique_ptr<OrtTensorRTProviderOptions> get_default_trt_provider_options() {
tensorrt_options->trt_engine_decryption_enable = false;
tensorrt_options->trt_engine_decryption_lib_path = "";
tensorrt_options->trt_force_sequential_engine_build = false;
tensorrt_options->trt_context_memory_sharing_enable = false;
return tensorrt_options;
}

1
onnxruntime/test/perftest/command_args_parser.cc
View file

@ -78,6 +78,7 @@ namespace perftest {
"\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n"
"\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n"
"\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n"
"\t [TensorRT only] [trt_context_memory_sharing_enable]: Enable TensorRT context memory sharing between subgraphs.\n"
"\t [Usage]: -e <provider_name> -i '<key1>|<value1> <key2>|<value2>'\n\n"
"\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n"
"\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n"

12
onnxruntime/test/perftest/ort_test_session.cc
View file

@ -76,6 +76,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
bool trt_engine_decryption_enable = false;
std::string trt_engine_decryption_lib_path = "";
bool trt_force_sequential_engine_build = false;
bool trt_context_memory_sharing_enable = false;
#ifdef _MSC_VER
std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string);
@ -207,8 +208,16 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. true or false. Default value is false.\n");
}
} else if (key == "trt_context_memory_sharing_enable") {
if (value == "true" || value == "True") {
trt_context_memory_sharing_enable = true;
} else if (value == "false" || value == "False") {
trt_context_memory_sharing_enable = false;
} else {
ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_context_memory_sharing_enable' should be a boolean i.e. true or false. Default value is false.\n");
}
} else {
ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build'] \n");
ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build', 'trt_context_memory_sharing_enable'] \n");
}
}
OrtTensorRTProviderOptionsV2 tensorrt_options;
@ -230,6 +239,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
tensorrt_options.trt_engine_decryption_enable = trt_engine_decryption_enable;
tensorrt_options.trt_engine_decryption_lib_path = trt_engine_decryption_lib_path.c_str();
tensorrt_options.trt_force_sequential_engine_build = trt_force_sequential_engine_build;
tensorrt_options.trt_context_memory_sharing_enable = trt_context_memory_sharing_enable;
session_options.AppendExecutionProvider_TensorRT_V2(tensorrt_options);
OrtCUDAProviderOptions cuda_options;

2
onnxruntime/test/providers/cpu/model_tests.cc
View file

@ -663,7 +663,7 @@ TEST_P(ModelTest, Run) {
if (test_case_name.find(ORT_TSTR("FLOAT16")) != std::string::npos) {
OrtTensorRTProviderOptionsV2 params{0, 0, nullptr, 1000, 1, 1 << 30,
1, // enable fp16
0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0};
0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0, 0};
ASSERT_ORT_STATUS_OK(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2(ortso, &params));
} else {
OrtTensorRTProviderOptionsV2* ep_option;

4
onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc
View file

@ -149,6 +149,7 @@ void RunWithOneSessionSingleThreadInference(std::string model_name, std::string
nullptr,
0,
nullptr,
0,
0};
params.trt_engine_cache_enable = 1;
@ -218,6 +219,7 @@ void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string
nullptr,
0,
nullptr,
0,
0};
params.trt_engine_cache_enable = 1;
@ -356,6 +358,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
nullptr,
0,
nullptr,
0,
0};
if (cache_type.compare("engine") == 0) {
@ -483,6 +486,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) {
nullptr,
0,
nullptr,
0,
0};
if (cache_type.compare("engine") == 0) {