From bd65acd08d06f430c73cbc1372f2aea13141130f Mon Sep 17 00:00:00 2001 From: stevenlix <38092805+stevenlix@users.noreply.github.com> Date: Thu, 16 Jun 2022 22:42:40 -0700 Subject: [PATCH] Share execution context memory between TensorRT subgraphs (#11859) * share trt context memory * update parser to 8.4-EA * update parser to 8.4-GA * add context memory sharing enable option * update parser to 8.2-GA * fix format issue * reverse orders * fix format * fix format * fix issues --- .../tensorrt/tensorrt_provider_options.h | 1 + .../tensorrt/tensorrt_execution_provider.cc | 56 ++++++++++++++++--- .../tensorrt/tensorrt_execution_provider.h | 7 +++ .../tensorrt_execution_provider_info.cc | 3 + .../tensorrt_execution_provider_info.h | 1 + .../tensorrt/tensorrt_provider_factory.cc | 2 + .../core/session/provider_bridge_ort.cc | 3 +- .../python/onnxruntime_pybind_state.cc | 25 ++++++--- .../tools/tensorrt/perf/mem_test/main.cpp | 1 + .../test/perftest/command_args_parser.cc | 1 + onnxruntime/test/perftest/ort_test_session.cc | 12 +++- onnxruntime/test/providers/cpu/model_tests.cc | 2 +- .../providers/tensorrt/tensorrt_basic_test.cc | 4 ++ 13 files changed, 100 insertions(+), 18 deletions(-) diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h index fd4ad0b6d9..b7c7d8984d 100644 --- a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h +++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h @@ -29,4 +29,5 @@ struct OrtTensorRTProviderOptionsV2 { int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true const char* trt_engine_decryption_lib_path; // specify engine decryption library path int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true + int trt_context_memory_sharing_enable; // enable context memory sharing between subgraphs. Default 0 = false, nonzero = true }; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc index d64a2b669b..9805ad45e6 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc @@ -289,6 +289,7 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv engine_decryption_lib_path_ = info.engine_decryption_lib_path; } force_sequential_engine_build_ = info.force_sequential_engine_build; + context_memory_sharing_enable_ = info.context_memory_sharing_enable; } else { const std::string max_partition_iterations_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kMaxPartitionIterations); if (!max_partition_iterations_env.empty()) { @@ -373,6 +374,11 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv if (!force_sequential_engine_build_env.empty()) { force_sequential_engine_build_ = (std::stoi(force_sequential_engine_build_env) == 0 ? false : true); } + + const std::string context_memory_sharing_enable_env = onnxruntime::GetEnvironmentVar(tensorrt_env_vars::kContextMemorySharingEnable); + if (!context_memory_sharing_enable_env.empty()) { + context_memory_sharing_enable_ = (std::stoi(context_memory_sharing_enable_env) == 0 ? false : true); + } } // Validate setting @@ -436,7 +442,8 @@ TensorrtExecutionProvider::TensorrtExecutionProvider(const TensorrtExecutionProv << ", trt_cache_path: " << cache_path_ << ", trt_engine_decryption_enable: " << engine_decryption_enable_ << ", trt_engine_decryption_lib_path: " << engine_decryption_lib_path_ - << ", trt_force_sequential_engine_build: " << force_sequential_engine_build_; + << ", trt_force_sequential_engine_build: " << force_sequential_engine_build_ + << ", trt_context_memory_sharing_enable: " << context_memory_sharing_enable_; } TensorrtExecutionProvider::~TensorrtExecutionProvider() { @@ -1199,7 +1206,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(trt_engine->createExecutionContext()); + if (context_memory_sharing_enable_) { + size_t mem_size = trt_engine->getDeviceMemorySize(); + if (mem_size > max_ctx_mem_size_) { + max_ctx_mem_size_ = mem_size; + context_memory_ = IAllocator::MakeUniquePtr(allocator_, max_ctx_mem_size_); + } + trt_context = tensorrt_ptr::unique_pointer(trt_engine->createExecutionContextWithoutDeviceMemory()); + } else { + trt_context = tensorrt_ptr::unique_pointer(trt_engine->createExecutionContext()); + } if (trt_context == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP could not build execution context for fused node: " + fused_node.Name()); @@ -1245,7 +1261,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vector(trt_engine->createExecutionContext()); + if (context_memory_sharing_enable_) { + size_t mem_size = trt_engine->getDeviceMemorySize(); + if (mem_size > max_ctx_mem_size_) { + max_ctx_mem_size_ = mem_size; + context_memory_ = IAllocator::MakeUniquePtr(allocator_, max_ctx_mem_size_); + } + trt_context = tensorrt_ptr::unique_pointer(trt_engine->createExecutionContextWithoutDeviceMemory()); + } else { + trt_context = tensorrt_ptr::unique_pointer(trt_engine->createExecutionContext()); + } if (trt_context == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP could not build execution context for fused node: " + fused_node.Name()); @@ -1296,8 +1321,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectornode_name], input_info_[context->node_name], output_info_[context->node_name], input_shape_ranges_[context->node_name], &tensorrt_mu_, fp16_enable_, int8_enable_, int8_calibration_cache_available_, dla_enable_, dla_core_, &max_workspace_size_, trt_node_name_with_precision, engine_cache_enable_, cache_path_, - runtime_.get(), nullptr, allocator_, dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_, - update_engine_cache}; + runtime_.get(), nullptr, allocator_, context_memory_sharing_enable_, &max_ctx_mem_size_, &context_memory_, + dynamic_range_map, engine_decryption_enable_, engine_decryption_, engine_encryption_, update_engine_cache}; *state = p.release(); return 0; }; @@ -1356,6 +1381,8 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorcontext->get(); auto trt_profile = &(trt_state->trt_profile); auto alloc = trt_state->scratch_allocator; + auto context_memory = trt_state->context_memory; + auto max_context_mem_size_ptr = trt_state->max_context_mem_size_ptr; int num_inputs = static_cast(input_indexes.size()); int num_outputs = static_cast(output_indexes.size()); bool engine_update = false; @@ -1542,8 +1569,13 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorengine->get(); // Build context - *(trt_state->context) = tensorrt_ptr::unique_pointer( - trt_state->engine->get()->createExecutionContext()); + if (trt_state->context_memory_sharing_enable) { + *(trt_state->context) = tensorrt_ptr::unique_pointer( + trt_state->engine->get()->createExecutionContextWithoutDeviceMemory()); + } else { + *(trt_state->context) = tensorrt_ptr::unique_pointer( + trt_state->engine->get()->createExecutionContext()); + } if (trt_state->context == nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, "TensorRT EP failed to create context."); } @@ -1835,6 +1867,16 @@ common::Status TensorrtExecutionProvider::Compile(const std::vectorcontext_memory_sharing_enable) { + size_t mem_size = trt_engine->getDeviceMemorySize(); + if (mem_size > *max_context_mem_size_ptr) { + *max_context_mem_size_ptr = mem_size; + *context_memory = IAllocator::MakeUniquePtr(alloc, *max_context_mem_size_ptr); + } + trt_context->setDeviceMemory((*context_memory).get()); + } + // Run TRT inference if (!trt_context->enqueueV2(&buffers[0], stream, nullptr)) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "TensorRT EP execution context enqueue failed."); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index d8c3c37113..27a53b1b8f 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -26,6 +26,7 @@ static const std::string kCachePath = "ORT_TENSORRT_CACHE_PATH"; static const std::string kDecryptionEnable = "ORT_TENSORRT_ENGINE_DECRYPTION_ENABLE"; static const std::string kDecryptionLibPath = "ORT_TENSORRT_ENGINE_DECRYPTION_LIB_PATH"; static const std::string kForceSequentialEngineBuild= "ORT_TENSORRT_FORCE_SEQUENTIAL_ENGINE_BUILD"; +static const std::string kContextMemorySharingEnable= "ORT_TENSORRT_CONTEXT_MEMORY_SHARING_ENABLE"; // Old env variable for backward compatibility static const std::string kEngineCachePath = "ORT_TENSORRT_ENGINE_CACHE_PATH"; } // namespace tensorrt_env_vars @@ -103,6 +104,9 @@ struct TensorrtFuncState { nvinfer1::IRuntime* runtime = nullptr; nvinfer1::IOptimizationProfile* trt_profile = nullptr; AllocatorPtr scratch_allocator; + bool context_memory_sharing_enable; + size_t* max_context_mem_size_ptr = nullptr; + IAllocatorUniquePtr* context_memory = nullptr; std::unordered_map dynamic_range_map; bool engine_decryption_enable; int (*engine_decryption)(const char*, char*, size_t*); @@ -166,6 +170,9 @@ class TensorrtExecutionProvider : public IExecutionProvider { OrtMutex tensorrt_mu_; int device_id_; AllocatorPtr allocator_; + bool context_memory_sharing_enable_ = false; + size_t max_ctx_mem_size_ = 0; + IAllocatorUniquePtr context_memory_ = nullptr; mutable char model_path_[4096]; // Reserved for max path length bool engine_decryption_enable_ = false; int (*engine_decryption_)(const char*, char*, size_t*); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index 7386ce6c88..6d45608d0b 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -28,6 +28,7 @@ constexpr const char* kDecryptionEnable = "trt_engine_decryption_enable"; constexpr const char* kDecryptionLibPath = "trt_engine_decryption_lib_path"; constexpr const char* kForceSequentialEngineBuild = "trt_force_sequential_engine_build"; // add new provider option name here. +constexpr const char* kContextMemorySharingEnable = "trt_context_memory_sharing_enable"; } // namespace provider_option_names } // namespace tensorrt @@ -64,6 +65,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions .AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionEnable, info.engine_decryption_enable) .AddAssignmentToReference(tensorrt::provider_option_names::kDecryptionLibPath, info.engine_decryption_lib_path) .AddAssignmentToReference(tensorrt::provider_option_names::kForceSequentialEngineBuild, info.force_sequential_engine_build) + .AddAssignmentToReference(tensorrt::provider_option_names::kContextMemorySharingEnable, info.context_memory_sharing_enable) .Parse(options)); // add new provider option here. return info; @@ -89,6 +91,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE {tensorrt::provider_option_names::kDecryptionLibPath, MakeStringWithClassicLocale(info.engine_decryption_lib_path)}, {tensorrt::provider_option_names::kForceSequentialEngineBuild, MakeStringWithClassicLocale(info.force_sequential_engine_build)}, // add new provider option here. + {tensorrt::provider_option_names::kContextMemorySharingEnable, MakeStringWithClassicLocale(info.context_memory_sharing_enable)}, }; return options; } diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h index b6d879bf72..3637c2991d 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.h @@ -31,6 +31,7 @@ struct TensorrtExecutionProviderInfo { bool engine_decryption_enable{false}; std::string engine_decryption_lib_path{""}; bool force_sequential_engine_build{false}; + bool context_memory_sharing_enable{false}; static TensorrtExecutionProviderInfo FromProviderOptions(const ProviderOptions& options); static ProviderOptions ToProviderOptions(const TensorrtExecutionProviderInfo& info); diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc index 848c0d0f92..148017bca1 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -68,6 +68,7 @@ struct Tensorrt_Provider : Provider { info.engine_decryption_enable = options.trt_engine_decryption_enable != 0; info.engine_decryption_lib_path = options.trt_engine_decryption_lib_path == nullptr ? "" : options.trt_engine_decryption_lib_path; info.force_sequential_engine_build = options.trt_force_sequential_engine_build != 0; + info.context_memory_sharing_enable = options.trt_context_memory_sharing_enable != 0; return std::make_shared(info); } @@ -133,6 +134,7 @@ struct Tensorrt_Provider : Provider { } trt_options.trt_force_sequential_engine_build = internal_options.force_sequential_engine_build; + trt_options.trt_context_memory_sharing_enable = internal_options.context_memory_sharing_enable; } ProviderOptions GetProviderOptions(const void* provider_options) override { diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index daed0e0e8c..86a2642c29 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1179,7 +1179,7 @@ OrtTensorRTProviderOptionsV2 OrtTensorRTProviderOptionsToOrtTensorRTProviderOpti trt_options_converted.trt_force_sequential_engine_build = legacy_trt_options->trt_force_sequential_engine_build; // Add new provider option below // Use default value as this field is not available in OrtTensorRTProviderOptionsV - + trt_options_converted.trt_context_memory_sharing_enable = 0; return trt_options_converted; } @@ -1470,6 +1470,7 @@ ORT_API_STATUS_IMPL(OrtApis::CreateTensorRTProviderOptions, _Outptr_ OrtTensorRT (*out)->trt_engine_decryption_enable = false; (*out)->trt_engine_decryption_lib_path = nullptr; (*out)->trt_force_sequential_engine_build = false; + (*out)->trt_context_memory_sharing_enable = false; return nullptr; #else ORT_UNUSED_PARAMETER(out); diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index f8e1681200..72b0ed284e 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -398,6 +398,7 @@ std::unique_ptr CreateExecutionProviderInstance( nullptr, 0, nullptr, + 0, 0}; for (auto option : it->second) { if (option.first == "device_id") { @@ -430,7 +431,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_fp16_enable = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_fp16_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_fp16_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_int8_enable") { if (option.second == "True" || option.second == "true") { @@ -438,7 +439,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_int8_enable = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_int8_calibration_table_name") { if (!option.second.empty()) { @@ -453,7 +454,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_int8_use_native_calibration_table = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_int8_use_native_calibration_table' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_dla_enable") { if (option.second == "True" || option.second == "true") { @@ -461,7 +462,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_dla_enable = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dla_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_dla_core") { if (!option.second.empty()) { @@ -475,7 +476,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_dump_subgraphs = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_dump_subgraphs' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_engine_cache_enable") { if (option.second == "True" || option.second == "true") { @@ -483,7 +484,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_engine_cache_enable = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_cache_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_engine_cache_path") { if (!option.second.empty()) { @@ -498,7 +499,7 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_engine_decryption_enable = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_engine_decryption_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else if (option.first == "trt_engine_decryption_lib_path") { if (!option.second.empty()) { @@ -513,7 +514,15 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.second == "False" || option.second == "false") { params.trt_force_sequential_engine_build = false; } else { - ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. 'True' or 'False'. Default value is False.\n"); + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be 'True' or 'False'. Default value is 'False'.\n"); + } + } else if (option.first == "trt_context_memory_sharing_enable") { + if (option.second == "True" || option.second == "true") { + params.trt_context_memory_sharing_enable = true; + } else if (option.second == "False" || option.second == "false") { + params.trt_context_memory_sharing_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_context_memory_sharing_enable' should be 'True' or 'False'. Default value is 'False'.\n"); } } else { ORT_THROW("Invalid TensorRT EP option: ", option.first); diff --git a/onnxruntime/python/tools/tensorrt/perf/mem_test/main.cpp b/onnxruntime/python/tools/tensorrt/perf/mem_test/main.cpp index a3fd958941..74ee766190 100644 --- a/onnxruntime/python/tools/tensorrt/perf/mem_test/main.cpp +++ b/onnxruntime/python/tools/tensorrt/perf/mem_test/main.cpp @@ -27,6 +27,7 @@ std::unique_ptr get_default_trt_provider_options() { tensorrt_options->trt_engine_decryption_enable = false; tensorrt_options->trt_engine_decryption_lib_path = ""; tensorrt_options->trt_force_sequential_engine_build = false; + tensorrt_options->trt_context_memory_sharing_enable = false; return tensorrt_options; } diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 57ec50e580..5fab642ff4 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -78,6 +78,7 @@ namespace perftest { "\t [TensorRT only] [trt_engine_cache_enable]: Enable engine caching.\n" "\t [TensorRT only] [trt_engine_cache_path]: Specify engine cache path.\n" "\t [TensorRT only] [trt_force_sequential_engine_build]: Force TensorRT engines to be built sequentially.\n" + "\t [TensorRT only] [trt_context_memory_sharing_enable]: Enable TensorRT context memory sharing between subgraphs.\n" "\t [Usage]: -e -i '| |'\n\n" "\t [Example] [For TensorRT EP] -e tensorrt -i 'trt_fp16_enable|true trt_int8_enable|true trt_int8_calibration_table_name|calibration.flatbuffers trt_int8_use_native_calibration_table|false trt_force_sequential_engine_build|false'\n" "\t [NNAPI only] [NNAPI_FLAG_USE_FP16]: Use fp16 relaxation in NNAPI EP..\n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 5bf9f62e2b..c97343b293 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -76,6 +76,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device bool trt_engine_decryption_enable = false; std::string trt_engine_decryption_lib_path = ""; bool trt_force_sequential_engine_build = false; + bool trt_context_memory_sharing_enable = false; #ifdef _MSC_VER std::string ov_string = ToUTF8String(performance_test_config.run_config.ep_runtime_config_string); @@ -207,8 +208,16 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_force_sequential_engine_build' should be a boolean i.e. true or false. Default value is false.\n"); } + } else if (key == "trt_context_memory_sharing_enable") { + if (value == "true" || value == "True") { + trt_context_memory_sharing_enable = true; + } else if (value == "false" || value == "False") { + trt_context_memory_sharing_enable = false; + } else { + ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_context_memory_sharing_enable' should be a boolean i.e. true or false. Default value is false.\n"); + } } else { - ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build'] \n"); + ORT_THROW("[ERROR] [TensorRT] wrong key type entered. Choose from the following runtime key options that are available for TensorRT. ['device_id', 'trt_max_partition_iterations', 'trt_min_subgraph_size', 'trt_max_workspace_size', 'trt_fp16_enable', 'trt_int8_enable', 'trt_int8_calibration_table_name', 'trt_int8_use_native_calibration_table', 'trt_dla_enable', 'trt_dla_core', 'trt_dump_subgraphs', 'trt_engine_cache_enable', 'trt_engine_cache_path', 'trt_engine_decryption_enable', 'trt_engine_decryption_lib_path', 'trt_force_sequential_engine_build', 'trt_context_memory_sharing_enable'] \n"); } } OrtTensorRTProviderOptionsV2 tensorrt_options; @@ -230,6 +239,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device tensorrt_options.trt_engine_decryption_enable = trt_engine_decryption_enable; tensorrt_options.trt_engine_decryption_lib_path = trt_engine_decryption_lib_path.c_str(); tensorrt_options.trt_force_sequential_engine_build = trt_force_sequential_engine_build; + tensorrt_options.trt_context_memory_sharing_enable = trt_context_memory_sharing_enable; session_options.AppendExecutionProvider_TensorRT_V2(tensorrt_options); OrtCUDAProviderOptions cuda_options; diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 07d1ddcf6f..4961718e64 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -663,7 +663,7 @@ TEST_P(ModelTest, Run) { if (test_case_name.find(ORT_TSTR("FLOAT16")) != std::string::npos) { OrtTensorRTProviderOptionsV2 params{0, 0, nullptr, 1000, 1, 1 << 30, 1, // enable fp16 - 0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0}; + 0, nullptr, 0, 0, 0, 0, 0, nullptr, 0, nullptr, 0, 0}; ASSERT_ORT_STATUS_OK(OrtApis::SessionOptionsAppendExecutionProvider_TensorRT_V2(ortso, ¶ms)); } else { OrtTensorRTProviderOptionsV2* ep_option; diff --git a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc index 3d7483c42e..8dd7c78c49 100644 --- a/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc +++ b/onnxruntime/test/providers/tensorrt/tensorrt_basic_test.cc @@ -149,6 +149,7 @@ void RunWithOneSessionSingleThreadInference(std::string model_name, std::string nullptr, 0, nullptr, + 0, 0}; params.trt_engine_cache_enable = 1; @@ -218,6 +219,7 @@ void RunWithOneSessionMultiThreadsInference(std::string model_name, std::string nullptr, 0, nullptr, + 0, 0}; params.trt_engine_cache_enable = 1; @@ -356,6 +358,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) { nullptr, 0, nullptr, + 0, 0}; if (cache_type.compare("engine") == 0) { @@ -483,6 +486,7 @@ TEST_P(TensorrtExecutionProviderCacheTest, Run) { nullptr, 0, nullptr, + 0, 0}; if (cache_type.compare("engine") == 0) {