diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 75359602f8..cf50681153 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -4346,6 +4346,32 @@ struct OrtApi { _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, _Inout_updates_all_(output_names_len) OrtValue** output, _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); + + /** + * Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'. + * If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions. + * + * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param value - A pointer to the instance that will be assigned to this provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); + + /** + * Get TensorRT EP provider option where its data type is pointer. + * If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param ptr - A pointer to the instance that is kept by the provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); }; /* diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index d47af40b4c..515fc1c62c 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -13,6 +13,7 @@ namespace onnxruntime { namespace tensorrt { namespace provider_option_names { constexpr const char* kDeviceId = "device_id"; +constexpr const char* kHasUserComputeStream = "has_user_compute_stream"; constexpr const char* kMaxPartitionIterations = "trt_max_partition_iterations"; constexpr const char* kMinSubgraphSize = "trt_min_subgraph_size"; constexpr const char* kMaxWorkspaceSize = "trt_max_workspace_size"; @@ -64,6 +65,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions return Status::OK(); }) .AddAssignmentToReference(tensorrt::provider_option_names::kMaxPartitionIterations, info.max_partition_iterations) + .AddAssignmentToReference(tensorrt::provider_option_names::kHasUserComputeStream, info.has_user_compute_stream) .AddAssignmentToReference(tensorrt::provider_option_names::kMinSubgraphSize, info.min_subgraph_size) .AddAssignmentToReference(tensorrt::provider_option_names::kMaxWorkspaceSize, info.max_workspace_size) .AddAssignmentToReference(tensorrt::provider_option_names::kFp16Enable, info.fp16_enable) @@ -102,6 +104,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE const ProviderOptions options{ {tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, {tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.max_partition_iterations)}, + {tensorrt::provider_option_names::kHasUserComputeStream, MakeStringWithClassicLocale(info.has_user_compute_stream)}, {tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.min_subgraph_size)}, {tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.max_workspace_size)}, {tensorrt::provider_option_names::kFp16Enable, MakeStringWithClassicLocale(info.fp16_enable)}, @@ -149,6 +152,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const OrtTensor const ProviderOptions options{ {tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)}, + {tensorrt::provider_option_names::kHasUserComputeStream, MakeStringWithClassicLocale(info.has_user_compute_stream)}, {tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.trt_max_partition_iterations)}, {tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.trt_min_subgraph_size)}, {tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.trt_max_workspace_size)}, diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc index ed5cca93f7..90aeeb64c9 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc @@ -104,10 +104,23 @@ struct Tensorrt_Provider : Provider { return std::make_shared(info); } + /** + * This function will be called by the C API UpdateTensorRTProviderOptions(). + * + * Please note that it will reset the OrtProviderOptionsV2 instance first and then set up the provided provider options + * See TensorrtExecutionProviderInfo::FromProviderOptions() for more details + */ void UpdateProviderOptions(void* provider_options, const ProviderOptions& options) override { auto internal_options = onnxruntime::TensorrtExecutionProviderInfo::FromProviderOptions(options); auto& trt_options = *reinterpret_cast(provider_options); trt_options.device_id = internal_options.device_id; + + // The 'has_user_compute_stream' of the OrtTensorRTProviderOptionsV2 instance can be set by C API UpdateTensorRTProviderOptionsWithValue() as well + // We only set the 'has_user_compute_stream' of the OrtTensorRTProviderOptionsV2 instance if it is provided in options + if (options.find("has_user_compute_stream") != options.end()) { + trt_options.has_user_compute_stream = internal_options.has_user_compute_stream; + } + trt_options.trt_max_partition_iterations = internal_options.max_partition_iterations; trt_options.trt_min_subgraph_size = internal_options.min_subgraph_size; trt_options.trt_max_workspace_size = internal_options.max_workspace_size; diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index e6e17390dd..808e7c3c3c 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -2711,6 +2711,8 @@ static constexpr OrtApi ort_api_1_to_16 = { &OrtApis::ReleaseROCMProviderOptions, &OrtApis::CreateAndRegisterAllocatorV2, &OrtApis::RunAsync, + &OrtApis::UpdateTensorRTProviderOptionsWithValue, + &OrtApis::GetTensorRTProviderOptionsByName, }; // OrtApiBase can never change as there is no way to know what version of OrtApiBase is returned by OrtGetApiBase. diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index 6a18581bf6..4cefe3fb96 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -485,4 +485,6 @@ ORT_API_STATUS_IMPL(RunAsync, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOpt _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, _Inout_updates_all_(output_names_len) OrtValue** outputs, _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); +ORT_API_STATUS_IMPL(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); +ORT_API_STATUS_IMPL(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); } // namespace OrtApis diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index be549f5e66..fe3fae718d 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1826,6 +1826,49 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString, _In_ const OrtT API_IMPL_END } +ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptionsWithValue, + _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_ const char* key, + _In_ void* value) { + API_IMPL_BEGIN +#ifdef USE_TENSORRT + // current provider option that has pointer data type (excluding const char*) is 'user_compute_stream' + if (strcmp(key, "user_compute_stream") == 0) { + tensorrt_options->has_user_compute_stream = 1; + tensorrt_options->user_compute_stream = value; + } + return nullptr; +#else + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(value); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsByName, + _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_ const char* key, + _Outptr_ void** ptr) { + API_IMPL_BEGIN +#ifdef USE_TENSORRT + // current provider option that has pointer data type (excluding const char*) is 'user_compute_stream' + if (strcmp(key, "user_compute_stream") == 0) { + *ptr = tensorrt_options->user_compute_stream; + } else { + *ptr = nullptr; + } + return nullptr; +#else + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(ptr); + return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build."); +#endif + API_IMPL_END +} + ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) { #ifdef USE_TENSORRT if (ptr != nullptr) { diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 8b32ec0571..bfd66463eb 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -289,6 +289,26 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString, return CreateNotEnabledStatus("TensorRT"); } +ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptionsWithValue, + _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_ const char* key, + _In_ void* value) { + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(value); + return CreateNotEnabledStatus("TensorRT"); +} + +ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsByName, + _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_ const char* key, + _Outptr_ void** ptr) { + ORT_UNUSED_PARAMETER(tensorrt_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(ptr); + return CreateNotEnabledStatus("TensorRT"); +} + ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) { ORT_UNUSED_PARAMETER(ptr); } diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index f0457facd8..fd50a1e4bd 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -2754,12 +2754,21 @@ TEST_P(CApiTensorRTTest, TestConfigureTensorRTProviderOptions) { ASSERT_TRUE(api.CreateTensorRTProviderOptions(&trt_options) == nullptr); std::unique_ptr rel_trt_options(trt_options, api.ReleaseTensorRTProviderOptions); + // Only test updating provider option with user provided compute stream + cudaStream_t compute_stream = nullptr; + void* user_compute_stream = nullptr; + cudaStreamCreateWithFlags(&compute_stream, cudaStreamNonBlocking); + ASSERT_TRUE(api.UpdateTensorRTProviderOptionsWithValue(rel_trt_options.get(), "user_compute_stream", compute_stream) == nullptr); + ASSERT_TRUE(api.GetTensorRTProviderOptionsByName(rel_trt_options.get(), "user_compute_stream", &user_compute_stream) == nullptr); + ASSERT_TRUE(user_compute_stream == (void*)compute_stream); + cudaStreamDestroy(compute_stream); + const char* engine_cache_path = "./trt_engine_folder"; - std::vector keys{"device_id", "trt_fp16_enable", "trt_int8_enable", "trt_engine_cache_enable", + std::vector keys{"device_id", "has_user_compute_stream", "trt_fp16_enable", "trt_int8_enable", "trt_engine_cache_enable", "trt_engine_cache_path", option_name.c_str()}; - std::vector values{"0", "1", "0", "1", + std::vector values{"0", "0", "1", "0", "1", engine_cache_path, option_value.c_str()}; ASSERT_TRUE(api.UpdateTensorRTProviderOptions(rel_trt_options.get(), keys.data(), values.data(), keys.size()) == nullptr); @@ -3422,4 +3431,4 @@ TEST(CApiTest, RunAsyncFail) { Ort::RunOptions run_options; EXPECT_THROW(session.RunAsync(run_options, input_names, input_tensors, 1, output_names, output_values, 1, CallbackFail, nullptr), std::exception); -} \ No newline at end of file +}