saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add API for updating TRT EP provider option user compute stream (#16965)

Browse source 
Add a generic `UpdateTensorRTProviderOptionsWithValue()` C API to update
TensorRT provider options where its data type is pointer that can't be
represented by string.
This commit is contained in:
Chi Lo 2023-08-04 15:14:43 -07:00 committed by GitHub
parent 9ea0a3129b
commit fc8003349e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 122 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
include/onnxruntime/core/session/onnxruntime_c_api.h
View file

@ -4346,6 +4346,32 @@ struct OrtApi {
_In_reads_(output_names_len) const char* const* output_names, size_t output_names_len,
_Inout_updates_all_(output_names_len) OrtValue** output,
_In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data);
/**
* Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'.
* If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions.
*
* Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer.
*
* \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance
* \param key - Name of the provider option
* \param value - A pointer to the instance that will be assigned to this provider option
*
* \since Version 1.16.
*/
ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value);
/**
* Get TensorRT EP provider option where its data type is pointer.
* If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString.
*
* \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance
* \param key - Name of the provider option
* \param ptr - A pointer to the instance that is kept by the provider option
*
* \since Version 1.16.
*/
ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr);
};
/*

4
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc
View file

@ -13,6 +13,7 @@ namespace onnxruntime {
namespace tensorrt {
namespace provider_option_names {
constexpr const char* kDeviceId = "device_id";
constexpr const char* kHasUserComputeStream = "has_user_compute_stream";
constexpr const char* kMaxPartitionIterations = "trt_max_partition_iterations";
constexpr const char* kMinSubgraphSize = "trt_min_subgraph_size";
constexpr const char* kMaxWorkspaceSize = "trt_max_workspace_size";
@ -64,6 +65,7 @@ TensorrtExecutionProviderInfo TensorrtExecutionProviderInfo::FromProviderOptions
return Status::OK();
})
.AddAssignmentToReference(tensorrt::provider_option_names::kMaxPartitionIterations, info.max_partition_iterations)
.AddAssignmentToReference(tensorrt::provider_option_names::kHasUserComputeStream, info.has_user_compute_stream)
.AddAssignmentToReference(tensorrt::provider_option_names::kMinSubgraphSize, info.min_subgraph_size)
.AddAssignmentToReference(tensorrt::provider_option_names::kMaxWorkspaceSize, info.max_workspace_size)
.AddAssignmentToReference(tensorrt::provider_option_names::kFp16Enable, info.fp16_enable)
@ -102,6 +104,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const TensorrtE
const ProviderOptions options{
{tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)},
{tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.max_partition_iterations)},
{tensorrt::provider_option_names::kHasUserComputeStream, MakeStringWithClassicLocale(info.has_user_compute_stream)},
{tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.min_subgraph_size)},
{tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.max_workspace_size)},
{tensorrt::provider_option_names::kFp16Enable, MakeStringWithClassicLocale(info.fp16_enable)},
@ -149,6 +152,7 @@ ProviderOptions TensorrtExecutionProviderInfo::ToProviderOptions(const OrtTensor
const ProviderOptions options{
{tensorrt::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)},
{tensorrt::provider_option_names::kHasUserComputeStream, MakeStringWithClassicLocale(info.has_user_compute_stream)},
{tensorrt::provider_option_names::kMaxPartitionIterations, MakeStringWithClassicLocale(info.trt_max_partition_iterations)},
{tensorrt::provider_option_names::kMinSubgraphSize, MakeStringWithClassicLocale(info.trt_min_subgraph_size)},
{tensorrt::provider_option_names::kMaxWorkspaceSize, MakeStringWithClassicLocale(info.trt_max_workspace_size)},

13
onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.cc
View file

@ -104,10 +104,23 @@ struct Tensorrt_Provider : Provider {
return std::make_shared<TensorrtProviderFactory>(info);
}
/**
* This function will be called by the C API UpdateTensorRTProviderOptions().
*
* Please note that it will reset the OrtProviderOptionsV2 instance first and then set up the provided provider options
* See TensorrtExecutionProviderInfo::FromProviderOptions() for more details
*/
void UpdateProviderOptions(void* provider_options, const ProviderOptions& options) override {
auto internal_options = onnxruntime::TensorrtExecutionProviderInfo::FromProviderOptions(options);
auto& trt_options = *reinterpret_cast<OrtTensorRTProviderOptionsV2*>(provider_options);
trt_options.device_id = internal_options.device_id;
// The 'has_user_compute_stream' of the OrtTensorRTProviderOptionsV2 instance can be set by C API UpdateTensorRTProviderOptionsWithValue() as well
// We only set the 'has_user_compute_stream' of the OrtTensorRTProviderOptionsV2 instance if it is provided in options
if (options.find("has_user_compute_stream") != options.end()) {
trt_options.has_user_compute_stream = internal_options.has_user_compute_stream;
}
trt_options.trt_max_partition_iterations = internal_options.max_partition_iterations;
trt_options.trt_min_subgraph_size = internal_options.min_subgraph_size;
trt_options.trt_max_workspace_size = internal_options.max_workspace_size;

2
onnxruntime/core/session/onnxruntime_c_api.cc
View file

@ -2711,6 +2711,8 @@ static constexpr OrtApi ort_api_1_to_16 = {
&OrtApis::ReleaseROCMProviderOptions,
&OrtApis::CreateAndRegisterAllocatorV2,
&OrtApis::RunAsync,
&OrtApis::UpdateTensorRTProviderOptionsWithValue,
&OrtApis::GetTensorRTProviderOptionsByName,
};
// OrtApiBase can never change as there is no way to know what version of OrtApiBase is returned by OrtGetApiBase.

2
onnxruntime/core/session/ort_apis.h
View file

@ -485,4 +485,6 @@ ORT_API_STATUS_IMPL(RunAsync, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOpt
_In_reads_(output_names_len) const char* const* output_names, size_t output_names_len,
_Inout_updates_all_(output_names_len) OrtValue** outputs,
_In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data);
ORT_API_STATUS_IMPL(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value);
ORT_API_STATUS_IMPL(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr);
} // namespace OrtApis

43
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -1826,6 +1826,49 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString, _In_ const OrtT
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptionsWithValue,
_Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options,
_In_ const char* key,
_In_ void* value) {
API_IMPL_BEGIN
#ifdef USE_TENSORRT
// current provider option that has pointer data type (excluding const char*) is 'user_compute_stream'
if (strcmp(key, "user_compute_stream") == 0) {
tensorrt_options->has_user_compute_stream = 1;
tensorrt_options->user_compute_stream = value;
}
return nullptr;
#else
ORT_UNUSED_PARAMETER(tensorrt_options);
ORT_UNUSED_PARAMETER(key);
ORT_UNUSED_PARAMETER(value);
return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build.");
#endif
API_IMPL_END
}
ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsByName,
_In_ const OrtTensorRTProviderOptionsV2* tensorrt_options,
_In_ const char* key,
_Outptr_ void** ptr) {
API_IMPL_BEGIN
#ifdef USE_TENSORRT
// current provider option that has pointer data type (excluding const char*) is 'user_compute_stream'
if (strcmp(key, "user_compute_stream") == 0) {
*ptr = tensorrt_options->user_compute_stream;
} else {
*ptr = nullptr;
}
return nullptr;
#else
ORT_UNUSED_PARAMETER(tensorrt_options);
ORT_UNUSED_PARAMETER(key);
ORT_UNUSED_PARAMETER(ptr);
return CreateStatus(ORT_FAIL, "TensorRT execution provider is not enabled in this build.");
#endif
API_IMPL_END
}
ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) {
#ifdef USE_TENSORRT
if (ptr != nullptr) {

20
onnxruntime/core/session/provider_registration.cc
View file

@ -289,6 +289,26 @@ ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsAsString,
return CreateNotEnabledStatus("TensorRT");
}
ORT_API_STATUS_IMPL(OrtApis::UpdateTensorRTProviderOptionsWithValue,
_Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options,
_In_ const char* key,
_In_ void* value) {
ORT_UNUSED_PARAMETER(tensorrt_options);
ORT_UNUSED_PARAMETER(key);
ORT_UNUSED_PARAMETER(value);
return CreateNotEnabledStatus("TensorRT");
}
ORT_API_STATUS_IMPL(OrtApis::GetTensorRTProviderOptionsByName,
_In_ const OrtTensorRTProviderOptionsV2* tensorrt_options,
_In_ const char* key,
_Outptr_ void** ptr) {
ORT_UNUSED_PARAMETER(tensorrt_options);
ORT_UNUSED_PARAMETER(key);
ORT_UNUSED_PARAMETER(ptr);
return CreateNotEnabledStatus("TensorRT");
}
ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* ptr) {
ORT_UNUSED_PARAMETER(ptr);
}

15
onnxruntime/test/shared_lib/test_inference.cc
View file

@ -2754,12 +2754,21 @@ TEST_P(CApiTensorRTTest, TestConfigureTensorRTProviderOptions) {
ASSERT_TRUE(api.CreateTensorRTProviderOptions(&trt_options) == nullptr);
std::unique_ptr<OrtTensorRTProviderOptionsV2, decltype(api.ReleaseTensorRTProviderOptions)> rel_trt_options(trt_options, api.ReleaseTensorRTProviderOptions);
// Only test updating provider option with user provided compute stream
cudaStream_t compute_stream = nullptr;
void* user_compute_stream = nullptr;
cudaStreamCreateWithFlags(&compute_stream, cudaStreamNonBlocking);
ASSERT_TRUE(api.UpdateTensorRTProviderOptionsWithValue(rel_trt_options.get(), "user_compute_stream", compute_stream) == nullptr);
ASSERT_TRUE(api.GetTensorRTProviderOptionsByName(rel_trt_options.get(), "user_compute_stream", &user_compute_stream) == nullptr);
ASSERT_TRUE(user_compute_stream == (void*)compute_stream);
cudaStreamDestroy(compute_stream);
const char* engine_cache_path = "./trt_engine_folder";
std::vector<const char*> keys{"device_id", "trt_fp16_enable", "trt_int8_enable", "trt_engine_cache_enable",
std::vector<const char*> keys{"device_id", "has_user_compute_stream", "trt_fp16_enable", "trt_int8_enable", "trt_engine_cache_enable",
"trt_engine_cache_path", option_name.c_str()};
std::vector<const char*> values{"0", "1", "0", "1",
std::vector<const char*> values{"0", "0", "1", "0", "1",
engine_cache_path, option_value.c_str()};
ASSERT_TRUE(api.UpdateTensorRTProviderOptions(rel_trt_options.get(), keys.data(), values.data(), keys.size()) == nullptr);
@ -3422,4 +3431,4 @@ TEST(CApiTest, RunAsyncFail) {
Ort::RunOptions run_options;
EXPECT_THROW(session.RunAsync(run_options, input_names, input_tensors, 1, output_names, output_values, 1, CallbackFail, nullptr), std::exception);
}
}