mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-26 19:52:38 +00:00
Enable CUDA provider option configuration for C# (#10188)
This commit is contained in:
Hariharan Seshadri
GitHub
parent
08f512b25e
commit
0552a47ec2
18 changed files with 683 additions and 15 deletions
|
|
@ -235,6 +235,11 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
public IntPtr SetGlobalCustomJoinThreadFn;
|
||||
public IntPtr SynchronizeBoundInputs;
|
||||
public IntPtr SynchronizeBoundOutputs;
|
||||
public IntPtr SessionOptionsAppendExecutionProvider_CUDA_V2;
|
||||
public IntPtr CreateCUDAProviderOptions;
|
||||
public IntPtr UpdateCUDAProviderOptions;
|
||||
public IntPtr GetCUDAProviderOptionsAsString;
|
||||
public IntPtr ReleaseCUDAProviderOptions;
|
||||
}
|
||||
|
||||
internal static class NativeMethods
|
||||
|
|
@ -397,6 +402,15 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
OrtUpdateTensorRTProviderOptions = (DOrtUpdateTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.UpdateTensorRTProviderOptions, typeof(DOrtUpdateTensorRTProviderOptions));
|
||||
OrtGetTensorRTProviderOptionsAsString = (DOrtGetTensorRTProviderOptionsAsString)Marshal.GetDelegateForFunctionPointer(api_.GetTensorRTProviderOptionsAsString, typeof(DOrtGetTensorRTProviderOptionsAsString));
|
||||
OrtReleaseTensorRTProviderOptions = (DOrtReleaseTensorRTProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseTensorRTProviderOptions, typeof(DOrtReleaseTensorRTProviderOptions));
|
||||
|
||||
SessionOptionsAppendExecutionProvider_CUDA = (DSessionOptionsAppendExecutionProvider_CUDA)Marshal.GetDelegateForFunctionPointer(
|
||||
api_.SessionOptionsAppendExecutionProvider_CUDA, typeof(DSessionOptionsAppendExecutionProvider_CUDA));
|
||||
SessionOptionsAppendExecutionProvider_CUDA_V2 = (DSessionOptionsAppendExecutionProvider_CUDA_V2)Marshal.GetDelegateForFunctionPointer(
|
||||
api_.SessionOptionsAppendExecutionProvider_CUDA_V2, typeof(DSessionOptionsAppendExecutionProvider_CUDA_V2));
|
||||
OrtCreateCUDAProviderOptions = (DOrtCreateCUDAProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.CreateCUDAProviderOptions, typeof(DOrtCreateCUDAProviderOptions));
|
||||
OrtUpdateCUDAProviderOptions = (DOrtUpdateCUDAProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.UpdateCUDAProviderOptions, typeof(DOrtUpdateCUDAProviderOptions));
|
||||
OrtGetCUDAProviderOptionsAsString = (DOrtGetCUDAProviderOptionsAsString)Marshal.GetDelegateForFunctionPointer(api_.GetCUDAProviderOptionsAsString, typeof(DOrtGetCUDAProviderOptionsAsString));
|
||||
OrtReleaseCUDAProviderOptions = (DOrtReleaseCUDAProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseCUDAProviderOptions, typeof(DOrtReleaseCUDAProviderOptions));
|
||||
}
|
||||
|
||||
[DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
|
||||
|
|
@ -469,6 +483,49 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
public delegate void DOrtReleaseTensorRTProviderOptions(IntPtr /*(OrtTensorRTProviderOptions*)*/ trtProviderOptionsInstance);
|
||||
public static DOrtReleaseTensorRTProviderOptions OrtReleaseTensorRTProviderOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Creates native OrtCUDAProviderOptions instance
|
||||
/// </summary>
|
||||
/// <param name="cudaProviderOptionsInstance">(output) native instance of OrtCUDAProviderOptions</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate IntPtr /* OrtStatus* */DOrtCreateCUDAProviderOptions(
|
||||
out IntPtr /*(OrtCUDAProviderOptions**)*/ cudaProviderOptionsInstance);
|
||||
public static DOrtCreateCUDAProviderOptions OrtCreateCUDAProviderOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Updates native OrtCUDAProviderOptions instance using given key/value pairs
|
||||
/// </summary>
|
||||
/// <param name="cudaProviderOptionsInstance">native instance of OrtCUDAProviderOptions</param>
|
||||
/// <param name="providerOptionsKeys">configuration keys of OrtCUDAProviderOptions</param>
|
||||
/// <param name="providerOptionsValues">configuration values of OrtCUDAProviderOptions</param>
|
||||
/// <param name="numKeys">number of configuration keys</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate IntPtr /* OrtStatus* */DOrtUpdateCUDAProviderOptions(
|
||||
IntPtr /*(OrtCUDAProviderOptions*)*/ cudaProviderOptionsInstance,
|
||||
IntPtr[] /*(const char* const *)*/ providerOptionsKeys,
|
||||
IntPtr[] /*(const char* const *)*/ providerOptionsValues,
|
||||
UIntPtr /*(size_t)*/ numKeys);
|
||||
public static DOrtUpdateCUDAProviderOptions OrtUpdateCUDAProviderOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Get native OrtCUDAProviderOptionsV2 in serialized string
|
||||
/// </summary>
|
||||
/// <param name="allocator">instance of OrtAllocator</param>
|
||||
/// <param name="ptr">is a UTF-8 null terminated string allocated using 'allocator'</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate IntPtr /* OrtStatus* */DOrtGetCUDAProviderOptionsAsString(
|
||||
IntPtr /*(OrtCUDAProviderOptionsV2**)*/ cudaProviderOptionsInstance,
|
||||
IntPtr /*(OrtAllocator*)*/ allocator,
|
||||
out IntPtr /*(char**)*/ptr);
|
||||
public static DOrtGetCUDAProviderOptionsAsString OrtGetCUDAProviderOptionsAsString;
|
||||
|
||||
/// <summary>
|
||||
/// Releases native OrtCUDAProviderOptions instance
|
||||
/// </summary>
|
||||
/// <param name="cudaProviderOptionsInstance">native instance of OrtCUDAProviderOptions to be released</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate void DOrtReleaseCUDAProviderOptions(IntPtr /*(OrtCUDAProviderOptions*)*/ cudaProviderOptionsInstance);
|
||||
public static DOrtReleaseCUDAProviderOptions OrtReleaseCUDAProviderOptions;
|
||||
#endregion
|
||||
|
||||
#region Status API
|
||||
|
|
@ -820,6 +877,30 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
|
||||
public static DSessionOptionsAppendExecutionProvider_TensorRT_V2 SessionOptionsAppendExecutionProvider_TensorRT_V2;
|
||||
|
||||
/// <summary>
|
||||
/// Append a CUDA EP instance (configured based on given provider options) to the native OrtSessionOptions instance
|
||||
/// </summary>
|
||||
/// <param name="options">Native OrtSessionOptions instance</param>
|
||||
/// <param name="cudaProviderOptions">Native OrtCUDAProviderOptions instance</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_CUDA(
|
||||
IntPtr /*(OrtSessionOptions*)*/ options,
|
||||
IntPtr /*(const OrtCUDAProviderOptions*)*/ cudaProviderOptions);
|
||||
|
||||
public static DSessionOptionsAppendExecutionProvider_CUDA SessionOptionsAppendExecutionProvider_CUDA;
|
||||
|
||||
/// <summary>
|
||||
/// Append a CUDA EP instance (configured based on given provider options) to the native OrtSessionOptions instance
|
||||
/// </summary>
|
||||
/// <param name="options">Native OrtSessionOptions instance</param>
|
||||
/// <param name="cudaProviderOptions">Native OrtCUDAProviderOptionsV2 instance</param>
|
||||
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
|
||||
public delegate IntPtr /*(OrtStatus*)*/DSessionOptionsAppendExecutionProvider_CUDA_V2(
|
||||
IntPtr /*(OrtSessionOptions*)*/ options,
|
||||
IntPtr /*(const OrtCUDAProviderOptionsV2*)*/ cudaProviderOptions);
|
||||
|
||||
public static DSessionOptionsAppendExecutionProvider_CUDA_V2 SessionOptionsAppendExecutionProvider_CUDA_V2;
|
||||
|
||||
/// <summary>
|
||||
/// Free Dimension override (by denotation)
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -120,6 +120,104 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
#endregion
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Holds the options for configuring a CUDA Execution Provider instance
|
||||
/// </summary>
|
||||
public class OrtCUDAProviderOptions : SafeHandle
|
||||
{
|
||||
internal IntPtr Handle
|
||||
{
|
||||
get
|
||||
{
|
||||
return handle;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#region Constructor
|
||||
|
||||
/// <summary>
|
||||
/// Constructs an empty OrtCUDAroviderOptions instance
|
||||
/// </summary>
|
||||
public OrtCUDAProviderOptions() : base(IntPtr.Zero, true)
|
||||
{
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateCUDAProviderOptions(out handle));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Public Methods
|
||||
|
||||
/// <summary>
|
||||
/// Get CUDA EP provider options
|
||||
/// </summary>
|
||||
/// <returns> return C# UTF-16 encoded string </returns>
|
||||
public string GetOptions()
|
||||
{
|
||||
var allocator = OrtAllocator.DefaultInstance;
|
||||
|
||||
// Process provider options string
|
||||
IntPtr providerOptions = IntPtr.Zero;
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetCUDAProviderOptionsAsString(handle, allocator.Pointer, out providerOptions));
|
||||
using (var ortAllocation = new OrtMemoryAllocation(allocator, providerOptions, 0))
|
||||
{
|
||||
return NativeOnnxValueHelper.StringFromNativeUtf8(providerOptions);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the configuration knobs of OrtCUDAProviderOptions that will eventually be used to configure a CUDA EP
|
||||
/// Please refer to the following on different key/value pairs to configure a CUDA EP and their meaning:
|
||||
/// https://www.onnxruntime.ai/docs/reference/execution-providers/CUDA-ExecutionProvider.html
|
||||
/// </summary>
|
||||
/// <param name="providerOptions">key/value pairs used to configure a CUDA Execution Provider</param>
|
||||
public void UpdateOptions(Dictionary<string, string> providerOptions)
|
||||
{
|
||||
|
||||
using (var cleanupList = new DisposableList<IDisposable>())
|
||||
{
|
||||
var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Keys.ToArray(), n => n, cleanupList);
|
||||
var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8(providerOptions.Values.ToArray(), n => n, cleanupList);
|
||||
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateCUDAProviderOptions(handle, keysArray, valuesArray, (UIntPtr)providerOptions.Count));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Public Properties
|
||||
|
||||
/// <summary>
|
||||
/// Overrides SafeHandle.IsInvalid
|
||||
/// </summary>
|
||||
/// <value>returns true if handle is equal to Zero</value>
|
||||
public override bool IsInvalid { get { return handle == IntPtr.Zero; } }
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Methods
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
#region SafeHandle
|
||||
/// <summary>
|
||||
/// Overrides SafeHandle.ReleaseHandle() to properly dispose of
|
||||
/// the native instance of OrtCUDAProviderOptions
|
||||
/// </summary>
|
||||
/// <returns>always returns true</returns>
|
||||
protected override bool ReleaseHandle()
|
||||
{
|
||||
NativeMethods.OrtReleaseCUDAProviderOptions(handle);
|
||||
handle = IntPtr.Zero;
|
||||
return true;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// This helper class contains methods to handle values of provider options
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -66,6 +66,28 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
return options;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A helper method to construct a SessionOptions object for CUDA execution provider.
|
||||
/// Use only if CUDA is installed and you have the onnxruntime package specific to this Execution Provider.
|
||||
/// </summary>
|
||||
/// <param name="cudaProviderOptions">CUDA EP provider options</param>
|
||||
/// <returns>A SessionsOptions() object configured for execution on provider options</returns>
|
||||
public static SessionOptions MakeSessionOptionWithCudaProvider(OrtCUDAProviderOptions cudaProviderOptions)
|
||||
{
|
||||
CheckCudaExecutionProviderDLLs();
|
||||
SessionOptions options = new SessionOptions();
|
||||
try
|
||||
{
|
||||
options.AppendExecutionProvider_CUDA(cudaProviderOptions);
|
||||
return options;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
options.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A helper method to construct a SessionOptions object for TensorRT execution.
|
||||
/// Use only if CUDA/TensorRT are installed and you have the onnxruntime package specific to this Execution Provider.
|
||||
|
|
@ -191,6 +213,20 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Append a CUDA EP instance (based on specified configuration) to the SessionOptions instance.
|
||||
/// Use only if you have the onnxruntime package specific to this Execution Provider.
|
||||
/// </summary>
|
||||
/// <param name="cudaProviderOptions">CUDA EP provider options</param>
|
||||
public void AppendExecutionProvider_CUDA(OrtCUDAProviderOptions cudaProviderOptions)
|
||||
{
|
||||
#if __MOBILE__
|
||||
throw new NotSupportedException("The CUDA Execution Provider is not supported in this build");
|
||||
#else
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_CUDA_V2(handle, cudaProviderOptions.Handle));
|
||||
#endif
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Use only if you have the onnxruntime package specific to this Execution Provider.
|
||||
/// </summary>
|
||||
|
|
@ -245,7 +281,7 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
#if __MOBILE__
|
||||
throw new NotSupportedException("The TensorRT Execution Provider is not supported in this build");
|
||||
#else
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT(handle, trtProviderOptions.Handle));
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider_TensorRT_V2(handle, trtProviderOptions.Handle));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,68 @@ namespace Microsoft.ML.OnnxRuntime.Tests
|
|||
}
|
||||
}
|
||||
|
||||
#if USE_CUDA
|
||||
|
||||
[Fact(DisplayName = "TestCUDAProviderOptions")]
|
||||
private void TestCUDAProviderOptions()
|
||||
{
|
||||
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
|
||||
|
||||
using (var cleanUp = new DisposableListTest<IDisposable>())
|
||||
{
|
||||
var cudaProviderOptions = new OrtCUDAProviderOptions();
|
||||
cleanUp.Add(cudaProviderOptions);
|
||||
|
||||
var providerOptionsDict = new Dictionary<string, string>();
|
||||
providerOptionsDict["device_id"] = "0";
|
||||
providerOptionsDict["gpu_mem_limit"] = "20971520";
|
||||
providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested";
|
||||
providerOptionsDict["cudnn_conv_algo_search"] = "DEFAULT";
|
||||
providerOptionsDict["do_copy_in_default_stream"] = "1";
|
||||
providerOptionsDict["cudnn_conv_use_max_workspace"] = "1";
|
||||
cudaProviderOptions.UpdateOptions(providerOptionsDict);
|
||||
|
||||
var resultProviderOptionsDict = new Dictionary<string, string>();
|
||||
ProviderOptionsValueHelper.StringToDict(cudaProviderOptions.GetOptions(), resultProviderOptionsDict);
|
||||
|
||||
// test provider options configuration
|
||||
string value;
|
||||
value = resultProviderOptionsDict["device_id"];
|
||||
Assert.Equal("0", value);
|
||||
value = resultProviderOptionsDict["gpu_mem_limit"];
|
||||
Assert.Equal("20971520", value);
|
||||
value = resultProviderOptionsDict["arena_extend_strategy"];
|
||||
Assert.Equal("kSameAsRequested", value);
|
||||
value = resultProviderOptionsDict["cudnn_conv_algo_search"];
|
||||
Assert.Equal("DEFAULT", value);
|
||||
value = resultProviderOptionsDict["do_copy_in_default_stream"];
|
||||
Assert.Equal("1", value);
|
||||
value = resultProviderOptionsDict["cudnn_conv_use_max_workspace"];
|
||||
Assert.Equal("1", value);
|
||||
|
||||
// test correctness of provider options
|
||||
SessionOptions options = SessionOptions.MakeSessionOptionWithCudaProvider(cudaProviderOptions);
|
||||
cleanUp.Add(options);
|
||||
|
||||
var session = new InferenceSession(modelPath, options);
|
||||
cleanUp.Add(session);
|
||||
|
||||
var inputMeta = session.InputMetadata;
|
||||
var container = new List<NamedOnnxValue>();
|
||||
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
|
||||
foreach (var name in inputMeta.Keys)
|
||||
{
|
||||
Assert.Equal(typeof(float), inputMeta[name].ElementType);
|
||||
Assert.True(inputMeta[name].IsTensor);
|
||||
var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
|
||||
container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
|
||||
}
|
||||
|
||||
session.Run(container);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#if USE_TENSORRT
|
||||
[Fact(DisplayName = "CanRunInferenceOnAModelWithTensorRT")]
|
||||
private void CanRunInferenceOnAModelWithTensorRT()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
#include "core/session/onnxruntime_c_api.h"
|
||||
|
||||
namespace onnxruntime {
|
||||
#ifdef USE_TENSORRT
|
||||
#if defined(USE_TENSORRT) || defined(USE_CUDA)
|
||||
static char* StrDup(const std::string& str, _Inout_ OrtAllocator* allocator) {
|
||||
char* output_string = reinterpret_cast<char*>(allocator->Alloc(allocator, str.size() + 1));
|
||||
memcpy(output_string, str.c_str(), str.size());
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "onnxruntime_c_api.h"
|
||||
#include "core/framework/arena_extend_strategy.h"
|
||||
|
||||
/// <summary>
|
||||
/// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA_V2.
|
||||
/// Please note that this struct is *similar* to OrtCUDAProviderOptions but only to be used internally.
|
||||
/// Going forward, new cuda provider options are to be supported via this struct and usage of the publicly defined
|
||||
/// OrtCUDAProviderOptions will be deprecated over time.
|
||||
/// User can only get the instance of OrtCUDAProviderOptionsV2 via CreateCUDAProviderOptions.
|
||||
/// </summary>
|
||||
struct OrtCUDAProviderOptionsV2 {
|
||||
int device_id; // cuda device id.
|
||||
int has_user_compute_stream; // indicator of user specified CUDA compute stream.
|
||||
void* user_compute_stream; // user specified CUDA compute stream.
|
||||
int do_copy_in_default_stream; // flag specifying if the default stream is to be used for copying.
|
||||
OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn algo search enum.
|
||||
size_t gpu_mem_limit; // BFC Arena memory limit for CUDA.
|
||||
// (will be overridden by contents of `default_memory_arena_cfg` is it exists)
|
||||
onnxruntime::ArenaExtendStrategy arena_extend_strategy; // BFC Arena extension strategy.
|
||||
// (will be overridden by contents of `default_memory_arena_cfg` is it exists)
|
||||
OrtArenaCfg* default_memory_arena_cfg; // BFC Arena config flags.
|
||||
int cudnn_conv_use_max_workspace; // flag specifying if maximum workspace can be used in cudnn conv algo search.
|
||||
};
|
||||
|
|
@ -5,8 +5,8 @@
|
|||
|
||||
/// <summary>
|
||||
/// Options for the TensorRT provider that are passed to SessionOptionsAppendExecutionProvider_TensorRT_V2.
|
||||
/// Please note that this sturct is identical to OrtTensorRTProviderOptions but only to be used internally.
|
||||
// User can only get the instance of OrtTensorRTProviderOptionsV2 via CreateTensorRTProviderOptions.
|
||||
/// Please note that this struct is identical to OrtTensorRTProviderOptions but only to be used internally.
|
||||
/// User can only get the instance of OrtTensorRTProviderOptionsV2 via CreateTensorRTProviderOptions.
|
||||
/// </summary>
|
||||
struct OrtTensorRTProviderOptionsV2 {
|
||||
int device_id; // cuda device id.
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
*
|
||||
* This value is used by some API functions to behave as this version of the header expects.
|
||||
*/
|
||||
#define ORT_API_VERSION 10
|
||||
#define ORT_API_VERSION 11
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -256,6 +256,7 @@ ORT_RUNTIME_CLASS(ThreadingOptions);
|
|||
ORT_RUNTIME_CLASS(ArenaCfg);
|
||||
ORT_RUNTIME_CLASS(PrepackedWeightsContainer);
|
||||
ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2);
|
||||
ORT_RUNTIME_CLASS(CUDAProviderOptionsV2);
|
||||
|
||||
#ifdef _WIN32
|
||||
typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
|
||||
|
|
@ -536,7 +537,7 @@ ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION;
|
|||
*/
|
||||
typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param);
|
||||
|
||||
typedef const struct OrtCustomHandleType{ char __place_holder; }* OrtCustomThreadHandle;
|
||||
typedef const struct OrtCustomHandleType { char __place_holder; } * OrtCustomThreadHandle;
|
||||
|
||||
/** \brief Ort custom thread creation function
|
||||
*
|
||||
|
|
@ -3185,6 +3186,81 @@ struct OrtApi {
|
|||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*/
|
||||
ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr);
|
||||
|
||||
/// \name OrtSessionOptions
|
||||
/// @{
|
||||
|
||||
/** \brief Append CUDA execution provider to the session options
|
||||
*
|
||||
* If CUDA is not available (due to a non CUDA enabled build), this function will return failure.
|
||||
*
|
||||
* This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an
|
||||
* ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2
|
||||
* which must be created with OrtApi::CreateCUDAProviderOptions.
|
||||
*
|
||||
* For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions
|
||||
* as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions.
|
||||
* Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you.
|
||||
*
|
||||
* \param[in] options
|
||||
* \param[in] cuda_options
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*/
|
||||
ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2,
|
||||
_In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options);
|
||||
|
||||
/// @}
|
||||
/// \name OrtCUDAProviderOptionsV2
|
||||
/// @{
|
||||
|
||||
/** \brief Create an OrtCUDAProviderOptionsV2
|
||||
*
|
||||
* \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*/
|
||||
ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out);
|
||||
|
||||
/** \brief Set options in a CUDA Execution Provider.
|
||||
*
|
||||
* Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options
|
||||
* to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2
|
||||
* and value should be its related range.
|
||||
*
|
||||
* For example, key="device_id" and value="0"
|
||||
*
|
||||
* \param[in] cuda_options
|
||||
* \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys
|
||||
* \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values
|
||||
* \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays
|
||||
*
|
||||
* \snippet{doc} snippets.dox OrtStatus Return Value
|
||||
*/
|
||||
ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options,
|
||||
_In_reads_(num_keys) const char* const* provider_options_keys,
|
||||
_In_reads_(num_keys) const char* const* provider_options_values,
|
||||
_In_ size_t num_keys);
|
||||
|
||||
/**
|
||||
* Get serialized CUDA provider options string.
|
||||
*
|
||||
* For example, "device_id=0;arena_extend_strategy=0;......"
|
||||
*
|
||||
* \param cuda_options - OrtCUDAProviderOptionsV2 instance
|
||||
* \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions()
|
||||
* the specified allocator will be used to allocate continuous buffers for output strings and lengths.
|
||||
* \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it.
|
||||
*/
|
||||
ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr);
|
||||
|
||||
/** \brief Release an ::OrtCUDAProviderOptionsV2
|
||||
*
|
||||
* \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does
|
||||
*/
|
||||
void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input);
|
||||
|
||||
/// @}
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "core/providers/cuda/cuda_execution_provider_info.h"
|
||||
#include "core/providers/cuda/cuda_provider_options.h"
|
||||
|
||||
#include "core/common/make_string.h"
|
||||
#include "core/common/parse_string.h"
|
||||
|
|
@ -116,4 +117,17 @@ ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const CUDAExecution
|
|||
return options;
|
||||
}
|
||||
|
||||
ProviderOptions CUDAExecutionProviderInfo::ToProviderOptions(const OrtCUDAProviderOptionsV2& info) {
|
||||
const ProviderOptions options{
|
||||
{cuda::provider_option_names::kDeviceId, MakeStringWithClassicLocale(info.device_id)},
|
||||
{cuda::provider_option_names::kMemLimit, MakeStringWithClassicLocale(info.gpu_mem_limit)},
|
||||
{cuda::provider_option_names::kArenaExtendStrategy, EnumToName(*arena_extend_strategy_mapping, info.arena_extend_strategy)},
|
||||
{cuda::provider_option_names::kCudnnConvAlgoSearch, EnumToName(*ort_cudnn_conv_algo_search_mapping, info.cudnn_conv_algo_search)},
|
||||
{cuda::provider_option_names::kDoCopyInDefaultStream, MakeStringWithClassicLocale(info.do_copy_in_default_stream)},
|
||||
{cuda::provider_option_names::kCudnnConvUseMaxWorkspace, MakeStringWithClassicLocale(info.cudnn_conv_use_max_workspace)},
|
||||
};
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -54,5 +54,6 @@ struct CUDAExecutionProviderInfo {
|
|||
|
||||
static CUDAExecutionProviderInfo FromProviderOptions(const ProviderOptions& options);
|
||||
static ProviderOptions ToProviderOptions(const CUDAExecutionProviderInfo& info);
|
||||
static ProviderOptions ToProviderOptions(const OrtCUDAProviderOptionsV2& info);
|
||||
};
|
||||
} // namespace onnxruntime
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "core/providers/shared_library/provider_api.h"
|
||||
#include "core/providers/cuda/cuda_provider_factory_creator.h"
|
||||
#include "core/providers/cuda/cuda_provider_factory.h"
|
||||
#include "core/providers/cuda/cuda_provider_options.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
|
|
@ -186,21 +187,42 @@ struct CUDA_Provider : Provider {
|
|||
void* GetInfo() override { return &g_info; }
|
||||
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory(const void* void_params) override {
|
||||
auto params = reinterpret_cast<const OrtCUDAProviderOptions*>(void_params);
|
||||
auto params = reinterpret_cast<const OrtCUDAProviderOptionsV2*>(void_params);
|
||||
|
||||
CUDAExecutionProviderInfo info{};
|
||||
info.device_id = gsl::narrow<OrtDevice::DeviceId>(params->device_id);
|
||||
info.gpu_mem_limit = params->gpu_mem_limit;
|
||||
info.arena_extend_strategy = static_cast<onnxruntime::ArenaExtendStrategy>(params->arena_extend_strategy);
|
||||
info.arena_extend_strategy = params->arena_extend_strategy;
|
||||
info.cudnn_conv_algo_search = params->cudnn_conv_algo_search;
|
||||
info.do_copy_in_default_stream = params->do_copy_in_default_stream != 0;
|
||||
info.has_user_compute_stream = params->has_user_compute_stream != 0;
|
||||
info.user_compute_stream = params->user_compute_stream;
|
||||
info.default_memory_arena_cfg = params->default_memory_arena_cfg;
|
||||
info.cudnn_conv_use_max_workspace = params->cudnn_conv_use_max_workspace != 0;
|
||||
|
||||
return std::make_shared<CUDAProviderFactory>(info);
|
||||
}
|
||||
|
||||
void UpdateProviderOptions(void* provider_options, const ProviderOptions& options) override {
|
||||
auto internal_options = onnxruntime::CUDAExecutionProviderInfo::FromProviderOptions(options);
|
||||
auto& cuda_options = *reinterpret_cast<OrtCUDAProviderOptionsV2*>(provider_options);
|
||||
|
||||
cuda_options.device_id = internal_options.device_id;
|
||||
cuda_options.cudnn_conv_algo_search = internal_options.cudnn_conv_algo_search;
|
||||
cuda_options.gpu_mem_limit = internal_options.gpu_mem_limit;
|
||||
cuda_options.arena_extend_strategy = internal_options.arena_extend_strategy;
|
||||
cuda_options.do_copy_in_default_stream = internal_options.do_copy_in_default_stream;
|
||||
cuda_options.has_user_compute_stream = internal_options.has_user_compute_stream;
|
||||
cuda_options.user_compute_stream = internal_options.user_compute_stream;
|
||||
cuda_options.default_memory_arena_cfg = internal_options.default_memory_arena_cfg;
|
||||
cuda_options.cudnn_conv_use_max_workspace = internal_options.cudnn_conv_use_max_workspace;
|
||||
}
|
||||
|
||||
ProviderOptions GetProviderOptions(const void* provider_options) override {
|
||||
auto& options = *reinterpret_cast<const OrtCUDAProviderOptionsV2*>(provider_options);
|
||||
return onnxruntime::CUDAExecutionProviderInfo::ToProviderOptions(options);
|
||||
}
|
||||
|
||||
void Shutdown() override {
|
||||
Shutdown_DeleteRegistry();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2271,7 +2271,7 @@ Second example, if we wanted to add and remove some members, we'd do this:
|
|||
In GetApi we now make it return ort_api_3 for version 3.
|
||||
*/
|
||||
|
||||
static constexpr OrtApi ort_api_1_to_10 = {
|
||||
static constexpr OrtApi ort_api_1_to_11 = {
|
||||
// NOTE: The ordering of these fields MUST not change after that version has shipped since existing binaries depend on this ordering.
|
||||
|
||||
// Shipped as version 1 - DO NOT MODIFY (see above text for more information)
|
||||
|
|
@ -2511,10 +2511,15 @@ static constexpr OrtApi ort_api_1_to_10 = {
|
|||
&OrtApis::SetGlobalCustomThreadCreationOptions,
|
||||
&OrtApis::SetGlobalCustomJoinThreadFn,
|
||||
&OrtApis::SynchronizeBoundInputs,
|
||||
&OrtApis::SynchronizeBoundOutputs
|
||||
&OrtApis::SynchronizeBoundOutputs,
|
||||
// End of Version 10 - DO NOT MODIFY ABOVE (see above text for more information)
|
||||
|
||||
// Version 11 - In development, feel free to add/remove/rearrange here
|
||||
&OrtApis::SessionOptionsAppendExecutionProvider_CUDA_V2,
|
||||
&OrtApis::CreateCUDAProviderOptions,
|
||||
&OrtApis::UpdateCUDAProviderOptions,
|
||||
&OrtApis::GetCUDAProviderOptionsAsString,
|
||||
&OrtApis::ReleaseCUDAProviderOptions,
|
||||
};
|
||||
|
||||
// Asserts to do a some checks to ensure older Versions of the OrtApi never change (will detect an addition or deletion but not if they cancel out each other)
|
||||
|
|
@ -2527,17 +2532,19 @@ static_assert(offsetof(OrtApi, SetGlobalSpinControl) / sizeof(void*) == 149, "Si
|
|||
static_assert(offsetof(OrtApi, ReleaseArenaCfg) / sizeof(void*) == 157, "Size of version 6 API cannot change");
|
||||
static_assert(offsetof(OrtApi, GetCurrentGpuDeviceId) / sizeof(void*) == 161, "Size of version 7 API cannot change");
|
||||
static_assert(offsetof(OrtApi, CreateSessionFromArrayWithPrepackedWeightsContainer) / sizeof(void*) == 169, "Size of version 8 API cannot change");
|
||||
static_assert(offsetof(OrtApi, GetSparseTensorIndices) / sizeof(void*) == 191, "Size of version 9 API cannot change");
|
||||
static_assert(offsetof(OrtApi, SynchronizeBoundOutputs) / sizeof(void*) == 203, "Size of version 10 API cannot change");
|
||||
|
||||
// So that nobody forgets to finish an API version, this check will serve as a reminder:
|
||||
static_assert(std::string_view(ORT_VERSION) == "1.11.0", "ORT_Version change detected, please follow below steps to ensure OrtApi is updated properly");
|
||||
// 1. Update the hardcoded version string in above static_assert to silence it
|
||||
// 2. If there were any APIs added to ort_api_1_to_10 above:
|
||||
// 2. If there were any APIs added to ort_api_1_to_11 above:
|
||||
// a. Add the 'End of version #' markers (pattern above should be obvious)
|
||||
// b. Add a static_assert in the directly above list of version sizes to ensure nobody adds any more functions to the just shipped API version
|
||||
|
||||
ORT_API(const OrtApi*, OrtApis::GetApi, uint32_t version) {
|
||||
if (version >= 1 && version <= ORT_API_VERSION)
|
||||
return &ort_api_1_to_10;
|
||||
return &ort_api_1_to_11;
|
||||
|
||||
fprintf(stderr, "The given version [%u] is not supported, only version 1 to %u is supported in this build.\n",
|
||||
version, ORT_API_VERSION);
|
||||
|
|
|
|||
|
|
@ -327,4 +327,13 @@ ORT_API_STATUS_IMPL(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOp
|
|||
ORT_API_STATUS_IMPL(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn);
|
||||
ORT_API_STATUS_IMPL(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr);
|
||||
ORT_API_STATUS_IMPL(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr);
|
||||
ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_CUDA_V2,
|
||||
_In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options);
|
||||
ORT_API_STATUS_IMPL(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out);
|
||||
ORT_API_STATUS_IMPL(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options,
|
||||
_In_reads_(num_keys) const char* const* provider_options_keys,
|
||||
_In_reads_(num_keys) const char* const* provider_options_values,
|
||||
size_t num_keys);
|
||||
ORT_API_STATUS_IMPL(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr);
|
||||
ORT_API(void, ReleaseCUDAProviderOptions, _Frees_ptr_opt_ OrtCUDAProviderOptionsV2*);
|
||||
} // namespace OrtApis
|
||||
|
|
|
|||
|
|
@ -65,9 +65,10 @@ using IndexedSubGraph_MetaDef = IndexedSubGraph::MetaDef;
|
|||
#include "core/providers/cuda/cuda_provider_factory.h"
|
||||
#include "core/providers/rocm/rocm_provider_factory.h"
|
||||
#include "core/providers/dnnl/dnnl_provider_factory.h"
|
||||
#include "core/providers/tensorrt/tensorrt_provider_factory.h"
|
||||
#include "core/providers/openvino/openvino_provider_factory.h"
|
||||
#include "core/platform/tensorrt_provider_options.h"
|
||||
#include "core/providers/tensorrt/tensorrt_provider_factory.h"
|
||||
#include "core/providers/tensorrt/tensorrt_provider_options.h"
|
||||
#include "core/providers/cuda/cuda_provider_options.h"
|
||||
|
||||
// The filename extension for a shared library is different per platform
|
||||
#ifdef _WIN32
|
||||
|
|
@ -1076,7 +1077,33 @@ std::unique_ptr<IAllocator> CreateROCMPinnedAllocator(int16_t device_id, const c
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
// Adapter to convert the legacy OrtCUDAProviderOptions to the latest OrtCUDAProviderOptionsV2
|
||||
OrtCUDAProviderOptionsV2 OrtCUDAProviderOptionsToOrtCUDAProviderOptionsV2(const OrtCUDAProviderOptions* legacy_cuda_options) {
|
||||
OrtCUDAProviderOptionsV2 cuda_options_converted;
|
||||
|
||||
cuda_options_converted.device_id = legacy_cuda_options->device_id;
|
||||
cuda_options_converted.cudnn_conv_algo_search = legacy_cuda_options->cudnn_conv_algo_search;
|
||||
cuda_options_converted.gpu_mem_limit = legacy_cuda_options->gpu_mem_limit;
|
||||
cuda_options_converted.arena_extend_strategy = static_cast<onnxruntime::ArenaExtendStrategy>(legacy_cuda_options->arena_extend_strategy);
|
||||
cuda_options_converted.do_copy_in_default_stream = legacy_cuda_options->do_copy_in_default_stream;
|
||||
cuda_options_converted.has_user_compute_stream = legacy_cuda_options->has_user_compute_stream;
|
||||
cuda_options_converted.user_compute_stream = legacy_cuda_options->user_compute_stream;
|
||||
cuda_options_converted.default_memory_arena_cfg = legacy_cuda_options->default_memory_arena_cfg;
|
||||
// Use default value as this field is not available in OrtCUDAProviderOptions
|
||||
cuda_options_converted.cudnn_conv_use_max_workspace = 0;
|
||||
|
||||
return cuda_options_converted;
|
||||
}
|
||||
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options) {
|
||||
OrtCUDAProviderOptionsV2 cuda_options_converted = onnxruntime::OrtCUDAProviderOptionsToOrtCUDAProviderOptionsV2(provider_options);
|
||||
if (auto* provider = s_library_cuda.Get())
|
||||
return provider->CreateExecutionProviderFactory(&cuda_options_converted);
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptionsV2* provider_options) {
|
||||
if (auto* provider = s_library_cuda.Get())
|
||||
return provider->CreateExecutionProviderFactory(provider_options);
|
||||
|
||||
|
|
@ -1215,6 +1242,20 @@ ProviderOptions GetProviderInfo_Tensorrt(const OrtTensorRTProviderOptions* provi
|
|||
return {};
|
||||
}
|
||||
|
||||
void UpdateProviderInfo_Cuda(OrtCUDAProviderOptionsV2* provider_options, const ProviderOptions& options) {
|
||||
if (auto provider = s_library_cuda.Get()) {
|
||||
provider->UpdateProviderOptions(reinterpret_cast<void*>(provider_options), options);
|
||||
}
|
||||
}
|
||||
|
||||
ProviderOptions GetProviderInfo_Cuda(const OrtCUDAProviderOptionsV2* provider_options) {
|
||||
if (auto provider = s_library_cuda.Get()) {
|
||||
return provider->GetProviderOptions(reinterpret_cast<const void*>(provider_options));
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
} // namespace onnxruntime
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena) {
|
||||
|
|
@ -1448,3 +1489,107 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor
|
|||
ORT_UNUSED_PARAMETER(ptr);
|
||||
#endif
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA_V2, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options) {
|
||||
API_IMPL_BEGIN
|
||||
auto factory = onnxruntime::CreateExecutionProviderFactory_Cuda(cuda_options);
|
||||
if (!factory) {
|
||||
return OrtApis::CreateStatus(ORT_FAIL, "OrtSessionOptionsAppendExecutionProvider_Cuda: Failed to load shared library");
|
||||
}
|
||||
|
||||
options->provider_factories.push_back(factory);
|
||||
return nullptr;
|
||||
API_IMPL_END
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out) {
|
||||
API_IMPL_BEGIN
|
||||
#ifdef USE_CUDA
|
||||
*out = new OrtCUDAProviderOptionsV2();
|
||||
(*out)->device_id = 0;
|
||||
(*out)->cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::OrtCudnnConvAlgoSearchExhaustive;
|
||||
(*out)->gpu_mem_limit = std::numeric_limits<size_t>::max();
|
||||
(*out)->arena_extend_strategy = static_cast<onnxruntime::ArenaExtendStrategy>(0);
|
||||
(*out)->do_copy_in_default_stream = 1;
|
||||
(*out)->has_user_compute_stream = 0;
|
||||
(*out)->user_compute_stream = nullptr;
|
||||
(*out)->default_memory_arena_cfg = nullptr;
|
||||
(*out)->cudnn_conv_use_max_workspace = 0;
|
||||
return nullptr;
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(out);
|
||||
return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled in this build.");
|
||||
#endif
|
||||
API_IMPL_END
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::UpdateCUDAProviderOptions,
|
||||
_Inout_ OrtCUDAProviderOptionsV2* cuda_options,
|
||||
_In_reads_(num_keys) const char* const* provider_options_keys,
|
||||
_In_reads_(num_keys) const char* const* provider_options_values,
|
||||
size_t num_keys) {
|
||||
API_IMPL_BEGIN
|
||||
#ifdef USE_CUDA
|
||||
onnxruntime::ProviderOptions provider_options_map;
|
||||
for (size_t i = 0; i != num_keys; ++i) {
|
||||
if (provider_options_keys[i] == nullptr || provider_options_keys[i][0] == '\0' ||
|
||||
provider_options_values[i] == nullptr || provider_options_values[i][0] == '\0') {
|
||||
return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "key/value cannot be empty");
|
||||
}
|
||||
|
||||
provider_options_map[provider_options_keys[i]] = provider_options_values[i];
|
||||
}
|
||||
|
||||
onnxruntime::UpdateProviderInfo_Cuda(cuda_options,
|
||||
reinterpret_cast<const onnxruntime::ProviderOptions&>(provider_options_map));
|
||||
return nullptr;
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(cuda_options);
|
||||
ORT_UNUSED_PARAMETER(provider_options_keys);
|
||||
ORT_UNUSED_PARAMETER(provider_options_values);
|
||||
ORT_UNUSED_PARAMETER(num_keys);
|
||||
return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled in this build.");
|
||||
#endif
|
||||
API_IMPL_END
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator,
|
||||
_Outptr_ char** ptr) {
|
||||
API_IMPL_BEGIN
|
||||
#ifdef USE_CUDA
|
||||
onnxruntime::ProviderOptions options = onnxruntime::GetProviderInfo_Cuda(cuda_options);
|
||||
onnxruntime::ProviderOptions::iterator it = options.begin();
|
||||
std::string options_str = "";
|
||||
|
||||
while (it != options.end()) {
|
||||
if (options_str == "") {
|
||||
options_str += it->first;
|
||||
options_str += "=";
|
||||
options_str += it->second;
|
||||
} else {
|
||||
options_str += ";";
|
||||
options_str += it->first;
|
||||
options_str += "=";
|
||||
options_str += it->second;
|
||||
}
|
||||
it++;
|
||||
}
|
||||
|
||||
*ptr = onnxruntime::StrDup(options_str, allocator);
|
||||
return nullptr;
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(cuda_options);
|
||||
ORT_UNUSED_PARAMETER(allocator);
|
||||
ORT_UNUSED_PARAMETER(ptr);
|
||||
return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled in this build.");
|
||||
#endif
|
||||
API_IMPL_END
|
||||
}
|
||||
|
||||
ORT_API(void, OrtApis::ReleaseCUDAProviderOptions, _Frees_ptr_opt_ OrtCUDAProviderOptionsV2* ptr) {
|
||||
#ifdef USE_CUDA
|
||||
delete ptr;
|
||||
#else
|
||||
ORT_UNUSED_PARAMETER(ptr);
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,6 +97,42 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA,
|
|||
return CreateNotEnabledStatus("CUDA");
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_CUDA_V2,
|
||||
_In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options) {
|
||||
ORT_UNUSED_PARAMETER(options);
|
||||
ORT_UNUSED_PARAMETER(cuda_options);
|
||||
return CreateNotEnabledStatus("CUDA");
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out) {
|
||||
ORT_UNUSED_PARAMETER(out);
|
||||
return CreateNotEnabledStatus("CUDA");
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::UpdateCUDAProviderOptions,
|
||||
_Inout_ OrtCUDAProviderOptionsV2* cuda_options,
|
||||
_In_reads_(num_keys) const char* const* provider_options_keys,
|
||||
_In_reads_(num_keys) const char* const* provider_options_values,
|
||||
size_t num_keys) {
|
||||
ORT_UNUSED_PARAMETER(cuda_options);
|
||||
ORT_UNUSED_PARAMETER(provider_options_keys);
|
||||
ORT_UNUSED_PARAMETER(provider_options_values);
|
||||
ORT_UNUSED_PARAMETER(num_keys);
|
||||
return CreateNotEnabledStatus("CUDA");
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator,
|
||||
_Outptr_ char** ptr) {
|
||||
ORT_UNUSED_PARAMETER(cuda_options);
|
||||
ORT_UNUSED_PARAMETER(allocator);
|
||||
ORT_UNUSED_PARAMETER(ptr);
|
||||
return CreateStatus(ORT_FAIL, "CUDA execution provider is not enabled in this build.");
|
||||
}
|
||||
|
||||
ORT_API(void, OrtApis::ReleaseCUDAProviderOptions, _Frees_ptr_opt_ OrtCUDAProviderOptionsV2* ptr) {
|
||||
ORT_UNUSED_PARAMETER(ptr);
|
||||
}
|
||||
|
||||
ORT_API_STATUS_IMPL(OrtApis::GetCurrentGpuDeviceId, _In_ int* device_id) {
|
||||
ORT_UNUSED_PARAMETER(device_id);
|
||||
return CreateNotEnabledStatus("CUDA");
|
||||
|
|
|
|||
|
|
@ -1852,6 +1852,51 @@ TEST(CApiTest, TestConfigureTensorRTProviderOptions) {
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef USE_CUDA
|
||||
|
||||
// This test uses CreateCUDAProviderOptions/UpdateCUDAProviderOptions APIs to configure and create a CUDA Execution Provider instance
|
||||
TEST(CApiTest, TestConfigureCUDAProviderOptions) {
|
||||
const auto& api = Ort::GetApi();
|
||||
|
||||
OrtCUDAProviderOptionsV2* cuda_options = nullptr;
|
||||
ASSERT_TRUE(api.CreateCUDAProviderOptions(&cuda_options) == nullptr);
|
||||
std::unique_ptr<OrtCUDAProviderOptionsV2, decltype(api.ReleaseCUDAProviderOptions)> rel_cuda_options(cuda_options, api.ReleaseCUDAProviderOptions);
|
||||
|
||||
std::vector<const char*> keys{
|
||||
"device_id", "gpu_mem_limit", "arena_extend_strategy",
|
||||
"cudnn_conv_algo_search", "do_copy_in_default_stream", "cudnn_conv_use_max_workspace"};
|
||||
|
||||
std::vector<const char*> values{
|
||||
"0", "1024", "kSameAsRequested",
|
||||
"DEFAULT", "1", "1"};
|
||||
|
||||
ASSERT_TRUE(api.UpdateCUDAProviderOptions(rel_cuda_options.get(), keys.data(), values.data(), 6) == nullptr);
|
||||
|
||||
OrtAllocator* allocator;
|
||||
ASSERT_TRUE(api.GetAllocatorWithDefaultOptions(&allocator) == nullptr);
|
||||
|
||||
char* cuda_options_str = nullptr;
|
||||
ASSERT_TRUE(api.GetCUDAProviderOptionsAsString(rel_cuda_options.get(), allocator, &cuda_options_str) == nullptr);
|
||||
std::string s(cuda_options_str, strnlen(cuda_options_str, 2048));
|
||||
ASSERT_TRUE(s.find("device_id=0") != std::string::npos);
|
||||
ASSERT_TRUE(s.find("gpu_mem_limit=1024") != std::string::npos);
|
||||
ASSERT_TRUE(s.find("arena_extend_strategy=kSameAsRequested") != std::string::npos);
|
||||
ASSERT_TRUE(s.find("cudnn_conv_algo_search=DEFAULT") != std::string::npos);
|
||||
ASSERT_TRUE(s.find("do_copy_in_default_stream=1") != std::string::npos);
|
||||
ASSERT_TRUE(s.find("cudnn_conv_use_max_workspace=1") != std::string::npos);
|
||||
|
||||
ASSERT_TRUE(api.AllocatorFree(allocator, (void*)cuda_options_str) == nullptr);
|
||||
|
||||
Ort::SessionOptions session_options;
|
||||
ASSERT_TRUE(api.SessionOptionsAppendExecutionProvider_CUDA_V2(static_cast<OrtSessionOptions*>(session_options), rel_cuda_options.get()) == nullptr);
|
||||
|
||||
// if session creation passes, model loads fine
|
||||
std::basic_string<ORTCHAR_T> model_uri = MODEL_URI;
|
||||
Ort::Session session(*ort_env, model_uri.c_str(), session_options);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
namespace TestPerSessionCustomThreadHooks {
|
||||
|
||||
std::vector<std::thread> threads;
|
||||
|
|
@ -1933,4 +1978,3 @@ TEST(CApiTest, crop_and_resize) {
|
|||
#endif
|
||||
|
||||
} // namespace TestPerSessionCustomThreadHooks
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,19 @@
|
|||
// Licensed under the MIT License.
|
||||
|
||||
#include "utils.h"
|
||||
#include <limits>
|
||||
|
||||
OrtCUDAProviderOptions CreateDefaultOrtCudaProviderOptionsWithCustomStream(void* cuda_compute_stream) {
|
||||
OrtCUDAProviderOptions cuda_options;
|
||||
|
||||
cuda_options.device_id = 0;
|
||||
cuda_options.cudnn_conv_algo_search = OrtCudnnConvAlgoSearch::OrtCudnnConvAlgoSearchExhaustive;
|
||||
cuda_options.gpu_mem_limit = std::numeric_limits<size_t>::max();
|
||||
cuda_options.arena_extend_strategy = 0;
|
||||
cuda_options.do_copy_in_default_stream = true;
|
||||
cuda_options.has_user_compute_stream = cuda_compute_stream != nullptr ? 1 : 0;
|
||||
cuda_options.user_compute_stream = cuda_compute_stream;
|
||||
cuda_options.default_memory_arena_cfg = nullptr;
|
||||
|
||||
return cuda_options;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ACL(in
|
|||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_ArmNN(int use_arena);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_CoreML(uint32_t);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptions* provider_options);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Cuda(const OrtCUDAProviderOptionsV2* provider_options);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Dnnl(int use_arena);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_MIGraphX(int device_id);
|
||||
std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory_Nnapi(
|
||||
|
|
|
|||
Loading…
Reference in a new issue