saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-24 19:43:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add ability to register custom ops from ORT extensions nuget package (#15696)

Browse source 
### Description
<!-- Describe your changes. -->
Add infrastructure so it's easy for a user to add the ORT extensions
nuget package and register the custom ops for C# apps.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
Need to be able to use extensions on mobile platforms with Xamarin/MAUI
This commit is contained in:
Scott McKay 2023-04-28 18:53:02 +10:00 committed by GitHub
parent 94c9a31f83
commit 7e6331d5c7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 271 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs
View file

@ -305,7 +305,7 @@ namespace Microsoft.ML.OnnxRuntime
DOrtGetApi OrtGetApi = (DOrtGetApi)Marshal.GetDelegateForFunctionPointer(OrtGetApiBase().GetApi, typeof(DOrtGetApi));
// TODO: Make this save the pointer, and not copy the whole structure across
api_ = (OrtApi)OrtGetApi(4 /*ORT_API_VERSION*/);
api_ = (OrtApi)OrtGetApi(14 /*ORT_API_VERSION*/);
OrtGetVersionString = (DOrtGetVersionString)Marshal.GetDelegateForFunctionPointer(OrtGetApiBase().GetVersionString, typeof(DOrtGetVersionString));
OrtCreateEnv = (DOrtCreateEnv)Marshal.GetDelegateForFunctionPointer(api_.CreateEnv, typeof(DOrtCreateEnv));
@ -361,6 +361,8 @@ namespace Microsoft.ML.OnnxRuntime
OrtSetIntraOpNumThreads = (DOrtSetIntraOpNumThreads)Marshal.GetDelegateForFunctionPointer(api_.SetIntraOpNumThreads, typeof(DOrtSetIntraOpNumThreads));
OrtSetSessionGraphOptimizationLevel = (DOrtSetSessionGraphOptimizationLevel)Marshal.GetDelegateForFunctionPointer(api_.SetSessionGraphOptimizationLevel, typeof(DOrtSetSessionGraphOptimizationLevel));
OrtRegisterCustomOpsLibrary = (DOrtRegisterCustomOpsLibrary)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary, typeof(DOrtRegisterCustomOpsLibrary));
OrtRegisterCustomOpsLibrary_V2 = (DOrtRegisterCustomOpsLibrary_V2)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary_V2, typeof(DOrtRegisterCustomOpsLibrary_V2));
OrtRegisterCustomOpsUsingFunction = (DOrtRegisterCustomOpsUsingFunction)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsUsingFunction, typeof(DOrtRegisterCustomOpsUsingFunction));
OrtAddSessionConfigEntry = (DOrtAddSessionConfigEntry)Marshal.GetDelegateForFunctionPointer(api_.AddSessionConfigEntry, typeof(DOrtAddSessionConfigEntry));
OrtAddInitializer = (DOrtAddInitializer)Marshal.GetDelegateForFunctionPointer(api_.AddInitializer, typeof(DOrtAddInitializer));
SessionOptionsAppendExecutionProvider_TensorRT = (DSessionOptionsAppendExecutionProvider_TensorRT)Marshal.GetDelegateForFunctionPointer(
@ -496,18 +498,16 @@ namespace Microsoft.ML.OnnxRuntime
internal class NativeLib
{
#if __ANDROID__
// define the library name required for android
internal const string DllName = "libonnxruntime.so";
// define the library name required for android
internal const string DllName = "libonnxruntime.so";
#elif __IOS__
// define the library name required for iOS
internal const string DllName = "__Internal";
// define the library name required for iOS
internal const string DllName = "__Internal";
#else
internal const string DllName = "onnxruntime";
#endif
// TODO: Does macos need special handling or will 'onnxruntime' -> libonnxruntime.dylib?
}
[DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
public static extern ref OrtApiBase OrtGetApiBase();
@ -1079,6 +1079,29 @@ namespace Microsoft.ML.OnnxRuntime
public static DOrtRegisterCustomOpsLibrary OrtRegisterCustomOpsLibrary;
/// <summary>
/// Register custom op library. ORT will manage freeing the library.
/// </summary>
/// <param name="options">Native SessionOptions instance</param>
/// <param name="libraryPath">Library path</param>
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
public delegate IntPtr /*(OrtStatus*)*/DOrtRegisterCustomOpsLibrary_V2(IntPtr /*(OrtSessionOptions*) */ options,
byte[] /*(const ORTCHAR_T*)*/ libraryPath);
public static DOrtRegisterCustomOpsLibrary_V2 OrtRegisterCustomOpsLibrary_V2;
/// <summary>
/// Register custom op library using a function name. ORT will lookup the function name using dlsym.
/// Library containing the function must be loaded and the function name must be globally visible.
/// </summary>
/// <param name="options">Native SessionOptions instance</param>
/// <param name="functionName">Function name to call to register the custom ops.</param>
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
public delegate IntPtr /*(OrtStatus*)*/DOrtRegisterCustomOpsUsingFunction(IntPtr /*(OrtSessionOptions*) */ options,
byte[] /*(const char*)*/ functionName);
public static DOrtRegisterCustomOpsUsingFunction OrtRegisterCustomOpsUsingFunction;
/// <summary>
/// Add initializer that is shared across Sessions using this SessionOptions (by denotation)
/// </summary>
@ -1907,4 +1930,27 @@ namespace Microsoft.ML.OnnxRuntime
#endregion
} //class NativeMethods
// onnxruntime-extensions helpers to make usage simpler.
// The onnxruntime-extensions nuget package containing the native library can be optionally added to the app.
// If added, SessionOptions.RegisterOrtExtensions can be called to add the custom ops to the session options.
// We handle the DllImport and platform specific aspects so the user code doesn't require that.
// adjust the library name based on platform.
internal static class OrtExtensionsNativeMethods
{
#if __ANDROID__
internal const string ExtensionsDllName = "libortextensions.so";
#elif __IOS__
internal const string ExtensionsDllName = "__Internal";
#else
internal const string ExtensionsDllName = "ortextensions";
#endif
[DllImport(ExtensionsDllName, CharSet = CharSet.Ansi,
CallingConvention = CallingConvention.Winapi)]
public static extern IntPtr /* OrtStatus* */ RegisterCustomOps(IntPtr /* OrtSessionOptions* */ sessionOptions,
ref OrtApiBase /* OrtApiBase* */ ortApiBase);
}
} //namespace

83
csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs
View file

@ -394,19 +394,22 @@ namespace Microsoft.ML.OnnxRuntime
#region Public Methods
/// <summary>
/// (Deprecated) Loads a DLL named 'libraryPath' and looks for this entry point:
/// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
/// Loads a DLL named 'libraryPath' and looks for this entry point:
/// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
/// It then passes in the provided session options to this function along with the api base.
/// Deprecated in favor of RegisterCustomOpLibraryV2() because it provides users with the library handle
/// to release when all sessions relying on it are destroyed
///
/// Prior to v1.15 this leaked the library handle and RegisterCustomOpLibraryV2
/// was added to resolve that.
///
/// From v1.15 on ONNX Runtime will manage the lifetime of the handle.
/// </summary>
/// <param name="libraryPath">path to the custom op library</param>
[ObsoleteAttribute("RegisterCustomOpLibrary(...) is obsolete. Use RegisterCustomOpLibraryV2(...) instead.", false)]
public void RegisterCustomOpLibrary(string libraryPath)
{
var utf8Path = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath);
// The handle is leaking in this version
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path, out IntPtr libraryHandle));
NativeApiStatus.VerifySuccess(
NativeMethods.OrtRegisterCustomOpsLibrary_V2(
handle, NativeOnnxValueHelper.GetPlatformSerializedString(libraryPath))
);
}
/// <summary>
@ -422,8 +425,70 @@ namespace Microsoft.ML.OnnxRuntime
/// <param name="libraryHandle">out parameter, library handle</param>
public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle)
{
// NOTE: This is confusing due to the history.
// SessionOptions.RegisterCustomOpLibrary initially called NativeMethods.OrtRegisterCustomOpsLibrary
// and leaked the handle.
// SessionOptions.RegisterCustomOpLibraryV2 was added to resolve that by returning the handle.
// Later, NativeMethods.OrtRegisterCustomOpsLibrary_V2 was added with ORT owning the handle.
//
// End result of that is
// SessionOptions.RegisterCustomOpLibrary calls NativeMethods.OrtRegisterCustomOpsLibrary_V2
// SessionOptions.RegisterCustomOpLibraryV2 calls NativeMethods.OrtRegisterCustomOpsLibrary
var utf8Path = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath);
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path, out libraryHandle));
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path,
out libraryHandle));
}
/// <summary>
/// Register custom ops by calling the C function with functionName.
/// The C function has the signature
/// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api);
///
/// The function must be available in the global symbols so ONNX Runtime can find it using GetProcAddress or
/// dlsym. This will require your app to 'DllImport' the native library containing the function and use
/// Marshal.Prelink to ensure the function is loaded.
///
/// Example usage:
/// static class NativeCustomOps {
/// [DllImport("YourLibraryName", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Winapi)]
/// public static extern IntPtr /* OrtStatus* */
/// YourRegisterCustomOpsFunctionName(IntPtr /* OrtSessionOptions* */ sessionOptions,
/// IntPtr /* OrtApiBase* */ ortApiBase);
/// }
/// ...
/// var sessionOptions = new SessionOptions();
/// Marshal.Prelink(typeof(NativeCustomOps).GetMethod("YourRegisterCustomOpsFunctionName"));
/// sessionOptions.RegisterCustomOpsUsingFunction("YourRegisterCustomOpsFunctionName");
///
/// </summary>
/// <param name="functionName">Function name to call to register custom operators.</param>
public void RegisterCustomOpsUsingFunction(string functionName)
{
var utf8FuncName = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(functionName);
NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsUsingFunction(this.handle, utf8FuncName));
}
/// <summary>
/// Register the custom operators from the Microsoft.ML.OnnxRuntime.Extensions NuGet package.
/// A reference to Microsoft.ML.OnnxRuntime.Extensions must be manually added to your project.
/// </summary>
/// <exception cref="OnnxRuntimeException">Throws if the extensions library is not found.</exception>
public void RegisterOrtExtensions()
{
try
{
var ortApiBase = NativeMethods.OrtGetApiBase();
NativeApiStatus.VerifySuccess(
OrtExtensionsNativeMethods.RegisterCustomOps(this.handle, ref ortApiBase)
);
}
catch (DllNotFoundException)
{
throw new OnnxRuntimeException(
ErrorCode.NoSuchFile,
"The ONNX Runtime extensions library was not found. The Microsoft.ML.OnnxRuntime.Extensions " +
"NuGet package must be referenced by the project to use 'OrtExtensions.RegisterCustomOps.");
}
}
/// <summary>

5
csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs
View file

@ -92,6 +92,11 @@ namespace Microsoft.ML.OnnxRuntime.Tests
var ex = Assert.Throws<OnnxRuntimeException>(() => { opt.AddSessionConfigEntry("", "invalid key"); });
Assert.Contains("[ErrorCode:InvalidArgument] Config key is empty", ex.Message);
// SessionOptions.RegisterOrtExtensions can be manually tested by referencing the
// Microsoft.ML.OnnxRuntime.Extensions nuget package. After that is done, this should not throw.
ex = Assert.Throws<OnnxRuntimeException>(() => { opt.RegisterOrtExtensions(); });
Assert.Contains("Microsoft.ML.OnnxRuntime.Extensions NuGet package must be referenced", ex.Message);
#if USE_CUDA
opt.AppendExecutionProvider_CUDA(0);
#endif

5
csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Microsoft.ML.OnnxRuntime.Tests.Droid.csproj
View file

@ -27,7 +27,7 @@
<AndroidHttpClientHandlerType>Xamarin.Android.Net.AndroidClientHandler</AndroidHttpClientHandlerType>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>..\..\OnnxRuntime.snk</AssemblyOriginatorKeyFile>
<AndroidSupportedAbis>armeabi-v7a;arm64-v8a</AndroidSupportedAbis>
<AndroidSupportedAbis>armeabi-v7a;arm64-v8a;x86;x86_64</AndroidSupportedAbis>
</PropertyGroup>
<PropertyGroup>
<AutoGenerateBindingRedirects>True</AutoGenerateBindingRedirects>
@ -65,7 +65,6 @@
<AndroidEnableProfiledAot>true</AndroidEnableProfiledAot>
<EnableLLVM>true</EnableLLVM>
<AndroidLinkTool>r8</AndroidLinkTool>
<AndroidSupportedAbis>arm64-v8a;armeabi-v7a;x86</AndroidSupportedAbis>
<AndroidLinkSkip>Microsoft.ML.OnnxRuntime.Tests.Common;Microsoft.ML.OnnxRuntime.Tests.Devices;</AndroidLinkSkip>
</PropertyGroup>
<ItemGroup>
@ -174,4 +173,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>

193
csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs
View file

@ -28,6 +28,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
return str.IndexOf(substring, comp) >= 0;
}
}
public partial class InferenceTest
{
private const string module = "onnxruntime.dll";
@ -67,7 +68,6 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
#if USE_CUDA
[Fact(DisplayName = "TestCUDAProviderOptions")]
private void TestCUDAProviderOptions()
{
@ -805,28 +805,110 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
private string GetCustomOpLibFullPath()
{
string libName = "custom_op_library.dll";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
libName = "custom_op_library.dll";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
libName = "libcustom_op_library.so";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
libName = "libcustom_op_library.dylib";
}
string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName);
Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist.");
return libFullPath;
}
private void ValidateModelWithCustomOps(SessionOptions options)
{
string modelPath = "custom_op_test.onnx";
using (var session = new InferenceSession(modelPath, options))
{
var inputContainer = new List<NamedOnnxValue>();
inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_1",
new DenseTensor<float>(
new float[]
{
1.1f, 2.2f, 3.3f, 4.4f, 5.5f,
6.6f, 7.7f, 8.8f, 9.9f, 10.0f,
11.1f, 12.2f, 13.3f, 14.4f, 15.5f
},
new int[] { 3, 5 }
)));
inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_2",
new DenseTensor<float>(
new float[]
{
15.5f, 14.4f, 13.3f, 12.2f, 11.1f,
10.0f, 9.9f, 8.8f, 7.7f, 6.6f,
5.5f, 4.4f, 3.3f, 2.2f, 1.1f
},
new int[] { 3, 5 }
)));
using (var result = session.Run(inputContainer))
{
Assert.Equal("output", result.First().Name);
var tensorOut = result.First().AsTensor<int>();
var expectedOut = new DenseTensor<int>(
new int[]
{
17, 17, 17, 17, 17,
17, 18, 18, 18, 17,
17, 17, 17, 17, 17
},
new int[] { 3, 5 }
);
Assert.True(tensorOut.SequenceEqual(expectedOut));
}
}
}
[SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibrary")]
private void TestRegisterCustomOpLibrary()
{
using (var option = new SessionOptions())
{
string libName = "custom_op_library.dll";
string modelPath = "custom_op_test.onnx";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
string libFullPath = GetCustomOpLibFullPath();
try
{
libName = "custom_op_library.dll";
option.RegisterCustomOpLibrary(libFullPath);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
catch (Exception ex)
{
libName = "libcustom_op_library.so";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
libName = "libcustom_op_library.dylib";
var msg = $"Failed to load custom op library {libFullPath}, error = {ex.Message}";
throw new Exception(msg + "\n" + ex.StackTrace);
}
string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName);
Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist.");
var ortEnvInstance = OrtEnv.Instance();
string[] providers = ortEnvInstance.GetAvailableProviders();
if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider"))
{
option.AppendExecutionProvider_CUDA(0);
}
ValidateModelWithCustomOps(option);
}
}
[SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibraryV2")]
private void TestRegisterCustomOpLibraryV2()
{
using (var option = new SessionOptions())
{
string libFullPath = GetCustomOpLibFullPath();
var ortEnvInstance = OrtEnv.Instance();
string[] providers = ortEnvInstance.GetAvailableProviders();
@ -846,55 +928,54 @@ namespace Microsoft.ML.OnnxRuntime.Tests
throw new Exception(msg + "\n" + ex.StackTrace);
}
using (var session = new InferenceSession(modelPath, option))
{
var inputContainer = new List<NamedOnnxValue>();
inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_1",
new DenseTensor<float>(
new float[]
{
1.1f, 2.2f, 3.3f, 4.4f, 5.5f,
6.6f, 7.7f, 8.8f, 9.9f, 10.0f,
11.1f, 12.2f, 13.3f, 14.4f, 15.5f
},
new int[] { 3, 5 }
)));
inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_2",
new DenseTensor<float>(
new float[]
{
15.5f, 14.4f, 13.3f, 12.2f, 11.1f,
10.0f, 9.9f, 8.8f, 7.7f, 6.6f,
5.5f, 4.4f, 3.3f, 2.2f, 1.1f
},
new int[] { 3, 5 }
)));
using (var result = session.Run(inputContainer))
{
Assert.Equal("output", result.First().Name);
var tensorOut = result.First().AsTensor<int>();
var expectedOut = new DenseTensor<int>(
new int[]
{
17, 17, 17, 17, 17,
17, 18, 18, 18, 17,
17, 17, 17, 17, 17
},
new int[] { 3, 5 }
);
Assert.True(tensorOut.SequenceEqual(expectedOut));
}
}
ValidateModelWithCustomOps(option);
// Safe to unload the custom op shared library now
UnloadLibrary(libraryHandle);
}
}
// test registration by the alternative function name in the custom ops library.
// We use DllImport and Marshal.Prelink to make sure the library is loaded and
// the symbol is available.
internal static class CustomOpLibrary
{
internal const string ExtensionsDllName = "custom_op_library";
[DllImport(ExtensionsDllName, CharSet = CharSet.Ansi,
CallingConvention = CallingConvention.Winapi)]
public static extern IntPtr /* OrtStatus* */ RegisterCustomOpsAltName(
IntPtr /* OrtSessionOptions* */ sessionOptions,
ref OrtApiBase /* OrtApiBase* */ ortApiBase);
}
[SkipNonPackageTests(DisplayName = "TestRegisterCustomOpsWithFunction")]
private void TestRegisterCustomOpsWithFunction()
{
using (var option = new SessionOptions())
{
try
{
Marshal.Prelink(typeof(CustomOpLibrary).GetMethod("RegisterCustomOpsAltName"));
option.RegisterCustomOpsUsingFunction("RegisterCustomOpsAltName");
}
catch (Exception ex)
{
var msg = $"Failed to load custom op library, error = {ex.Message}";
throw new Exception(msg + "\n" + ex.StackTrace);
}
var ortEnvInstance = OrtEnv.Instance();
string[] providers = ortEnvInstance.GetAvailableProviders();
if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider"))
{
option.AppendExecutionProvider_CUDA(0);
}
ValidateModelWithCustomOps(option);
}
}
[Fact(DisplayName = "TestModelSerialization")]
private void TestModelSerialization()
{