diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs index 4926cd4752..b44e94f313 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs @@ -305,7 +305,7 @@ namespace Microsoft.ML.OnnxRuntime DOrtGetApi OrtGetApi = (DOrtGetApi)Marshal.GetDelegateForFunctionPointer(OrtGetApiBase().GetApi, typeof(DOrtGetApi)); // TODO: Make this save the pointer, and not copy the whole structure across - api_ = (OrtApi)OrtGetApi(4 /*ORT_API_VERSION*/); + api_ = (OrtApi)OrtGetApi(14 /*ORT_API_VERSION*/); OrtGetVersionString = (DOrtGetVersionString)Marshal.GetDelegateForFunctionPointer(OrtGetApiBase().GetVersionString, typeof(DOrtGetVersionString)); OrtCreateEnv = (DOrtCreateEnv)Marshal.GetDelegateForFunctionPointer(api_.CreateEnv, typeof(DOrtCreateEnv)); @@ -361,6 +361,8 @@ namespace Microsoft.ML.OnnxRuntime OrtSetIntraOpNumThreads = (DOrtSetIntraOpNumThreads)Marshal.GetDelegateForFunctionPointer(api_.SetIntraOpNumThreads, typeof(DOrtSetIntraOpNumThreads)); OrtSetSessionGraphOptimizationLevel = (DOrtSetSessionGraphOptimizationLevel)Marshal.GetDelegateForFunctionPointer(api_.SetSessionGraphOptimizationLevel, typeof(DOrtSetSessionGraphOptimizationLevel)); OrtRegisterCustomOpsLibrary = (DOrtRegisterCustomOpsLibrary)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary, typeof(DOrtRegisterCustomOpsLibrary)); + OrtRegisterCustomOpsLibrary_V2 = (DOrtRegisterCustomOpsLibrary_V2)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsLibrary_V2, typeof(DOrtRegisterCustomOpsLibrary_V2)); + OrtRegisterCustomOpsUsingFunction = (DOrtRegisterCustomOpsUsingFunction)Marshal.GetDelegateForFunctionPointer(api_.RegisterCustomOpsUsingFunction, typeof(DOrtRegisterCustomOpsUsingFunction)); OrtAddSessionConfigEntry = (DOrtAddSessionConfigEntry)Marshal.GetDelegateForFunctionPointer(api_.AddSessionConfigEntry, typeof(DOrtAddSessionConfigEntry)); OrtAddInitializer = (DOrtAddInitializer)Marshal.GetDelegateForFunctionPointer(api_.AddInitializer, typeof(DOrtAddInitializer)); SessionOptionsAppendExecutionProvider_TensorRT = (DSessionOptionsAppendExecutionProvider_TensorRT)Marshal.GetDelegateForFunctionPointer( @@ -496,18 +498,16 @@ namespace Microsoft.ML.OnnxRuntime internal class NativeLib { #if __ANDROID__ - // define the library name required for android - internal const string DllName = "libonnxruntime.so"; + // define the library name required for android + internal const string DllName = "libonnxruntime.so"; #elif __IOS__ - // define the library name required for iOS - internal const string DllName = "__Internal"; + // define the library name required for iOS + internal const string DllName = "__Internal"; #else internal const string DllName = "onnxruntime"; #endif - // TODO: Does macos need special handling or will 'onnxruntime' -> libonnxruntime.dylib? } - [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)] public static extern ref OrtApiBase OrtGetApiBase(); @@ -1079,6 +1079,29 @@ namespace Microsoft.ML.OnnxRuntime public static DOrtRegisterCustomOpsLibrary OrtRegisterCustomOpsLibrary; + /// + /// Register custom op library. ORT will manage freeing the library. + /// + /// Native SessionOptions instance + /// Library path + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/DOrtRegisterCustomOpsLibrary_V2(IntPtr /*(OrtSessionOptions*) */ options, + byte[] /*(const ORTCHAR_T*)*/ libraryPath); + + public static DOrtRegisterCustomOpsLibrary_V2 OrtRegisterCustomOpsLibrary_V2; + + /// + /// Register custom op library using a function name. ORT will lookup the function name using dlsym. + /// Library containing the function must be loaded and the function name must be globally visible. + /// + /// Native SessionOptions instance + /// Function name to call to register the custom ops. + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/DOrtRegisterCustomOpsUsingFunction(IntPtr /*(OrtSessionOptions*) */ options, + byte[] /*(const char*)*/ functionName); + + public static DOrtRegisterCustomOpsUsingFunction OrtRegisterCustomOpsUsingFunction; + /// /// Add initializer that is shared across Sessions using this SessionOptions (by denotation) /// @@ -1907,4 +1930,27 @@ namespace Microsoft.ML.OnnxRuntime #endregion } //class NativeMethods + + // onnxruntime-extensions helpers to make usage simpler. + // The onnxruntime-extensions nuget package containing the native library can be optionally added to the app. + // If added, SessionOptions.RegisterOrtExtensions can be called to add the custom ops to the session options. + // We handle the DllImport and platform specific aspects so the user code doesn't require that. + // adjust the library name based on platform. + internal static class OrtExtensionsNativeMethods + { +#if __ANDROID__ + internal const string ExtensionsDllName = "libortextensions.so"; +#elif __IOS__ + internal const string ExtensionsDllName = "__Internal"; +#else + internal const string ExtensionsDllName = "ortextensions"; +#endif + + [DllImport(ExtensionsDllName, CharSet = CharSet.Ansi, + CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OrtStatus* */ RegisterCustomOps(IntPtr /* OrtSessionOptions* */ sessionOptions, + ref OrtApiBase /* OrtApiBase* */ ortApiBase); + + + } } //namespace diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs index 7590299000..c7e92e7e5c 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs @@ -394,19 +394,22 @@ namespace Microsoft.ML.OnnxRuntime #region Public Methods /// - /// (Deprecated) Loads a DLL named 'libraryPath' and looks for this entry point: - /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api); + /// Loads a DLL named 'libraryPath' and looks for this entry point: + /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api); /// It then passes in the provided session options to this function along with the api base. - /// Deprecated in favor of RegisterCustomOpLibraryV2() because it provides users with the library handle - /// to release when all sessions relying on it are destroyed + /// + /// Prior to v1.15 this leaked the library handle and RegisterCustomOpLibraryV2 + /// was added to resolve that. + /// + /// From v1.15 on ONNX Runtime will manage the lifetime of the handle. /// /// path to the custom op library - [ObsoleteAttribute("RegisterCustomOpLibrary(...) is obsolete. Use RegisterCustomOpLibraryV2(...) instead.", false)] public void RegisterCustomOpLibrary(string libraryPath) { - var utf8Path = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath); - // The handle is leaking in this version - NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path, out IntPtr libraryHandle)); + NativeApiStatus.VerifySuccess( + NativeMethods.OrtRegisterCustomOpsLibrary_V2( + handle, NativeOnnxValueHelper.GetPlatformSerializedString(libraryPath)) + ); } /// @@ -422,8 +425,70 @@ namespace Microsoft.ML.OnnxRuntime /// out parameter, library handle public void RegisterCustomOpLibraryV2(string libraryPath, out IntPtr libraryHandle) { + // NOTE: This is confusing due to the history. + // SessionOptions.RegisterCustomOpLibrary initially called NativeMethods.OrtRegisterCustomOpsLibrary + // and leaked the handle. + // SessionOptions.RegisterCustomOpLibraryV2 was added to resolve that by returning the handle. + // Later, NativeMethods.OrtRegisterCustomOpsLibrary_V2 was added with ORT owning the handle. + // + // End result of that is + // SessionOptions.RegisterCustomOpLibrary calls NativeMethods.OrtRegisterCustomOpsLibrary_V2 + // SessionOptions.RegisterCustomOpLibraryV2 calls NativeMethods.OrtRegisterCustomOpsLibrary var utf8Path = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(libraryPath); - NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path, out libraryHandle)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsLibrary(handle, utf8Path, + out libraryHandle)); + } + + /// + /// Register custom ops by calling the C function with functionName. + /// The C function has the signature + /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api); + /// + /// The function must be available in the global symbols so ONNX Runtime can find it using GetProcAddress or + /// dlsym. This will require your app to 'DllImport' the native library containing the function and use + /// Marshal.Prelink to ensure the function is loaded. + /// + /// Example usage: + /// static class NativeCustomOps { + /// [DllImport("YourLibraryName", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Winapi)] + /// public static extern IntPtr /* OrtStatus* */ + /// YourRegisterCustomOpsFunctionName(IntPtr /* OrtSessionOptions* */ sessionOptions, + /// IntPtr /* OrtApiBase* */ ortApiBase); + /// } + /// ... + /// var sessionOptions = new SessionOptions(); + /// Marshal.Prelink(typeof(NativeCustomOps).GetMethod("YourRegisterCustomOpsFunctionName")); + /// sessionOptions.RegisterCustomOpsUsingFunction("YourRegisterCustomOpsFunctionName"); + /// + /// + /// Function name to call to register custom operators. + public void RegisterCustomOpsUsingFunction(string functionName) + { + var utf8FuncName = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(functionName); + NativeApiStatus.VerifySuccess(NativeMethods.OrtRegisterCustomOpsUsingFunction(this.handle, utf8FuncName)); + } + + /// + /// Register the custom operators from the Microsoft.ML.OnnxRuntime.Extensions NuGet package. + /// A reference to Microsoft.ML.OnnxRuntime.Extensions must be manually added to your project. + /// + /// Throws if the extensions library is not found. + public void RegisterOrtExtensions() + { + try + { + var ortApiBase = NativeMethods.OrtGetApiBase(); + NativeApiStatus.VerifySuccess( + OrtExtensionsNativeMethods.RegisterCustomOps(this.handle, ref ortApiBase) + ); + } + catch (DllNotFoundException) + { + throw new OnnxRuntimeException( + ErrorCode.NoSuchFile, + "The ONNX Runtime extensions library was not found. The Microsoft.ML.OnnxRuntime.Extensions " + + "NuGet package must be referenced by the project to use 'OrtExtensions.RegisterCustomOps."); + } } /// diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs index 61205b7412..e662676da9 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs @@ -92,6 +92,11 @@ namespace Microsoft.ML.OnnxRuntime.Tests var ex = Assert.Throws(() => { opt.AddSessionConfigEntry("", "invalid key"); }); Assert.Contains("[ErrorCode:InvalidArgument] Config key is empty", ex.Message); + // SessionOptions.RegisterOrtExtensions can be manually tested by referencing the + // Microsoft.ML.OnnxRuntime.Extensions nuget package. After that is done, this should not throw. + ex = Assert.Throws(() => { opt.RegisterOrtExtensions(); }); + Assert.Contains("Microsoft.ML.OnnxRuntime.Extensions NuGet package must be referenced", ex.Message); + #if USE_CUDA opt.AppendExecutionProvider_CUDA(0); #endif diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Microsoft.ML.OnnxRuntime.Tests.Droid.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Microsoft.ML.OnnxRuntime.Tests.Droid.csproj index b2a20e3d4e..78cfc5ca80 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Microsoft.ML.OnnxRuntime.Tests.Droid.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Microsoft.ML.OnnxRuntime.Tests.Droid.csproj @@ -27,7 +27,7 @@ Xamarin.Android.Net.AndroidClientHandler true ..\..\OnnxRuntime.snk - armeabi-v7a;arm64-v8a + armeabi-v7a;arm64-v8a;x86;x86_64 True @@ -65,7 +65,6 @@ true true r8 - arm64-v8a;armeabi-v7a;x86 Microsoft.ML.OnnxRuntime.Tests.Common;Microsoft.ML.OnnxRuntime.Tests.Devices; @@ -174,4 +173,4 @@ --> - \ No newline at end of file + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index 1495bcc459..bebd53a538 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -28,6 +28,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests return str.IndexOf(substring, comp) >= 0; } } + public partial class InferenceTest { private const string module = "onnxruntime.dll"; @@ -67,7 +68,6 @@ namespace Microsoft.ML.OnnxRuntime.Tests } #if USE_CUDA - [Fact(DisplayName = "TestCUDAProviderOptions")] private void TestCUDAProviderOptions() { @@ -805,28 +805,110 @@ namespace Microsoft.ML.OnnxRuntime.Tests } } + private string GetCustomOpLibFullPath() + { + string libName = "custom_op_library.dll"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + libName = "custom_op_library.dll"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + libName = "libcustom_op_library.so"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + libName = "libcustom_op_library.dylib"; + } + + string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName); + Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist."); + + return libFullPath; + } + + private void ValidateModelWithCustomOps(SessionOptions options) + { + string modelPath = "custom_op_test.onnx"; + + using (var session = new InferenceSession(modelPath, options)) + { + var inputContainer = new List(); + inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_1", + new DenseTensor( + new float[] + { + 1.1f, 2.2f, 3.3f, 4.4f, 5.5f, + 6.6f, 7.7f, 8.8f, 9.9f, 10.0f, + 11.1f, 12.2f, 13.3f, 14.4f, 15.5f + }, + new int[] { 3, 5 } + ))); + + inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_2", + new DenseTensor( + new float[] + { + 15.5f, 14.4f, 13.3f, 12.2f, 11.1f, + 10.0f, 9.9f, 8.8f, 7.7f, 6.6f, + 5.5f, 4.4f, 3.3f, 2.2f, 1.1f + }, + new int[] { 3, 5 } + ))); + + using (var result = session.Run(inputContainer)) + { + Assert.Equal("output", result.First().Name); + var tensorOut = result.First().AsTensor(); + + var expectedOut = new DenseTensor( + new int[] + { + 17, 17, 17, 17, 17, + 17, 18, 18, 18, 17, + 17, 17, 17, 17, 17 + }, + new int[] { 3, 5 } + ); + Assert.True(tensorOut.SequenceEqual(expectedOut)); + } + } + } + [SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibrary")] private void TestRegisterCustomOpLibrary() { using (var option = new SessionOptions()) { - string libName = "custom_op_library.dll"; - string modelPath = "custom_op_test.onnx"; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + string libFullPath = GetCustomOpLibFullPath(); + + try { - libName = "custom_op_library.dll"; + option.RegisterCustomOpLibrary(libFullPath); } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + catch (Exception ex) { - libName = "libcustom_op_library.so"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - libName = "libcustom_op_library.dylib"; + var msg = $"Failed to load custom op library {libFullPath}, error = {ex.Message}"; + throw new Exception(msg + "\n" + ex.StackTrace); } - string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName); - Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist."); + var ortEnvInstance = OrtEnv.Instance(); + string[] providers = ortEnvInstance.GetAvailableProviders(); + if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider")) + { + option.AppendExecutionProvider_CUDA(0); + } + + ValidateModelWithCustomOps(option); + } + } + + [SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibraryV2")] + private void TestRegisterCustomOpLibraryV2() + { + using (var option = new SessionOptions()) + { + string libFullPath = GetCustomOpLibFullPath(); var ortEnvInstance = OrtEnv.Instance(); string[] providers = ortEnvInstance.GetAvailableProviders(); @@ -846,55 +928,54 @@ namespace Microsoft.ML.OnnxRuntime.Tests throw new Exception(msg + "\n" + ex.StackTrace); } - - using (var session = new InferenceSession(modelPath, option)) - { - var inputContainer = new List(); - inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_1", - new DenseTensor( - new float[] - { - 1.1f, 2.2f, 3.3f, 4.4f, 5.5f, - 6.6f, 7.7f, 8.8f, 9.9f, 10.0f, - 11.1f, 12.2f, 13.3f, 14.4f, 15.5f - }, - new int[] { 3, 5 } - ))); - - inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_2", - new DenseTensor( - new float[] - { - 15.5f, 14.4f, 13.3f, 12.2f, 11.1f, - 10.0f, 9.9f, 8.8f, 7.7f, 6.6f, - 5.5f, 4.4f, 3.3f, 2.2f, 1.1f - }, - new int[] { 3, 5 } - ))); - - using (var result = session.Run(inputContainer)) - { - Assert.Equal("output", result.First().Name); - var tensorOut = result.First().AsTensor(); - - var expectedOut = new DenseTensor( - new int[] - { - 17, 17, 17, 17, 17, - 17, 18, 18, 18, 17, - 17, 17, 17, 17, 17 - }, - new int[] { 3, 5 } - ); - Assert.True(tensorOut.SequenceEqual(expectedOut)); - } - } + ValidateModelWithCustomOps(option); // Safe to unload the custom op shared library now UnloadLibrary(libraryHandle); } } + // test registration by the alternative function name in the custom ops library. + // We use DllImport and Marshal.Prelink to make sure the library is loaded and + // the symbol is available. + internal static class CustomOpLibrary + { + internal const string ExtensionsDllName = "custom_op_library"; + + [DllImport(ExtensionsDllName, CharSet = CharSet.Ansi, + CallingConvention = CallingConvention.Winapi)] + public static extern IntPtr /* OrtStatus* */ RegisterCustomOpsAltName( + IntPtr /* OrtSessionOptions* */ sessionOptions, + ref OrtApiBase /* OrtApiBase* */ ortApiBase); + } + + [SkipNonPackageTests(DisplayName = "TestRegisterCustomOpsWithFunction")] + private void TestRegisterCustomOpsWithFunction() + { + using (var option = new SessionOptions()) + { + try + { + Marshal.Prelink(typeof(CustomOpLibrary).GetMethod("RegisterCustomOpsAltName")); + option.RegisterCustomOpsUsingFunction("RegisterCustomOpsAltName"); + } + catch (Exception ex) + { + var msg = $"Failed to load custom op library, error = {ex.Message}"; + throw new Exception(msg + "\n" + ex.StackTrace); + } + + var ortEnvInstance = OrtEnv.Instance(); + string[] providers = ortEnvInstance.GetAvailableProviders(); + if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider")) + { + option.AppendExecutionProvider_CUDA(0); + } + + ValidateModelWithCustomOps(option); + } + } + [Fact(DisplayName = "TestModelSerialization")] private void TestModelSerialization() {