// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; using System.Runtime.InteropServices; namespace Microsoft.ML.OnnxRuntime { /// /// Delegate for logging function callback. /// Supply your function and register it with the environment to receive logging callbacks via /// EnvironmentCreationOptions /// /// Pointer to data passed into Constructor `log_param` parameter. /// Log severity level. /// Log category /// Log Id. /// Code location detail. /// Log message. public delegate void DOrtLoggingFunction(IntPtr param, OrtLoggingLevel severity, string category, string logId, string codeLocation, string message); /// /// Options you might want to supply when creating the environment. /// Everything is optional. /// public struct EnvironmentCreationOptions { /// /// Supply a log id to identify the application using ORT, otherwise, a default one will be used /// public string logId; /// /// Initial logging level so that you can see what is going on during environment creation /// Default is LogLevel.Warning /// public OrtLoggingLevel? logLevel; /// /// Supply OrtThreadingOptions instance, otherwise null /// public OrtThreadingOptions threadOptions; /// /// Supply IntPtr logging param when registering logging function, otherwise IntPtr.Zero /// This param will be passed to the logging function when called, it is opaque for the API /// public IntPtr? loggingParam; /// /// Supply custom logging function otherwise null /// public DOrtLoggingFunction loggingFunction; } /// /// The singleton class OrtEnv contains the process-global ONNX Runtime environment. /// It sets up logging, creates system wide thread-pools (if Thread Pool options are provided) /// and other necessary things for OnnxRuntime to function. /// /// Create or access OrtEnv by calling the Instance() method. Instance() can be called multiple times. /// It would return the same instance. /// /// CreateInstanceWithOptions() provides a way to create environment with options. /// It must be called once before Instance() is called, otherwise it would not have effect. /// /// If the environment is not explicitly created, it will be created as needed, e.g., /// when creating a SessionOptions instance. /// public sealed class OrtEnv : SafeHandle { #region Static members private static readonly int ORT_PROJECTION_CSHARP = 2; private static readonly byte[] _defaultLogId = NativeOnnxValueHelper.StringToZeroTerminatedUtf8(@"CSharpOnnxRuntime"); // This must be static and set before the first creation call, otherwise, has no effect. private static EnvironmentCreationOptions? _createOptions; // Lazy instantiation. _createOptions must be set before the first creation of the instance. private static Lazy _instance = new Lazy(CreateInstance); // Internal logging function that will be called from native code private delegate void DOrtLoggingFunctionInternal(IntPtr param, IntPtr severity, IntPtr /* utf-8 const char* */ category, IntPtr /* utf-8 const char* */ logid, IntPtr /* utf-8 const char* */ codeLocation, IntPtr /* utf-8 const char* */ message); // Must keep this delegate alive, otherwise GC will collect it and native code will call into freed memory private static readonly DOrtLoggingFunctionInternal _loggingFunctionInternal = LoggingFunctionThunk; // Customer supplied logging function (if specified) private static DOrtLoggingFunction _userLoggingFunction; #endregion #region Instance members private OrtLoggingLevel _envLogLevel; #endregion #region Private methods /// /// The only __ctor__ for OrtEnv. /// /// /// private OrtEnv(IntPtr handle, OrtLoggingLevel logLevel) : base(handle, true) { _envLogLevel = logLevel; } /// /// The actual logging callback to the native code /// /// /// /// /// /// /// private static void LoggingFunctionThunk(IntPtr param, IntPtr severity, IntPtr /* utf-8 const char* */ category, IntPtr /* utf-8 const char* */ logid, IntPtr /* utf-8 const char* */ codeLocation, IntPtr /* utf-8 const char* */ message) { var categoryStr = NativeOnnxValueHelper.StringFromNativeUtf8(category); var logidStr = NativeOnnxValueHelper.StringFromNativeUtf8(logid); var codeLocationStr = NativeOnnxValueHelper.StringFromNativeUtf8(codeLocation); var messageStr = NativeOnnxValueHelper.StringFromNativeUtf8(message); _userLoggingFunction(param, (OrtLoggingLevel)severity, categoryStr, logidStr, codeLocationStr, messageStr); } /// /// This is invoked only once when the first call refers _instance.Value. /// private static OrtEnv CreateInstance() { OrtEnv result = null; if (!_createOptions.HasValue) { // Default creation result = CreateDefaultEnv(OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING, _defaultLogId); } else { var opts = _createOptions.Value; var logId = (string.IsNullOrEmpty(opts.logId)) ? _defaultLogId : NativeOnnxValueHelper.StringToZeroTerminatedUtf8(opts.logId); var logLevel = opts.logLevel ?? OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING; var threadOps = opts.threadOptions; var loggingFunc = opts.loggingFunction; var logParam = opts.loggingParam ?? IntPtr.Zero; if (threadOps is null && loggingFunc is null) { result = CreateDefaultEnv(logLevel, logId); } else if (threadOps == null) { result = CreateWithCustomLogger(logLevel, logId, logParam, loggingFunc); } else if (loggingFunc == null) { result = CreateWithThreadingOptions(logLevel, logId, threadOps); } else { result = CreateEnvWithCustomLoggerAndGlobalThreadPools(logLevel, logId, logParam, threadOps, loggingFunc); } } return result; } private static OrtEnv CreateDefaultEnv(OrtLoggingLevel logLevel, byte[] logIdUtf8) { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnv(logLevel, logIdUtf8, out IntPtr handle)); var result = new OrtEnv(handle, logLevel); SetLanguageProjection(result); return result; } private static OrtEnv CreateWithCustomLogger(OrtLoggingLevel logLevel, byte[] logIdUtf8, IntPtr loggerParam, DOrtLoggingFunction loggingFunction) { System.Diagnostics.Debug.Assert(loggingFunction != null); // We pass _loggingFunctionInternal which then call user supplied _userLoggingFunction _userLoggingFunction = loggingFunction; var nativeFuncPtr = Marshal.GetFunctionPointerForDelegate(_loggingFunctionInternal); NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnvWithCustomLogger( nativeFuncPtr, loggerParam, logLevel, logIdUtf8, out IntPtr handle)); var result = new OrtEnv(handle, logLevel); SetLanguageProjection(result); return result; } private static OrtEnv CreateWithThreadingOptions(OrtLoggingLevel logLevel, byte[] logIdUtf8, OrtThreadingOptions threadingOptions) { System.Diagnostics.Debug.Assert(threadingOptions != null); NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnvWithGlobalThreadPools( logLevel, logIdUtf8, threadingOptions.Handle, out IntPtr handle)); var result = new OrtEnv(handle, logLevel); SetLanguageProjection(result); return result; } private static OrtEnv CreateEnvWithCustomLoggerAndGlobalThreadPools(OrtLoggingLevel logLevel, byte[] logIdUtf8, IntPtr logParam, OrtThreadingOptions threadingOptions, DOrtLoggingFunction loggingFunction) { System.Diagnostics.Debug.Assert(threadingOptions != null); System.Diagnostics.Debug.Assert(loggingFunction != null); // We pass _loggingFunctionInternal which then call user supplied _userLoggingFunction _userLoggingFunction = loggingFunction; var nativeFuncPtr = Marshal.GetFunctionPointerForDelegate(_loggingFunctionInternal); NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnvWithCustomLoggerAndGlobalThreadPools(nativeFuncPtr, logParam, logLevel, logIdUtf8, threadingOptions.Handle, out IntPtr handle)); var result = new OrtEnv(handle, logLevel); SetLanguageProjection(result); return result; } /// /// To be called only from constructor /// private static void SetLanguageProjection(OrtEnv env) { try { NativeApiStatus.VerifySuccess(NativeMethods.OrtSetLanguageProjection(env.Handle, ORT_PROJECTION_CSHARP)); } catch (Exception) { env.Dispose(); throw; } } #endregion #region Public methods /// /// Instantiates (if not already done so) a new OrtEnv instance with the default logging level /// and no other options. Otherwise returns the existing instance. /// /// It returns the same instance on every call - `OrtEnv` is singleton /// /// Returns a singleton instance of OrtEnv that represents native OrtEnv object public static OrtEnv Instance() { return _instance.Value; } /// /// Provides a way to create an instance with options. /// It throws if the instance already exists and the specified options /// not have effect. /// /// /// /// if the singleton has already been created public static OrtEnv CreateInstanceWithOptions(ref EnvironmentCreationOptions options) { // Non-thread safe, best effort hopefully helpful check. // Environment is usually created once per process, so this should be fine. if (_instance.IsValueCreated) { throw new OnnxRuntimeException(ErrorCode.RuntimeException, "OrtEnv singleton instance already exists, supplied options would not have effect"); } _createOptions = options; return _instance.Value; } /// /// Provides visibility if singleton already been instantiated /// public static bool IsCreated { get { return _instance.IsValueCreated; } } /// /// Enable platform telemetry collection where applicable /// (currently only official Windows ORT builds have telemetry collection capabilities) /// public void EnableTelemetryEvents() { NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableTelemetryEvents(Handle)); } /// /// Disable platform telemetry collection /// public void DisableTelemetryEvents() { NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableTelemetryEvents(Handle)); } /// /// Create and register an allocator to the OrtEnv instance /// so as to enable sharing across all sessions using the OrtEnv instance /// OrtMemoryInfo instance to be used for allocator creation /// OrtArenaCfg instance that will be used to define the behavior of the arena based allocator /// public void CreateAndRegisterAllocator(OrtMemoryInfo memInfo, OrtArenaCfg arenaCfg) { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateAndRegisterAllocator(Handle, memInfo.Pointer, arenaCfg.Pointer)); } /// /// This function returns the onnxruntime version string /// /// version string public string GetVersionString() { IntPtr versionString = NativeMethods.OrtGetVersionString(); return NativeOnnxValueHelper.StringFromNativeUtf8(versionString); } /// /// Queries all the execution providers supported in the native onnxruntime shared library /// /// an array of strings that represent execution provider names public string[] GetAvailableProviders() { IntPtr availableProvidersHandle = IntPtr.Zero; int numProviders; NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAvailableProviders(out availableProvidersHandle, out numProviders)); try { var availableProviders = new string[numProviders]; for (int i = 0; i < numProviders; ++i) { availableProviders[i] = NativeOnnxValueHelper.StringFromNativeUtf8(Marshal.ReadIntPtr(availableProvidersHandle, IntPtr.Size * i)); } return availableProviders; } finally { // This should never throw. The original C API should have never returned status in the first place. // If it does, it is BUG and we would like to propagate that to the user in the form of an exception NativeApiStatus.VerifySuccess(NativeMethods.OrtReleaseAvailableProviders(availableProvidersHandle, numProviders)); } } /// /// Get/Set log level property of OrtEnv instance /// Default LogLevel.Warning /// /// env log level public OrtLoggingLevel EnvLogLevel { get { return _envLogLevel; } set { NativeApiStatus.VerifySuccess(NativeMethods.OrtUpdateEnvWithCustomLogLevel(Handle, value)); _envLogLevel = value; } } #endregion #region SafeHandle overrides /// /// Returns a handle to the native `OrtEnv` instance held by the singleton C# `OrtEnv` instance /// Exception caching: May throw an exception on every call, if the `OrtEnv` constructor threw an exception /// during lazy initialization /// internal IntPtr Handle { get { return handle; } } /// /// Overrides SafeHandle.IsInvalid /// /// returns true if handle is equal to Zero public override bool IsInvalid { get { return handle == IntPtr.Zero; } } /// /// Destroys native object /// /// always returns true protected override bool ReleaseHandle() { NativeMethods.OrtReleaseEnv(handle); handle = IntPtr.Zero; // Re-create empty Lazy initializer // This is great for tests _instance = new Lazy(CreateInstance); return true; } #endregion } }