// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System; using System.Runtime.InteropServices; namespace Microsoft.ML.OnnxRuntime { /// /// NamedOnnxValue type, must match the native enum /// internal static class NativeMethods { private const string nativeLib = "onnxruntime.dll"; internal const CharSet charSet = CharSet.Ansi; #region Runtime/Environment API [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* ONNXStatus* */ONNXRuntimeInitialize( LogLevel default_warning_level, string logId, out IntPtr /*(ONNXEnv*)*/ env); // ReleaseONNXEnv should not be used [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXEnv(IntPtr /*(ONNXEnv*)*/ env); #endregion Runtime/Environment API #region Status API [DllImport(nativeLib, CharSet = charSet)] public static extern ErrorCode ONNXRuntimeGetErrorCode(IntPtr /*(ONNXStatus*)*/status); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* char* */ONNXRuntimeGetErrorMessage(IntPtr /* (ONNXStatus*) */status); // returns char*, need to convert to string by the caller. // does not free the underlying ONNXStatus* [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXStatus(IntPtr /*(ONNXStatus*)*/ statusPtr); #endregion Status API #region InferenceSession API [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* ONNXStatus* */ONNXRuntimeCreateInferenceSession( IntPtr /* (ONNXEnv*) */ environment, [MarshalAs(UnmanagedType.LPWStr)]string modelPath, //the model path is consumed as a wchar* in the C-api IntPtr /* (ONNXRuntimeSessionOptions*) */sessopnOptions, out IntPtr /**/ session); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInference( IntPtr /*(ONNXSession*)*/ session, IntPtr /*(ONNXSessionRunOptions*)*/ runOptions, // can be null to use the default options string[] inputNames, IntPtr[] /* (ONNXValue*[])*/ inputValues, ulong inputCount, /* TODO: size_t, make it portable for x86 arm */ string[] outputNames, ulong outputCount, /* TODO: size_t, make it portable for x86 and arm */ [MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 5 /*index of outputCount*/)][In, Out] IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */ ); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeInferenceSessionGetInputCount( IntPtr /*(ONNXSession*)*/ session, out ulong /* TODO: size_t */ count); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeInferenceSessionGetOutputCount( IntPtr /*(ONNXSession*)*/ session, out ulong /*TODO: size_t port*/ count); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetInputName( IntPtr /*(ONNXSession*)*/ session, ulong index, //TODO: port size_t IntPtr /*(ONNXRuntimeAllocator*)*/ allocator, out IntPtr /*(char**)*/name); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetOutputName( IntPtr /*(ONNXSession*)*/ session, ulong index, //TODO: port size_t IntPtr /*(ONNXRuntimeAllocator*)*/ allocator, out IntPtr /*(char**)*/name); // release the typeinfo using ONNXRuntimeReleaseObject [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetInputTypeInfo( IntPtr /*(const ONNXSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /*(struct ONNXRuntimeTypeInfo**)*/ typeInfo); // release the typeinfo using ONNXRuntimeReleaseObject [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeInferenceSessionGetOutputTypeInfo( IntPtr /*(const ONNXSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /* (struct ONNXRuntimeTypeInfo**)*/ typeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/session); #endregion InferenceSession API #region SessionOptions API //Release using ONNXRuntimeReleaseObject [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*ONNXRuntimeSessionOptions* */ ONNXRuntimeCreateSessionOptions(); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXRuntimeSessionOptions*)*/ONNXRuntimeCloneSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeEnableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options, string profilePathPrefix); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeDisableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeEnableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeDisableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeEnableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeDisableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeSetSessionLogId(IntPtr /* ONNXRuntimeSessionOptions* */ options, string logId); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeSetSessionLogVerbosityLevel(IntPtr /* ONNXRuntimeSessionOptions* */ options, LogLevel sessionLogVerbosityLevel); [DllImport(nativeLib, CharSet = charSet)] public static extern int ONNXRuntimeSetSessionThreadPoolSize(IntPtr /* ONNXRuntimeSessionOptions* */ options, int sessionThreadPoolSize); ///** // * The order of invocation indicates the preference order as well. In other words call this method // * on your most preferred execution provider first followed by the less preferred ones. // * Calling this API is optional in which case onnxruntime will use its internal CPU execution provider. // */ [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeSessionOptionsAppendExecutionProvider(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options, IntPtr /* (ONNXRuntimeProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateCpuExecutionProviderFactory(int use_arena, out IntPtr /*(ONNXRuntimeProviderFactoryPtr*)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateMkldnnExecutionProviderFactory(int use_arena, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateCUDAExecutionProviderFactory(int device_id, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeCreateNupharExecutionProviderFactory(int device_id, string target_str, out IntPtr /*(ONNXRuntimeProviderFactoryPtr**)*/ factory); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeAddCustomOp(IntPtr /*(ONNXRuntimeSessionOptions*)*/ options, string custom_op_path); #endregion #region Allocator/AllocatorInfo API //TODO: consider exposing them publicly, when allocator API is exposed public enum AllocatorType { DeviceAllocator = 0, ArenaAllocator = 1 } //TODO: consider exposing them publicly when allocator API is exposed public enum MemoryType { CpuInput = -2, // Any CPU memory used by non-CPU execution provider CpuOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED Cpu = CpuOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED Default = 0, // the default allocator for execution provider } [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateAllocatorInfo( IntPtr /*(const char*) */name, AllocatorType allocatorType, int identifier, MemoryType memType, out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo // memory ownership transfered to caller ); //ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out) [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateCpuAllocatorInfo( AllocatorType allocatorType, MemoryType memoryType, out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo ); [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXRuntimeAllocatorInfo(IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeCreateDefaultAllocator(out IntPtr /*(ONNXRuntimeAllocator**)*/ allocator); /// /// Releases/Unrefs any object, including the Allocator /// /// /// remaining ref count [DllImport(nativeLib, CharSet = charSet)] public static extern uint /*remaining ref count*/ ONNXRuntimeReleaseObject(IntPtr /*(void*)*/ ptr); /// /// Release any object allocated by an allocator /// /// /// [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeAllocatorFree(IntPtr allocator, IntPtr memory); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(const struct ONNXRuntimeAllocatorInfo*)*/ ONNXRuntimeAllocatorGetInfo(IntPtr /*(const ONNXRuntimeAllocator*)*/ ptr); #endregion Allocator/AllocatorInfo API #region Tensor/OnnxValue API [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* ONNXStatus */ ONNXRuntimeCreateTensorWithDataAsONNXValue( IntPtr /* (const ONNXRuntimeAllocatorInfo*) */ allocatorInfo, IntPtr /* (void*) */dataBufferHandle, ulong dataLength, //size_t, TODO: make it portable for x86, arm ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm ulong shapeLength, //size_t, TODO: make it portable for x86, arm TensorElementType type, out IntPtr /* ONNXValuePtr* */ outputValue); /// This function doesn't work with string tensor /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); //[DllImport(nativeLib, CharSet = charSet)] //public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm //[DllImport(nativeLib, CharSet = charSet)] //public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ ONNXRuntimeCastTypeInfoToTensorInfo(IntPtr /*(struct ONNXRuntimeTypeInfo*)*/ typeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeAndType(IntPtr /*(ONNXValue*)*/ value, out IntPtr /*(struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern TensorElementType ONNXRuntimeGetTensorElementType(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern ulong /*TODO: port for size_t */ONNXRuntimeGetNumOfDimensions(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern void ONNXRuntimeGetDimensions( IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo, long[] dim_values, ulong dim_values_length); /** * How many elements does this tensor have. * May return a negative value * e.g. * [] -> 1 * [1,3,4] -> 12 * [2,0,4] -> 0 * [-1,3,4] -> -1 */ [DllImport(nativeLib, CharSet = charSet)] public static extern long ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(const struct ONNXRuntimeTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); #endregion } //class NativeMethods } //namespace