// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Microsoft.ML.OnnxRuntime.Tensors; using System; using System.Buffers; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; using System.Text; namespace Microsoft.ML.OnnxRuntime { /// /// A type of data that OrtValue encapsulates. /// public enum OnnxValueType { ONNX_TYPE_UNKNOWN = 0, // Not set ONNX_TYPE_TENSOR = 1, // It's a Tensor ONNX_TYPE_SEQUENCE = 2, // It's an Onnx sequence which may be a sequence of Tensors/Maps/Sequences ONNX_TYPE_MAP = 3, // It's a map ONNX_TYPE_OPAQUE = 4, // It's an experimental Opaque object ONNX_TYPE_SPARSETENSOR = 5, // It's a Sparse Tensor ONNX_TYPE_OPTIONAL = 6, // It's an optional type that designates anything above (except UNKOWN) } /// /// Represents a disposable OrtValue. /// This class exposes a native instance of OrtValue. /// The class implements IDisposable and must /// be disposed of, otherwise native resources will leak /// and will eventually cause the application to slow down or crash. /// /// If the OrtValue instance is constructed over a managed memory, and it is not /// disposed properly, the pinned memory will continue to be pinned and interfere /// with GC operation. /// public class OrtValue : IOrtValueOwner, IDisposable { // OrtValues that are members of Sequences or Maps that map. They potentially map managed memory and we need to keep them around. // this exists only when we deal with compose ML types. private DisposableList _compositeMembers; private IntPtr _handle; private MemoryHandle? _memHandle; // Present when the OrtValue is created on top of managed memory private bool _disposed; internal OrtValue(IntPtr handle) { _handle = handle; InitOnnxType(); } /// /// Constructor. The newly constructed OrtValue takes ownership of the native OrtValue instance /// /// /// /// thrown when onnxValue type is not known internal OrtValue(IntPtr handle, OnnxValueType onnxValueType) { if (onnxValueType == OnnxValueType.ONNX_TYPE_UNKNOWN) { throw new ArgumentException("onnxValueType argument is passed as unknown"); } _handle = handle; OnnxType = onnxValueType; } /// /// Constructor. The newly constructed OrtValue takes ownership of the native OrtValue instance /// and disposes of it when the OrtValue instance is disposed. The instance will take ownership and will /// dispose of compositeMembers instances. /// /// This constructor can only throw if OnnxType is not specified. /// /// native ortValue handle /// must one of the valid types /// For composite types this contains dependent ortValues such as members of a sequence /// or keys/values for the map, that may have been created on top of the managed memory and must be disposed /// with the new ortValue. This container will be taken the ownership of and the argument will be set to null. /// throws when onnxValueType is not specified internal OrtValue(IntPtr handle, OnnxValueType onnxValueType, ref DisposableList compositeMembers) { if (onnxValueType == OnnxValueType.ONNX_TYPE_UNKNOWN) { throw new ArgumentException("onnxValueType argument is passed as unknown"); } _handle = handle; OnnxType = onnxValueType; _compositeMembers = compositeMembers; compositeMembers = null; } /// /// Constructor to construct OrtValue over managed memory. /// We pin the memory and unpin it at the disposal time. /// The newly constructed OrtValue takes ownership of the native OrtValue instance /// and disposes of it when the OrtValue instance is disposed. /// /// Pointer to a native instance of OrtValue /// memory handle to a pinned user supplied (usually managed) memory /// It is disposed of (unpinned) when OrtValue is disposed. /// private OrtValue(IntPtr handle, MemoryHandle memHandle) { _handle = handle; _memHandle = memHandle; // OrtValue on top of the pinned memory is always a tensor OnnxType = OnnxValueType.ONNX_TYPE_TENSOR; } /// /// Native handle to OrtValue for internal use. /// internal IntPtr Handle { get { return _handle; } } /// /// Implement IOrtValueOwner interface /// /// returns this public OrtValue Value => this; /// /// Fetches OrtValue type if it has one. /// /// OnnxValueType public OnnxValueType OnnxType { get; private set; } private void InitOnnxType() { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueType(Handle, out IntPtr onnxType)); OnnxType = (OnnxValueType)onnxType; } /// /// Returns true if OrtValue contains a tensor /// /// true if tensor public bool IsTensor { get { return OnnxType == OnnxValueType.ONNX_TYPE_TENSOR; } } /// /// Returns true if OrtValue contains a sparse tensor /// /// true if sparse tensor public bool IsSparseTensor { get { return OnnxType == OnnxValueType.ONNX_TYPE_SPARSETENSOR; } } /// /// Valid for composite ML types like map, sequence. /// Returns 2 for map (keys, values) and N for sequence, where N is the number of elements /// in the sequence. /// /// Element count public int GetValueCount() { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueCount(Handle, out IntPtr count)); return (int)count; } /// /// For non tensors return OrtValue element at the specified index. /// For maps only indices 0 and 1 are valid. For sequences, [0..N) are valid. /// See GetValueCount() to determine the valid range. /// /// /// allocator to use /// OrtValue disposable instance that points to the corresponding element of the composite type public OrtValue GetValue(int index, OrtAllocator allocator) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(Handle, index, allocator.Pointer, out IntPtr ortValueHandle)); return new OrtValue(ortValueHandle); } /// /// Returns a ReadOnlySpan over tensor native buffer that /// provides a read-only view. /// /// Note, that the memory may be device allocated and, therefore, not accessible from the CPU. /// To get memory descriptor use GetTensorMemoryInfo(). /// /// OrtValue must contain a non-string tensor. /// The span is valid as long as the OrtValue instance is alive (not disposed). /// /// /// ReadOnlySpan /// public ReadOnlySpan GetTensorDataAsSpan() where T : unmanaged { var byteSpan = GetTensorBufferRawData(typeof(T)); return MemoryMarshal.Cast(byteSpan); } /// /// Returns a Span over tensor native buffer. /// This enables you to safely and efficiently modify the underlying /// native buffer in a type-safe manner. This is useful for example in IOBinding scenarios /// where you want to modify results of the inference and feed it back as input. /// /// Note, that the memory may be device allocated. /// To get memory descriptor use GetTensorMemoryInfo(). /// /// OrtValue must contain a non-string tensor. /// The span is valid as long as the OrtValue instance is alive (not disposed). /// /// /// Typed Span over the native buffer public Span GetTensorMutableDataAsSpan() where T : unmanaged { var byteSpan = GetTensorBufferRawData(typeof(T)); return MemoryMarshal.Cast(byteSpan); } /// /// Provides mutable raw native buffer access. /// /// Span over the native buffer bytes public Span GetTensorMutableRawData() { return GetTensorBufferRawData(typeof(byte)); } /// /// Fetch string tensor element buffer pointer at the specified index, /// convert/copy to UTF-16 char[] and return a ReadOnlyMemory instance. /// /// Obtain TensorTypeAndShape to get shape and element count. /// /// flat string tensor element index /// ReadOnlyMemory backed by a managed char[]. Its lifespan is not /// tied to the native buffer of OrtValue. public ReadOnlyMemory GetStringElementAsMemory(int index) { var chars = GetStringTensorElementChars(index); if (chars.Length == 0) { return ReadOnlyMemory.Empty; } return new ReadOnlyMemory(chars); } /// /// Fetch string tensor element buffer pointer at the specified index, /// copy/convert UTF-8 into a UTF-16 string and return it. /// /// Obtain TensorTypeAndShape to get shape and element count. /// /// flat string tensor element index /// UTF-16 string instance public string GetStringElement(int index) { var chars = GetStringTensorElementChars(index); if (chars.Length == 0) { return string.Empty; } return new string(chars); } /// /// Get a span over the native memory of the string tensor element. /// The span is valid as long as the OrtValue is valid. /// /// This is useful if you want to perform your own UTF-8 decoding or /// you do not care about decoding. /// Obtain TensorTypeAndShape to get shape and element count. /// /// flat element index /// ReadOnlySpan over UTF-8 bytes of the string tensor element public ReadOnlySpan GetStringElementAsSpan(int index) { GetStringTensorElementBuffer((UIntPtr)index, out uint bytesLen, out IntPtr bufferPtr); if (bytesLen == 0) { return ReadOnlySpan.Empty; } unsafe { return new ReadOnlySpan((bufferPtr).ToPointer(), (int)bytesLen); } } /// /// Convenience method to obtain all string tensor elements as a string array. /// /// string[] /// public string[] GetStringTensorAsArray() { GetTensorElementTypeAndCount(out long count, out TensorElementType elementType); if (elementType != TensorElementType.String) { throw new OnnxRuntimeException( ErrorCode.Fail, $"GetStringTensorAsArray() is only supported for string tensors. This OrtValue contains a {elementType} tensor."); } var strings = new string[count]; for (int i = 0; i < count; i++) { strings[i] = GetStringElement(i); } return strings; } /// /// Creates and fetches Type information about the contained OnnxValue. /// /// a disposable instance of OrtTypeInfo public OrtTypeInfo GetTypeInfo() { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTypeInfo(Handle, out IntPtr typeInfo)); try { return new OrtTypeInfo(typeInfo); } finally { NativeMethods.OrtReleaseTypeInfo(typeInfo); } } /// /// Obtains Tensor And Type Information from the OrtValue iff it contains a tensor. /// Valid only for OrtValues that contain a tensor. /// /// A disposable instance of OrtTensorTypeAndShapeInfo public OrtTensorTypeAndShapeInfo GetTensorTypeAndShape() { var onnxType = OnnxType; if (onnxType != OnnxValueType.ONNX_TYPE_TENSOR && onnxType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"This OrtValue type contains: {onnxType}, not a tensor or sparse tensor"); } NativeMethods.OrtGetTensorTypeAndShape(Handle, out IntPtr typeAndShapeInfo); try { return new OrtTensorTypeAndShapeInfo(typeAndShapeInfo); } finally { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShapeInfo); } } /// /// Returns OrtMemoryInfo iff this OrtValue contains a tensor or a sparse tensor. /// /// OrtMemoryInfo that describes the underlying memory allocation /// public OrtMemoryInfo GetTensorMemoryInfo() { var onnxType = OnnxType; if (onnxType != OnnxValueType.ONNX_TYPE_TENSOR && onnxType != OnnxValueType.ONNX_TYPE_SPARSETENSOR) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"This OrtValue type contains: {onnxType}, not a tensor or sparse tensor"); } NativeMethods.OrtGetTensorMemoryInfo(Handle, out IntPtr memoryInfo); return new OrtMemoryInfo(memoryInfo, false); } private void GetTensorElementTypeAndCount(out long count, out TensorElementType elementType) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorTypeAndShape(Handle, out IntPtr typeAndShapeInfo)); try { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorElementType(typeAndShapeInfo, out IntPtr elType)); NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeElementCount(typeAndShapeInfo, out UIntPtr cnt)); elementType = (TensorElementType)elType; count = (long)cnt; } finally { NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShapeInfo); } } private char[] GetStringTensorElementChars(int index) { GetStringTensorElementBuffer((UIntPtr)index, out uint bytesLen, out IntPtr bufferPtr); if (bytesLen == 0) { return Array.Empty(); } unsafe { int charCount = Encoding.UTF8.GetCharCount((byte*)(bufferPtr).ToPointer(), (int)bytesLen); var chars = new char[charCount]; fixed (char* ch = chars) { Encoding.UTF8.GetChars((byte*)(bufferPtr).ToPointer(), (int)bytesLen, (char*)ch, charCount); } return chars; } } private void GetStringTensorElementBuffer(UIntPtr index, out uint bytesLen, out IntPtr bufferPtr) { // Length is in UTF-8 bytes. Strings are not zero terminated, so length is required. NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorElementLength(Handle, index, out UIntPtr bufferLen)); bytesLen = (uint)bufferLen; if (bytesLen == 0) { bufferPtr = IntPtr.Zero; return; } // XXX: We lack the API (at the moment) that simply gives access to string element buffer. So we get the resized one // to the same length which leaves it unchanged. NativeApiStatus.VerifySuccess(NativeMethods.OrtGetResizedStringTensorElementBuffer(Handle, (UIntPtr)index, bufferLen, out bufferPtr)); } private Span GetTensorBufferRawData(Type requestedType) { if (OnnxType != OnnxValueType.ONNX_TYPE_TENSOR) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"This OrtValue type contains: {OnnxType}, not a tensor"); } GetTensorElementTypeAndCount(out long count, out TensorElementType elementType); if (elementType == TensorElementType.String) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Strings are not supported by this API"); } var typeInfo = TensorBase.GetElementTypeInfo(elementType) ?? throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Element type: {elementType} is not registered type."); // We are always Ok with byte if (requestedType != typeof(byte) && requestedType != typeInfo.TensorType) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Requested type: {requestedType} does not match the actual type: {typeInfo.TensorType}"); } if (count == 0) { return Span.Empty; } NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(Handle, out IntPtr tensorData)); var bufferLenInBytes = count * typeInfo.TypeSize; unsafe { return new Span(tensorData.ToPointer(), (int)bufferLenInBytes); } } /// /// Factory method to construct an OrtValue of Tensor type on top of pre-allocated memory. /// This can be a piece of arbitrary memory that may be allocated by OrtAllocator (possibly on a device), /// a chunk of managed memory (must be pinned for the duration of OrtValue lifetime) or a memory that is allocated /// natively allocated using Marshal.AllocHGlobal(), stackalloc or other means (may be on a device). /// /// The resulting OrtValue does not own the underlying memory buffer and will not attempt to /// deallocate it. The caller must make sure that the memory remains valid for the duration of OrtValue lifetime. /// /// Memory Info. For managed memory its default is cpu. /// For other kinds of memory, one must construct as appropriate. /// DataType for the Tensor /// shape of the tensor to create. The size required by the shape /// must be less of equal of the memory.Length /// Pointer to a raw memory buffer which may reside on a device /// Buffer length in bytes /// A disposable instance of OrtValue public static OrtValue CreateTensorValueWithData(OrtMemoryInfo memInfo, TensorElementType elementType, long[] shape, IntPtr dataBufferPtr, long bufferLengthInBytes) { var typeInfo = TensorBase.GetElementTypeInfo(elementType) ?? throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Tensor element type: {elementType} is not supported"); if (typeInfo.IsString) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Cannot map managed strings buffer to native OrtValue. Use string specific interfaces"); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); var requiredBufferSizeInBytes = shapeSize * typeInfo.TypeSize; // We allow creating a tensor over part of the buffer if (requiredBufferSizeInBytes > bufferLengthInBytes) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Shape: {shape} has: {shapeSize} elements requires a buffer of at least {requiredBufferSizeInBytes} bytes. Provided: {bufferLengthInBytes} bytes"); } NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memInfo.Pointer, dataBufferPtr, (UIntPtr)bufferLengthInBytes, shape, (UIntPtr)shape.Length, elementType, out IntPtr ortValueHandle )); return new OrtValue(ortValueHandle, OnnxValueType.ONNX_TYPE_TENSOR); } /// /// This is a factory method that creates an OrtValue of Tensor type on top of Memory memory. /// The API pins the memory for the duration of the OrtValue lifetime. /// It is unpinned at disposal time. /// /// T must be one of the supported types /// Memory information that describes memory location /// contiguous region of memory /// shape of the tensor to create. The size required by the shape /// must be less of equal of the memory.Length /// A disposable OrtValue instance /// public static OrtValue CreateTensorValueFromMemory(OrtMemoryInfo memoryInfo, Memory memory, long[] shape) where T : unmanaged { var typeInfo = TensorBase.GetTypeInfo(typeof(T)) ?? throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Tensor of type: {typeof(T)} is not supported"); if (typeInfo.IsString) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Cannot map managed strings buffer to native OrtValue. Use string specific interfaces."); } var shapeSize = ArrayUtilities.GetSizeForShape(shape); // We allow creating a tensor over part of the buffer if (shapeSize > memory.Length) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Managed memory size: {memory.Length} elements is less than shape size: {shapeSize} elements"); } var bufferLengthInBytes = memory.Length * typeInfo.TypeSize; var memoryHandle = memory.Pin(); try { IntPtr bufferPtr; unsafe { bufferPtr = new IntPtr(memoryHandle.Pointer); } NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( memoryInfo.Pointer, bufferPtr, (UIntPtr)bufferLengthInBytes, shape, (UIntPtr)shape.Length, typeInfo.ElementType, out IntPtr ortValueHandle )); return new OrtValue(ortValueHandle, memoryHandle); } catch (Exception) { memoryHandle.Dispose(); throw; } } /// /// This is a factory method that creates an OrtValue of Tensor type on top managed data array. /// The API pins the memory for the duration of the OrtValue lifetime. /// It is unpinned at disposal time. /// /// /// managed data buffer /// shape that describes the buffer /// A disposable OrtValue instance public static OrtValue CreateTensorValueFromMemory(T[] data, long[] shape) where T : unmanaged { return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, new Memory(data), shape); } /// /// The factory API creates an OrtValue with memory allocated using the given allocator /// according to the specified shape and element type. The memory will be released when OrtValue /// is disposed. Use GetTensorMutableDataAsSpan<T>() API to fill in the data. /// /// /// /// /// A disposable OrtValue public static OrtValue CreateAllocatedTensorValue(OrtAllocator allocator, TensorElementType elementType, long[] shape) { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(allocator.Pointer, shape, (UIntPtr)shape.Length, elementType, out IntPtr ortValueHandle)); return new OrtValue(ortValueHandle, OnnxValueType.ONNX_TYPE_TENSOR); } /// /// This is a factory method creates a native Onnxruntime OrtValue containing a tensor. /// The method will attempt to pin managed memory so no copying occurs when data is passed down /// to native code. /// /// Tensor object /// discovered tensor element type /// And instance of OrtValue constructed on top of the object internal static OrtValue CreateFromTensorObject(TensorBase value, out TensorElementType elementType) { var typeInfo = value.GetTypeInfo(); OrtValue ortValue = null; TensorElementType elType = typeInfo.ElementType; var typeSize = typeInfo.TypeSize; if (typeInfo.IsString) { ortValue = CreateFromStringTensor(value as Tensor); } else { int dataBufferLength; long[] shape; int rank; MemoryHandle memHandle; switch (elType) { case TensorElementType.Float: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Double: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Int32: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.UInt32: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Int64: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.UInt64: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Int16: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.UInt16: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.UInt8: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Int8: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Bool: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.Float16: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; case TensorElementType.BFloat16: PinAsTensor(value as Tensor, typeSize, out memHandle, out dataBufferLength, out shape, out rank); break; default: throw new NotSupportedException("Element type: " + elType + " is not of a supported type"); } try { IntPtr dataBufferPointer = IntPtr.Zero; unsafe { dataBufferPointer = (IntPtr)memHandle.Pointer; } NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue( OrtMemoryInfo.DefaultInstance.Pointer, dataBufferPointer, (UIntPtr)(dataBufferLength), shape, (UIntPtr)rank, elType, out IntPtr nativeValue)); ortValue = new OrtValue(nativeValue, memHandle); } catch (Exception) { memHandle.Dispose(); throw; } } elementType = elType; return ortValue; } /// /// Creates an OrtValue that contains a string tensor of specified shape, and /// containing empty strings. String tensors are always on CPU. /// Use FillStringTensorElement to assign individual elements values. /// /// /// disposable OrtValue /// tensor shape public static OrtValue CreateTensorWithEmptyStrings(OrtAllocator allocator, long[] shape) { // allocate the native tensor NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue( allocator.Pointer, shape, (UIntPtr)(shape.Length), TensorElementType.String, out IntPtr valueHandle )); return new OrtValue(valueHandle, OnnxValueType.ONNX_TYPE_TENSOR); } /// /// Converts the string argument represented by ReadOnlySpan to UTF-8, /// allocates space in the native tensor and copies it into the native tensor memory. /// Typically, this is used to populate a new empty string tensor element. /// /// The number of elements is according to the shape supplied to CreateTensorWithEmptyStrings(). /// However, this API can also be used to overwrite any existing element within the string tensor. /// /// In general, to obtain the number of elements for any tensor, use GetTensorTypeAndShape() which /// would return a disposable instance of TensorTypeAndShapeInfo. /// Then call GetElementCount() or GetShape(). /// /// ReadOnlySpan over chars /// index of the string element within the tensor /// must be within bounds of [0, N) public void FillStringTensorElement(ReadOnlySpan str, int index) { unsafe { fixed (char* strPtr = str) { FillStringTensorElement(strPtr, str.Length, index); } } } /// /// Converts the string argument represented by ReadOnlyMemory to UTF-8, /// allocates space in the native tensor and copies it into the native tensor memory. /// Typically, this is used to populate a new empty string tensor element. /// /// The number of elements is according to the shape supplied to CreateTensorWithEmptyStrings(). /// However, this API can also be used to overwrite any existing element within the string tensor. /// /// In general, to obtain the number of elements for any tensor, use GetTensorTypeAndShape() which /// would return a disposable instance of TensorTypeAndShapeInfo. /// Then call GetElementCount() or GetShape(). /// /// /// ReadOnlyMemory instance over an array of chars /// index of the string element within the tensor /// must be within bounds of [0, N) public void FillStringTensorElement(ReadOnlyMemory rom, int index) { FillStringTensorElement(rom.Span, index); } /// /// This API resizes String Tensor element to the requested amount of bytes (UTF-8) /// and copies the bytes from the supplied ReadOnlySpan into the native tensor memory (resized buffer). /// /// The API is useful for quick loading of utf8 data into the native tensor memory. /// /// read only span of bytes /// flat index of the element in the string tensor public void FillStringTensorElement(ReadOnlySpan utf8Bytes, int index) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetResizedStringTensorElementBuffer(Handle, (UIntPtr)index, (UIntPtr)utf8Bytes.Length, out IntPtr buffer)); if (utf8Bytes.Length == 0) { return; } unsafe { var destSpan = new Span(buffer.ToPointer(), utf8Bytes.Length); utf8Bytes.CopyTo(destSpan); } } /// /// Creates an OrtValue that contains a string tensor. /// String tensors are always allocated on CPU. /// String data will be converted to UTF-8 and copied to native memory. /// /// Note, this is different from creating an OrtValue from other primitive data types /// where memory is pinned (if necessary) and the OrtValue points to that chunk of memory. /// /// Tensor /// A disposable OrtValue instance /// public static OrtValue CreateFromStringTensor(Tensor tensor) { if (tensor == null) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "Expecting a valid string tensor"); } long[] shape = Array.ConvertAll(tensor.Dimensions.ToArray(), Convert.ToInt64); var ortValue = CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, shape); try { var len = tensor.Length; for (int i = 0; i < len; ++i) { var str = tensor.GetValue(i) ?? throw new ArgumentNullException($"Tensor contains null reference at index:{i}"); unsafe { fixed (char* strPtr = str) { ortValue.FillStringTensorElement(strPtr, str.Length, i); } } } } catch (Exception) { ortValue.Dispose(); throw; } return ortValue; } /// /// Creates a sequence of OrtValues from a collection of OrtValues. /// All OrtValues in the collection must be of the same Onnx type. /// I.e. (Tensor, SparseTensor, Map, Sequence, etc.) /// /// The ortValues that are passed as argument are taken possession of by the newly /// created OrtValue. The caller should not dispose them, unless this call fails. /// /// The ortValues would be empty on successful return. /// /// a collection of OrtValues. On success the ortValues contained in the list /// are taken ownership of and the list is cleared. /// A disposable instance of OrtValues /// public static OrtValue CreateSequence(ICollection ortValues) { if (ortValues is null) { throw new ArgumentNullException(nameof(ortValues)); } if (ortValues.IsReadOnly) { throw new ArgumentException("ortValues argument can not be a readonly collection"); } var compositeMembers = new DisposableList(ortValues); try { var result = CreateSequence(ref compositeMembers); Debug.Assert(compositeMembers is null, "Must be null on success"); ortValues.Clear(); return result; } catch (Exception) { // The caller is responsible for disposing the ortValues compositeMembers?.Clear(); throw; } } /// /// Creates a sequence from the values in compositeMembers /// The argument is taken possession of and is nullified on successful return. /// /// sequence ortValues /// OrtValue instance representing a Sequence internal static OrtValue CreateSequence(ref DisposableList compositeMembers) { var handles = new IntPtr[compositeMembers.Count]; for (int i = 0; i < compositeMembers.Count; i++) { handles[i] = compositeMembers[i].Handle; } NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateValue(handles, (UIntPtr)handles.Length, (IntPtr)OnnxValueType.ONNX_TYPE_SEQUENCE, out IntPtr sequenceHandle)); return new OrtValue(sequenceHandle, OnnxValueType.ONNX_TYPE_SEQUENCE, ref compositeMembers); } /// /// A delegate type that is expected to process each OrtValue in a sequence. /// /// OrtValue that holds sequence element /// ordinal of the value public delegate void SequenceElementVisitor(OrtValue ortValue, int index); /// /// Feeds each OrtValue in a sequence to the visitor delegate. /// This helps users to avoid dealing each value life-span /// /// visitor delegate /// allocator to use for intermediate ort values /// public void ProcessSequence(SequenceElementVisitor visitor, OrtAllocator allocator) { if (OnnxType != OnnxValueType.ONNX_TYPE_SEQUENCE) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"OrtValue.OnnxType of {OnnxType} is not a sequence"); } int count = GetValueCount(); for (int i = 0; i < count; i++) { using var ortValue = GetValue(i, allocator); visitor(ortValue, i); } } /// /// Creates a map OrtValue with keys and values. /// On a high level the Onnxruntime representation of the map always consists of two /// OrtValues, keys and values. /// /// According to ONNX standard map keys can be unmanaged types only (or strings). /// Those keys are contained in a single tensor within OrtValue keys. /// /// Map values, on the other hand, can be composite types. The values parameter /// can either contain a single tensor with unmanaged map values with the same number of /// elements as the keys, or it can be a sequence of OrtValues, /// each of those can be a composite type (tensor, sequence, map). If it is a sequence, /// then the number of elements must match the number of elements in keys. /// /// Keys and values must be in the same order. /// /// ORT supports only a subset of types for keys and values, however, this API does not /// restrict it. /// /// The ortValues that are passed as argument are taken possession of by the newly /// created OrtValue. The caller should not dispose them, unless this call fails. /// /// Keys and values arguments will be set to null on success. /// /// Contains keys /// Contains values /// A disposable OrtValue /// public static OrtValue CreateMap(ref OrtValue keys, ref OrtValue values) { if (keys is null || values is null) { throw new ArgumentNullException("keys or/and values are null"); } IntPtr[] handles = { keys.Handle, values.Handle }; NativeApiStatus.VerifySuccess( NativeMethods.OrtCreateValue(handles, (UIntPtr)handles.Length, (IntPtr)OnnxValueType.ONNX_TYPE_MAP, out IntPtr mapHandle)); var compositeMembers = new DisposableList { keys, values }; keys = null; values = null; // This constructor will not throw. return new OrtValue(mapHandle, OnnxValueType.ONNX_TYPE_MAP, ref compositeMembers); } /// /// This API helps to quickly creates a map OrtValue with unmanaged (primitive) keys and values specified as arrays. /// This helps the user not to create OrtValues for keys and values separately and deal only with the final result. /// The map would consist of two tensors, one for keys and one for values. /// /// The OrtValues would be created on top of the managed memory arrays and use it directly. /// The number of elements in keys and values must be the same and they must be in order. /// /// The types must be unmanaged. /// /// keys type /// values type /// array of keys of K type /// array of values of V type /// OrtValue instance /// /// public static OrtValue CreateMap(K[] keys, V[] values) where K : unmanaged where V : unmanaged { if (keys is null || values is null) { throw new ArgumentNullException("Keys or/and values are null"); } if (keys.Length != values.Length) { throw new ArgumentException("Expecting keys and values same len. " + $"Received keys: {keys.Length}, Values: {values.Length}"); } long[] shape = { keys.Length }; Span ortValues = new OrtValue[2]; var disposableGuard = new DisposableArray(ortValues); try { ortValues[0] = CreateTensorValueFromMemory(keys, shape); ortValues[1] = CreateTensorValueFromMemory(values, shape); return CreateMap(ref ortValues[0], ref ortValues[1]); } catch (Exception) { disposableGuard.Dispose(); throw; } } /// /// Creates a map OrtValue with string keys and non-string values. /// This helps the user not to create OrtValues for keys and values separately. /// The number of elements in keys and values must be the same and they must be in order. /// The map would consist of two tensors, one for keys and one for values. /// /// string keys would be converted to UTF-8 encoding and copied to an allocated native memory. /// The OrtValue for values would be created on top of the managed memory using it directly. /// /// The values type must be unmanaged. /// /// /// Collection of strings /// /// OrtValue instance /// /// public static OrtValue CreateMapWithStringKeys(IReadOnlyCollection keys, V[] values) where V : unmanaged { if (keys is null || values is null) { throw new ArgumentNullException("Keys or/and values are null"); } if (keys.Count != values.Length) { throw new ArgumentException("Expecting keys and values same len. " + $"Received keys: {keys.Count}, Values: {values.Length}"); } long[] shape = { keys.Count }; Span ortValues = new OrtValue[2]; var disposableGuard = new DisposableArray(ortValues); try { ortValues[0] = CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, shape); int count = 0; foreach (var key in keys) { ortValues[0].FillStringTensorElement(key.AsSpan(), count++); } ortValues[1] = CreateTensorValueFromMemory(values, shape); return CreateMap(ref ortValues[0], ref ortValues[1]); } catch (Exception) { disposableGuard.Dispose(); throw; } } /// /// Creates a map OrtValue with non-string keys and string values. /// /// This helps the user not to create OrtValues for keys and values separately. /// The number of elements in keys and values must be the same and they must be in order. /// /// The OrtValue for keys would be created on top of the managed memory using it directly. /// string values would be converted to UTF-8 encoding and copied to an allocated native memory. /// /// /// unmanaged type of keys /// /// collection of string values /// Instance of OrtValue /// /// public static OrtValue CreateMapWithStringValues(K[] keys, IReadOnlyCollection values) where K : unmanaged { if (keys is null || values is null) { throw new ArgumentNullException("Keys or/and values are null"); } if (keys.Length != values.Count) { throw new ArgumentException("Expecting keys and values same len. " + $"Received keys: {keys.Length}, Values: {values.Count}"); } long[] shape = { keys.Length }; Span ortValues = new OrtValue[2]; var disposableGuard = new DisposableArray(ortValues); try { ortValues[0] = CreateTensorValueFromMemory(keys, shape); ortValues[1] = CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, shape); int count = 0; foreach (var value in values) { ortValues[1].FillStringTensorElement(value.AsSpan(), count++); } return CreateMap(ref ortValues[0], ref ortValues[1]); } catch (Exception) { disposableGuard.Dispose(); throw; } } /// /// A public delegate that will be invoked once with map keys and values. /// The delegate helps not to deal with the lifespan of intermediate OrtValues. /// Typically, when one uses GetValue() API, it creates a copy of OrtValue /// that points to the same buffer as keys or values. This API helps to deal with those /// temporary instances and avoid leaks. /// /// According to ONNX standard map keys can be unmanaged types only (or strings). /// Those keys are contained in a single tensor within OrtValue keys. So you can query those /// directly from keys argument. /// /// Map values, on the other hand, can be composite types. The values parameter /// can either contain a single tensor with unmanaged map values with the same number of /// elements as the keys, or it can be a sequence of OrtValues, /// each of those can be a composite type (tensor, sequence, map). If it is a sequence, /// then the number of elements must match the number of elements in keys. /// /// Depending on the structure of the values, one will either directly query a single tensor /// from values, or will have to iterate over the sequence of OrtValues and visit each of those /// resulting in a recursive visitation. /// /// This would always represent a tensor /// Can be any of the Onnx types, but they would all reduce to tensors eventually public delegate void MapVisitor(OrtValue keys, OrtValue values); /// /// This API helps the user to process a map OrtValue without /// having to deal with the lifespan of intermediate OrtValues. /// /// each API value is fed to the vistor functor. /// /// visitor function /// Allocator to use for intermediate values /// public void ProcessMap(MapVisitor visitor, OrtAllocator allocator) { if (OnnxType != OnnxValueType.ONNX_TYPE_MAP) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "This OrtValue does not represent a map"); } using var keys = GetValue(0, allocator); using var values = GetValue(1, allocator); visitor(keys, values); } private unsafe void FillStringTensorElement(char* strPtr, int strLength, int index) { IntPtr buffer; if (strLength == 0) { NativeApiStatus.VerifySuccess(NativeMethods.OrtGetResizedStringTensorElementBuffer(Handle, (UIntPtr)index, UIntPtr.Zero, out buffer)); return; } var bytesCount = Encoding.UTF8.GetByteCount(strPtr, strLength); NativeApiStatus.VerifySuccess(NativeMethods.OrtGetResizedStringTensorElementBuffer(Handle, (UIntPtr)index, (UIntPtr)bytesCount, out buffer)); NativeOnnxValueHelper.StringToUtf8NativeMemory(strPtr, strLength, buffer, bytesCount); } private static void PinAsTensor( Tensor tensor, int elementSize, out MemoryHandle pinnedHandle, out int dataBufferLength, out long[] shape, out int rank) { if (tensor == null) { throw new OnnxRuntimeException(ErrorCode.Fail, "Cast to Tensor failed. BUG check!"); } if (tensor.IsReversedStride) { //TODO: not sure how to support reverse stride. may be able to calculate the shape differently throw new NotSupportedException(nameof(Tensor) + " of reverseStride is not supported"); } DenseTensor dt = tensor as DenseTensor ?? tensor.ToDenseTensor(); shape = Array.ConvertAll(dt.Dimensions.ToArray(), Convert.ToInt64); rank = dt.Rank; dataBufferLength = dt.Buffer.Length * elementSize; pinnedHandle = dt.Buffer.Pin(); } #region IDisposable Support ~OrtValue() { Dispose(false); } /// /// IDisposable implementation /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// IDisposable implementation /// /// true if invoked from Dispose() method protected virtual void Dispose(bool disposing) { if (_disposed) { return; } if (disposing) { _memHandle?.Dispose(); _memHandle = null; _compositeMembers?.Dispose(); _compositeMembers = null; } Debug.Assert(_handle != IntPtr.Zero); NativeMethods.OrtReleaseValue(_handle); _handle = IntPtr.Zero; _disposed = true; } #endregion } }