// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Microsoft.ML.OnnxRuntime.Tensors; using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Linq; namespace Microsoft.ML.OnnxRuntime { /// /// The class helps to feed the NamedOnnxValue as inference input. /// It projects managed classes to OrtValues so they can be consumed /// by the native onnxruntime library. if possible, it will avoid copying data. /// The NamedOnnxValue can be a tensor, sequence or map. /// For recursive structures, create nested NamedOnnxValue instances. /// For example, a sequence instance would contain a list of NamedOnnxValue instances /// that in turn may represent tensors or other ONNX values. /// internal class ManagedTypeProjection : IDisposable { readonly DisposableList _disposables; readonly OrtValue _ortValue; bool _disposed = false; /// /// Provides access to non-owning instance of OrtValue /// /// Provides access to the OrtValue to be used as input internal OrtValue Value { get { return new OrtValue(_ortValue.Handle, false); } } /// /// Constructor to create an input OrtValue projection from managed data /// /// /// /// internal ManagedTypeProjection(NamedOnnxValue namedOnnxValue, NodeMetadata metadata) { int requiredCapacity = 32; var disposables = new DisposableList(requiredCapacity); try { _ortValue = CreateDispatchProjection(namedOnnxValue, metadata, disposables); } catch (Exception) { disposables.Dispose(); throw; } _disposables = disposables; } /// /// Dispatches the creation of the projection /// /// /// /// /// private OrtValue CreateDispatchProjection(NamedOnnxValue namedOnnxValue, NodeMetadata metadata, DisposableList disposables) { OrtValue result; NodeMetadata meta = metadata; // Use element meta to create types if (metadata.OnnxValueType == OnnxValueType.ONNX_TYPE_OPTIONAL) { meta = metadata.AsOptionalMetadata().ElementMeta; } if (namedOnnxValue.ValueType != meta.OnnxValueType) { throw new OnnxRuntimeException(ErrorCode.RuntimeException, $"NamedOnnxValue: {namedOnnxValue.Name} has value type: {namedOnnxValue.ValueType}" + $" expected: {meta.OnnxValueType} after optional type adjustment"); } switch (namedOnnxValue.ValueType) { case OnnxValueType.ONNX_TYPE_TENSOR: result = CreateTensorProjection(namedOnnxValue, meta, disposables); break; case OnnxValueType.ONNX_TYPE_SEQUENCE: result = CreateSequenceProjection(namedOnnxValue, meta, disposables); break; case OnnxValueType.ONNX_TYPE_MAP: result = CreateMapProjection(namedOnnxValue, meta, disposables); break; default: throw new OnnxRuntimeException(ErrorCode.InvalidArgument, "ManagedTypeProjection can only project tensors, sequences, maps and optional types"); } return result; } /// /// The function creates OrtValue objects for each element of the sequence /// and then creates an OrtValue for the whole sequence. /// /// NamedOnnxValue containing a IEnumeralbe /// sequence metadata /// cleanup list /// /// private OrtValue CreateSequenceProjection(NamedOnnxValue namedOnnxValue, NodeMetadata metadata, DisposableList disposables) { OrtValue result = null; var elementMeta = metadata.AsSequenceMetadata().ElementMeta; var elementOnnxValue = elementMeta.OnnxValueType; var seqContainer = namedOnnxValue.AsEnumerable(); if (seqContainer is null) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"NamedOnnxValue: {namedOnnxValue.Name} sequence does not contain NamedOnnxValue elements"); } int capacity = 0; if (seqContainer is ICollection) { capacity = ((ICollection)seqContainer).Count; } // Record all the ortValues belonging to the sequence locally var sequenceOrtValues = new List(capacity); foreach (var element in seqContainer) { if (elementOnnxValue != element.ValueType) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"NamedOnnxValue: {namedOnnxValue.Name} sequence element expected to be {elementOnnxValue}, received {element.ValueType}"); } sequenceOrtValues.Add(CreateDispatchProjection(element, elementMeta, disposables)); } IntPtr[] ortValHandles = new IntPtr[sequenceOrtValues.Count]; for (int i = 0; i < sequenceOrtValues.Count; i++) { ortValHandles[i] = sequenceOrtValues[i].Handle; } NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateValue(ortValHandles, (UIntPtr)sequenceOrtValues.Count, (IntPtr)OnnxValueType.ONNX_TYPE_SEQUENCE, out IntPtr sequenceHandle)); result = new OrtValue(sequenceHandle); disposables.Add(result); return result; } /// /// Creates map projection. Since we support only primitive types in maps /// we map two tensors (keys and values) /// /// /// /// /// OrtValue /// private OrtValue CreateMapProjection(NamedOnnxValue node, NodeMetadata elementMeta, DisposableList disposables) { OrtValue result = null; var mapMeta = elementMeta.AsMapMetadata(); Debug.Assert(mapMeta != null); // Maps currently support only primitive types expressed as two parallel tensors and not nested Sequences or Maps var mapValuesMeta = mapMeta.ValueMetadata; if (mapValuesMeta.OnnxValueType != OnnxValueType.ONNX_TYPE_TENSOR) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Node: {node.Name} onnxruntime only supports maps with primitive types values"); } var keys = node.GetDictionaryKeys(); var ortValueKeys = OrtValue.CreateFromTensorObject(keys, out MemoryHandle? memoryHandleKeys, out TensorElementType elementTypeKeys); disposables.Add(ortValueKeys); if (memoryHandleKeys.HasValue) { disposables.Add(memoryHandleKeys); } if (elementTypeKeys != mapMeta.KeyDataType) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Map key data type supplied: {elementTypeKeys} metadata expected: {mapMeta.KeyDataType}"); } var values = node.GetDictionaryValues(); var ortValueValues = OrtValue.CreateFromTensorObject(values, out MemoryHandle? memoryHandleValues, out TensorElementType elementTypeValues); disposables.Add(ortValueValues); if (memoryHandleValues.HasValue) { disposables.Add(memoryHandleValues); } if (elementTypeValues != mapValuesMeta.ElementDataType) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Map value data type supplied: {elementTypeValues} metadata expected: {mapValuesMeta.ElementDataType}"); } // Create Map OrtValue IntPtr[] ortValHandles = { ortValueKeys.Handle, ortValueValues.Handle }; NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateValue(ortValHandles, (UIntPtr)2, (IntPtr)OnnxValueType.ONNX_TYPE_MAP, out IntPtr ortValueMap)); result = new OrtValue(ortValueMap); disposables.Add(result); return result; } /// /// This pins memory that is contained within DenseTensor. /// /// NodeOnnxValue containing DenseTensor /// /// cleanup list /// /// private OrtValue CreateTensorProjection(NamedOnnxValue node, NodeMetadata elementMeta, DisposableList disposables) { var ortValue = OrtValue.CreateFromTensorObject(node.Value, out MemoryHandle? memoryHandle, out TensorElementType elementType); disposables.Add(ortValue); if (memoryHandle.HasValue) { disposables.Add(memoryHandle); } if (elementType != elementMeta.ElementDataType) { throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Tensor element data type discovered: {elementType} metadata expected: {elementMeta.ElementDataType}"); } return ortValue; } #region IDisposable /// /// IDisposable implementation /// /// true if invoked by Dispose() protected virtual void Dispose(bool disposing) { if (_disposed) { return; } // dispose managed state (managed objects). if (disposing) { _disposables.Dispose(); } _disposed = true; } public void Dispose() { Dispose(true); } #endregion IDisposable } }