From 1288a8caed7f473d42a3b2cc6f9d2c31011d328d Mon Sep 17 00:00:00 2001 From: jignparm Date: Tue, 5 Mar 2019 16:00:40 -0800 Subject: [PATCH] Initial check-in to support non-tensor (sequence/map) types (#527) * Initial check-in to support non-tensor (sequence/map) types * Added support for String tensors * address PR comments --- .../DisposableNamedOnnxValue.cs | 143 +++++++++++++++--- .../InferenceSession.cs | 9 +- .../NamedOnnxValue.cs | 47 ++++++ .../Microsoft.ML.OnnxRuntime/NativeMethods.cs | 43 +++++- .../NativeOnnxTensorMemory.cs | 91 ++++++++--- .../InferenceTest.cs | 101 ++++++++++++- .../testdata/test_sequence_map_int_float.pb | Bin 0 -> 4209 bytes .../test_sequence_map_string_float.pb | Bin 0 -> 4217 bytes 8 files changed, 376 insertions(+), 58 deletions(-) create mode 100644 csharp/testdata/test_sequence_map_int_float.pb create mode 100644 csharp/testdata/test_sequence_map_string_float.pb diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs index f230363e6e..187a44efcd 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs @@ -4,17 +4,18 @@ using System; using System.Collections.Generic; using System.Numerics.Tensors; +using System.Runtime.InteropServices; namespace Microsoft.ML.OnnxRuntime { - public interface IDisposableReadOnlyCollection: IReadOnlyCollection, IDisposable + public interface IDisposableReadOnlyCollection : IReadOnlyCollection, IDisposable { } internal class DisposableList : List, IDisposableReadOnlyCollection - where T: IDisposable + where T : IDisposable { #region IDisposable Support @@ -57,18 +58,16 @@ namespace Microsoft.ML.OnnxRuntime #endregion } - - public class DisposableNamedOnnxValue: NamedOnnxValue, IDisposable + public class DisposableNamedOnnxValue : NamedOnnxValue, IDisposable { protected IDisposable _nativeMemoryManager; protected DisposableNamedOnnxValue(string name, Object value, IDisposable nativeMemoryManager) - :base(name, value) + : base(name, value) { _nativeMemoryManager = nativeMemoryManager; } - - internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) + internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, IntPtr nativeOnnxValue) { DisposableNamedOnnxValue result = null; @@ -91,31 +90,34 @@ namespace Microsoft.ML.OnnxRuntime switch (elemType) { case TensorElementType.Float: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.Double: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.Int16: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.UInt16: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.Int32: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.UInt32: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.Int64: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.UInt64: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; case TensorElementType.UInt8: - result = NameOnnxValueFromNativeTensor(name, nativeOnnxValue); + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); + break; + case TensorElementType.String: + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; default: throw new NotSupportedException("Tensor of element type: " + elemType + " is not supported"); @@ -125,15 +127,114 @@ namespace Microsoft.ML.OnnxRuntime return result; } - - private static DisposableNamedOnnxValue NameOnnxValueFromNativeTensor(string name, IntPtr nativeOnnxValue) + internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) { - NativeOnnxTensorMemory nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValue); - DenseTensor dt = new DenseTensor(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions); - return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper); + IntPtr allocator = IntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateDefaultAllocator(out allocator)); + var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator); + NativeMethods.OrtReleaseAllocator(allocator); + return (DisposableNamedOnnxValue)ret; } + internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue, IntPtr allocator) + { + var onnxValueType = NativeMethods.OrtGetValueType(nativeOnnxValue); + switch (onnxValueType) + { + case OnnxValueType.ONNX_TYPE_TENSOR: + return CreateTensorFromOnnxValue(name, nativeOnnxValue); + case OnnxValueType.ONNX_TYPE_SEQUENCE: + IntPtr count = IntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueCount(nativeOnnxValue, out count)); + var sequence = new DisposableList(); + for (long i = 0; i < count.ToInt64(); i++) + { + IntPtr nativeOnnxValueSeq; + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 0, allocator, out nativeOnnxValueSeq)); + sequence.Add(CreateFromOnnxValue(string.Empty, nativeOnnxValueSeq, allocator)); + NativeMethods.OrtReleaseValue(nativeOnnxValueSeq); + } + return new DisposableNamedOnnxValue(name, sequence, null); + + case OnnxValueType.ONNX_TYPE_MAP: + IntPtr typeAndShape = IntPtr.Zero; + IntPtr nativeOnnxValueMapKeys = IntPtr.Zero; + IntPtr nativeOnnxValueMapValues = IntPtr.Zero; + TensorElementType elemType = TensorElementType.DataTypeMax; + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 0, allocator, out nativeOnnxValueMapKeys)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 1, allocator, out nativeOnnxValueMapValues)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(nativeOnnxValueMapKeys, out typeAndShape)); + + elemType = NativeMethods.OrtGetTensorElementType(typeAndShape); + if (typeAndShape != IntPtr.Zero) + { + NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape); + } + switch (elemType) + { + case TensorElementType.Int64: + return DisposableNamedOnnxValueFromNativeMap(string.Empty, nativeOnnxValueMapKeys, nativeOnnxValueMapValues); + case TensorElementType.String: + return DisposableNamedOnnxValueFromNativeMap(string.Empty, nativeOnnxValueMapKeys, nativeOnnxValueMapValues); + default: + throw new NotSupportedException("Map of element type: " + elemType + " is not supported"); + } + default: + throw new NotSupportedException("OnnxValueType : " + onnxValueType + " is not supported"); + } + } + + private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeTensor(string name, IntPtr nativeOnnxValue) + { + if (typeof(T) == typeof(string)) + { + var nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValue, true); + var dt = new DenseTensor(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions); + return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper); + } + else + { + NativeOnnxTensorMemory nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValue); + DenseTensor dt = new DenseTensor(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions); + return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper); + } + } + + private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeMap(string name, IntPtr nativeOnnxValueKeys, IntPtr nativeOnnxValueValues) + { + var nativeTensorWrapperValues = new NativeOnnxTensorMemory(nativeOnnxValueValues); + var denseTensorValues = new DenseTensor(nativeTensorWrapperValues.Memory, nativeTensorWrapperValues.Dimensions); + + if (typeof(K) == typeof(string)) + { + var map = new Dictionary(); + var nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValueKeys, true); + var denseTensorKeys = new DenseTensor(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions); + for (var i = 0; i < denseTensorKeys.Length; i++) + { + map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i)); + } + // release native memory + nativeTensorWrapperValues.Dispose(); + nativeTensorWrapper.Dispose(); + return new DisposableNamedOnnxValue(string.Empty, map, null); + } + else + { + var map = new Dictionary(); + var nativeTensorWrapper = new NativeOnnxTensorMemory(nativeOnnxValueKeys); + var denseTensorKeys = new DenseTensor(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions); + for (var i = 0; i < denseTensorKeys.Length; i++) + { + map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i)); + } + // release native memory + nativeTensorWrapperValues.Dispose(); + nativeTensorWrapper.Dispose(); + return new DisposableNamedOnnxValue(string.Empty, map, null); + } + } #region IDisposable Support private bool disposedValue = false; // To detect redundant calls diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index c94c4d5db1..9e2422e820 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -169,7 +169,7 @@ namespace Microsoft.ML.OnnxRuntime var result = new DisposableList(); for (uint i = 0; i < outputValueArray.Length; i++) { - result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i])); + result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i])); } return result; @@ -297,10 +297,13 @@ namespace Microsoft.ML.OnnxRuntime } } - private NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) + internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo) { IntPtr tensorInfo = NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo); - // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* + // Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo* + + if (tensorInfo == IntPtr.Zero) + return null; TensorElementType type = NativeMethods.OrtGetTensorElementType(tensorInfo); Type dotnetType = null; diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs index 5659d25c64..3c9fa43c83 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs @@ -29,11 +29,39 @@ namespace Microsoft.ML.OnnxRuntime } public string Name { get { return _name; } } + + /// + /// Try-get value as a Tensor<T>. + /// + /// Type + /// Tensor object if contained value is a Tensor. Null otherwise public Tensor AsTensor() { return _value as Tensor; // will return null if not castable } + /// + /// Try-get value as an Enumerable<T>. + /// + /// Type + /// Enumerable object if contained value is a Enumerable. Null otherwise + public IEnumerable AsEnumerable () + { + var x = _value as IEnumerable; + return x; + } + + /// + /// Try-get value as an Dictionary<K,V>. + /// + /// Key type + /// Value type + /// Dictionary object if contained value is a Dictionary. Null otherwise + public IDictionary AsDictionary() + { + return _value as IDictionary; + } + /// /// Attempts to Pin the buffer, and create a native OnnxValue out of it. the pinned MemoryHandle is passed to output. /// In this case, the pinnedHandle should be kept alive till the native OnnxValue is used, then dispose it. @@ -271,6 +299,11 @@ namespace Microsoft.ML.OnnxRuntime nativeElementType = TensorElementType.UInt8; dataBufferLength = dt.Buffer.Length * sizeof(byte); } + else if (typeof(T) == typeof(string)) + { + nativeElementType = TensorElementType.String; + dataBufferLength = dt.Buffer.Length * IntPtr.Size; + } //TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy //else if (typeof(T) == typeof(bool)) //{ @@ -312,6 +345,16 @@ namespace Microsoft.ML.OnnxRuntime DataTypeMax = 17 } + internal enum OnnxValueType + { + ONNX_TYPE_UNKNOWN = 0, + ONNX_TYPE_TENSOR = 1, + ONNX_TYPE_SEQUENCE = 2, + ONNX_TYPE_MAP = 3, + ONNX_TYPE_OPAQUE = 4, + ONNX_TYPE_SPARSETENSOR = 5, + } + internal static class TensorElementTypeConverter { public static void GetTypeAndWidth(TensorElementType elemType, out Type type, out int width) @@ -354,6 +397,10 @@ namespace Microsoft.ML.OnnxRuntime type = typeof(byte); width = sizeof(byte); break; + case TensorElementType.String: + type = typeof(byte); + width = sizeof(byte); + break; default: type = null; width = 0; diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 69538df3e7..10f391d297 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -30,10 +30,11 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern ErrorCode OrtGetErrorCode(IntPtr /*(OrtStatus*)*/status); + // returns char*, need to convert to string by the caller. + // does not free the underlying OrtStatus* [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* char* */OrtGetErrorMessage(IntPtr /* (OrtStatus*) */status); - // returns char*, need to convert to string by the caller. - // does not free the underlying OrtStatus* + [DllImport(nativeLib, CharSet = charSet)] public static extern void OrtReleaseStatus(IntPtr /*(OrtStatus*)*/ statusPtr); @@ -67,7 +68,7 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtSessionGetInputCount( - IntPtr /*(OrtSession*)*/ session, + IntPtr /*(OrtSession*)*/ session, out ulong /* TODO: size_t */ count); @@ -80,7 +81,7 @@ namespace Microsoft.ML.OnnxRuntime public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetInputName( IntPtr /*(OrtSession*)*/ session, ulong index, //TODO: port size_t - IntPtr /*(OrtAllocator*)*/ allocator, + IntPtr /*(OrtAllocator*)*/ allocator, out IntPtr /*(char**)*/name); [DllImport(nativeLib, CharSet = charSet)] @@ -93,14 +94,14 @@ namespace Microsoft.ML.OnnxRuntime // release the typeinfo using OrtReleaseTypeInfo [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetInputTypeInfo( - IntPtr /*(const OrtSession*)*/ session, + IntPtr /*(const OrtSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /*(struct OrtTypeInfo**)*/ typeInfo); // release the typeinfo using OrtReleaseTypeInfo [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetOutputTypeInfo( - IntPtr /*(const OrtSession*)*/ session, + IntPtr /*(const OrtSession*)*/ session, ulong index, //TODO: port for size_t out IntPtr /* (struct OrtTypeInfo**)*/ typeInfo); @@ -239,6 +240,21 @@ namespace Microsoft.ML.OnnxRuntime #region Tensor/OnnxValue API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtGetValue(IntPtr /*(OrtValue*)*/ value, + int index, + IntPtr /*(OrtAllocator*)*/ allocator, + out IntPtr /*(OrtValue**)*/ outputValue); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern OnnxValueType /*Onnxtype*/ OrtGetValueType(IntPtr /*(OrtValue*)*/ value); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtGetValueCount(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(size_t*)*/ count); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtGetTypeInfo(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(OrtValue**)*/ typeInfo); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue( IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo, @@ -254,6 +270,17 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorContent( + IntPtr /*(OrtValue*)*/ value, + IntPtr /*(void*)*/ dst_buffer, + ulong dst_buffer_len, //size_t, TODO: make it portable for x86, arm + IntPtr offsets, + ulong offsets_len); //size_t, TODO: make it portable for x86, arm + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorDataLength(IntPtr /*(OrtValue*)*/ value, out ulong /*(size_t*)*/ len); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ OrtCastTypeInfoToTensorInfo(IntPtr /*(struct OrtTypeInfo*)*/ typeInfo); @@ -273,8 +300,8 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern void OrtGetDimensions( - IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo, - long[] dim_values, + IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo, + long[] dim_values, ulong dim_values_length); /** diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs index 31e354ab04..10dc1b4701 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs @@ -16,32 +16,30 @@ namespace Microsoft.ML.OnnxRuntime { private bool _disposed; private int _referenceCount; - private IntPtr _onnxValueHandle; - private IntPtr _dataBufferHandle; + private IntPtr _onnxValueHandle; // pointer to onnxvalue object in native + private IntPtr _dataBufferPointer; // pointer to mutable tensor data in native memory + private string[] _dataBufferAsString; // string tensor values copied into managed memory private int _elementCount; private int _elementWidth; private int[] _dimensions; - public NativeOnnxTensorMemory(IntPtr onnxValueHandle) + public NativeOnnxTensorMemory(IntPtr onnxValueHandle, bool isStringTensor = false) { IntPtr typeAndShape = IntPtr.Zero; try { - NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(onnxValueHandle, out typeAndShape)); - - TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape); - Type type = null; int width = 0; - TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); - if (typeof(T) != type) - throw new NotSupportedException(nameof(NativeOnnxTensorMemory)+" does not support T = "+nameof(T)); - _elementWidth = width; - _onnxValueHandle = onnxValueHandle; - // derive the databuffer pointer, element_count, element_width, and shape - NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle)); - // throws OnnxRuntimeException if native call failed + + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(onnxValueHandle, out typeAndShape)); + TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape); + TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width); + + if (typeof(T) != type) + throw new NotSupportedException(nameof(NativeOnnxTensorMemory) + " does not support T = " + nameof(T)); + + _elementWidth = width; ulong dimension = NativeMethods.OrtGetNumOfDimensions(typeAndShape); long count = NativeMethods.OrtGetTensorShapeElementCount(typeAndShape); // count can be negative. @@ -59,6 +57,48 @@ namespace Microsoft.ML.OnnxRuntime { _dimensions[i] = (int)shape[i]; } + + if (!isStringTensor) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferPointer)); + } + else + { + if (typeof(T) != typeof(byte)) + throw new NotSupportedException(nameof(NativeOnnxTensorMemory) + " T = " + nameof(T) + ". Should = byte, when isStringTensor is true"); + ulong strLen; + var offsets = new ulong[_elementCount]; + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(_onnxValueHandle, out strLen)); + var dataBuffer = new byte[strLen]; + var dataBufferMemory = new Memory(dataBuffer); + var dataBufferHandle = dataBufferMemory.Pin(); + IntPtr dataBufferPointer = IntPtr.Zero; + + var offsetMemory = new Memory(offsets); + var offsetMemoryHandle = offsetMemory.Pin(); + IntPtr offsetBufferPointer = IntPtr.Zero; + unsafe + { + dataBufferPointer = (IntPtr)dataBufferHandle.Pointer; + offsetBufferPointer = (IntPtr)offsetMemoryHandle.Pointer; + } + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorContent(_onnxValueHandle, dataBufferPointer, strLen, offsetBufferPointer, Convert.ToUInt64(_elementCount))); + _dataBufferPointer = dataBufferPointer; + _dataBufferAsString = new string[_elementCount]; + + for (var i = 0; i < offsets.Length; i++) + { + var length = (i == offsets.Length - 1) + ? strLen - offsets[i] + : offsets[i + 1] - offsets[i]; + // Onnx specifies strings always in UTF-8, no trailing null, no leading BOM + _dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length); + } + + // unpin memory + offsetMemoryHandle.Dispose(); + dataBufferHandle.Dispose(); + } } catch (Exception e) { @@ -74,6 +114,7 @@ namespace Microsoft.ML.OnnxRuntime } } } + ~NativeOnnxTensorMemory() { Dispose(false); @@ -128,12 +169,22 @@ namespace Microsoft.ML.OnnxRuntime Span span = null; unsafe { - span = new Span((void*)_dataBufferHandle, _elementCount); + span = new Span((void*)_dataBufferPointer, _elementCount); } return span; } + public Memory GetBytesAsStringMemory() + { + if (IsDisposed) + throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory)); + + if (typeof(T) != typeof(byte)) + throw new NotSupportedException(nameof(NativeOnnxTensorMemory.GetBytesAsStringMemory) + ": T must be byte"); + + return (_dataBufferAsString == null) ? new Memory() : new Memory(_dataBufferAsString); + } public override MemoryHandle Pin(int elementIndex = 0) { @@ -146,17 +197,15 @@ namespace Microsoft.ML.OnnxRuntime } Retain(); - return new MemoryHandle((void*)((int)_dataBufferHandle + elementIndex*_elementWidth)); //could not use Unsafe.Add + return new MemoryHandle((void*)((int)_dataBufferPointer + elementIndex * _elementWidth)); //could not use Unsafe.Add } } - public override void Unpin() { Release(); } - private bool Release() { int newRefCount = Interlocked.Decrement(ref _referenceCount); @@ -202,9 +251,5 @@ namespace Microsoft.ML.OnnxRuntime arraySegment = default(ArraySegment); return false; } - - - - } } diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index dccb4a1629..1eb8a85051 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -232,7 +232,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests validModelFound = true; } } - + if (!validModelFound) { var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString())); @@ -351,7 +351,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests Assert.True(tensorOut.SequenceEqual(tensorIn)); } } - + } [Fact(Skip = "String tensor not supported yet")] @@ -524,6 +524,101 @@ namespace Microsoft.ML.OnnxRuntime.Tests } } + [Fact] + private void TestModelSequenceOfMapIntFloat() + { + // test model trained using lightgbm classifier + // produces 2 named outputs + // "label" is a tensor, + // "probabilities" is a sequence> + // https://github.com/onnx/sklearn-onnx/blob/master/docs/examples/plot_pipeline_lightgbm.py + + string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_sequence_map_int_float.pb"); + using (var session = new InferenceSession(modelPath)) + { + var container = new List(); + var tensorIn = new DenseTensor(new float[] { 5.8f, 2.8f }, new int[] { 1, 2 }); + var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn); + container.Add(nov); + + using (var outputs = session.Run(container)) + { + // first output is a tensor containing label + var outNode1 = outputs.ElementAtOrDefault(0); + Assert.Equal("label", outNode1.Name); + + // try-cast as a tensor + var outLabelTensor = outNode1.AsTensor(); + + // Label 1 should have highest probaility + Assert.Equal(1, outLabelTensor[0]); + + // second output is a sequence> + // try-cast to an sequence of NOV + var outNode2 = outputs.ElementAtOrDefault(1); + Assert.Equal("probabilities", outNode2.Name); + + // try-cast to an sequence of NOV + var seq = outNode2.AsEnumerable(); + + // try-cast first element in sequence to map/dictionary type + var map = seq.First().AsDictionary(); + //verify values are valid + Assert.Equal(0.25938290, map[0], 6); + Assert.Equal(0.40904793, map[1], 6); + Assert.Equal(0.33156919, map[2], 6); + } + } + } + + [Fact] + private void TestModelSequenceOfMapStringFloat() + { + // test model trained using lightgbm classifier + // produces 2 named outputs + // "label" is a tensor, + // "probabilities" is a sequence> + // https://github.com/onnx/sklearn-onnx/blob/master/docs/examples/plot_pipeline_lightgbm.py + + string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_sequence_map_string_float.pb"); + + using (var session = new InferenceSession(modelPath)) + { + var container = new List(); + var tensorIn = new DenseTensor(new float[] { 5.8f, 2.8f }, new int[] { 1, 2 }); + var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn); + container.Add(nov); + + using (var outputs = session.Run(container)) + { + // first output is a tensor containing label + var outNode1 = outputs.ElementAtOrDefault(0); + Assert.Equal("label", outNode1.Name); + + // try-cast as a tensor + var outLabelTensor = outNode1.AsTensor(); + + // Label 1 should have highest probaility + Assert.Equal("1", outLabelTensor[0]); + + // second output is a sequence> + // try-cast to an sequence of NOV + var outNode2 = outputs.ElementAtOrDefault(1); + Assert.Equal("probabilities", outNode2.Name); + + // try-cast to an sequence of NOV + var seq = outNode2.AsEnumerable(); + + // try-cast first element in sequence to map/dictionary type + var map = seq.First().AsDictionary(); + //verify values are valid + Assert.Equal(0.25938290, map["0"], 6); + Assert.Equal(0.40904793, map["1"], 6); + Assert.Equal(0.33156919, map["2"], 6); + } + } + } + [GpuFact] private void TestGpu() { @@ -619,7 +714,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); var session = (cudaDeviceId.HasValue) - ? new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(cudaDeviceId.Value)) + ? new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(cudaDeviceId.Value)) : new InferenceSession(modelPath); float[] inputData = LoadTensorFromFile(@"bench.in"); float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); diff --git a/csharp/testdata/test_sequence_map_int_float.pb b/csharp/testdata/test_sequence_map_int_float.pb new file mode 100644 index 0000000000000000000000000000000000000000..16729df202d2ffac805a0557d7f2d30e3c72439e GIT binary patch literal 4209 zcmeHKU1%It6yC}HO>dK!Ow+1Cjcg0*8X_jGV%@WuB&5-h_?Id*)Go8hY&&N5$DK(U zD?;@_tV+}152T?rVBd_sn7ovoje@?#pGu4R(u#@-T0!)|Y7p<4*}F5l(;-O(L6OVe zbMJS)bMCow&&;07`vqPbQ`@td%w%)$_O|V9?UAsgwBh__U%PK-@kTOCf=VVgt_%Ex zlvkvrDmRCQMkH0vM@llfnS`>blqT!&+RAAW9(}V%oZ~{#CKA+;2>sX0c0z7w6QVmw z$$wi~B`0nmWv@a|m6Eb52)TSVDJ2zE(VtA{az@MMo3{)o`w!@S$@D;9_h64IX_}H! zXaHMw=ViG!qseI+mW@YsvS9>S31w6heWk{UcB>ItV_KB9o-6B-w+7A}7miMnI&0NT zc2vfd{bE22iXl^mMPA$>5_299afaHkiB&TECdbmOfm5-vPM^o>d-l_{Sqso8SPRmX zSqss)SqsxRS>tKsrG_g{*RV|=l<|x-_;7h1;P}k7aoWuo@W8^k0(5k|190hu+hP{B zcj{RK;^ea!B`+6X<+&Y(rKlgfa5pS$d>-Jbrk?;pug?Iy-5HF-=Vx9q;N1^iHefzQ zNAo|wUw}*J=Ky*-qA@slmSQiq9x~wE*7suY%lCx>JaX|C12V^6HK6ZS)Q+~s;La1z z0BqZO7GSD)1mNKdKLT7ed(42FemH8tzAvT>SeRZcz~ZOV0H?m)7K10IsP=GU&Vcsg zZ^dBgxA_8m6l*bH@0ZgCypf|8cO8nvpmpqVfYiw)z`Vbn2xcCb`Al;#qu;R|(~u?= zKC&|$5imovL`qUMnI%gNJ%&&@)$AEhlZGWAqP&CUqshsvc#~yuooKbSuWaw;;yP}0 z6%V+lk2%s%+LJ8lYV zq{|xke1|*#@BTZ_0N>T~9BHwRIjzutmCXLcgrttk33}usV;U9XGR-|_KG`!j>dZqY z&(2^cYO@*25KA(YWxpF`d}Lr_%#y@BJ{=+(OB=**PCw~>sO!F-dlCb^4|XDxiUtsC+3(KM(cF&GtEwj zqo)%zYX?ojVpz1e%}fhp_B#WPo_whj zOfQ$?nV4r7f9?r5qpI?9g`t^InYO3yT`9TWG1=cTtU0bD^|`F3V~vr~ zQrUbu5+1sL*Y3ftJ#(DD;(ol3T;?_`)Y&&=Mb?@_!%A*Y%0)_YmFtJoztEUdaK!)Jpcdz literal 0 HcmV?d00001 diff --git a/csharp/testdata/test_sequence_map_string_float.pb b/csharp/testdata/test_sequence_map_string_float.pb new file mode 100644 index 0000000000000000000000000000000000000000..30596711b7cce804b33cd0b3be6ec0d46113466f GIT binary patch literal 4217 zcmeHKT}&KR6y9O~rni8w(5l3UZkwnpK?^pfp=bp3o%*?sGH^i&sN=G)6nQHOxXuqkwBODavcAQ`9>G14&?H-?DW;9^0?=C({G{J%hc9sH$>G zrU7i-lb58vj4GvRSQZ}9$fglwCFD_6@Dv9pSffT@gK1G(dak5LUKN~qE*P02b>^y> z?5Kn*dj+517XpS13cRpMAjaG$;0(1v11o2E4UVa)f>XA#M*okC=j^3xv*Dvru;HgG zvk{A34R55LStrNQsInv4W|gTM;Ogn>FKU;jkh^l((@wv_P2^Z?H_R9j&zX zlSXha71r<|K*QM7~&v_)xLftviNTvRr4k!Uq$1<%~+pO6}kDfEj(| zCG}=T`(-UJYLW^b&!MjWFaMqY0H4>h9BDPTIW5yKOJ*!FDJm0Ef^PZ9xJt!@RQxos znlnEXW=&dkXQI}eeNe30HFbYxZB@CGYxl#dn@JW6TCoTWiFJH3KsFZ>#CJ}d^xV^Z zNAGQkfxf$4?|lP(-Miz?>qc$e_)gCz;>LHw-TYMV&F;az%0_X!t^MB3j~k~NzSGu; zxcPCz?H=s+Zv2hncAxfpH$QHiYWPlDC*tPEhU2Gtl_PcL!-H3|xx)64HFx;DSu@t= zIZ-Eq?chI(;=8~~r?OYXR|tl)-eBFbC0YMiE?Fm8E?G}mE?IY3E?J*hE=xTp>pf9o zIv;VQ(cB$;O|u>1=Tm*2R@mf?KYatt2$b!pkAo!Ty#`@YD-{P z&JBvWa8Xuq1hKCXYRt(wNs%*BLcv4aSTa3)DLeMplUkk&@uY_5=&vT(M2ZJ>LU^7J b`B0H;CFKEy>ch2!L`sC7dSW~GH*kLd*LT2y literal 0 HcmV?d00001