using Microsoft.ML.OnnxRuntime.Tensors; using System; using System.Collections.Generic; using System.Runtime.InteropServices; using System.Text; using Xunit; namespace Microsoft.ML.OnnxRuntime.Tests { [Collection("OrtValueTests")] public class OrtValueTests { public OrtValueTests() { } [Fact(DisplayName = "PopulateAndReadStringTensor")] public void PopulateAndReadStringTensor() { OrtEnv.Instance(); string[] strsRom = { "HelloR", "OrtR", "WorldR" }; string[] strs = { "Hello", "Ort", "World" }; long[] shape = { 1, 1, 3 }; var elementsNum = ArrayUtilities.GetSizeForShape(shape); Assert.Equal(elementsNum, strs.Length); Assert.Equal(elementsNum, strsRom.Length); using (var strTensor = OrtValue.CreateTensorWithEmptyStrings(OrtAllocator.DefaultInstance, shape)) { Assert.True(strTensor.IsTensor); Assert.False(strTensor.IsSparseTensor); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, strTensor.OnnxType); var typeShape = strTensor.GetTensorTypeAndShape(); { Assert.True(typeShape.IsString); Assert.Equal(shape.Length, typeShape.DimensionsCount); var fetchedShape = typeShape.Shape; Assert.Equal(shape.Length, fetchedShape.Length); Assert.Equal(shape, fetchedShape); Assert.Equal(elementsNum, typeShape.ElementCount); } using (var memInfo = strTensor.GetTensorMemoryInfo()) { Assert.Equal("Cpu", memInfo.Name); Assert.Equal(OrtMemType.Default, memInfo.GetMemoryType()); Assert.Equal(OrtAllocatorType.DeviceAllocator, memInfo.GetAllocatorType()); } // Verify that everything is empty now. for (int i = 0; i < elementsNum; ++i) { var str = strTensor.GetStringElement(i); Assert.Empty(str); var rom = strTensor.GetStringElementAsMemory(i); Assert.Equal(0, rom.Length); var bytes = strTensor.GetStringElementAsSpan(i); Assert.Equal(0, bytes.Length); } // Let's populate the tensor with strings. for (int i = 0; i < elementsNum; ++i) { // First populate via ROM strTensor.FillStringTensorElement(strsRom[i].AsMemory(), i); Assert.Equal(strsRom[i], strTensor.GetStringElement(i)); Assert.Equal(strsRom[i], strTensor.GetStringElementAsMemory(i).ToString()); Assert.Equal(Encoding.UTF8.GetBytes(strsRom[i]), strTensor.GetStringElementAsSpan(i).ToArray()); // Fill via Span strTensor.FillStringTensorElement(strs[i].AsSpan(), i); Assert.Equal(strs[i], strTensor.GetStringElement(i)); Assert.Equal(strs[i], strTensor.GetStringElementAsMemory(i).ToString()); Assert.Equal(Encoding.UTF8.GetBytes(strs[i]), strTensor.GetStringElementAsSpan(i).ToArray()); } } } [Fact(DisplayName = "PopulateAndReadStringTensorViaTensor")] public void PopulateAndReadStringTensorViaTensor() { OrtEnv.Instance(); string[] strs = { "Hello", "Ort", "World" }; int[] shape = { 1, 1, 3 }; var tensor = new DenseTensor(strs, shape); using (var strTensor = OrtValue.CreateFromStringTensor(tensor)) { Assert.True(strTensor.IsTensor); Assert.False(strTensor.IsSparseTensor); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, strTensor.OnnxType); var typeShape = strTensor.GetTensorTypeAndShape(); { Assert.True(typeShape.IsString); Assert.Equal(shape.Length, typeShape.DimensionsCount); var fetchedShape = typeShape.Shape; Assert.Equal(shape.Length, fetchedShape.Length); Assert.Equal(strs.Length, typeShape.ElementCount); } using (var memInfo = strTensor.GetTensorMemoryInfo()) { Assert.Equal("Cpu", memInfo.Name); Assert.Equal(OrtMemType.Default, memInfo.GetMemoryType()); Assert.Equal(OrtAllocatorType.DeviceAllocator, memInfo.GetAllocatorType()); } for (int i = 0; i < strs.Length; ++i) { // Fill via Span Assert.Equal(strs[i], strTensor.GetStringElement(i)); Assert.Equal(strs[i], strTensor.GetStringElementAsMemory(i).ToString()); Assert.Equal(Encoding.UTF8.GetBytes(strs[i]), strTensor.GetStringElementAsSpan(i).ToArray()); } } } static void VerifyTensorCreateWithData(OrtValue tensor, TensorElementType dataType, long[] shape, ReadOnlySpan originalData) where T : struct { // Verify invocation var dataTypeInfo = TensorBase.GetTypeInfo(typeof(T)); Assert.NotNull(dataTypeInfo); Assert.Equal(dataType, dataTypeInfo.ElementType); var elementsNum = ArrayUtilities.GetSizeForShape(shape); Assert.True(tensor.IsTensor); Assert.False(tensor.IsSparseTensor); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, tensor.OnnxType); var typeInfo = tensor.GetTypeInfo(); { Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, typeInfo.OnnxType); var typeShape = typeInfo.TensorTypeAndShapeInfo; Assert.Equal(shape.Length, typeShape.DimensionsCount); var fetchedShape = typeShape.Shape; Assert.Equal(shape.Length, fetchedShape.Length); Assert.Equal(shape, fetchedShape); Assert.Equal(elementsNum, typeShape.ElementCount); } using (var memInfo = tensor.GetTensorMemoryInfo()) { Assert.Equal("Cpu", memInfo.Name); Assert.Equal(OrtMemType.CpuOutput, memInfo.GetMemoryType()); Assert.Equal(OrtAllocatorType.DeviceAllocator, memInfo.GetAllocatorType()); } // Verify contained data Assert.Equal(originalData.ToArray(), tensor.GetTensorDataAsSpan().ToArray()); } [Fact(DisplayName = "CreateTensorOverManagedBuffer")] public void CreateTensorOverManagedBuffer() { int[] data = { 1, 2, 3 }; var mem = new Memory(data); long[] shape = { 1, 1, 3 }; var elementsNum = ArrayUtilities.GetSizeForShape(shape); Assert.Equal(elementsNum, data.Length); var typeInfo = TensorBase.GetElementTypeInfo(TensorElementType.Int32); Assert.NotNull(typeInfo); // The tensor will be created on top of the managed memory. No copy is made. // The memory should stay pinned until the OrtValue instance is disposed. This means // stayed pinned until the end of Run() method when you are actually running inference. using(var tensor = OrtValue.CreateTensorValueFromMemory(data, shape)) { VerifyTensorCreateWithData(tensor, TensorElementType.Int32, shape, data); } } // One can do create an OrtValue over a device memory and used as input. // Just make sure that OrtMemoryInfo is created for GPU. [Fact(DisplayName = "CreateTensorOverUnManagedBuffer")] public void CreateTensorOverUnmangedBuffer() { const int Elements = 3; // One can use stackalloc as well var bufferLen = Elements * sizeof(int); var dataPtr = Marshal.AllocHGlobal(bufferLen); try { // Use span to populate chunk of native memory Span data; unsafe { data = new Span(dataPtr.ToPointer(), Elements); } data[0] = 1; data[1] = 2; data[2] = 3; long[] shape = { 1, 1, 3 }; var elementsNum = ArrayUtilities.GetSizeForShape(shape); Assert.Equal(elementsNum, Elements); using (var tensor = OrtValue.CreateTensorValueWithData(OrtMemoryInfo.DefaultInstance, TensorElementType.Int32, shape, dataPtr, bufferLen)) { VerifyTensorCreateWithData(tensor, TensorElementType.Int32, shape, data); } } finally { Marshal.FreeHGlobal(dataPtr); } } private static void PopulateAndCheck(T[] data) where T : struct { var typeInfo = TensorBase.GetTypeInfo(typeof(T)); Assert.NotNull(typeInfo); long[] shape = { data.LongLength }; using (var ortValue = OrtValue.CreateAllocatedTensorValue(OrtAllocator.DefaultInstance, typeInfo.ElementType, shape)) { var dst = ortValue.GetTensorMutableDataAsSpan(); Assert.Equal(data.Length, dst.Length); var src = new Span(data); src.CopyTo(dst); Assert.Equal(data, ortValue.GetTensorDataAsSpan().ToArray()); } } // Create Tensor with allocated memory so we can test copying of the data [Fact(DisplayName = "CreateAllocatedTensor")] public void CreateAllocatedTensor() { float[] float_data = { 1, 2, 3, 4, 5, 6, 7, 8 }; int[] int_data = { 1, 2, 3, 4, 5, 6, 7, 8 }; ushort[] ushort_data = { 1, 2, 3, 4, 5, 6, 7, 8 }; double[] dbl_data = { 1, 2, 3, 4, 5, 6, 7, 8 }; var fp16_data = Array.ConvertAll(ushort_data, sh => new Float16(sh)); PopulateAndCheck(float_data); PopulateAndCheck(int_data); PopulateAndCheck(ushort_data); PopulateAndCheck(dbl_data); PopulateAndCheck(fp16_data); } private static readonly long[] ml_data_1 = { 1, 2 }; private static readonly long[] ml_data_2 = { 3, 4 }; // Use this utility method to create two tensors for Map and Sequence tests private static Tuple CreateTwoTensors(IList cleanup) { const int ml_data_dim = 2; // For map tensors they must be single dimensional long[] shape = { ml_data_dim }; unsafe { var ortValue_1 = OrtValue.CreateTensorValueFromMemory(ml_data_1, shape); cleanup.Add(ortValue_1); var ortValue_2 = OrtValue.CreateTensorValueFromMemory(ml_data_2, shape); cleanup.Add(ortValue_2); return Tuple.Create(ortValue_1, ortValue_2); } } [Fact(DisplayName = "CreateMap")] public void CreateMap() { using (var cleanUp = new DisposableListTest()) { var valTuple = CreateTwoTensors(cleanUp); using (var map = OrtValue.CreateMap(valTuple.Item1, valTuple.Item2)) { Assert.Equal(OnnxValueType.ONNX_TYPE_MAP, map.OnnxType); var typeInfo = map.GetTypeInfo(); var mapInfo = typeInfo.MapTypeInfo; Assert.Equal(TensorElementType.Int64, mapInfo.KeyType); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, mapInfo.ValueType.OnnxType); // Must return always 2 for map since we have two ort values Assert.Equal(2, map.GetValueCount()); var keys = map.GetValue(0, OrtAllocator.DefaultInstance); cleanUp.Add(keys); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, keys.OnnxType); Assert.Equal(ml_data_1, keys.GetTensorDataAsSpan().ToArray()); var vals = map.GetValue(1, OrtAllocator.DefaultInstance); cleanUp.Add(vals); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, vals.OnnxType); Assert.Equal(ml_data_2, vals.GetTensorDataAsSpan().ToArray()); } } } [Fact(DisplayName = "CreateSequence")] public void CreateSequence() { using (var cleanUp = new DisposableListTest()) { var valTuple = CreateTwoTensors(cleanUp); OrtValue[] seqVals = { valTuple.Item1, valTuple.Item2 }; using (var seq = OrtValue.CreateSequence(seqVals)) { Assert.Equal(OnnxValueType.ONNX_TYPE_SEQUENCE, seq.OnnxType); var typeInfo = seq.GetTypeInfo(); var seqInfo = typeInfo.SequenceTypeInfo; Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, seqInfo.ElementType.OnnxType); // Will return 2 because we put 2 values in the sequence Assert.Equal(2, seq.GetValueCount()); var item_0 = seq.GetValue(0, OrtAllocator.DefaultInstance); cleanUp.Add(item_0); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, item_0.OnnxType); Assert.Equal(ml_data_1, item_0.GetTensorDataAsSpan().ToArray()); var item_1 = seq.GetValue(1, OrtAllocator.DefaultInstance); cleanUp.Add(item_1); Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, item_1.OnnxType); Assert.Equal(ml_data_2, item_1.GetTensorDataAsSpan().ToArray()); } } } } }