diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs index c6fc8a3080..c85b6aded2 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs @@ -120,9 +120,15 @@ namespace Microsoft.ML.OnnxRuntime case TensorElementType.UInt8: result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; + case TensorElementType.Int8: + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); + break; case TensorElementType.String: result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); break; + case TensorElementType.Bool: + result = DisposableNamedOnnxValueFromNativeTensor(name, nativeOnnxValue); + break; default: throw new NotSupportedException("Tensor of element type: " + elemType + " is not supported"); diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index d20a89b14c..7964302956 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -124,8 +124,7 @@ namespace Microsoft.ML.OnnxRuntime IntPtr status = NativeMethods.OrtRun( this._nativeHandle, - IntPtr.Zero, // TODO: use Run options when Run options creation API is available - // Passing null uses the default run options in the C-api + options.Handle, inputNames, inputTensors, (UIntPtr)(inputTensors.Length), @@ -162,7 +161,8 @@ namespace Microsoft.ML.OnnxRuntime // always unpin the input buffers, and delete the native Onnx value objects for (int i = 0; i < inputs.Count; i++) { - NativeMethods.OrtReleaseValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object + NativeMethods.OrtReleaseValue(inputTensors[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object. + // For string tensors, this releases the native memory allocated for the tensor, including the buffer pinnedBufferHandles[i].Dispose(); } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs index eef56e68e8..3ac360e67e 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs @@ -162,6 +162,15 @@ namespace Microsoft.ML.OnnxRuntime )) { } + else if (TryPinAsTensor(out pinnedMemoryHandle, + out dataBufferPointer, + out dataBufferLength, + out shape, + out rank, + out nativeElementType + )) + { + } else if (TryPinAsTensor(out pinnedMemoryHandle, out dataBufferPointer, out dataBufferLength, @@ -171,41 +180,93 @@ namespace Microsoft.ML.OnnxRuntime )) { } - //TODO: add other types - else + // special case for string Tensor, data needs to be copied to the native buffer + else if (!(_value is Tensor)) { // nothing to cleanup here, since no memory has been pinned throw new NotSupportedException("The inference value " + nameof(_value) + " is not of a supported type"); } - Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer"); + if (_value is Tensor) + { + // calculate native tensor length (sum of string lengths in utf-8) + var tensorValue = _value as Tensor; + int totalLength = 0; + for (int i = 0; i < tensorValue.Length; i++) + { + totalLength += Encoding.UTF8.GetByteCount(tensorValue.GetValue(i)); + } - // copy to an ulong[] shape to match size_t[] - long[] longShape = new long[rank]; - for (int i = 0; i < rank; i++) - { - longShape[i] = shape[i]; - } + long[] longShape = new long[tensorValue.Dimensions.Length]; + for (int i = 0; i < tensorValue.Dimensions.Length; i++) + { + longShape[i] = tensorValue.Dimensions[i]; + } - IntPtr status = NativeMethods.OrtCreateTensorWithDataAsOrtValue( - NativeMemoryAllocatorInfo.DefaultInstance.Handle, - dataBufferPointer, - (UIntPtr)(dataBufferLength), - longShape, - (UIntPtr)rank, - nativeElementType, - out onnxValue - ); - try - { - NativeApiStatus.VerifySuccess(status); + // allocate the native tensor + IntPtr nativeTensor = IntPtr.Zero; + try + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue( + NativeMemoryAllocator.DefaultInstance.Handle, + longShape, + (UIntPtr)(longShape.Length), + TensorElementType.String, + out nativeTensor + )); + + // fill the native tensor, using GetValue(index) from the Tensor + string[] stringsInTensor = new string[tensorValue.Length]; + for (int i = 0; i < tensorValue.Length; i++) + { + stringsInTensor[i] = tensorValue.GetValue(i); + } + NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(nativeTensor, stringsInTensor, (UIntPtr)tensorValue.Length)); + } + catch (OnnxRuntimeException e) + { + if (nativeTensor != IntPtr.Zero) + { + NativeMethods.OrtReleaseValue(nativeTensor); + throw e; + } + } + + onnxValue = nativeTensor; // set the output + pinnedMemoryHandle = default; // dummy value for the output } - catch (OnnxRuntimeException e) + else { - pinnedMemoryHandle.Dispose(); - throw e; + Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer"); + + // copy to an ulong[] shape to match size_t[] + long[] longShape = new long[rank]; + for (int i = 0; i < rank; i++) + { + longShape[i] = shape[i]; + } + + IntPtr status = NativeMethods.OrtCreateTensorWithDataAsOrtValue( + NativeMemoryAllocatorInfo.DefaultInstance.Handle, + dataBufferPointer, + (UIntPtr)(dataBufferLength), + longShape, + (UIntPtr)rank, + nativeElementType, + out onnxValue + ); + try + { + NativeApiStatus.VerifySuccess(status); + } + catch (OnnxRuntimeException e) + { + pinnedMemoryHandle.Dispose(); + throw e; + } + } } @@ -224,7 +285,9 @@ namespace Microsoft.ML.OnnxRuntime dataBufferLength = 0; shape = null; rank = 0; - pinnedMemoryHandle = default(MemoryHandle); + pinnedMemoryHandle = default; + + Debug.Assert(typeof(T) != typeof(string), "NamedOnnxValue.TryPinAsTensor() must not be called with a string Tensor value"); if (_value is Tensor) { @@ -299,15 +362,21 @@ namespace Microsoft.ML.OnnxRuntime nativeElementType = TensorElementType.UInt8; dataBufferLength = dt.Buffer.Length * sizeof(byte); } + else if (typeof(T) == typeof(sbyte)) + { + nativeElementType = TensorElementType.Int8; + dataBufferLength = dt.Buffer.Length * sizeof(sbyte); + } else if (typeof(T) == typeof(string)) { nativeElementType = TensorElementType.String; dataBufferLength = dt.Buffer.Length * IntPtr.Size; } - //TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy - //else if (typeof(T) == typeof(bool)) - //{ - //} + else if (typeof(T) == typeof(bool)) + { + nativeElementType = TensorElementType.Bool; + dataBufferLength = dt.Buffer.Length * sizeof(bool); // Assumes sizeof(BOOL) is always 1 byte in native + } else { //TODO: may extend the supported types @@ -397,10 +466,18 @@ namespace Microsoft.ML.OnnxRuntime type = typeof(byte); width = sizeof(byte); break; + case TensorElementType.Int8: + type = typeof(sbyte); + width = sizeof(sbyte); + break; case TensorElementType.String: type = typeof(byte); width = sizeof(byte); break; + case TensorElementType.Bool: + type = typeof(bool); + width = sizeof(bool); + break; default: type = null; width = 0; diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index e1b57b4acf..1c9971cc57 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -295,6 +295,14 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtGetTypeInfo(IntPtr /*(OrtValue*)*/ value, IntPtr /*(OrtValue**)*/ typeInfo); + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateTensorAsOrtValue( + IntPtr /*_Inout_ OrtAllocator* */ allocator, + long[] /*_In_ const int64_t* */ shape, + UIntPtr /*size_t*/ shape_len, + TensorElementType type, + out IntPtr /* OrtValue** */ outputValue); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue( IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo, @@ -310,6 +318,15 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + + /// \param value A tensor created from OrtCreateTensor... function. + /// \param len total data length, not including the trailing '\0' chars. + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtFillStringTensor( + IntPtr /* OrtValue */ value, + string[] /* const char* const* */s, + UIntPtr /* size_t */ s_len); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorContent( IntPtr /*(OrtValue*)*/ value, diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs index 6037503e70..b40c795757 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs @@ -9,6 +9,14 @@ namespace Microsoft.ML.OnnxRuntime public class RunOptions: IDisposable { private IntPtr _nativePtr; + internal IntPtr Handle + { + get + { + return _nativePtr; + } + } + public RunOptions() { diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index fd8ddc45da..5fa81b9ae0 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -164,7 +164,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests using (var runOptions = new RunOptions()) { runOptions.LogTag = "CsharpTest"; - runOptions.Terminate = true; + runOptions.Terminate = false; // TODO: Test terminate = true, it currently crashes runOptions.LogVerbosityLevel = LogLevel.Error; IReadOnlyCollection outputNames = session.OutputMetadata.Keys.ToList(); @@ -391,7 +391,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests } } - [Fact(Skip = "Boolean tensor not supported yet")] + [Fact] private void TestModelInputBOOL() { // model takes 1x5 input of fixed type, echoes back @@ -449,15 +449,15 @@ namespace Microsoft.ML.OnnxRuntime.Tests } - [Fact(Skip = "String tensor not supported yet")] + [Fact] private void TestModelInputSTRING() { // model takes 1x5 input of fixed type, echoes back - string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_types_STRING.onnx"); + string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_types_STRING.pb"); using (var session = new InferenceSession(modelPath)) { var container = new List(); - var tensorIn = new DenseTensor(new string[] { "a", "c", "d", "z", "f" }, new int[] { 1, 5 }); + var tensorIn = new DenseTensor(new string[] { "abc", "ced", "def", "", "frozen" }, new int[] { 1, 5 }); var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn); container.Add(nov); using (var res = session.Run(container)) @@ -468,7 +468,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests } } - [Fact(Skip = "Int8 not supported yet")] + [Fact] private void TestModelInputINT8() { // model takes 1x5 input of fixed type, echoes back