saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support Tensor<bool> and Tensor<Int8> in C# API. Support Tensor<string> as input. Fix a bug in the InferenceSession Run() with RunOptions (#1671)

Browse source 
- Support bool-Tensor and int8-Tensor in input-output of C# api
- Support string-tensor as input in C# api
- Fix a bug in InferenceSession.Run() -- RunOptions was not passed into the native call
This commit is contained in:
shahasad 2019-08-22 10:14:50 -07:00 committed by GitHub
parent b53f40a886
commit a818740d91
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 146 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs
View file

@ -120,9 +120,15 @@ namespace Microsoft.ML.OnnxRuntime
case TensorElementType.UInt8:
result = DisposableNamedOnnxValueFromNativeTensor<byte>(name, nativeOnnxValue);
break;
case TensorElementType.Int8:
result = DisposableNamedOnnxValueFromNativeTensor<sbyte>(name, nativeOnnxValue);
break;
case TensorElementType.String:
result = DisposableNamedOnnxValueFromNativeTensor<string>(name, nativeOnnxValue);
break;
case TensorElementType.Bool:
result = DisposableNamedOnnxValueFromNativeTensor<bool>(name, nativeOnnxValue);
break;
default:
throw new NotSupportedException("Tensor of element type: " + elemType + " is not supported");

6
csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs
View file

@ -124,8 +124,7 @@ namespace Microsoft.ML.OnnxRuntime
IntPtr status = NativeMethods.OrtRun(
this._nativeHandle,
IntPtr.Zero, // TODO: use Run options when Run options creation API is available
// Passing null uses the default run options in the C-api
options.Handle,
inputNames,
inputTensors,
(UIntPtr)(inputTensors.Length),
@ -162,7 +161,8 @@ namespace Microsoft.ML.OnnxRuntime
// always unpin the input buffers, and delete the native Onnx value objects
for (int i = 0; i < inputs.Count; i++)
{
NativeMethods.OrtReleaseValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object
NativeMethods.OrtReleaseValue(inputTensors[i]); // For elementary type Tensors, this should not release the buffer, but should delete the native tensor object.
// For string tensors, this releases the native memory allocated for the tensor, including the buffer
pinnedBufferHandles[i].Dispose();
}
}

135
csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs
View file

@ -162,6 +162,15 @@ namespace Microsoft.ML.OnnxRuntime
))
{
}
else if (TryPinAsTensor<sbyte>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<bool>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
@ -171,41 +180,93 @@ namespace Microsoft.ML.OnnxRuntime
))
{
}
//TODO: add other types
else
// special case for string Tensor, data needs to be copied to the native buffer
else if (!(_value is Tensor<string>))
{
// nothing to cleanup here, since no memory has been pinned
throw new NotSupportedException("The inference value " + nameof(_value) + " is not of a supported type");
}
Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer");
if (_value is Tensor<string>)
{
// calculate native tensor length (sum of string lengths in utf-8)
var tensorValue = _value as Tensor<string>;
int totalLength = 0;
for (int i = 0; i < tensorValue.Length; i++)
{
totalLength += Encoding.UTF8.GetByteCount(tensorValue.GetValue(i));
}
// copy to an ulong[] shape to match size_t[]
long[] longShape = new long[rank];
for (int i = 0; i < rank; i++)
{
longShape[i] = shape[i];
}
long[] longShape = new long[tensorValue.Dimensions.Length];
for (int i = 0; i < tensorValue.Dimensions.Length; i++)
{
longShape[i] = tensorValue.Dimensions[i];
}
IntPtr status = NativeMethods.OrtCreateTensorWithDataAsOrtValue(
NativeMemoryAllocatorInfo.DefaultInstance.Handle,
dataBufferPointer,
(UIntPtr)(dataBufferLength),
longShape,
(UIntPtr)rank,
nativeElementType,
out onnxValue
);
try
{
NativeApiStatus.VerifySuccess(status);
// allocate the native tensor
IntPtr nativeTensor = IntPtr.Zero;
try
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorAsOrtValue(
NativeMemoryAllocator.DefaultInstance.Handle,
longShape,
(UIntPtr)(longShape.Length),
TensorElementType.String,
out nativeTensor
));
// fill the native tensor, using GetValue(index) from the Tensor<string>
string[] stringsInTensor = new string[tensorValue.Length];
for (int i = 0; i < tensorValue.Length; i++)
{
stringsInTensor[i] = tensorValue.GetValue(i);
}
NativeApiStatus.VerifySuccess(NativeMethods.OrtFillStringTensor(nativeTensor, stringsInTensor, (UIntPtr)tensorValue.Length));
}
catch (OnnxRuntimeException e)
{
if (nativeTensor != IntPtr.Zero)
{
NativeMethods.OrtReleaseValue(nativeTensor);
throw e;
}
}
onnxValue = nativeTensor; // set the output
pinnedMemoryHandle = default; // dummy value for the output
}
catch (OnnxRuntimeException e)
else
{
pinnedMemoryHandle.Dispose();
throw e;
Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer");
// copy to an ulong[] shape to match size_t[]
long[] longShape = new long[rank];
for (int i = 0; i < rank; i++)
{
longShape[i] = shape[i];
}
IntPtr status = NativeMethods.OrtCreateTensorWithDataAsOrtValue(
NativeMemoryAllocatorInfo.DefaultInstance.Handle,
dataBufferPointer,
(UIntPtr)(dataBufferLength),
longShape,
(UIntPtr)rank,
nativeElementType,
out onnxValue
);
try
{
NativeApiStatus.VerifySuccess(status);
}
catch (OnnxRuntimeException e)
{
pinnedMemoryHandle.Dispose();
throw e;
}
}
}
@ -224,7 +285,9 @@ namespace Microsoft.ML.OnnxRuntime
dataBufferLength = 0;
shape = null;
rank = 0;
pinnedMemoryHandle = default(MemoryHandle);
pinnedMemoryHandle = default;
Debug.Assert(typeof(T) != typeof(string), "NamedOnnxValue.TryPinAsTensor() must not be called with a string Tensor value");
if (_value is Tensor<T>)
{
@ -299,15 +362,21 @@ namespace Microsoft.ML.OnnxRuntime
nativeElementType = TensorElementType.UInt8;
dataBufferLength = dt.Buffer.Length * sizeof(byte);
}
else if (typeof(T) == typeof(sbyte))
{
nativeElementType = TensorElementType.Int8;
dataBufferLength = dt.Buffer.Length * sizeof(sbyte);
}
else if (typeof(T) == typeof(string))
{
nativeElementType = TensorElementType.String;
dataBufferLength = dt.Buffer.Length * IntPtr.Size;
}
//TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy
//else if (typeof(T) == typeof(bool))
//{
//}
else if (typeof(T) == typeof(bool))
{
nativeElementType = TensorElementType.Bool;
dataBufferLength = dt.Buffer.Length * sizeof(bool); // Assumes sizeof(BOOL) is always 1 byte in native
}
else
{
//TODO: may extend the supported types
@ -397,10 +466,18 @@ namespace Microsoft.ML.OnnxRuntime
type = typeof(byte);
width = sizeof(byte);
break;
case TensorElementType.Int8:
type = typeof(sbyte);
width = sizeof(sbyte);
break;
case TensorElementType.String:
type = typeof(byte);
width = sizeof(byte);
break;
case TensorElementType.Bool:
type = typeof(bool);
width = sizeof(bool);
break;
default:
type = null;
width = 0;

17
csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs
View file

@ -295,6 +295,14 @@ namespace Microsoft.ML.OnnxRuntime
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtGetTypeInfo(IntPtr /*(OrtValue*)*/ value, IntPtr /*(OrtValue**)*/ typeInfo);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtCreateTensorAsOrtValue(
IntPtr /*_Inout_ OrtAllocator* */ allocator,
long[] /*_In_ const int64_t* */ shape,
UIntPtr /*size_t*/ shape_len,
TensorElementType type,
out IntPtr /* OrtValue** */ outputValue);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue(
IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo,
@ -310,6 +318,15 @@ namespace Microsoft.ML.OnnxRuntime
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle);
/// \param value A tensor created from OrtCreateTensor... function.
/// \param len total data length, not including the trailing '\0' chars.
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtFillStringTensor(
IntPtr /* OrtValue */ value,
string[] /* const char* const* */s,
UIntPtr /* size_t */ s_len);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorContent(
IntPtr /*(OrtValue*)*/ value,

8
csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs
View file

@ -9,6 +9,14 @@ namespace Microsoft.ML.OnnxRuntime
public class RunOptions: IDisposable
{
private IntPtr _nativePtr;
internal IntPtr Handle
{
get
{
return _nativePtr;
}
}
public RunOptions()
{

12
csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
View file

@ -164,7 +164,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
using (var runOptions = new RunOptions())
{
runOptions.LogTag = "CsharpTest";
runOptions.Terminate = true;
runOptions.Terminate = false; // TODO: Test terminate = true, it currently crashes
runOptions.LogVerbosityLevel = LogLevel.Error;
IReadOnlyCollection<string> outputNames = session.OutputMetadata.Keys.ToList();
@ -391,7 +391,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
[Fact(Skip = "Boolean tensor not supported yet")]
[Fact]
private void TestModelInputBOOL()
{
// model takes 1x5 input of fixed type, echoes back
@ -449,15 +449,15 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
[Fact(Skip = "String tensor not supported yet")]
[Fact]
private void TestModelInputSTRING()
{
// model takes 1x5 input of fixed type, echoes back
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_types_STRING.onnx");
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_types_STRING.pb");
using (var session = new InferenceSession(modelPath))
{
var container = new List<NamedOnnxValue>();
var tensorIn = new DenseTensor<string>(new string[] { "a", "c", "d", "z", "f" }, new int[] { 1, 5 });
var tensorIn = new DenseTensor<string>(new string[] { "abc", "ced", "def", "", "frozen" }, new int[] { 1, 5 });
var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn);
container.Add(nov);
using (var res = session.Run(container))
@ -468,7 +468,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
[Fact(Skip = "Int8 not supported yet")]
[Fact]
private void TestModelInputINT8()
{
// model takes 1x5 input of fixed type, echoes back