mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Initial check-in to support non-tensor (sequence/map) types (#527)
* Initial check-in to support non-tensor (sequence/map) types * Added support for String tensors * address PR comments
This commit is contained in:
jignparm
GitHub
parent
1d3fcc525a
commit
1288a8caed
8 changed files with 376 additions and 58 deletions
|
|
@ -4,17 +4,18 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Numerics.Tensors;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
|
||||
namespace Microsoft.ML.OnnxRuntime
|
||||
{
|
||||
public interface IDisposableReadOnlyCollection<T>: IReadOnlyCollection<T>, IDisposable
|
||||
public interface IDisposableReadOnlyCollection<T> : IReadOnlyCollection<T>, IDisposable
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
internal class DisposableList<T> : List<T>, IDisposableReadOnlyCollection<T>
|
||||
where T: IDisposable
|
||||
where T : IDisposable
|
||||
{
|
||||
|
||||
#region IDisposable Support
|
||||
|
|
@ -57,18 +58,16 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
#endregion
|
||||
}
|
||||
|
||||
|
||||
public class DisposableNamedOnnxValue: NamedOnnxValue, IDisposable
|
||||
public class DisposableNamedOnnxValue : NamedOnnxValue, IDisposable
|
||||
{
|
||||
protected IDisposable _nativeMemoryManager;
|
||||
protected DisposableNamedOnnxValue(string name, Object value, IDisposable nativeMemoryManager)
|
||||
:base(name, value)
|
||||
: base(name, value)
|
||||
{
|
||||
_nativeMemoryManager = nativeMemoryManager;
|
||||
}
|
||||
|
||||
|
||||
internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue)
|
||||
internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, IntPtr nativeOnnxValue)
|
||||
{
|
||||
DisposableNamedOnnxValue result = null;
|
||||
|
||||
|
|
@ -91,31 +90,34 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
switch (elemType)
|
||||
{
|
||||
case TensorElementType.Float:
|
||||
result = NameOnnxValueFromNativeTensor<float>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<float>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.Double:
|
||||
result = NameOnnxValueFromNativeTensor<double>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<double>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.Int16:
|
||||
result = NameOnnxValueFromNativeTensor<short>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<short>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.UInt16:
|
||||
result = NameOnnxValueFromNativeTensor<ushort>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<ushort>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.Int32:
|
||||
result = NameOnnxValueFromNativeTensor<int>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<int>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.UInt32:
|
||||
result = NameOnnxValueFromNativeTensor<uint>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<uint>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.Int64:
|
||||
result = NameOnnxValueFromNativeTensor<long>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<long>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.UInt64:
|
||||
result = NameOnnxValueFromNativeTensor<ulong>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<ulong>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.UInt8:
|
||||
result = NameOnnxValueFromNativeTensor<byte>(name, nativeOnnxValue);
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<byte>(name, nativeOnnxValue);
|
||||
break;
|
||||
case TensorElementType.String:
|
||||
result = DisposableNamedOnnxValueFromNativeTensor<string>(name, nativeOnnxValue);
|
||||
break;
|
||||
default:
|
||||
throw new NotSupportedException("Tensor of element type: " + elemType + " is not supported");
|
||||
|
|
@ -125,15 +127,114 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
return result;
|
||||
}
|
||||
|
||||
|
||||
private static DisposableNamedOnnxValue NameOnnxValueFromNativeTensor<T>(string name, IntPtr nativeOnnxValue)
|
||||
internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue)
|
||||
{
|
||||
NativeOnnxTensorMemory<T> nativeTensorWrapper = new NativeOnnxTensorMemory<T>(nativeOnnxValue);
|
||||
DenseTensor<T> dt = new DenseTensor<T>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
|
||||
return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper);
|
||||
IntPtr allocator = IntPtr.Zero;
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateDefaultAllocator(out allocator));
|
||||
var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator);
|
||||
NativeMethods.OrtReleaseAllocator(allocator);
|
||||
return (DisposableNamedOnnxValue)ret;
|
||||
}
|
||||
|
||||
internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue, IntPtr allocator)
|
||||
{
|
||||
var onnxValueType = NativeMethods.OrtGetValueType(nativeOnnxValue);
|
||||
switch (onnxValueType)
|
||||
{
|
||||
case OnnxValueType.ONNX_TYPE_TENSOR:
|
||||
return CreateTensorFromOnnxValue(name, nativeOnnxValue);
|
||||
|
||||
case OnnxValueType.ONNX_TYPE_SEQUENCE:
|
||||
IntPtr count = IntPtr.Zero;
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValueCount(nativeOnnxValue, out count));
|
||||
var sequence = new DisposableList<DisposableNamedOnnxValue>();
|
||||
for (long i = 0; i < count.ToInt64(); i++)
|
||||
{
|
||||
IntPtr nativeOnnxValueSeq;
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 0, allocator, out nativeOnnxValueSeq));
|
||||
sequence.Add(CreateFromOnnxValue(string.Empty, nativeOnnxValueSeq, allocator));
|
||||
NativeMethods.OrtReleaseValue(nativeOnnxValueSeq);
|
||||
}
|
||||
return new DisposableNamedOnnxValue(name, sequence, null);
|
||||
|
||||
case OnnxValueType.ONNX_TYPE_MAP:
|
||||
IntPtr typeAndShape = IntPtr.Zero;
|
||||
IntPtr nativeOnnxValueMapKeys = IntPtr.Zero;
|
||||
IntPtr nativeOnnxValueMapValues = IntPtr.Zero;
|
||||
TensorElementType elemType = TensorElementType.DataTypeMax;
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 0, allocator, out nativeOnnxValueMapKeys));
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetValue(nativeOnnxValue, 1, allocator, out nativeOnnxValueMapValues));
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(nativeOnnxValueMapKeys, out typeAndShape));
|
||||
|
||||
elemType = NativeMethods.OrtGetTensorElementType(typeAndShape);
|
||||
if (typeAndShape != IntPtr.Zero)
|
||||
{
|
||||
NativeMethods.OrtReleaseTensorTypeAndShapeInfo(typeAndShape);
|
||||
}
|
||||
switch (elemType)
|
||||
{
|
||||
case TensorElementType.Int64:
|
||||
return DisposableNamedOnnxValueFromNativeMap<Int64, float>(string.Empty, nativeOnnxValueMapKeys, nativeOnnxValueMapValues);
|
||||
case TensorElementType.String:
|
||||
return DisposableNamedOnnxValueFromNativeMap<string, float>(string.Empty, nativeOnnxValueMapKeys, nativeOnnxValueMapValues);
|
||||
default:
|
||||
throw new NotSupportedException("Map of element type: " + elemType + " is not supported");
|
||||
}
|
||||
default:
|
||||
throw new NotSupportedException("OnnxValueType : " + onnxValueType + " is not supported");
|
||||
}
|
||||
}
|
||||
|
||||
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeTensor<T>(string name, IntPtr nativeOnnxValue)
|
||||
{
|
||||
if (typeof(T) == typeof(string))
|
||||
{
|
||||
var nativeTensorWrapper = new NativeOnnxTensorMemory<byte>(nativeOnnxValue, true);
|
||||
var dt = new DenseTensor<string>(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions);
|
||||
return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper);
|
||||
}
|
||||
else
|
||||
{
|
||||
NativeOnnxTensorMemory<T> nativeTensorWrapper = new NativeOnnxTensorMemory<T>(nativeOnnxValue);
|
||||
DenseTensor<T> dt = new DenseTensor<T>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
|
||||
return new DisposableNamedOnnxValue(name, dt, nativeTensorWrapper);
|
||||
}
|
||||
}
|
||||
|
||||
private static DisposableNamedOnnxValue DisposableNamedOnnxValueFromNativeMap<K, V>(string name, IntPtr nativeOnnxValueKeys, IntPtr nativeOnnxValueValues)
|
||||
{
|
||||
var nativeTensorWrapperValues = new NativeOnnxTensorMemory<V>(nativeOnnxValueValues);
|
||||
var denseTensorValues = new DenseTensor<V>(nativeTensorWrapperValues.Memory, nativeTensorWrapperValues.Dimensions);
|
||||
|
||||
if (typeof(K) == typeof(string))
|
||||
{
|
||||
var map = new Dictionary<string, V>();
|
||||
var nativeTensorWrapper = new NativeOnnxTensorMemory<byte>(nativeOnnxValueKeys, true);
|
||||
var denseTensorKeys = new DenseTensor<string>(nativeTensorWrapper.GetBytesAsStringMemory(), nativeTensorWrapper.Dimensions);
|
||||
for (var i = 0; i < denseTensorKeys.Length; i++)
|
||||
{
|
||||
map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i));
|
||||
}
|
||||
// release native memory
|
||||
nativeTensorWrapperValues.Dispose();
|
||||
nativeTensorWrapper.Dispose();
|
||||
return new DisposableNamedOnnxValue(string.Empty, map, null);
|
||||
}
|
||||
else
|
||||
{
|
||||
var map = new Dictionary<K, V>();
|
||||
var nativeTensorWrapper = new NativeOnnxTensorMemory<K>(nativeOnnxValueKeys);
|
||||
var denseTensorKeys = new DenseTensor<K>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
|
||||
for (var i = 0; i < denseTensorKeys.Length; i++)
|
||||
{
|
||||
map.Add(denseTensorKeys.GetValue(i), denseTensorValues.GetValue(i));
|
||||
}
|
||||
// release native memory
|
||||
nativeTensorWrapperValues.Dispose();
|
||||
nativeTensorWrapper.Dispose();
|
||||
return new DisposableNamedOnnxValue(string.Empty, map, null);
|
||||
}
|
||||
}
|
||||
|
||||
#region IDisposable Support
|
||||
private bool disposedValue = false; // To detect redundant calls
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
var result = new DisposableList<DisposableNamedOnnxValue>();
|
||||
for (uint i = 0; i < outputValueArray.Length; i++)
|
||||
{
|
||||
result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i]));
|
||||
result.Add(DisposableNamedOnnxValue.CreateFromOnnxValue(outputNamesArray[i], outputValueArray[i]));
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
@ -297,10 +297,13 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
}
|
||||
}
|
||||
|
||||
private NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo)
|
||||
internal static NodeMetadata GetMetadataFromTypeInfo(IntPtr typeInfo)
|
||||
{
|
||||
IntPtr tensorInfo = NativeMethods.OrtCastTypeInfoToTensorInfo(typeInfo);
|
||||
// Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo*
|
||||
// Convert the newly introduced OrtTypeInfo* to the older OrtTypeAndShapeInfo*
|
||||
|
||||
if (tensorInfo == IntPtr.Zero)
|
||||
return null;
|
||||
|
||||
TensorElementType type = NativeMethods.OrtGetTensorElementType(tensorInfo);
|
||||
Type dotnetType = null;
|
||||
|
|
|
|||
|
|
@ -29,11 +29,39 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
}
|
||||
|
||||
public string Name { get { return _name; } }
|
||||
|
||||
/// <summary>
|
||||
/// Try-get value as a Tensor<T>.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Type</typeparam>
|
||||
/// <returns>Tensor object if contained value is a Tensor. Null otherwise</returns>
|
||||
public Tensor<T> AsTensor<T>()
|
||||
{
|
||||
return _value as Tensor<T>; // will return null if not castable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Try-get value as an Enumerable<T>.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Type</typeparam>
|
||||
/// <returns>Enumerable object if contained value is a Enumerable. Null otherwise</returns>
|
||||
public IEnumerable<T> AsEnumerable<T> ()
|
||||
{
|
||||
var x = _value as IEnumerable<T>;
|
||||
return x;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Try-get value as an Dictionary<K,V>.
|
||||
/// </summary>
|
||||
/// <typeparam name="K">Key type</typeparam>
|
||||
/// <typeparam name="V">Value type</typeparam>
|
||||
/// <returns>Dictionary object if contained value is a Dictionary. Null otherwise</returns>
|
||||
public IDictionary<K, V> AsDictionary<K, V>()
|
||||
{
|
||||
return _value as IDictionary<K, V>;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to Pin the buffer, and create a native OnnxValue out of it. the pinned MemoryHandle is passed to output.
|
||||
/// In this case, the pinnedHandle should be kept alive till the native OnnxValue is used, then dispose it.
|
||||
|
|
@ -271,6 +299,11 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
nativeElementType = TensorElementType.UInt8;
|
||||
dataBufferLength = dt.Buffer.Length * sizeof(byte);
|
||||
}
|
||||
else if (typeof(T) == typeof(string))
|
||||
{
|
||||
nativeElementType = TensorElementType.String;
|
||||
dataBufferLength = dt.Buffer.Length * IntPtr.Size;
|
||||
}
|
||||
//TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy
|
||||
//else if (typeof(T) == typeof(bool))
|
||||
//{
|
||||
|
|
@ -312,6 +345,16 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
DataTypeMax = 17
|
||||
}
|
||||
|
||||
internal enum OnnxValueType
|
||||
{
|
||||
ONNX_TYPE_UNKNOWN = 0,
|
||||
ONNX_TYPE_TENSOR = 1,
|
||||
ONNX_TYPE_SEQUENCE = 2,
|
||||
ONNX_TYPE_MAP = 3,
|
||||
ONNX_TYPE_OPAQUE = 4,
|
||||
ONNX_TYPE_SPARSETENSOR = 5,
|
||||
}
|
||||
|
||||
internal static class TensorElementTypeConverter
|
||||
{
|
||||
public static void GetTypeAndWidth(TensorElementType elemType, out Type type, out int width)
|
||||
|
|
@ -354,6 +397,10 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
type = typeof(byte);
|
||||
width = sizeof(byte);
|
||||
break;
|
||||
case TensorElementType.String:
|
||||
type = typeof(byte);
|
||||
width = sizeof(byte);
|
||||
break;
|
||||
default:
|
||||
type = null;
|
||||
width = 0;
|
||||
|
|
|
|||
|
|
@ -30,10 +30,11 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern ErrorCode OrtGetErrorCode(IntPtr /*(OrtStatus*)*/status);
|
||||
|
||||
// returns char*, need to convert to string by the caller.
|
||||
// does not free the underlying OrtStatus*
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /* char* */OrtGetErrorMessage(IntPtr /* (OrtStatus*) */status);
|
||||
// returns char*, need to convert to string by the caller.
|
||||
// does not free the underlying OrtStatus*
|
||||
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern void OrtReleaseStatus(IntPtr /*(OrtStatus*)*/ statusPtr);
|
||||
|
|
@ -67,7 +68,7 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtSessionGetInputCount(
|
||||
IntPtr /*(OrtSession*)*/ session,
|
||||
IntPtr /*(OrtSession*)*/ session,
|
||||
out ulong /* TODO: size_t */ count);
|
||||
|
||||
|
||||
|
|
@ -80,7 +81,7 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetInputName(
|
||||
IntPtr /*(OrtSession*)*/ session,
|
||||
ulong index, //TODO: port size_t
|
||||
IntPtr /*(OrtAllocator*)*/ allocator,
|
||||
IntPtr /*(OrtAllocator*)*/ allocator,
|
||||
out IntPtr /*(char**)*/name);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
|
|
@ -93,14 +94,14 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
// release the typeinfo using OrtReleaseTypeInfo
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetInputTypeInfo(
|
||||
IntPtr /*(const OrtSession*)*/ session,
|
||||
IntPtr /*(const OrtSession*)*/ session,
|
||||
ulong index, //TODO: port for size_t
|
||||
out IntPtr /*(struct OrtTypeInfo**)*/ typeInfo);
|
||||
|
||||
// release the typeinfo using OrtReleaseTypeInfo
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/OrtSessionGetOutputTypeInfo(
|
||||
IntPtr /*(const OrtSession*)*/ session,
|
||||
IntPtr /*(const OrtSession*)*/ session,
|
||||
ulong index, //TODO: port for size_t
|
||||
out IntPtr /* (struct OrtTypeInfo**)*/ typeInfo);
|
||||
|
||||
|
|
@ -239,6 +240,21 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
|
||||
#region Tensor/OnnxValue API
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetValue(IntPtr /*(OrtValue*)*/ value,
|
||||
int index,
|
||||
IntPtr /*(OrtAllocator*)*/ allocator,
|
||||
out IntPtr /*(OrtValue**)*/ outputValue);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern OnnxValueType /*Onnxtype*/ OrtGetValueType(IntPtr /*(OrtValue*)*/ value);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetValueCount(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(size_t*)*/ count);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetTypeInfo(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(OrtValue**)*/ typeInfo);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue(
|
||||
IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo,
|
||||
|
|
@ -254,6 +270,17 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorContent(
|
||||
IntPtr /*(OrtValue*)*/ value,
|
||||
IntPtr /*(void*)*/ dst_buffer,
|
||||
ulong dst_buffer_len, //size_t, TODO: make it portable for x86, arm
|
||||
IntPtr offsets,
|
||||
ulong offsets_len); //size_t, TODO: make it portable for x86, arm
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(OrtStatus*)*/ OrtGetStringTensorDataLength(IntPtr /*(OrtValue*)*/ value, out ulong /*(size_t*)*/ len);
|
||||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/
|
||||
OrtCastTypeInfoToTensorInfo(IntPtr /*(struct OrtTypeInfo*)*/ typeInfo);
|
||||
|
|
@ -273,8 +300,8 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
|
||||
[DllImport(nativeLib, CharSet = charSet)]
|
||||
public static extern void OrtGetDimensions(
|
||||
IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo,
|
||||
long[] dim_values,
|
||||
IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo,
|
||||
long[] dim_values,
|
||||
ulong dim_values_length);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -16,32 +16,30 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
{
|
||||
private bool _disposed;
|
||||
private int _referenceCount;
|
||||
private IntPtr _onnxValueHandle;
|
||||
private IntPtr _dataBufferHandle;
|
||||
private IntPtr _onnxValueHandle; // pointer to onnxvalue object in native
|
||||
private IntPtr _dataBufferPointer; // pointer to mutable tensor data in native memory
|
||||
private string[] _dataBufferAsString; // string tensor values copied into managed memory
|
||||
private int _elementCount;
|
||||
private int _elementWidth;
|
||||
private int[] _dimensions;
|
||||
|
||||
public NativeOnnxTensorMemory(IntPtr onnxValueHandle)
|
||||
public NativeOnnxTensorMemory(IntPtr onnxValueHandle, bool isStringTensor = false)
|
||||
{
|
||||
IntPtr typeAndShape = IntPtr.Zero;
|
||||
try
|
||||
{
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(onnxValueHandle, out typeAndShape));
|
||||
|
||||
TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape);
|
||||
|
||||
Type type = null;
|
||||
int width = 0;
|
||||
TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width);
|
||||
if (typeof(T) != type)
|
||||
throw new NotSupportedException(nameof(NativeOnnxTensorMemory<T>)+" does not support T = "+nameof(T));
|
||||
_elementWidth = width;
|
||||
|
||||
_onnxValueHandle = onnxValueHandle;
|
||||
// derive the databuffer pointer, element_count, element_width, and shape
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle));
|
||||
// throws OnnxRuntimeException if native call failed
|
||||
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorShapeAndType(onnxValueHandle, out typeAndShape));
|
||||
TensorElementType elemType = NativeMethods.OrtGetTensorElementType(typeAndShape);
|
||||
TensorElementTypeConverter.GetTypeAndWidth(elemType, out type, out width);
|
||||
|
||||
if (typeof(T) != type)
|
||||
throw new NotSupportedException(nameof(NativeOnnxTensorMemory<T>) + " does not support T = " + nameof(T));
|
||||
|
||||
_elementWidth = width;
|
||||
|
||||
ulong dimension = NativeMethods.OrtGetNumOfDimensions(typeAndShape);
|
||||
long count = NativeMethods.OrtGetTensorShapeElementCount(typeAndShape); // count can be negative.
|
||||
|
|
@ -59,6 +57,48 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
{
|
||||
_dimensions[i] = (int)shape[i];
|
||||
}
|
||||
|
||||
if (!isStringTensor)
|
||||
{
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetTensorMutableData(_onnxValueHandle, out _dataBufferPointer));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (typeof(T) != typeof(byte))
|
||||
throw new NotSupportedException(nameof(NativeOnnxTensorMemory<T>) + " T = " + nameof(T) + ". Should = byte, when isStringTensor is true");
|
||||
ulong strLen;
|
||||
var offsets = new ulong[_elementCount];
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorDataLength(_onnxValueHandle, out strLen));
|
||||
var dataBuffer = new byte[strLen];
|
||||
var dataBufferMemory = new Memory<byte>(dataBuffer);
|
||||
var dataBufferHandle = dataBufferMemory.Pin();
|
||||
IntPtr dataBufferPointer = IntPtr.Zero;
|
||||
|
||||
var offsetMemory = new Memory<ulong>(offsets);
|
||||
var offsetMemoryHandle = offsetMemory.Pin();
|
||||
IntPtr offsetBufferPointer = IntPtr.Zero;
|
||||
unsafe
|
||||
{
|
||||
dataBufferPointer = (IntPtr)dataBufferHandle.Pointer;
|
||||
offsetBufferPointer = (IntPtr)offsetMemoryHandle.Pointer;
|
||||
}
|
||||
NativeApiStatus.VerifySuccess(NativeMethods.OrtGetStringTensorContent(_onnxValueHandle, dataBufferPointer, strLen, offsetBufferPointer, Convert.ToUInt64(_elementCount)));
|
||||
_dataBufferPointer = dataBufferPointer;
|
||||
_dataBufferAsString = new string[_elementCount];
|
||||
|
||||
for (var i = 0; i < offsets.Length; i++)
|
||||
{
|
||||
var length = (i == offsets.Length - 1)
|
||||
? strLen - offsets[i]
|
||||
: offsets[i + 1] - offsets[i];
|
||||
// Onnx specifies strings always in UTF-8, no trailing null, no leading BOM
|
||||
_dataBufferAsString[i] = Encoding.UTF8.GetString(dataBuffer, (int)offsets[i], (int)length);
|
||||
}
|
||||
|
||||
// unpin memory
|
||||
offsetMemoryHandle.Dispose();
|
||||
dataBufferHandle.Dispose();
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
|
|
@ -74,6 +114,7 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
~NativeOnnxTensorMemory()
|
||||
{
|
||||
Dispose(false);
|
||||
|
|
@ -128,12 +169,22 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
Span<T> span = null;
|
||||
unsafe
|
||||
{
|
||||
span = new Span<T>((void*)_dataBufferHandle, _elementCount);
|
||||
span = new Span<T>((void*)_dataBufferPointer, _elementCount);
|
||||
}
|
||||
|
||||
return span;
|
||||
}
|
||||
|
||||
public Memory<String> GetBytesAsStringMemory()
|
||||
{
|
||||
if (IsDisposed)
|
||||
throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory<T>));
|
||||
|
||||
if (typeof(T) != typeof(byte))
|
||||
throw new NotSupportedException(nameof(NativeOnnxTensorMemory<T>.GetBytesAsStringMemory) + ": T must be byte");
|
||||
|
||||
return (_dataBufferAsString == null) ? new Memory<string>() : new Memory<string>(_dataBufferAsString);
|
||||
}
|
||||
|
||||
public override MemoryHandle Pin(int elementIndex = 0)
|
||||
{
|
||||
|
|
@ -146,17 +197,15 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
}
|
||||
Retain();
|
||||
|
||||
return new MemoryHandle((void*)((int)_dataBufferHandle + elementIndex*_elementWidth)); //could not use Unsafe.Add
|
||||
return new MemoryHandle((void*)((int)_dataBufferPointer + elementIndex * _elementWidth)); //could not use Unsafe.Add
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public override void Unpin()
|
||||
{
|
||||
Release();
|
||||
}
|
||||
|
||||
|
||||
private bool Release()
|
||||
{
|
||||
int newRefCount = Interlocked.Decrement(ref _referenceCount);
|
||||
|
|
@ -202,9 +251,5 @@ namespace Microsoft.ML.OnnxRuntime
|
|||
arraySegment = default(ArraySegment<T>);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
|
|||
validModelFound = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (!validModelFound)
|
||||
{
|
||||
var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
|
||||
|
|
@ -351,7 +351,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
|
|||
Assert.True(tensorOut.SequenceEqual(tensorIn));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
[Fact(Skip = "String tensor not supported yet")]
|
||||
|
|
@ -524,6 +524,101 @@ namespace Microsoft.ML.OnnxRuntime.Tests
|
|||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
private void TestModelSequenceOfMapIntFloat()
|
||||
{
|
||||
// test model trained using lightgbm classifier
|
||||
// produces 2 named outputs
|
||||
// "label" is a tensor,
|
||||
// "probabilities" is a sequence<map<int64, float>>
|
||||
// https://github.com/onnx/sklearn-onnx/blob/master/docs/examples/plot_pipeline_lightgbm.py
|
||||
|
||||
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_sequence_map_int_float.pb");
|
||||
using (var session = new InferenceSession(modelPath))
|
||||
{
|
||||
var container = new List<NamedOnnxValue>();
|
||||
var tensorIn = new DenseTensor<float>(new float[] { 5.8f, 2.8f }, new int[] { 1, 2 });
|
||||
var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn);
|
||||
container.Add(nov);
|
||||
|
||||
using (var outputs = session.Run(container))
|
||||
{
|
||||
// first output is a tensor containing label
|
||||
var outNode1 = outputs.ElementAtOrDefault(0);
|
||||
Assert.Equal("label", outNode1.Name);
|
||||
|
||||
// try-cast as a tensor
|
||||
var outLabelTensor = outNode1.AsTensor<Int64>();
|
||||
|
||||
// Label 1 should have highest probaility
|
||||
Assert.Equal(1, outLabelTensor[0]);
|
||||
|
||||
// second output is a sequence<map<int64, float>>
|
||||
// try-cast to an sequence of NOV
|
||||
var outNode2 = outputs.ElementAtOrDefault(1);
|
||||
Assert.Equal("probabilities", outNode2.Name);
|
||||
|
||||
// try-cast to an sequence of NOV
|
||||
var seq = outNode2.AsEnumerable<NamedOnnxValue>();
|
||||
|
||||
// try-cast first element in sequence to map/dictionary type
|
||||
var map = seq.First().AsDictionary<Int64, float>();
|
||||
//verify values are valid
|
||||
Assert.Equal(0.25938290, map[0], 6);
|
||||
Assert.Equal(0.40904793, map[1], 6);
|
||||
Assert.Equal(0.33156919, map[2], 6);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
private void TestModelSequenceOfMapStringFloat()
|
||||
{
|
||||
// test model trained using lightgbm classifier
|
||||
// produces 2 named outputs
|
||||
// "label" is a tensor,
|
||||
// "probabilities" is a sequence<map<int64, float>>
|
||||
// https://github.com/onnx/sklearn-onnx/blob/master/docs/examples/plot_pipeline_lightgbm.py
|
||||
|
||||
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "test_sequence_map_string_float.pb");
|
||||
|
||||
using (var session = new InferenceSession(modelPath))
|
||||
{
|
||||
var container = new List<NamedOnnxValue>();
|
||||
var tensorIn = new DenseTensor<float>(new float[] { 5.8f, 2.8f }, new int[] { 1, 2 });
|
||||
var nov = NamedOnnxValue.CreateFromTensor("input", tensorIn);
|
||||
container.Add(nov);
|
||||
|
||||
using (var outputs = session.Run(container))
|
||||
{
|
||||
// first output is a tensor containing label
|
||||
var outNode1 = outputs.ElementAtOrDefault(0);
|
||||
Assert.Equal("label", outNode1.Name);
|
||||
|
||||
// try-cast as a tensor
|
||||
var outLabelTensor = outNode1.AsTensor<string>();
|
||||
|
||||
// Label 1 should have highest probaility
|
||||
Assert.Equal("1", outLabelTensor[0]);
|
||||
|
||||
// second output is a sequence<map<int64, float>>
|
||||
// try-cast to an sequence of NOV
|
||||
var outNode2 = outputs.ElementAtOrDefault(1);
|
||||
Assert.Equal("probabilities", outNode2.Name);
|
||||
|
||||
// try-cast to an sequence of NOV
|
||||
var seq = outNode2.AsEnumerable<NamedOnnxValue>();
|
||||
|
||||
// try-cast first element in sequence to map/dictionary type
|
||||
var map = seq.First().AsDictionary<string, float>();
|
||||
//verify values are valid
|
||||
Assert.Equal(0.25938290, map["0"], 6);
|
||||
Assert.Equal(0.40904793, map["1"], 6);
|
||||
Assert.Equal(0.33156919, map["2"], 6);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[GpuFact]
|
||||
private void TestGpu()
|
||||
{
|
||||
|
|
@ -619,7 +714,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
|
|||
{
|
||||
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
|
||||
var session = (cudaDeviceId.HasValue)
|
||||
? new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(cudaDeviceId.Value))
|
||||
? new InferenceSession(modelPath, SessionOptions.MakeSessionOptionWithCudaProvider(cudaDeviceId.Value))
|
||||
: new InferenceSession(modelPath);
|
||||
float[] inputData = LoadTensorFromFile(@"bench.in");
|
||||
float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out");
|
||||
|
|
|
|||
BIN
csharp/testdata/test_sequence_map_int_float.pb
vendored
Normal file
BIN
csharp/testdata/test_sequence_map_int_float.pb
vendored
Normal file
Binary file not shown.
BIN
csharp/testdata/test_sequence_map_string_float.pb
vendored
Normal file
BIN
csharp/testdata/test_sequence_map_string_float.pb
vendored
Normal file
Binary file not shown.
Loading…
Reference in a new issue