saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-14 20:48:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds the new System.Numerics.Tensors as an input/output type when using dotnet 8.0 and up. (#23261)

Browse source 
### Description
Adds the new System.Numerics.Tensors as an input/output type when using
dotnet 8.0 and up. It does not change/remove any of the existing API,
only adds additional ones.


### Motivation and Context
Now that C#/Dotnet has an official tensor type built into the language,
we want to expand the places that it can be used.
This commit is contained in:
Michael Sharp 2025-01-27 11:58:38 -07:00 committed by GitHub
parent 97c2bbe3eb
commit 42f0c00f95
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 369 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
View file

@ -1,4 +1,4 @@
<Project Sdk="MSBuild.Sdk.Extras/3.0.22">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<!--- packaging properties -->
<OrtPackageId Condition="'$(OrtPackageId)' == ''">Microsoft.ML.OnnxRuntime</OrtPackageId>
@ -189,6 +189,10 @@
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
</ItemGroup>
<ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">
<PackageReference Include="System.Numerics.Tensors" Version="9.0.0" />
</ItemGroup>
<!-- debug output - makes finding/fixing any issues with the the conditions easy. -->
<Target Name="DumpValues" BeforeTargets="PreBuildEvent">
<Message Text="SolutionName='$(SolutionName)'" />

152
csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs
View file

@ -9,6 +9,14 @@ using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Text;
#if NET8_0_OR_GREATER
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
using System.Runtime.CompilerServices;
using SystemNumericsTensors = System.Numerics.Tensors;
using TensorPrimitives = System.Numerics.Tensors.TensorPrimitives;
#endif
namespace Microsoft.ML.OnnxRuntime
{
/// <summary>
@ -205,6 +213,33 @@ namespace Microsoft.ML.OnnxRuntime
return MemoryMarshal.Cast<byte, T>(byteSpan);
}
#if NET8_0_OR_GREATER
/// <summary>
/// Returns a ReadOnlyTensorSpan<typeparamref name="T"/> over tensor native buffer that
/// provides a read-only view.
///
/// Note, that the memory may be device allocated and, therefore, not accessible from the CPU.
/// To get memory descriptor use GetTensorMemoryInfo().
///
/// OrtValue must contain a non-string tensor.
/// The span is valid as long as the OrtValue instance is alive (not disposed).
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns>ReadOnlySpan<typeparamref name="T"/></returns>
/// <exception cref="OnnxRuntimeException"></exception>
[Experimental("SYSLIB5001")]
public SystemNumericsTensors.ReadOnlyTensorSpan<T> GetTensorDataAsTensorSpan<T>() where T : unmanaged
{
var byteSpan = GetTensorBufferRawData(typeof(T));
var typeSpan = MemoryMarshal.Cast<byte, T>(byteSpan);
var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape;
nint[] nArray = Array.ConvertAll(shape, new Converter<long, nint>(x => (nint)x));
return new SystemNumericsTensors.ReadOnlyTensorSpan<T>(typeSpan, nArray, []);
}
#endif
/// <summary>
/// Returns a Span<typeparamref name="T"/> over tensor native buffer.
/// This enables you to safely and efficiently modify the underlying
@ -225,6 +260,32 @@ namespace Microsoft.ML.OnnxRuntime
return MemoryMarshal.Cast<byte, T>(byteSpan);
}
#if NET8_0_OR_GREATER
/// <summary>
/// Returns a TensorSpan<typeparamref name="T"/> over tensor native buffer.
///
/// Note, that the memory may be device allocated and, therefore, not accessible from the CPU.
/// To get memory descriptor use GetTensorMemoryInfo().
///
/// OrtValue must contain a non-string tensor.
/// The span is valid as long as the OrtValue instance is alive (not disposed).
/// </summary>
/// <typeparam name="T"></typeparam>
/// <returns>ReadOnlySpan<typeparamref name="T"/></returns>
/// <exception cref="OnnxRuntimeException"></exception>
[Experimental("SYSLIB5001")]
public SystemNumericsTensors.TensorSpan<T> GetTensorMutableDataAsTensorSpan<T>() where T : unmanaged
{
var byteSpan = GetTensorBufferRawData(typeof(T));
var typeSpan = MemoryMarshal.Cast<byte, T>(byteSpan);
var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape;
nint[] nArray = Array.ConvertAll(shape, new Converter<long, nint>(x => (nint)x));
return new SystemNumericsTensors.TensorSpan<T>(typeSpan, nArray, []);
}
#endif
/// <summary>
/// Provides mutable raw native buffer access.
/// </summary>
@ -234,6 +295,23 @@ namespace Microsoft.ML.OnnxRuntime
return GetTensorBufferRawData(typeof(byte));
}
#if NET8_0_OR_GREATER
/// <summary>
/// Provides mutable raw native buffer access.
/// </summary>
/// <returns>TensorSpan over the native buffer bytes</returns>
[Experimental("SYSLIB5001")]
public SystemNumericsTensors.TensorSpan<byte> GetTensorSpanMutableRawData<T>() where T : unmanaged
{
var byteSpan = GetTensorBufferRawData(typeof(T));
var shape = GetTypeInfo().TensorTypeAndShapeInfo.Shape;
nint[] nArray = Array.ConvertAll(shape, new Converter<long, nint>(x => (nint)x));
return new SystemNumericsTensors.TensorSpan<byte>(byteSpan, nArray, []);
}
#endif
/// <summary>
/// Fetch string tensor element buffer pointer at the specified index,
/// convert/copy to UTF-16 char[] and return a ReadOnlyMemory{char} instance.
@ -605,6 +683,80 @@ namespace Microsoft.ML.OnnxRuntime
return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, new Memory<T>(data), shape);
}
#if NET8_0_OR_GREATER
/// <summary>
/// This is a factory method creates a native Onnxruntime OrtValue containing a tensor on top of the existing tensor managed memory.
/// The method will attempt to pin managed memory so no copying occurs when data is passed down
/// to native code.
/// </summary>
/// <param name="value">Tensor object</param>
/// <param name="elementType">discovered tensor element type</param>
/// <returns>And instance of OrtValue constructed on top of the object</returns>
[Experimental("SYSLIB5001")]
public static OrtValue CreateTensorValueFromSystemNumericsTensorObject<T>(SystemNumericsTensors.Tensor<T> tensor) where T : unmanaged
{
if (!IsContiguousAndDense(tensor))
{
var newTensor = SystemNumericsTensors.Tensor.Create<T>(tensor.Lengths);
tensor.CopyTo(newTensor);
tensor = newTensor;
}
unsafe
{
var backingData = (T[])tensor.GetType().GetField("_values", BindingFlags.Instance | BindingFlags.NonPublic).GetValue(tensor);
GCHandle handle = GCHandle.Alloc(backingData, GCHandleType.Pinned);
var memHandle = new MemoryHandle(Unsafe.AsPointer(ref tensor.GetPinnableReference()), handle);
try
{
IntPtr dataBufferPointer = IntPtr.Zero;
unsafe
{
dataBufferPointer = (IntPtr)memHandle.Pointer;
}
var bufferLengthInBytes = tensor.FlattenedLength * sizeof(T);
long[] shape = Array.ConvertAll(tensor.Lengths.ToArray(), new Converter<nint, long>(x => (long)x));
var typeInfo = TensorBase.GetTypeInfo(typeof(T)) ??
throw new OnnxRuntimeException(ErrorCode.InvalidArgument, $"Tensor of type: {typeof(T)} is not supported");
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateTensorWithDataAsOrtValue(
OrtMemoryInfo.DefaultInstance.Pointer,
dataBufferPointer,
(UIntPtr)(bufferLengthInBytes),
shape,
(UIntPtr)tensor.Rank,
typeInfo.ElementType,
out IntPtr nativeValue));
return new OrtValue(nativeValue, memHandle);
}
catch (Exception)
{
memHandle.Dispose();
throw;
}
}
}
[Experimental("SYSLIB5001")]
private static bool IsContiguousAndDense<T>(SystemNumericsTensors.Tensor<T> tensor) where T : unmanaged
{
// Right most dimension must be 1 for a dense tensor.
if (tensor.Strides[^1] != 1)
return false;
// For other dimensions, the stride must be equal to the product of the dimensions to the right.
for (int i = tensor.Rank - 2; i >= 0; i--)
{
if (tensor.Strides[i] != TensorPrimitives.Product(tensor.Lengths.Slice(i + 1, tensor.Lengths.Length - i - 1)))
return false;
}
return true;
}
#endif
/// <summary>
/// The factory API creates an OrtValue with memory allocated using the given allocator
/// according to the specified shape and element type. The memory will be released when OrtValue

211
csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs
View file

@ -7,6 +7,10 @@ using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Xunit;
#if NET8_0_OR_GREATER
using SystemNumericsTensors = System.Numerics.Tensors;
#endif
namespace Microsoft.ML.OnnxRuntime.Tests
{
/// <summary>
@ -67,6 +71,194 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
#if NET8_0_OR_GREATER
#pragma warning disable SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback
[Theory]
[InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, true)]
[InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, false)]
[InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, true)]
[InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, false)]
private void CanRunInferenceOnAModelDotnetTensors(GraphOptimizationLevel graphOptimizationLevel, bool enableParallelExecution)
{
var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx");
using (var cleanUp = new DisposableListTest<IDisposable>())
{
// Set the graph optimization level for this session.
SessionOptions options = new SessionOptions();
cleanUp.Add(options);
options.GraphOptimizationLevel = graphOptimizationLevel;
var session = new InferenceSession(model, options);
cleanUp.Add(session);
using var runOptions = new RunOptions();
var inputMeta = session.InputMetadata;
var outputMeta = session.OutputMetadata;
float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out");
long[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data
ReadOnlySpan<long> expectedOutputDimensions = expectedDimensions;
float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model
using var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count);
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, inputMeta[name].Dimensions.Select(x => (nint)x).ToArray());
inputOrtValues.Add(new DisposableTestPair<OrtValue>(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
}
runOptions.LogId = "CsharpTest";
runOptions.Terminate = false; // TODO: Test terminate = true, it currently crashes
runOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR;
// Run inference with named inputs and outputs created with in Run()
using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List<string>(["softmaxout_1"]))) // results is an IDisposableReadOnlyCollection<OrtValue> container
{
// validate the results
foreach (var r in results)
{
Assert.Single(results);
ValidateRunResult(r, expectedOutput, expectedDimensions);
}
}
}
}
[Fact]
public void InferenceSessionDisposedDotnetTensors()
{
var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx");
// Set the graph optimization level for this session.
using (SessionOptions options = new SessionOptions())
{
options.ProfileOutputPathPrefix = "Ort_P_";
options.EnableProfiling = true;
using (var session = new InferenceSession(model, options))
{
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model
using (var runOptions = new RunOptions())
using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
{
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, inputMeta[name].Dimensions.Select(x => (nint) x).ToArray());
inputOrtValues.Add(new DisposableTestPair<OrtValue>(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
}
// Run inference with named inputs and outputs created with in Run()
using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List<string>(["softmaxout_1"]))) // results is an IDisposableReadOnlyCollection<OrtValue> container
{
// validate the results
foreach (var r in results)
{
Assert.Single(results);
float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out");
long[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data
ValidateRunResult(r, expectedOutput, expectedDimensions);
}
}
}
string profile_file = session.EndProfiling();
// Profile file should have the output path prefix in it
Assert.Contains("Ort_P_", profile_file);
}
}
}
[Fact]
private void ThrowWrongOutputNameDotnetTensors()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputTensor = tuple.Item3;
using (var runOptions = new RunOptions())
using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
{
var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));
inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
outputOrtValues.Add(new DisposableTestPair<OrtValue>("bad_output_name", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor)));
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["bad_output_name"], [outputOrtValues[0].Value]));
Assert.Contains("Output name: 'bad_output_name' is not in the metadata", ex.Message);
}
session.Dispose();
}
[Fact]
private void ThrowWrongOutputDimensionDotnetTensors()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputTensor = tuple.Item3;
var outputTensor = SystemNumericsTensors.Tensor.Create<float>([1, 1001, 1, 1]);
using (var runOptions = new RunOptions())
using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
{
var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));
inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
outputOrtValues.Add(new DisposableTestPair<OrtValue>("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor)));
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], [outputOrtValues[0].Value]));
}
session.Dispose();
}
[Fact]
private void ThrowInconsistentPinnedOutputsDotnetTensors()
{
var tuple = OpenSessionSqueezeNet();
using var cleanUp = new DisposableListTest<IDisposable>();
cleanUp.Add(tuple.Item1);
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputTensor = tuple.Item3;
var outputTensor = SystemNumericsTensors.Tensor.Create([1, 1001, 1, 1], [4]);
using (var runOptions = new RunOptions())
using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
{
var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));
inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
outputOrtValues.Add(new DisposableTestPair<OrtValue>("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor)));
OrtValue[] outputs = [];
var ex = Assert.Throws<ArgumentException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], outputs));
Assert.StartsWith("Length of outputNames (1) must match that of outputValues (0).", ex.Message);
}
}
#pragma warning restore SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback
#endif
#if USE_CUDA
[Fact(DisplayName = "TestCUDAProviderOptions")]
private void TestCUDAProviderOptions()
@ -1416,6 +1608,25 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
#if NET8_0_OR_GREATER
#pragma warning disable SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
private void ValidateRunResultData(SystemNumericsTensors.Tensor<float> resultTensor, float[] expectedOutput, int[] expectedDimensions)
{
Assert.Equal(expectedDimensions.Length, resultTensor.Rank);
var resultDimensions = resultTensor.Lengths;
for (int i = 0; i < expectedDimensions.Length; i++)
{
Assert.Equal(expectedDimensions[i], resultDimensions[i]);
}
var resultArray = resultTensor.ToArray();
Assert.Equal(expectedOutput.Length, resultArray.Length);
Assert.Equal(expectedOutput, resultArray, new FloatComparer());
}
#pragma warning restore SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
#endif
static string GetTestModelsDir()
{
// get build directory, append downloaded models location

2
csharp/tools/linux_pack/LinuxPackNativeNuget.csproj
View file

@ -7,7 +7,7 @@
If you need a more sophisticated package for testing, you can run the production packaging pipeline against your
branch and download the resulting nuget package from the build artifacts.
-->
<Project Sdk="MSBuild.Sdk.Extras/3.0.22">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>netstandard2.0</TargetFrameworks>
<NuspecFile>$(OnnxRuntimeBuildDirectory)/NativeNuget.nuspec</NuspecFile>