saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #25 from Microsoft/dev/shahasad/merge-latest-few-changes-on-csharp-api

Browse source 
updated the files with the latest ones from VSTS repo
This commit is contained in:
shahasad 2018-11-28 01:50:00 -08:00 committed by GitHub
commit 75d329d845
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 662 additions and 1526 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
csharp/CSharpUsage/CSharpUsage.csproj
View file

@ -1,30 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<NativeBuildOutputDir>..\..\build\Windows\$(Configuration)\$(Configuration)</NativeBuildOutputDir>
</PropertyGroup>
<ItemGroup>
<None Include="$(NativeBuildOutputDir)\onnxruntime.???">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="$(NativeBuildOutputDir)\mkldnn.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="testdata\*">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj" />
</ItemGroup>
</Project>

88
csharp/CSharpUsage/Program.cs
View file

@ -1,88 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using Microsoft.ML.OnnxRuntime;
using System.Numerics.Tensors;
namespace CSharpUsage
{
class Program
{
public static void Main(string[] args)
{
Console.WriteLine("Using API");
UseApi();
Console.WriteLine("Done");
}
static void UseApi()
{
string modelPath = Directory.GetCurrentDirectory() + @"\testdata\squeezenet.onnx";
using (var session = new InferenceSession(modelPath))
{
var inputMeta = session.InputMetadata;
// User should be able to detect input name/type/shape from the metadata.
// Currently InputMetadata implementation is inclomplete, so assuming Tensor<flot> of predefined dimension.
var shape0 = new int[] { 1, 3, 224, 224 };
float[] inputData0 = LoadInputsFloat();
var tensor = new DenseTensor<float>(inputData0, shape0);
var container = new List<NamedOnnxValue>();
container.Add(new NamedOnnxValue("data_0", tensor));
// Run the inference
var results = session.Run(container); // results is an IReadOnlyList<NamedOnnxValue> container
// dump the results
foreach (var r in results)
{
Console.WriteLine("Output for {0}", r.Name);
Console.WriteLine(r.AsTensor<float>().GetArrayString());
}
// Just try some GC collect
results = null;
container = null;
GC.Collect();
GC.WaitForPendingFinalizers();
}
}
static int[] LoadInputsInt32()
{
return null;
}
static float[] LoadInputsFloat()
{
// input: data_0 = float32[1,3,224,224] for squeezenet model
// output: softmaxout_1 = float32[1,1000,1,1]
uint size = 1 * 3 * 224 * 224;
float[] tensor = new float[size];
// read data from file
using (var inputFile = new System.IO.StreamReader(@"testdata\bench.in"))
{
inputFile.ReadLine(); //skip the input name
string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < dataStr.Length; i++)
{
tensor[i] = Single.Parse(dataStr[i]);
}
}
return tensor;
}
}
}

2
csharp/CSharpUsage/testdata/bench.expected_out vendored
View file

File diff suppressed because one or more lines are too long

2
csharp/CSharpUsage/testdata/bench.in vendored
View file

File diff suppressed because one or more lines are too long

BIN
csharp/CSharpUsage/testdata/squeezenet.onnx vendored
View file

Binary file not shown.

33
csharp/OnnxRuntime.CSharp.proj
View file

@ -18,32 +18,53 @@ CMake creates a target to this project
<Message Importance="High" Text="Restoring NuGet packages for CSharp projects..." />
<MSBuild Projects="src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
Targets="Restore"
Properties="RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
Properties="Platform=AnyCPU;RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
/>
<MSBuild Projects="sample\Microsoft.ML.OnnxRuntime.InferenceSample\Microsoft.ML.OnnxRuntime.InferenceSample.csproj"
Targets="Restore"
Properties="RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
Properties="Platform=AnyCPU;RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
/>
<MSBuild Projects="test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj"
Targets="Restore"
Properties="RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
/>
<MSBuild Projects="tools\Microsoft.ML.OnnxRuntime.PerfTool\Microsoft.ML.OnnxRuntime.PerfTool.csproj"
Targets="Restore"
Properties="Platform=AnyCPU;RestoreConfigFile=$(MSBuildThisFileDirectory)\NuGet.CSharp.config;MSBuildWarningsAsMessages=NU1503;RestoreIgnoreFailedSource=true"
/>
</Target>
<Target Name="Build">
<Message Importance="High" Text="Building CSharp projects..." />
<MSBuild Projects="src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
Targets="Build" />
Targets="ObtainPackageVersion;Build"
Properties="Platform=AnyCPU"/>
<MSBuild Projects="sample\Microsoft.ML.OnnxRuntime.InferenceSample\Microsoft.ML.OnnxRuntime.InferenceSample.csproj"
Targets="Build" />
Targets="Build"
Properties="Platform=AnyCPU"
/>
<MSBuild Projects="test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj"
Targets="Build" />
Targets="Build"
/>
<MSBuild Projects="tools\Microsoft.ML.OnnxRuntime.PerfTool\Microsoft.ML.OnnxRuntime.PerfTool.csproj"
Targets="Build"
Properties="Platform=AnyCPU"
/>
</Target>
<Target Name="RunTest" AfterTargets="Build">
<Message Importance="High" Text="Running CSharp tests..." />
<Exec Command="dotnet test test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj -c $(Configuration) --no-build" />
</Target>
<Target Name="CreatePackage">
<Message Importance="High" Text="Bundling NuGet package ..." />
<MSBuild Projects="src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
Targets="ObtainPackageVersion;Pack"
Properties="NoBuild=true;Platform=AnyCPU"
/>
</Target>
</Project>

9
csharp/OnnxRuntime.CSharp.sln
View file

@ -9,7 +9,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.In
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.Tests", "test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj", "{50173D13-DF29-42E7-A30B-8B12D36C77B1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.PerfTool", "tools\Microsoft.ML.OnnxRuntime.PerfTool\Microsoft.ML.OnnxRuntime.PerfTool.csproj", "{310506FD-6E78-4D62-989B-25D69A85E8CF}"
EndProject
Global
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
@ -27,6 +32,10 @@ Global
{50173D13-DF29-42E7-A30B-8B12D36C77B1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.Build.0 = Release|Any CPU
{310506FD-6E78-4D62-989B-25D69A85E8CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{310506FD-6E78-4D62-989B-25D69A85E8CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{310506FD-6E78-4D62-989B-25D69A85E8CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{310506FD-6E78-4D62-989B-25D69A85E8CF}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

32
csharp/OnnxRuntime.proj
View file

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
This is the master msbuild project file for all csharp components.
This is created so that the NuGet dependencies are restored before the projects are built during a CI build.
CMake creates a target to this project
-->
<Project DefaultTargets="BuildProjects">
<Target Name="RestoreProjects" BeforeTargets="BuildProjects">
<Message Importance="High" Text="Restoring NuGet packages for CSharp projects..." />
<MSBuild Projects="OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
Targets="Restore"
Properties="MSBuildWarningsAsMessages=NU1503" />
<MSBuild Projects="CSharpUsage\CSharpUsage.csproj"
Targets="Restore"
Properties="MSBuildWarningsAsMessages=NU1503" />
</Target>
<Target Name="BuildProjects">
<Message Importance="High" Text="Building CSharp projects..." />
<MSBuild Projects="OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj"
Targets="Build" />
<MSBuild Projects="CSharpUsage\CSharpUsage.csproj"
Targets="Build" />
</Target>
</Project>

31
csharp/OnnxRuntime.sln
View file

@ -1,31 +0,0 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28010.2003
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime", "OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj", "{584B53B3-359D-4DC2-BCD8-530B5D4685AD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CSharpUsage", "CSharpUsage\CSharpUsage.csproj", "{1AA14958-9246-4163-9403-F650E65ADCBC}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{584B53B3-359D-4DC2-BCD8-530B5D4685AD}.Release|Any CPU.Build.0 = Release|Any CPU
{1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1AA14958-9246-4163-9403-F650E65ADCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1AA14958-9246-4163-9403-F650E65ADCBC}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {C3DBDA2B-F169-4EDE-9353-858904124B75}
EndGlobalSection
EndGlobal

66
csharp/OnnxRuntime/Exceptions.cs
View file

@ -1,66 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
namespace Microsoft.ML.OnnxRuntime
{
/// <summary>
/// Enum conresponding to native onnxruntime error codes. Must be in sync with the native API
/// </summary>
internal enum ErrorCode
{
Ok = 0,
Fail = 1,
InvalidArgument = 2,
NoSuchFile = 3,
NoModel = 4,
EngineError = 5,
RuntimeException = 6,
InvalidProtobuf = 7,
ModelLoaded = 8,
NotImplemented = 9,
InvalidGraph = 10,
ShapeInferenceNotRegistered = 11,
RequirementNotRegistered = 12
}
public class OnnxRuntimeException: Exception
{
public OnnxRuntimeException(string message)
:base(message)
{
}
}
public class CoreRuntimeException : OnnxRuntimeException
{
private static Dictionary<ErrorCode, string> errorCodeToString = new Dictionary<ErrorCode, string>()
{
{ ErrorCode.Ok, "Ok" },
{ ErrorCode.Fail, "Fail" },
{ ErrorCode.InvalidArgument, "InvalidArgument"} ,
{ ErrorCode.NoSuchFile, "NoSuchFile" },
{ ErrorCode.NoModel, "NoModel" },
{ ErrorCode.EngineError, "EngineError" },
{ ErrorCode.RuntimeException, "RuntimeException" },
{ ErrorCode.InvalidProtobuf, "InvalidProtobuf" },
{ ErrorCode.ModelLoaded, "ModelLoaded" },
{ ErrorCode.NotImplemented, "NotImplemented" },
{ ErrorCode.InvalidGraph, "InvalidGraph" },
{ ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" },
{ ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" }
};
internal CoreRuntimeException(ErrorCode errorCode, string message)
:base("[ErrorCode:" + errorCodeToString[errorCode] + "] " + message)
{
}
}
}

217
csharp/OnnxRuntime/InferenceSession.cs
View file

@ -1,217 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Runtime.InteropServices;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Microsoft.ML.OnnxRuntime
{
public struct RunOptions
{
// placeholder for RunOptions
}
/// <summary>
/// Represents an Inference Session against an ONNX Model
/// </summary>
public class InferenceSession: IDisposable
{
protected IntPtr _nativeHandle;
internal InferenceSession(IntPtr nativeHandle)
{
_nativeHandle = nativeHandle;
}
#region Public API
public InferenceSession(string modelPath)
: this(modelPath, SessionOptions.Default)
{
}
public InferenceSession(string modelPath, SessionOptions options)
{
var envHandle = OnnxRuntime.Instance.NativeHandle;
IntPtr outputHandle;
IntPtr status = NativeMethods.ONNXRuntimeCreateInferenceSession(envHandle, modelPath, options.NativeHandle, out outputHandle);
_nativeHandle = IntPtr.Zero;
NativeApiStatus.VerifySuccess(status);
_nativeHandle = outputHandle;
}
public IReadOnlyDictionary<string, NodeMetadata> InputMetadata
{
get
{
return null; // TODO: implement
}
}
public IReadOnlyDictionary<string, NodeMetadata> OutputMetadata
{
get
{
return null; // TODO: implement
}
}
public ModelMetadata ModelMetadata
{
get
{
return new ModelMetadata(); //TODO: implement
}
}
public IReadOnlyList<NamedOnnxValue> Run(IReadOnlyList<NamedOnnxValue> inputs, RunOptions options = new RunOptions())
{
var inputNames = new string[inputs.Count];
var inputTensors = new IntPtr[inputs.Count];
var pinnedBufferHandles = new System.Buffers.MemoryHandle[inputs.Count];
for (int i = 0; i < inputs.Count; i++)
{
inputNames[i] = inputs[i].Name;
// create Tensor fromt the inputs[i] if feasible, else throw notsupported exception for now
inputs[i].ToNativeOnnxValue(out inputTensors[i], out pinnedBufferHandles[i]);
}
IntPtr outputValueList = IntPtr.Zero;
ulong outputLength = 0;
IntPtr status = NativeMethods.ONNXRuntimeRunInferenceAndFetchAll(
this._nativeHandle,
inputNames,
inputTensors,
(uint)(inputTensors.Length),
out outputValueList,
out outputLength
); //Note: the inputTensors and pinnedBufferHandles must be alive for the duration of the call
try
{
NativeApiStatus.VerifySuccess(status);
var result = new List<NamedOnnxValue>();
for (uint i = 0; i < outputLength; i++)
{
IntPtr tensorValue = NativeMethods.ONNXRuntimeONNXValueListGetNthValue(outputValueList, i);
result.Add(NamedOnnxValue.CreateFromOnnxValue(Convert.ToString(i), tensorValue)); // TODO: currently Convert.ToString(i) is used instead of the output name, for the absense of C-api.
// Will be fixed as soon as the C-api for output name is available
}
return result;
}
catch (OnnxRuntimeException e)
{
//clean up the individual output tensors if it is not null;
if (outputValueList != IntPtr.Zero)
{
for (uint i = 0; i < outputLength; i++)
{
IntPtr tensorValue = NativeMethods.ONNXRuntimeONNXValueListGetNthValue(outputValueList, i);
NativeMethods.ReleaseONNXValue(tensorValue);
}
}
throw e;
}
finally
{
// always unpin the input buffers, and delete the native Onnx value objects
for (int i = 0; i < inputs.Count; i++)
{
NativeMethods.ReleaseONNXValue(inputTensors[i]); // this should not release the buffer, but should delete the native tensor object
pinnedBufferHandles[i].Dispose();
}
// always release the output value list, because the individual tensor pointers are already obtained.
if (outputValueList != IntPtr.Zero)
{
NativeMethods.ReleaseONNXValueList(outputValueList);
}
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>.
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Dictionary</returns>
public IReadOnlyList<NamedOnnxValue> Run(IReadOnlyList<NamedOnnxValue> inputs, ICollection<string> outputNames, RunOptions options = new RunOptions())
{
//TODO: implement
return null;
}
#endregion
#region private methods
#endregion
#region destructors disposers
~InferenceSession()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
// cleanup managed resources
}
// cleanup unmanaged resources
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.ReleaseONNXSession(_nativeHandle);
}
}
#endregion
}
public struct NodeMetadata
{
public uint[] Shape
{
get; internal set;
}
public System.Type Type
{
get; internal set;
}
}
public struct ModelMetadata
{
//placeholder for Model metadata. Python API has this
}
}

15
csharp/OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
View file

@ -1,15 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard1.1</TargetFramework>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<SignAssembly>true</SignAssembly>
<DelaySign>false</DelaySign>
<AssemblyOriginatorKeyFile>OnnxRuntime.snk</AssemblyOriginatorKeyFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Numerics.Tensors" Version="0.1.0-preview2-181101-1" />
</ItemGroup>
</Project>

322
csharp/OnnxRuntime/NamedOnnxValue.cs
View file

@ -1,322 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
using System.Numerics.Tensors;
using System.Buffers;
using System.Collections;
using System.Diagnostics;
namespace Microsoft.ML.OnnxRuntime
{
public class NamedOnnxValue
{
protected Object _value;
protected string _name;
public NamedOnnxValue(string name, Object value)
{
_name = name;
_value = value;
}
public string Name { get { return _name; } }
public Tensor<T> AsTensor<T>()
{
return _value as Tensor<T>; // will return null if not castable
}
/// <summary>
/// Attempts to Pin the buffer, and create a native OnnxValue out of it. the pinned MemoryHandle is passed to output.
/// In this case, the pinnedHandle should be kept alive till the native OnnxValue is used, then dispose it.
/// If it is not possible to Pin the buffer, then creates OnnxValue from the copy of the data. The output pinnedMemoryHandle
/// contains a default value in that case.
/// Attempts to infer the type of the value while creating the OnnxValue
/// </summary>
/// <param name="onnxValue"></param>
/// <param name="pinnedMemoryHandle"></param>
internal void ToNativeOnnxValue(out IntPtr onnxValue, out MemoryHandle pinnedMemoryHandle)
{
//try to cast _value to Tensor<T>
TensorElementType nativeElementType = TensorElementType.DataTypeMax; //invalid
IntPtr dataBufferPointer = IntPtr.Zero;
int dataBufferLength = 0;
ReadOnlySpan<int> shape = null;
int rank = 0;
onnxValue = IntPtr.Zero;
if (TryPinAsTensor<float>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<double>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<int>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<uint>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<long>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<ulong>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<short>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<ushort>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<byte>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
else if (TryPinAsTensor<bool>(out pinnedMemoryHandle,
out dataBufferPointer,
out dataBufferLength,
out shape,
out rank,
out nativeElementType
))
{
}
//TODO: add other types
else
{
// nothing to cleanup here, since no memory has been pinned
throw new NotSupportedException("The inference value " + nameof(_value) + " is not of a supported type");
}
Debug.Assert(dataBufferPointer != IntPtr.Zero, "dataBufferPointer must be non-null after obtaining the pinned buffer");
// copy to an ulong[] shape to match size_t[]
ulong[] longShape = new ulong[rank];
for (int i = 0; i < rank; i++)
{
longShape[i] = (ulong)shape[i];
}
IntPtr status = NativeMethods.ONNXRuntimeCreateTensorWithDataAsONNXValue(
NativeCpuAllocatorInfo.Handle,
dataBufferPointer,
(ulong)(dataBufferLength),
longShape,
(ulong)rank,
nativeElementType,
out onnxValue
);
try
{
NativeApiStatus.VerifySuccess(status);
}
catch (OnnxRuntimeException e)
{
pinnedMemoryHandle.Dispose();
throw e;
}
}
internal static NamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue)
{
NamedOnnxValue result = null;
if (true /* TODO: check native data type when API available. assuming Tensor<float> for now */)
{
NativeOnnxTensorMemory<float> nativeTensorWrapper = new NativeOnnxTensorMemory<float>(nativeOnnxValue);
DenseTensor<float> dt = new DenseTensor<float>(nativeTensorWrapper.Memory, nativeTensorWrapper.Dimensions);
result = new NamedOnnxValue(name, dt);
}
return result;
}
private bool TryPinAsTensor<T>(
out MemoryHandle pinnedMemoryHandle,
out IntPtr dataBufferPointer,
out int dataBufferLength,
out ReadOnlySpan<int> shape,
out int rank,
out TensorElementType nativeElementType
)
{
nativeElementType = TensorElementType.DataTypeMax; //invalid
dataBufferPointer = IntPtr.Zero;
dataBufferLength = 0;
shape = null;
rank = 0;
pinnedMemoryHandle = default(MemoryHandle);
if (_value is Tensor<T>)
{
Tensor<T> t = _value as Tensor<T>;
if (t.IsReversedStride)
{
//TODO: not sure how to support reverse stride. may be able to calculate the shape differently
throw new NotSupportedException(nameof(Tensor<T>) + " of reverseStride is not supported");
}
DenseTensor<T> dt = null;
if (_value is DenseTensor<T>)
{
dt = _value as DenseTensor<T>;
}
else
{
dt = t.ToDenseTensor();
}
shape = dt.Dimensions; // does not work for reverse stride
rank = dt.Rank;
pinnedMemoryHandle = dt.Buffer.Pin();
unsafe
{
dataBufferPointer = (IntPtr)pinnedMemoryHandle.Pointer;
}
// find the native type
if (typeof(T) == typeof(float))
{
nativeElementType = TensorElementType.Float;
dataBufferLength = dt.Buffer.Length * sizeof(float);
}
else if (typeof(T) == typeof(double))
{
nativeElementType = TensorElementType.Double;
dataBufferLength = dt.Buffer.Length * sizeof(double);
}
else if (typeof(T) == typeof(int))
{
nativeElementType = TensorElementType.Int32;
dataBufferLength = dt.Buffer.Length * sizeof(int);
}
else if (typeof(T) == typeof(uint))
{
nativeElementType = TensorElementType.UInt32;
dataBufferLength = dt.Buffer.Length * sizeof(uint);
}
else if (typeof(T) == typeof(long))
{
nativeElementType = TensorElementType.Int64;
dataBufferLength = dt.Buffer.Length * sizeof(long);
}
else if (typeof(T) == typeof(ulong))
{
nativeElementType = TensorElementType.UInt64;
dataBufferLength = dt.Buffer.Length * sizeof(ulong);
}
else if (typeof(T) == typeof(short))
{
nativeElementType = TensorElementType.Int16;
dataBufferLength = dt.Buffer.Length * sizeof(short);
}
else if (typeof(T) == typeof(ushort))
{
nativeElementType = TensorElementType.UInt16;
dataBufferLength = dt.Buffer.Length * sizeof(ushort);
}
else if (typeof(T) == typeof(byte))
{
nativeElementType = TensorElementType.UInt8;
dataBufferLength = dt.Buffer.Length * sizeof(byte);
}
//TODO: Not supporting boolean for now. bool is non-blittable, the interop needs some care, and possibly need to copy
//else if (typeof(T) == typeof(bool))
//{
//}
else
{
//TODO: may extend the supported types
// do not throw exception, rather assign the sentinel value
nativeElementType = TensorElementType.DataTypeMax;
}
return true;
}
return false;
}
// may expose different types of getters in future
}
internal enum TensorElementType
{
Float = 1,
UInt8 = 2,
Int8 = 3,
UInt16 = 4,
Int16 = 5,
Int32 = 6,
Int64 = 7,
String = 8,
Bool = 9,
Float16 = 10,
Double = 11,
UInt32 = 12,
UInt64 = 13,
Complex64 = 14,
Complex128 = 15,
BFloat16 = 16,
DataTypeMax = 17
}
}

35
csharp/OnnxRuntime/NativeApiStatus.cs
View file

@ -1,35 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Runtime.InteropServices;
namespace Microsoft.ML.OnnxRuntime
{
class NativeApiStatus
{
private static string GetErrorMessage(IntPtr /*(ONNXStatus*)*/status)
{
IntPtr nativeString = NativeMethods.ONNXRuntimeGetErrorMessage(status);
string str = Marshal.PtrToStringAnsi(nativeString); //assumes charset = ANSI
return str;
}
/// <summary>
/// Checks the native Status if the errocode is OK/Success. Otherwise constructs an appropriate exception and throws.
/// Releases the native status object, as needed.
/// </summary>
/// <param name="nativeStatus"></param>
/// <throws></throws>
public static void VerifySuccess(IntPtr nativeStatus)
{
if (nativeStatus != IntPtr.Zero)
{
ErrorCode statusCode = NativeMethods.ONNXRuntimeGetErrorCode(nativeStatus);
string errorMessage = GetErrorMessage(nativeStatus);
NativeMethods.ReleaseONNXStatus(nativeStatus);
throw new CoreRuntimeException(statusCode, errorMessage);
}
}
}
}

75
csharp/OnnxRuntime/NativeCpuAllocatorInfo.cs
View file

@ -1,75 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
namespace Microsoft.ML.OnnxRuntime
{
internal class NativeCpuAllocatorInfo : IDisposable
{
// static singleton
private static readonly Lazy<NativeCpuAllocatorInfo> _instance = new Lazy<NativeCpuAllocatorInfo>(() => new NativeCpuAllocatorInfo());
// member variables
private IntPtr _nativeHandle;
internal static IntPtr Handle // May throw exception in every access, if the constructor have thrown an exception
{
get
{
return _instance.Value._nativeHandle;
}
}
private NativeCpuAllocatorInfo()
{
_nativeHandle = CreateCPUAllocatorInfo();
}
private static IntPtr CreateCPUAllocatorInfo()
{
IntPtr allocInfo = IntPtr.Zero;
try
{
IntPtr status = NativeMethods.ONNXRuntimeCreateCpuAllocatorInfo(NativeMethods.AllocatorType.DeviceAllocator, NativeMethods.MemoryType.Cpu, out allocInfo);
NativeApiStatus.VerifySuccess(status);
return allocInfo;
}
catch (Exception e)
{
if (allocInfo != IntPtr.Zero)
{
NativeMethods.ReleaseONNXRuntimeAllocatorInfo(allocInfo);
}
throw e;
}
}
~NativeCpuAllocatorInfo()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
private void Dispose(bool disposing)
{
if (disposing)
{
//release managed resource
}
//release unmanaged resource
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.ReleaseONNXRuntimeAllocatorInfo(_nativeHandle);
}
}
}
}

236
csharp/OnnxRuntime/NativeMethods.cs
View file

@ -1,236 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Runtime.InteropServices;
namespace Microsoft.ML.OnnxRuntime
{
/// <summary>
/// NamedOnnxValue type, must match the native enum
/// </summary>
internal static class NativeMethods
{
private const string nativeLib = "onnxruntime.dll";
internal const CharSet charSet = CharSet.Ansi;
#region Runtime/Environment API
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* ONNXStatus* */ONNXRuntimeInitialize(
LogLevel default_warning_level,
string logId,
out IntPtr /*(ONNXEnv*)*/ env);
// ReleaseONNXEnv should not be used
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXEnv(IntPtr /*(ONNXEnv*)*/ env);
#endregion Runtime/Environment API
#region Status API
[DllImport(nativeLib, CharSet = charSet)]
public static extern ErrorCode ONNXRuntimeGetErrorCode(IntPtr /*(ONNXStatus*)*/status);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* char* */ONNXRuntimeGetErrorMessage(IntPtr /* (ONNXStatus*) */status);
// returns char*, need to convert to string by the caller.
// does not free the underlying ONNXStatus*
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXStatus(IntPtr /*(ONNXStatus*)*/ statusPtr);
#endregion Status API
#region InferenceSession API
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* ONNXStatus* */ONNXRuntimeCreateInferenceSession(
IntPtr /* (ONNXEnv*) */ environment,
[MarshalAs(UnmanagedType.LPWStr)]string modelPath, //the model path is consumed as a wchar* in the C-api
IntPtr /* (ONNXRuntimeSessionOptions*) */sessopnOptions,
out IntPtr /**/ session);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInferenceAndFetchAll(
IntPtr /*(ONNXSessionPtr)*/ session,
string[] inputNames,
IntPtr[] /*(ONNXValuePtr[])*/ inputValues,
ulong inputLength, // size_t, TODO: make it portable for x86, arm
out IntPtr /* (ONNXValueListPtr*)*/ outputValues,
out ulong /* (size_t*) */ outputLength); //TODO: make it portable for x86, arm
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNStatus*)*/ ONNXRuntimeRunInference(
IntPtr /*(ONNXSession*)*/ session,
string[] inputNames,
IntPtr[] /* (ONNXValue*[])*/ inputValues,
ulong inputCount, /* TODO: size_t, make it portable for x86 arm */
string[] outputNames,
ulong outputCount, /* TODO: size_t, make it portable for x86 and arm */
[MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 5 /*index of outputCount*/)][In, Out]
IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */
);
[DllImport(nativeLib, CharSet = charSet)]
public static extern int ONNXRuntimeInferenceSessionGetInputCount(IntPtr /*(ONNXSession*)*/ session);
[DllImport(nativeLib, CharSet = charSet)]
public static extern int ONNXRuntimeInferenceSessionGetOutputCount(IntPtr /*(ONNXSession*)*/ session);
//TODO: need the input/output names API
//TODO: need the input/output shape/type API
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXSession(IntPtr /*(ONNXSession*)*/ session);
#endregion InferenceSession API
#region SessionOptions API
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*ONNXRuntimeSessionOptions* */ ONNXRuntimeCreateSessionOptions();
//DEFINE_RUNTIME_CLASS(ONNXRuntimeSessionOptions)
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXRuntimeSessionOptions(IntPtr /*(ONNXRuntimeSessionOptions*)*/ sessionOptions);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeEnableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableSequentialExecution(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeEnableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options, string profilePathPrefix);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableProfiling(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeEnableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableMemPattern(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeEnableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableCpuMemArena(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeSetSessionLogId(IntPtr /* ONNXRuntimeSessionOptions* */ options, string logId);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeSetSessionLogVerbosityLevel(IntPtr /* ONNXRuntimeSessionOptions* */ options, LogLevel sessionLogVerbosityLevel);
[DllImport(nativeLib, CharSet = charSet)]
public static extern int ONNXRuntimeSetSessionThreadPoolSize(IntPtr /* ONNXRuntimeSessionOptions* */ options, int sessionThreadPoolSize);
[DllImport(nativeLib, CharSet = charSet)]
public static extern int ONNXRuntimeEnableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options, int deviceId);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableCudaProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern int ONNXRuntimeEnableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ONNXRuntimeDisableMklProvider(IntPtr /* ONNXRuntimeSessionOptions* */ options);
#endregion
#region Allocator/AllocatorInfo API
//TODO: consider exposing them publicly, when allocator API is exposed
public enum AllocatorType
{
DeviceAllocator = 0,
ArenaAllocator = 1
}
//TODO: consider exposing them publicly when allocator API is exposed
public enum MemoryType
{
CpuInput = -2, // Any CPU memory used by non-CPU execution provider
CpuOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED
Cpu = CpuOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED
Default = 0, // the default allocator for execution provider
}
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateAllocatorInfo(
IntPtr /*(const char*) */name,
AllocatorType allocatorType,
int identifier,
MemoryType memType,
out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo // memory ownership transfered to caller
);
//ONNXRUNTIME_API_STATUS(ONNXRuntimeCreateCpuAllocatorInfo, enum ONNXRuntimeAllocatorType type, enum ONNXRuntimeMemType mem_type1, _Out_ ONNXRuntimeAllocatorInfo** out)
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* (ONNXStatus*)*/ ONNXRuntimeCreateCpuAllocatorInfo(
AllocatorType allocatorType,
MemoryType memoryType,
out IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo
);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXRuntimeAllocatorInfo(IntPtr /*(ONNXRuntimeAllocatorInfo*)*/ allocatorInfo);
#endregion Allocator/AllocatorInfo API
#region Tensor/OnnxValue API
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /* ONNXStatus */ ONNXRuntimeCreateTensorWithDataAsONNXValue(
IntPtr /* (const ONNXRuntimeAllocatorInfo*) */ allocatorInfo,
IntPtr /* (void*) */dataBufferHandle,
ulong dataLength, //size_t, TODO: make it portable for x86, arm
ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm
ulong shapeLength, //size_t, TODO: make it portable for x86, arm
TensorElementType type,
out IntPtr /* ONNXValuePtr* */ outputValue);
/// This function doesn't work with string tensor
/// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd.
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNXStatus*)*/ ONNXRuntimeGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count);
///**
// * Generally, user should call ONNXRuntimeGetTensorShapeDimCount before calling this.
// * Unless they already have a good estimation on the dimension count
// * \param shape_array An array allocated by caller, with size of shape_array
// * \param shape_array_len the length of passed in shape_array.
// */
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNXStatus*)*/ONNXRuntimeGetTensorShape(
IntPtr /*(ONNXValue*)*/ value,
ulong[] shapeArray, //size_t[] TODO: make it portable for x86, arm
ulong shapeArrayLength); //size_t, TODO: make it portable for x86, arm
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(ONNXValuePtr)*/ ONNXRuntimeONNXValueListGetNthValue(IntPtr /*(ONNXValueListPtr)*/ list, ulong index); // 0-based index TODO: size_t, make it portable for x86, arm
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void ReleaseONNXValueList(IntPtr /*(ONNXValueList*)*/ valueList);
#endregion
} //class NativeMethods
} //namespace

170
csharp/OnnxRuntime/NativeOnnxTensorMemory.cs
View file

@ -1,170 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace Microsoft.ML.OnnxRuntime
{
internal class NativeOnnxTensorMemory<T> : MemoryManager<T>
{
private bool _disposed;
private int _referenceCount;
private IntPtr _onnxValueHandle;
private IntPtr _dataBufferHandle;
private int _elementCount;
private int _elementWidth;
private int[] _dimensions;
public NativeOnnxTensorMemory(IntPtr onnxValueHandle)
{
//TODO: check type param and the native tensor type
if (typeof(T) != typeof(float))
throw new NotSupportedException(nameof(NativeOnnxTensorMemory<T>)+" does not support T other than float");
_elementWidth = 4;
_onnxValueHandle = onnxValueHandle;
// derive the databuffer pointer, element_count, element_width, and shape
try
{
NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorMutableData(_onnxValueHandle, out _dataBufferHandle));
// throws OnnxRuntimeException if native call failed
ulong dimension;
NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShapeDimCount(_onnxValueHandle, out dimension));
ulong count;
NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShapeElementCount(_onnxValueHandle, out count));
ulong[] shape = new ulong[dimension];
NativeApiStatus.VerifySuccess(NativeMethods.ONNXRuntimeGetTensorShape(_onnxValueHandle, shape, dimension)); //Note: shape must be alive during the call
_elementCount = (int)count;
_dimensions = new int[dimension];
for (ulong i = 0; i < dimension; i++)
{
_dimensions[i] = (int)shape[i];
}
}
catch (OnnxRuntimeException e)
{
//TODO: cleanup any partially created state
//Do not call ReleaseTensor here. If the constructor has thrown exception, then this NativeOnnxTensorWrapper is not created, so caller should take appropriate action to dispose
throw e;
}
}
~NativeOnnxTensorMemory()
{
Dispose(false);
}
public bool IsDisposed => _disposed;
protected bool IsRetained => _referenceCount > 0;
public int[] Dimensions
{
get
{
return _dimensions;
}
}
public int Rank
{
get
{
return _dimensions.Length;
}
}
public override Span<T> GetSpan()
{
if (IsDisposed)
throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory<T>));
Span<T> span = null;
unsafe
{
span = new Span<T>((void*)_dataBufferHandle, _elementCount);
}
return span;
}
public override MemoryHandle Pin(int elementIndex = 0)
{
//Note: always pin the full buffer and return
unsafe
{
if (elementIndex >= _elementCount)
{
throw new ArgumentOutOfRangeException(nameof(elementIndex));
}
Retain();
return new MemoryHandle((void*)((int)_dataBufferHandle + elementIndex*_elementWidth)); //could not use Unsafe.Add
}
}
public override void Unpin()
{
Release();
}
private bool Release()
{
int newRefCount = Interlocked.Decrement(ref _referenceCount);
if (newRefCount < 0)
{
throw new InvalidOperationException("Unmatched Release/Retain");
}
return newRefCount != 0;
}
private void Retain()
{
if (IsDisposed)
{
throw new ObjectDisposedException(nameof(NativeOnnxTensorMemory<T>));
}
Interlocked.Increment(ref _referenceCount);
}
protected override void Dispose(bool disposing)
{
if (_disposed)
{
return;
}
if (disposing)
{
// do managed objects cleanup
}
// TODO Call nativeMethods.ReleaseTensor, once the corresponding native API is fixed
// Currently there will be memory leak
_disposed = true;
}
protected override bool TryGetArray(out ArraySegment<T> arraySegment)
{
// cannot expose managed array
arraySegment = default(ArraySegment<T>);
return false;
}
}
}

93
csharp/OnnxRuntime/OnnxRuntime.cs
View file

@ -1,93 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Runtime.CompilerServices;
using System.Collections.Generic;
namespace Microsoft.ML.OnnxRuntime
{
internal struct GlobalOptions //Options are currently not accessible to user
{
public string LogId { get; set; }
public LogLevel LogLevel { get; set; }
}
internal enum LogLevel
{
Verbose = 0,
Info = 1,
Warning = 2,
Error = 3,
Fatal = 4
}
/// <summary>
/// This class intializes the process-global ONNX runtime
/// C# API users do not need to access this, thus kept as internal
/// </summary>
internal sealed class OnnxRuntime : IDisposable
{
// static singleton
private static readonly Lazy<OnnxRuntime> _instance = new Lazy<OnnxRuntime>(() => new OnnxRuntime());
// member variables
private IntPtr _nativeHandle;
internal static OnnxRuntime Instance // May throw exception in every access, if the constructor have thrown an exception
{
get
{
return _instance.Value;
}
}
private OnnxRuntime() //Problem: it is not possible to pass any option for a Singleton
{
_nativeHandle = IntPtr.Zero;
IntPtr outPtr;
IntPtr status = NativeMethods.ONNXRuntimeInitialize(LogLevel.Warning, @"CSharpOnnxRuntime", out outPtr);
NativeApiStatus.VerifySuccess(status);
_nativeHandle = outPtr;
}
internal IntPtr NativeHandle
{
get
{
return _nativeHandle;
}
}
~OnnxRuntime()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
private void Dispose(bool disposing)
{
if (disposing)
{
//release managed resource
}
//release unmanaged resource
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.ReleaseONNXEnv(_nativeHandle);
}
}
}
}

BIN
csharp/OnnxRuntime/OnnxRuntime.snk
View file

Binary file not shown.

59
csharp/OnnxRuntime/SessionOptions.cs
View file

@ -1,59 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
namespace Microsoft.ML.OnnxRuntime
{
public class SessionOptions : IDisposable
{
private static SessionOptions _defaultOptions = new SessionOptions();
private IntPtr _nativeHandle;
public SessionOptions()
{
_nativeHandle = NativeMethods.ONNXRuntimeCreateSessionOptions();
}
internal IntPtr NativeHandle
{
get
{
return _nativeHandle;
}
}
public static SessionOptions Default
{
get
{
return _defaultOptions;
}
}
#region destructors disposers
~SessionOptions()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
// cleanup managed resources
}
// cleanup unmanaged resources
}
#endregion
}
}

2
csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.csproj
View file

@ -17,7 +17,7 @@
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="$(NativeBuildOutputDir)\mkldnn.dll">
<None Include="$(NativeBuildOutputDir)\mkldnn.dll" Condition="Exists('$(NativeBuildOutputDir)\mkldnn.dll')">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>

2
csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs
View file

@ -35,7 +35,7 @@ namespace CSharpUsage
foreach (var name in inputMeta.Keys)
{
var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
container.Add(new NamedOnnxValue(name, tensor));
container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
}
// Run the inference

14
csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs
View file

@ -28,16 +28,10 @@ namespace Microsoft.ML.OnnxRuntime
RequirementNotRegistered = 12
}
/// <summary>
/// The Exception that is thrown for errors related ton OnnxRuntime
/// </summary>
public class OnnxRuntimeException: Exception
{
public OnnxRuntimeException(string message)
:base(message)
{
}
}
public class CoreRuntimeException : OnnxRuntimeException
{
private static Dictionary<ErrorCode, string> errorCodeToString = new Dictionary<ErrorCode, string>()
{
@ -56,7 +50,7 @@ namespace Microsoft.ML.OnnxRuntime
{ ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" }
};
internal CoreRuntimeException(ErrorCode errorCode, string message)
internal OnnxRuntimeException(ErrorCode errorCode, string message)
:base("[ErrorCode:" + errorCodeToString[errorCode] + "] " + message)
{
}

75
csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs
View file

@ -21,17 +21,22 @@ namespace Microsoft.ML.OnnxRuntime
protected Dictionary<string, NodeMetadata> _inputMetadata, _outputMetadata;
internal InferenceSession(IntPtr nativeHandle)
{
_nativeHandle = nativeHandle;
}
#region Public API
/// <summary>
/// Constructs an InferenceSession from a model file
/// </summary>
/// <param name="modelPath"></param>
public InferenceSession(string modelPath)
: this(modelPath, SessionOptions.Default)
{
}
/// <summary>
/// Constructs an InferenceSession from a model file, with some additional session options
/// </summary>
/// <param name="modelPath"></param>
/// <param name="options"></param>
public InferenceSession(string modelPath, SessionOptions options)
{
var envHandle = OnnxRuntime.Handle;
@ -92,24 +97,27 @@ namespace Microsoft.ML.OnnxRuntime
}
}
public ModelMetadata ModelMetadata
{
get
{
return new ModelMetadata(); //TODO: implement
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches all the outputs.
/// </summary>
/// <param name="inputs"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue</returns>
public IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs)
{
return Run(inputs, RunOptions.Default);
}
public IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, RunOptions options)
{
string[] outputNames = new string[_outputMetadata.Count];
_outputMetadata.Keys.CopyTo(outputNames, 0);
return Run(inputs, outputNames, options);
return Run(inputs, outputNames);
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the outputs specified in <paramref name="outputNames"/>.
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue</returns>
public IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames)
{
return Run(inputs, outputNames, RunOptions.Default);
}
/// <summary>
@ -119,7 +127,8 @@ namespace Microsoft.ML.OnnxRuntime
/// <param name="outputNames"></param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Dictionary</returns>
public IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames, RunOptions options)
//TODO: kept internal until RunOptions is made public
internal IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames, RunOptions options)
{
var inputNames = new string[inputs.Count];
var inputTensors = new IntPtr[inputs.Count];
@ -186,6 +195,14 @@ namespace Microsoft.ML.OnnxRuntime
}
//TODO: kept internal until implemented
internal ModelMetadata ModelMetadata
{
get
{
return new ModelMetadata(); //TODO: implement
}
}
#endregion
@ -353,24 +370,32 @@ namespace Microsoft.ML.OnnxRuntime
return _dimensions;
}
}
public System.Type Type
public System.Type ElementType
{
get
{
return _type;
}
}
public bool IsTensor
{
get
{
return true; // currently only Tensor nodes are supported
}
}
}
public class ModelMetadata
internal class ModelMetadata
{
//TODO: placeholder for Model metadata. Currently C-API does not expose this
//TODO: placeholder for Model metadata. Currently C-API does not expose this.
}
/// Sets various runtime options.
/// TODO: currently uses Default options only
public class RunOptions
/// TODO: currently uses Default options only. kept internal until fully implemented
internal class RunOptions
{
protected static readonly Lazy<RunOptions> _default = new Lazy<RunOptions>(() => new RunOptions());

70
csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
View file

@ -6,8 +6,78 @@
<SignAssembly>true</SignAssembly>
<DelaySign>false</DelaySign>
<AssemblyOriginatorKeyFile>OnnxRuntime.snk</AssemblyOriginatorKeyFile>
<!--- packaging properties -->
<PackageId>Microsoft.ML.OnnxRuntime</PackageId>
<Authors>Microsoft Corporation</Authors>
<Description>This package contains Microsoft's implementation of ONNX runtime, usable in .Net platforms</Description>
<PackageTags>ONNX;ONNX Runtime;Machine Learning</PackageTags>
<Copyright>Microsoft Corporation</Copyright>
<IncludeSymbols>true</IncludeSymbols>
<PackageLicenseUrl>https://github.com/Microsoft/onnxruntime/blob/master/LICENSE</PackageLicenseUrl>
<RepositoryUrl>https://github.com/Microsoft/onnxruntime.git</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<!--internal build related properties-->
<OnnxRuntimeCsharpRoot>..\..</OnnxRuntimeCsharpRoot>
<buildDirectory Condition="'$(buildDirectory)'==''">$(OnnxRuntimeCsharpRoot)\..\build\Windows</buildDirectory>
<NativeBuildOutputDir>$(buildDirectory)\$(Configuration)\$(Configuration)</NativeBuildOutputDir>
</PropertyGroup>
<!--TODO: this works for single platform only. Need separate packaging scripts for multi-target packaging -->
<!--TODO: Find a way to bundle the native symbol files properly -->
<ItemGroup>
<None Include="$(NativeBuildOutputDir)\onnxruntime.dll"
PackagePath="\runtimes\win10-x64\native"
Pack="true"
CopyToOutputDirectory="Always"
Visible="false"
/>
<None Include="$(NativeBuildOutputDir)\onnxruntime.pdb"
Condition="Exists('$(NativeBuildOutputDir)\onnxruntime.pdb')"
PackagePath="\runtimes\win10-x64\native"
Pack="true"
CopyToOutputDirectory="Always"
Visible="false"
/>
<None Include="$(NativeBuildOutputDir)\mkldnn.dll"
Condition="Exists('$(NativeBuildOutputDir)\mkldnn.dll')"
PackagePath="\runtimes\win10-x64\native"
Pack="true"
CopyToOutputDirectory="Always"
Visible="false"
/>
<None Include="$(OnnxRuntimeCsharpRoot)\..\LICENSE;$(OnnxRuntimeCsharpRoot)\..\ThirdPartyNotices.txt"
PackagePath="\"
Pack="true"
Visible="false"
/>
<None Include="$(OnnxRuntimeCsharpRoot)\..\docs\CSharp_API.md"
PackagePath="\README.md"
Pack="true"
Visible="false"
/>
</ItemGroup>
<Target Name="ObtainPackageVersion" BeforeTargets="Build;Pack">
<ReadLinesFromFile File="$(OnnxRuntimeCsharpRoot)\..\VERSION_NUMBER">
<Output TaskParameter="Lines" ItemName="MajorVersionNumber"/>
</ReadLinesFromFile>
<Exec Command="git rev-parse --short HEAD" ConsoleToMSBuild="true">
<Output TaskParameter="ConsoleOutput" PropertyName="GitCommitHash" />
</Exec>
<PropertyGroup>
<RepositoryCommit>$(GitCommitHash)</RepositoryCommit>
<PackageVersion>@(MajorVersionNumber)</PackageVersion>
<Version>$(PackageVersion)</Version>
<PackageVersion Condition="'$(IsReleaseBuild)'==''">$(PackageVersion)-dev-$(GitCommitHash)</PackageVersion>
</PropertyGroup>
<Message Importance="High" Text="PackageVersion=$(PackageVersion)" />
</Target>
<ItemGroup>
<PackageReference Include="System.Numerics.Tensors" Version="0.1.0" />
</ItemGroup>

7
csharp/src/Microsoft.ML.OnnxRuntime/NamedOnnxValue.cs
View file

@ -16,12 +16,17 @@ namespace Microsoft.ML.OnnxRuntime
protected Object _value;
protected string _name;
public NamedOnnxValue(string name, Object value)
protected NamedOnnxValue(string name, Object value)
{
_name = name;
_value = value;
}
public static NamedOnnxValue CreateFromTensor<T>(string name, Tensor<T> value)
{
return new NamedOnnxValue(name, value);
}
public string Name { get { return _name; } }
public Tensor<T> AsTensor<T>()
{

2
csharp/src/Microsoft.ML.OnnxRuntime/NativeApiStatus.cs
View file

@ -28,7 +28,7 @@ namespace Microsoft.ML.OnnxRuntime
ErrorCode statusCode = NativeMethods.ONNXRuntimeGetErrorCode(nativeStatus);
string errorMessage = GetErrorMessage(nativeStatus);
NativeMethods.ReleaseONNXStatus(nativeStatus);
throw new CoreRuntimeException(statusCode, errorMessage);
throw new OnnxRuntimeException(statusCode, errorMessage);
}
}
}

43
csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs
View file

@ -7,6 +7,9 @@ using System.Runtime.InteropServices;
namespace Microsoft.ML.OnnxRuntime
{
/// <summary>
/// Various providers of ONNX operators
/// </summary>
public enum ExecutionProvider
{
Cpu,
@ -14,16 +17,25 @@ namespace Microsoft.ML.OnnxRuntime
//TODO: add more providers gradually
};
public class SessionOptions
/// <summary>
/// Holds the options for creating an InferenceSession
/// </summary>
public class SessionOptions:IDisposable
{
protected SafeHandle _nativeOption;
protected static readonly Lazy<SessionOptions> _default = new Lazy<SessionOptions>(MakeSessionOptionWithMklDnnProvider);
/// <summary>
/// Constructs an empty SessionOptions
/// </summary>
public SessionOptions()
{
_nativeOption = new NativeOnnxObjectHandle(NativeMethods.ONNXRuntimeCreateSessionOptions());
}
/// <summary>
/// Default instance
/// </summary>
public static SessionOptions Default
{
get
@ -32,6 +44,10 @@ namespace Microsoft.ML.OnnxRuntime
}
}
/// <summary>
/// Append an execution propvider. When any operator is evaluated, it is executed on the first execution provider that provides it
/// </summary>
/// <param name="provider"></param>
public void AppendExecutionProvider(ExecutionProvider provider)
{
switch (provider)
@ -80,5 +96,30 @@ namespace Microsoft.ML.OnnxRuntime
}
}
#region destructors disposers
~SessionOptions()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
// cleanup managed resources
}
_nativeOption.Dispose();
}
#endregion
}
}

103
csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
View file

@ -10,7 +10,6 @@ using System.Numerics.Tensors;
using Xunit;
using Microsoft.ML.OnnxRuntime;
namespace Microsoft.ML.OnnxRuntime.Tests
{
public class InfereceTest
@ -25,7 +24,8 @@ namespace Microsoft.ML.OnnxRuntime.Tests
Assert.NotNull(session.InputMetadata);
Assert.Equal(1, session.InputMetadata.Count); // 1 input node
Assert.True(session.InputMetadata.ContainsKey("data_0")); // input node name
Assert.Equal(typeof(float), session.InputMetadata["data_0"].Type);
Assert.Equal(typeof(float), session.InputMetadata["data_0"].ElementType);
Assert.True(session.InputMetadata["data_0"].IsTensor);
var expectedInputDimensions = new int[] { 1, 3, 224, 224 };
Assert.Equal(expectedInputDimensions.Length, session.InputMetadata["data_0"].Dimensions.Length);
for (int i = 0; i < expectedInputDimensions.Length; i++)
@ -36,7 +36,8 @@ namespace Microsoft.ML.OnnxRuntime.Tests
Assert.NotNull(session.OutputMetadata);
Assert.Equal(1, session.OutputMetadata.Count); // 1 output node
Assert.True(session.OutputMetadata.ContainsKey("softmaxout_1")); // output node name
Assert.Equal(typeof(float), session.OutputMetadata["softmaxout_1"].Type);
Assert.Equal(typeof(float), session.OutputMetadata["softmaxout_1"].ElementType);
Assert.True(session.OutputMetadata["softmaxout_1"].IsTensor);
var expectedOutputDimensions = new int[] { 1, 1000, 1, 1 };
Assert.Equal(expectedOutputDimensions.Length, session.OutputMetadata["softmaxout_1"].Dimensions.Length);
for (int i = 0; i < expectedOutputDimensions.Length; i++)
@ -60,9 +61,10 @@ namespace Microsoft.ML.OnnxRuntime.Tests
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].Type);
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
container.Add(new NamedOnnxValue(name, tensor));
container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
}
// Run the inference
@ -100,6 +102,88 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
[Fact]
private void ThrowWrongInputName()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var tensor = tuple.Item3;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
container.Add(NamedOnnxValue.CreateFromTensor<float>("wrong_name", tensor));
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(container));
Assert.Equal("[ErrorCode:InvalidArgument] Invalid Feed Input Names: wrong_name Valid input names are: data_0 ", ex.Message);
session.Dispose();
}
[Fact]
private void ThrowWrongInputType()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
int[] inputDataInt = inputData.Select(x => (int)x).ToArray();
var tensor = new DenseTensor<int>(inputDataInt, inputMeta["data_0"].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor<int>("data_0", tensor));
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(container));
Assert.Equal("[ErrorCode:InvalidArgument] Unexpected input data type. Actual: (class onnxruntime::NonOnnxType<int>) , expected: (class onnxruntime::NonOnnxType<float>)", ex.Message);
session.Dispose();
}
[Fact]
private void ThrowWrongDimensions()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
var inputData = new float[] { 0.1f, 0.2f, 0.3f };
var tensor = new DenseTensor<float>(inputData, new int[] { 1, 3 });
container.Add(NamedOnnxValue.CreateFromTensor<float>("data_0", tensor));
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(container));
Assert.Equal("[ErrorCode:Fail] X num_dims does not match W num_dims. X: {1,3} W: {64,3,3,3}", ex.Message);
session.Dispose();
}
[Fact]
private void ThrowDuplicateInput()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var tensor = tuple.Item3;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
var nov = NamedOnnxValue.CreateFromTensor<float>("data_0", tensor);
container.Add(nov);
container.Add(nov);
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(container));
Assert.Equal("[ErrorCode:InvalidArgument] duplicated input name", ex.Message);
session.Dispose();
}
[Fact]
private void ThrowExtraInputs()
{
var tuple = OpenSessionSqueezeNet();
var session = tuple.Item1;
var inputData = tuple.Item2;
var tensor = tuple.Item3;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
var nov1 = NamedOnnxValue.CreateFromTensor<float>("data_0", tensor);
var nov2 = NamedOnnxValue.CreateFromTensor<float>("extra", tensor);
container.Add(nov1);
container.Add(nov2);
var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(container));
Assert.Equal("[ErrorCode:InvalidArgument] The number of feeds is not same as the number of the model input, expect 1 got 2", ex.Message);
session.Dispose();
}
static float[] LoadTensorFromFile(string filename)
{
var tensorData = new List<float>();
@ -118,6 +202,15 @@ namespace Microsoft.ML.OnnxRuntime.Tests
return tensorData.ToArray();
}
static Tuple<InferenceSession, float[], DenseTensor<float>> OpenSessionSqueezeNet()
{
string modelPath = Directory.GetCurrentDirectory() + @"\squeezenet.onnx";
var session = new InferenceSession(modelPath);
float[] inputData = LoadTensorFromFile(@"bench.in");
var inputMeta = session.InputMetadata;
var tensor = new DenseTensor<float>(inputData, inputMeta["data_0"].Dimensions);
return new Tuple<InferenceSession, float[], DenseTensor<float>>(session, inputData, tensor);
}
}
}

2
csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj
View file

@ -4,6 +4,8 @@
<TargetFramework>netcoreapp2.0</TargetFramework>
<IsPackable>false</IsPackable>
<OnnxRuntimeCsharpRoot>..\..</OnnxRuntimeCsharpRoot>
<Platform>AnyCPU</Platform>
<OutputPath>bin\$(Configuration)\</OutputPath>
<buildDirectory Condition="'$(buildDirectory)'==''">$(OnnxRuntimeCsharpRoot)\..\build\Windows</buildDirectory>
<NativeBuildOutputDir>$(buildDirectory)\$(Configuration)\$(Configuration)</NativeBuildOutputDir>
</PropertyGroup>

38
csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj Normal file
View file

@ -0,0 +1,38 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.0</TargetFramework>
<OnnxRuntimeCsharpRoot>..\..</OnnxRuntimeCsharpRoot>
<buildDirectory Condition="'$(buildDirectory)'==''">$(OnnxRuntimeCsharpRoot)\..\build\Windows</buildDirectory>
<NativeBuildOutputDir>$(buildDirectory)\$(Configuration)\$(Configuration)</NativeBuildOutputDir>
<SignAssembly>false</SignAssembly>
</PropertyGroup>
<ItemGroup>
<None Include="$(NativeBuildOutputDir)\onnxruntime.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="$(NativeBuildOutputDir)\onnxruntime.pdb" Condition="Exists('$(NativeBuildOutputDir)\onnxruntime.pdb')">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="$(NativeBuildOutputDir)\mkldnn.dll">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
<None Include="$(OnnxRuntimeCSharpRoot)\testdata\*">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
<Visible>false</Visible>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="$(OnnxRuntimeCSharpRoot)\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj" />
<PackageReference Include="Microsoft.ML.Scoring" Version="1.1.0" />
</ItemGroup>
</Project>

139
csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs Normal file
View file

@ -0,0 +1,139 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using Microsoft.ML.OnnxRuntime;
using System.Numerics.Tensors;
using System.Diagnostics;
namespace Microsoft.ML.OnnxRuntime.PerfTool
{
public enum TimingPoint
{
Start = 0,
ModelLoaded = 1,
InputLoaded = 2,
RunComplete = 3,
TotalCount = 4
}
class Program
{
public static void Main(string[] args)
{
/*
args[0] = model-file-name
args[1] = input-file-name
args[2] = iteration count
*/
if (args.Length < 3)
{
PrintUsage();
Environment.Exit(1);
}
string modelPath = args[0];
string inputPath = args[1];
int iteration = Int32.Parse(args[2]);
Console.WriteLine("Running model {0} in OnnxRuntime with input {1} for {2} times", modelPath, inputPath, iteration);
DateTime[] timestamps = new DateTime[(int)TimingPoint.TotalCount];
RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps);
PrintReport(timestamps, iteration);
Console.WriteLine("Done");
Console.WriteLine("Running model {0} in Sonoma with input {1} for {2} times", modelPath, inputPath, iteration);
RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps);
PrintReport(timestamps, iteration);
Console.WriteLine("Done");
}
public static float[] LoadTensorFromFile(string filename)
{
var tensorData = new List<float>();
// read data from file
using (var inputFile = new System.IO.StreamReader(filename))
{
inputFile.ReadLine(); //skip the input name
string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < dataStr.Length; i++)
{
tensorData.Add(Single.Parse(dataStr[i]));
}
}
return tensorData.ToArray();
}
static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps)
{
if (timestamps.Length != (int)TimingPoint.TotalCount)
{
throw new ArgumentException("Timestamps array must have "+(int)TimingPoint.TotalCount+" size");
}
timestamps[(int)TimingPoint.Start] = DateTime.Now;
using (var session = new InferenceSession(modelPath))
{
timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;
var inputMeta = session.InputMetadata;
var container = new List<NamedOnnxValue>();
foreach (var name in inputMeta.Keys)
{
float[] rawData = LoadTensorFromFile(inputPath);
var tensor = new DenseTensor<float>(rawData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
}
timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now;
// Run the inference
for (int i=0; i < iteration; i++)
{
var results = session.Run(container); // results is an IReadOnlyList<NamedOnnxValue> container
Debug.Assert(results != null);
Debug.Assert(results.Count == 1);
//results = null;
//GC.Collect();
//GC.WaitForPendingFinalizers();
}
timestamps[(int)TimingPoint.RunComplete] = DateTime.Now;
}
}
static void PrintUsage()
{
Console.WriteLine("Usage:\n"
+"dotnet Microsoft.ML.OnnxRuntime.PerfTool <onnx-model-path> <input-file-path> <iteration-count>"
);
}
static void PrintReport(DateTime[] timestamps, int iterations)
{
Console.WriteLine("Model Load Time = " + (timestamps[(int)TimingPoint.ModelLoaded] - timestamps[(int)TimingPoint.Start]).TotalMilliseconds);
Console.WriteLine("Input Load Time = " + (timestamps[(int)TimingPoint.InputLoaded] - timestamps[(int)TimingPoint.ModelLoaded]).TotalMilliseconds);
double totalRuntime = (timestamps[(int)TimingPoint.RunComplete] - timestamps[(int)TimingPoint.InputLoaded]).TotalMilliseconds;
double perIterationTime = totalRuntime / iterations;
Console.WriteLine("Total Run time for {0} iterations = {1}", iterations, totalRuntime);
Console.WriteLine("Per iteration time = {0}", perIterationTime);
}
}
}

58
csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/SonomaRunner.cs Normal file
View file

@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.ML.Scoring;
using System.Diagnostics;
namespace Microsoft.ML.OnnxRuntime.PerfTool
{
public class SonomaRunner
{
public static void RunModelSonoma(string modelPath, string inputPath, int iteration, DateTime[] timestamps)
{
if (timestamps.Length != (int)TimingPoint.TotalCount)
{
throw new ArgumentException("Timestamps array must have " + (int)TimingPoint.TotalCount + " size");
}
timestamps[(int)TimingPoint.Start] = DateTime.Now;
var modelName = "lotusrt_squeezenet";
using (var modelManager = new ModelManager(modelPath, true))
{
modelManager.InitOnnxModel(modelName, int.MaxValue);
timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;
Tensor[] inputs = new Tensor[1];
var inputShape = new long[] { 1, 3, 224, 224 }; // hardcoded values
float[] inputData0 = Program.LoadTensorFromFile(inputPath);
inputs[0] = Tensor.Create(inputData0, inputShape);
string[] inputNames = new string[] {"data_0"};
string[] outputNames = new string[] { "softmaxout_1" };
timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now;
for (int i = 0; i < iteration; i++)
{
var outputs = modelManager.RunModel(
modelName,
int.MaxValue,
inputNames,
inputs,
outputNames
);
Debug.Assert(outputs != null);
Debug.Assert(outputs.Length == 1);
}
timestamps[(int)TimingPoint.RunComplete] = DateTime.Now;
}
}
}
}

116
docs/CSharp_API.md
View file

@ -1 +1,115 @@
# C# API
# ONNX Runtime C# API
The ONNX runtime provides a C# .Net binding for running inference on ONNX models in any of the .Net standard platforms. The API is .Net standard 1.1 compliant for maximum portability. This document describes the API.
## NuGet Package
There is a NuGet package Microsoft.ML.OnnxRuntime available for .Net consumers, which includes the prebuilt binaries for ONNX runtime. The API is portable across all platforms and architectures supported by the .Net standard, although currently the NuGet package contains the prebuilt binaries for Windows 10 platform on x64 CPUs only.
## Getting Started
Here is simple tutorial for getting started with running inference on an existing ONNX model for a given input data (a.k.a query). Say the model is trained using any of the well-known training frameworks and exported as an ONNX model into a file named `model.onnx`. The runtime incarnation of a model is an `InferenceSession` object. You simply construct an `InferenceSession` object using the model file as parameter --
var session = new InferenceSession("model.onnx");
Once a session is created, you can run queries on the session using your input data, using the `Run` method of the `InferenceSession`. Both input and output of `Run` method are represented as collections of .Net `Tensor` objects (as defined in [System.Numerics.Tensor](https://www.nuget.org/packages/System.Numerics.Tensors)) -
Tensor<float> t1, t2; // let's say data is fed into the Tensor objects
var inputs = new List<NamedOnnxValue>()
{
NamedOnnxValue.CreateFromTensor<float>("name1", t1),
NamedOnnxValue.CreateFromTensor<float>("name2", t2)
};
IReadOnlyCollection<NamedOnnxValue> results = session.Run(inputs);
You can load your input data into Tensor<T> objects in several ways. A simple example is to create the Tensor from arrays -
float[] sourceData; // assume your data is loaded into a flat float array
int[] dimensions; // and the dimensions of the input is stored here
Tensor<float> t1 = new DenseTensor<float>(sourceData, dimensions);
Here is a [complete sample code](https://github.com/Microsoft/onnxruntime/tree/master/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample) that runs inference on a pretrained model.
## API Reference
### InferenceSession
class InferenceSession: IDisposable
The runtime representation of an ONNX model
#### Constructor
InferenceSession(string modelPath);
InferenceSession(string modelPath, SesionOptions options);
#### Properties
IReadOnlyDictionary<NodeMetadata> InputMetadata;
Data types and shapes of the input nodes of the model.
IReadOnlyDictionary<NodeMetadata> OutputMetadata;
Data types and shapes of the output nodes of the model.
#### Methods
IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs);
Runs the model with the given input data to compute all the output nodes and returns the output node values. Both input and output are collection of NamedOnnxValue, which in turn is a name-value pair of string names and Tensor values.
IReadOnlyCollection<NamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> desiredOutputNodes);
Runs the model on given inputs for the given output nodes only.
### System.Numerics.Tensor
The primary .Net object that is used for holding input-output of the model inference. Details on this newly introduced data type can be found in its [open-source implementation](https://github.com/dotnet/corefx/tree/master/src/System.Numerics.Tensors). The binaries are available as a [.Net NuGet package](https://www.nuget.org/packages/System.Numerics.Tensors).
### NamedOnnxValue
class NamedOnnxValue;
Represents a name-value pair of string names and any type of value that ONNX runtime supports as input-output data. Currently, only Tensor objects are supported as input-output values.
#### Constructor
No public constructor available.
#### Properties
string Name; // read only
#### Methods
static NamedOnnxValue CreateFromTensor<T>(string name, Tensor<T>);
Creates a NamedOnnxValue from a name and a Tensor<T> object.
Tensor<T> AsTensor<T>();
Accesses the value as a Tensor<T>. Returns null if the value is not a Tensor<T>.
### SessionOptions
class SessionOptions: IDisposable;
A collection of properties to be set for configuring the OnnxRuntime session
#### Constructor
SessionOptions();
Constructs a SessionOptions will all options at default/unset values.
#### Properties
static SessionOptions Default; //read-only
Accessor to the default static option object
#### Methods
AppendExecutionProvider(ExecutionProvider provider);
Appends execution provider to the session. For any operator in the graph the first execution provider that implements the operator will be user. ExecutionProvider is defined as the following enum -
enum ExecutionProvider
{
Cpu,
MklDnn
}
### NodeMetadata
Container of metadata for a model graph node, used for communicating the shape and type of the input and output nodes.
#### Properties
int[] Dimensions;
Read-only shape of the node, when the node is a Tensor. Undefined if the node is not a Tensor.
System.Type ElementType;
Type of the elements of the node, when node is a Tensor. Undefined for non-Tensor nodes.
bool IsTensor;
Whether the node is a Tensor
### Exceptions
class OnnxRuntimeException: Exception;
The type of Exception that is thrown in most of the error conditions related to Onnx Runtime.

2
tools/ci_build/github/azure-pipelines/azure-pipelines.yml
View file

@ -44,7 +44,7 @@ jobs:
- task: BatchScript@1
inputs:
filename: build.bat
arguments: ' --enable_pybind --use_mkldnn --use_mklml --build_shared_lib --enable_onnx_tests'
arguments: ' --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests'
workingFolder: "$(Build.SourcesDirectory)"
- task: CmdLine@1