saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-25 22:26:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cleanup csharp API SessionOptions and RunOptions to be consistent with other APIs (#1570)

Browse source 
- Updated SessionOptions API to use properties instead of setter/getter methods. 
- Added missing APIs. 
- Added RunOptions.
This commit is contained in:
shahasad 2019-08-14 12:02:02 -07:00 committed by GitHub
parent bd64ca3019
commit a6a5acedda
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 633 additions and 210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs
View file

@ -26,7 +26,7 @@ namespace CSharpUsage
// Optional : Create session options and set the graph optimization level for the session
SessionOptions options = new SessionOptions();
options.SetSessionGraphOptimizationLevel(2);
options.GraphOptimizationLevel = 2;
using (var session = new InferenceSession(modelPath, options))
{

154
csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs
View file

@ -19,6 +19,8 @@ namespace Microsoft.ML.OnnxRuntime
{
protected IntPtr _nativeHandle;
protected Dictionary<string, NodeMetadata> _inputMetadata, _outputMetadata;
private SessionOptions _builtInSessionOptions = null;
private RunOptions _builtInRunOptions = null;
#region Public API
@ -28,10 +30,12 @@ namespace Microsoft.ML.OnnxRuntime
/// </summary>
/// <param name="modelPath"></param>
public InferenceSession(string modelPath)
: this(modelPath, SessionOptions.Default)
{
_builtInSessionOptions = new SessionOptions(); // need to be disposed
Init(modelPath, _builtInSessionOptions);
}
/// <summary>
/// Constructs an InferenceSession from a model file, with some additional session options
/// </summary>
@ -39,52 +43,13 @@ namespace Microsoft.ML.OnnxRuntime
/// <param name="options"></param>
public InferenceSession(string modelPath, SessionOptions options)
{
var envHandle = OnnxRuntime.Handle;
_nativeHandle = IntPtr.Zero;
try
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.Unicode.GetBytes(modelPath), options._nativePtr, out _nativeHandle));
else
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.UTF8.GetBytes(modelPath), options._nativePtr, out _nativeHandle));
// Initialize input/output metadata
_inputMetadata = new Dictionary<string, NodeMetadata>();
_outputMetadata = new Dictionary<string, NodeMetadata>();
// get input count
UIntPtr inputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount));
// get all the output names
for (ulong i = 0; i < (ulong)inputCount; i++)
{
var iname = GetInputName(i);
_inputMetadata[iname] = GetInputMetadata(i);
}
// get output count
UIntPtr outputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount));
// get all the output names
for (ulong i = 0; i < (ulong)outputCount; i++)
{
_outputMetadata[GetOutputName(i)] = GetOutputMetadata(i);
}
}
catch (OnnxRuntimeException e)
{
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.OrtReleaseSession(_nativeHandle);
_nativeHandle = IntPtr.Zero;
}
throw e;
}
Init(modelPath, options);
}
/// <summary>
/// Meta data regarding the input nodes, keyed by input names
/// </summary>
public IReadOnlyDictionary<string, NodeMetadata> InputMetadata
{
get
@ -93,6 +58,9 @@ namespace Microsoft.ML.OnnxRuntime
}
}
/// <summary>
/// Metadata regarding the output nodes, keyed by output names
/// </summary>
public IReadOnlyDictionary<string, NodeMetadata> OutputMetadata
{
get
@ -101,11 +69,12 @@ namespace Microsoft.ML.OnnxRuntime
}
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches all the outputs.
/// </summary>
/// <param name="inputs"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue</returns>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection<DisposableNamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs)
{
string[] outputNames = new string[_outputMetadata.Count];
@ -118,21 +87,22 @@ namespace Microsoft.ML.OnnxRuntime
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue</returns>
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection<DisposableNamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames)
{
return Run(inputs, outputNames, RunOptions.Default);
IDisposableReadOnlyCollection<DisposableNamedOnnxValue> result = null;
result = Run(inputs, outputNames, _builtInRunOptions);
return result;
}
/// <summary>
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames"/>.
/// Runs the loaded model for the given inputs, and fetches the specified outputs in <paramref name="outputNames". Uses the given RunOptions for this run./>.
/// </summary>
/// <param name="inputs"></param>
/// <param name="outputNames"></param>
/// <param name="options"></param>
/// <returns>Output Tensors in a Collection of NamedOnnxValue</returns>
//TODO: kept internal until RunOptions is made public
internal IDisposableReadOnlyCollection<DisposableNamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames, RunOptions options)
/// <returns>Output Tensors in a Collection of NamedOnnxValue. User must dispose the output.</returns>
public IDisposableReadOnlyCollection<DisposableNamedOnnxValue> Run(IReadOnlyCollection<NamedOnnxValue> inputs, IReadOnlyCollection<string> outputNames, RunOptions options)
{
var inputNames = new string[inputs.Count];
var inputTensors = new IntPtr[inputs.Count];
@ -211,6 +181,58 @@ namespace Microsoft.ML.OnnxRuntime
#endregion
#region private methods
protected void Init(string modelPath, SessionOptions options)
{
var envHandle = OnnxRuntime.Handle;
_nativeHandle = IntPtr.Zero;
try
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.Unicode.GetBytes(modelPath), options.Handle, out _nativeHandle));
else
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.UTF8.GetBytes(modelPath), options.Handle, out _nativeHandle));
// Initialize input/output metadata
_inputMetadata = new Dictionary<string, NodeMetadata>();
_outputMetadata = new Dictionary<string, NodeMetadata>();
// get input count
UIntPtr inputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount));
// get all the output names
for (ulong i = 0; i < (ulong)inputCount; i++)
{
var iname = GetInputName(i);
_inputMetadata[iname] = GetInputMetadata(i);
}
// get output count
UIntPtr outputCount = UIntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount));
// get all the output names
for (ulong i = 0; i < (ulong)outputCount; i++)
{
_outputMetadata[GetOutputName(i)] = GetOutputMetadata(i);
}
}
catch (OnnxRuntimeException e)
{
if (_nativeHandle != IntPtr.Zero)
{
NativeMethods.OrtReleaseSession(_nativeHandle);
_nativeHandle = IntPtr.Zero;
}
throw e;
}
_builtInRunOptions = new RunOptions(); // create a default built-in run option, and avoid creating a new one every run() call
}
private string GetOutputName(ulong index)
{
IntPtr nameHandle = IntPtr.Zero;
@ -358,6 +380,15 @@ namespace Microsoft.ML.OnnxRuntime
if (disposing)
{
// cleanup managed resources
if (_builtInSessionOptions != null)
{
_builtInSessionOptions.Dispose();
}
if (_builtInRunOptions != null)
{
_builtInRunOptions.Dispose();
}
}
// cleanup unmanaged resources
@ -426,24 +457,5 @@ namespace Microsoft.ML.OnnxRuntime
//TODO: placeholder for Model metadata. Currently C-API does not expose this.
}
/// Sets various runtime options.
/// TODO: currently uses Default options only. kept internal until fully implemented
internal class RunOptions
{
protected static readonly Lazy<RunOptions> _default = new Lazy<RunOptions>(() => new RunOptions());
public static RunOptions Default
{
get
{
return _default.Value;
}
}
private void RuntOptions()
{
}
}
}

31
csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs
View file

@ -184,6 +184,37 @@ namespace Microsoft.ML.OnnxRuntime
//[DllImport(nativeLib, CharSet = charSet)]
//public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path);
#endregion
#region RunOptions API
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtCreateRunOptions( out IntPtr /* OrtRunOptions** */ runOptions);
[DllImport(nativeLib, CharSet = charSet)]
public static extern void OrtReleaseRunOptions(IntPtr /*(OrtRunOptions*)*/options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsSetRunLogVerbosityLevel(IntPtr /* OrtRunOptions* */ options, LogLevel value);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsSetRunTag(IntPtr /* OrtRunOptions* */ options, string /* const char* */ runTag);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsGetRunLogVerbosityLevel(IntPtr /* OrtRunOptions* */ options, out LogLevel verbosityLevel);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsGetRunTag(IntPtr /* const OrtRunOptions* */options, out IntPtr /* const char** */ runtag);
// Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions
// will exit as soon as possible if the flag is true.
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsEnableTerminate(IntPtr /* OrtRunOptions* */ options);
[DllImport(nativeLib, CharSet = charSet)]
public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsDisableTerminate(IntPtr /* OrtRunOptions* */ options);
#endregion
#region Allocator/AllocatorInfo API

32
csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs
View file

@ -15,7 +15,7 @@ namespace Microsoft.ML.OnnxRuntime
public LogLevel LogLevel { get; set; }
}
internal enum LogLevel
public enum LogLevel
{
Verbose = 0,
Info = 1,
@ -51,6 +51,9 @@ namespace Microsoft.ML.OnnxRuntime
private OnnxRuntime() //Problem: it is not possible to pass any option for a Singleton
:base(IntPtr.Zero, true)
{
// Check LibC version on Linux, before doing any onnxruntime initialization
CheckLibcVersionGreaterThanMinimum();
handle = IntPtr.Zero;
try
{
@ -78,5 +81,32 @@ namespace Microsoft.ML.OnnxRuntime
Delete(handle);
return true;
}
[DllImport("libc", ExactSpelling = true, CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr gnu_get_libc_version();
private static void CheckLibcVersionGreaterThanMinimum()
{
// require libc version 2.23 or higher
var minVersion = new Version(2, 23);
var curVersion = new Version(0, 0);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
try
{
curVersion = Version.Parse(Marshal.PtrToStringAnsi(gnu_get_libc_version()));
if (curVersion >= minVersion)
return;
}
catch (Exception)
{
// trap any obscure exception
}
throw new OnnxRuntimeException(ErrorCode.RuntimeException,
$"libc.so version={curVersion} does not meet the minimun of 2.23 required by OnnxRuntime. " +
"Linux distribution should be similar to Ubuntu 16.04 or higher");
}
}
}
}

112
csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs Normal file
View file

@ -0,0 +1,112 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Runtime.InteropServices;
namespace Microsoft.ML.OnnxRuntime
{
/// Sets various runtime options.
public class RunOptions: IDisposable
{
private IntPtr _nativePtr;
public RunOptions()
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateRunOptions(out _nativePtr));
}
/// <summary>
/// LogVerbosityLevel for the Run
/// default == LogLevel.Verbose
/// </summary>
public LogLevel LogVerbosityLevel
{
get
{
LogLevel level;
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsGetRunLogVerbosityLevel(_nativePtr, out level));
return level;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsSetRunLogVerbosityLevel(_nativePtr, value));
}
}
/// <summary>
/// Log tag to be used during the run. default = ""
/// </summary>
public string LogTag
{
get
{
string tag = null;
IntPtr tagPtr = IntPtr.Zero;
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsGetRunTag(_nativePtr, out tagPtr));
tag = Marshal.PtrToStringAnsi(tagPtr); // assume ANSI string
// should not release the memory of the tagPtr, because it returns the c_str() of the std::string being used inside RunOptions C++ class
return tag;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsSetRunTag(_nativePtr, value));
}
}
/// <summary>
/// Sets a flag to terminate any other Run() call that is using the same RunOptions object
/// Default = false
/// </summary>
public bool Terminate
{
get
{
return _terminate;
}
set
{
if (!_terminate && value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsEnableTerminate(_nativePtr));
_terminate = true;
}
else if (_terminate && !value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsDisableTerminate(_nativePtr));
_terminate = false;
}
}
}
private bool _terminate = false; //value set to default value of the C++ RunOptions
#region destructors disposers
~RunOptions()
{
Dispose(false);
}
public void Dispose()
{
GC.SuppressFinalize(this);
Dispose(true);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
// cleanup managed resources
}
NativeMethods.OrtReleaseRunOptions(_nativePtr);
}
#endregion
}
}

349
csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs
View file

@ -4,102 +4,28 @@
using System;
using System.Text;
using System.Runtime.InteropServices;
using System.IO;
namespace Microsoft.ML.OnnxRuntime
{
/// <summary>
/// Holds the options for creating an InferenceSession
/// </summary>
public class SessionOptions:IDisposable
public class SessionOptions : IDisposable
{
public IntPtr _nativePtr;
protected static readonly Lazy<SessionOptions> _default = new Lazy<SessionOptions>(MakeSessionOptionWithCpuProvider);
private IntPtr _nativePtr;
private static string[] cudaDelayLoadedLibs = { "cublas64_100.dll", "cudnn64_7.dll" };
#region Constructor and Factory methods
/// <summary>
/// Constructs an empty SessionOptions
/// </summary>
public SessionOptions()
{
NativeMethods.OrtCreateSessionOptions(out _nativePtr);
NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSessionOptions(out _nativePtr));
}
/// <summary>
/// Sets the graph optimization level for the session. Default is set to 1.
/// </summary>
/// <param name="optimization_level">optimization level for the session</param>
/// Available options are : 0, 1, 2
/// 0 -> Disable all optimizations
/// 1 -> Enable basic optimizations
/// 2 -> Enable all optimizations
public void SetSessionGraphOptimizationLevel(uint optimization_level)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionGraphOptimizationLevel(_nativePtr, optimization_level));
}
/// <summary>
/// Set filepath to save optimized model after graph level transformations.
/// </summary>
/// <param name="optimizedModelFilepath">File path for saving optimized model.</param>
public void SetOptimizedModelFilePath(string optimizedModelFilepath)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(_nativePtr, optimizedModelFilepath));
}
/// <summary>
/// Enable Sequential Execution. By default, it is enabled.
/// </summary>
/// </param>
public void EnableSequentialExecution()
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableSequentialExecution(_nativePtr));
}
/// <summary>
/// Disable Sequential Execution and enable Parallel Execution.
/// </summary>
/// </param>
public void DisableSequentialExecution()
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableSequentialExecution(_nativePtr));
}
/// <summary>
/// Enable Mem Pattern. By default, it is enabled
/// </summary>
/// </param>
public void EnableMemPattern()
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableMemPattern(_nativePtr));
}
/// <summary>
/// Disable Mem Pattern.
/// </summary>
/// </param>
public void DisableMemPattern()
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableMemPattern(_nativePtr));
}
/// <summary>
/// Default instance
/// </summary>
public static SessionOptions Default
{
get
{
return _default.Value;
}
}
private static SessionOptions MakeSessionOptionWithCpuProvider()
{
CheckLibcVersionGreaterThanMinimum();
SessionOptions options = new SessionOptions();
NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options._nativePtr, 1);
return options;
}
/// <summary>
/// A helper method to constuct a SessionOptions object for CUDA execution
@ -110,14 +36,14 @@ namespace Microsoft.ML.OnnxRuntime
return MakeSessionOptionWithCudaProvider(0);
}
/// <summary>
/// A helper method to constuct a SessionOptions object for CUDA execution
/// </summary>
/// <param name="deviceId"></param>
/// <returns>A SessionsOptions() object configured for execution on deviceId</returns>
public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId=0)
public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId = 0)
{
CheckLibcVersionGreaterThanMinimum();
CheckCudaExecutionProviderDLLs();
SessionOptions options = new SessionOptions();
NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options._nativePtr, deviceId);
@ -125,6 +51,239 @@ namespace Microsoft.ML.OnnxRuntime
return options;
}
#endregion
#region Public Properties
internal IntPtr Handle
{
get
{
return _nativePtr;
}
}
/// <summary>
/// Enable Sequential Execution. Default = true.
/// </summary>
/// </param>
///
public bool EnableSequentialExecution
{
get
{
return _enableSequentialExecution;
}
set
{
if (!_enableSequentialExecution && value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableSequentialExecution(_nativePtr));
_enableSequentialExecution = true;
}
else if (_enableSequentialExecution && !value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableSequentialExecution(_nativePtr));
_enableSequentialExecution = false;
}
}
}
private bool _enableSequentialExecution = true;
/// <summary>
/// Enables the use of the memory allocation patterns in the first Run() call for subsequent runs. Default = true.
/// </summary>
public bool EnableMemoryPattern
{
get
{
return _enableMemoryPattern;
}
set
{
if (!_enableMemoryPattern && value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableMemPattern(_nativePtr));
_enableMemoryPattern = true;
}
else if (_enableMemoryPattern && !value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableMemPattern(_nativePtr));
_enableMemoryPattern = false;
}
}
}
private bool _enableMemoryPattern = true;
/// <summary>
/// Path prefix to use for output of profiling data
/// </summary>
public string ProfileOutputPathPrefix
{
get; set;
} = "onnxruntime_profile_"; // this is the same default in C++ implementation
/// <summary>
/// Enables profiling of InferenceSession.Run() calls. Default is false
/// </summary>
public bool EnableProfiling
{
get
{
return _enableProfiling;
}
set
{
if (!_enableProfiling && value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableProfiling(_nativePtr, ProfileOutputPathPrefix));
_enableProfiling = true;
}
else if (_enableProfiling && !value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableProfiling(_nativePtr));
_enableProfiling = false;
}
}
}
private bool _enableProfiling = false;
/// <summary>
/// Set filepath to save optimized model after graph level transformations. Default is empty, which implies saving is disabled.
/// </summary>
public string OptimizedModelFilePath
{
get
{
return _optimizedModelFilePath;
}
set
{
if (value != _optimizedModelFilePath)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(_nativePtr, value));
_optimizedModelFilePath = value;
}
}
}
private string _optimizedModelFilePath = "";
/// <summary>
/// Enables Arena allocator for the CPU memory allocations. Default is true.
/// </summary>
public bool EnableCpuMemArena
{
get
{
return _enableCpuMemArena;
}
set
{
if (!_enableCpuMemArena && value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableCpuMemArena(_nativePtr));
_enableCpuMemArena = true;
}
else if (_enableCpuMemArena && !value)
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableCpuMemArena(_nativePtr));
_enableCpuMemArena = false;
}
}
}
private bool _enableCpuMemArena = true;
/// <summary>
/// Log Id to be used for the session. Default is empty string.
/// TODO: Should it be named LogTag as in RunOptions?
/// </summary>
public string LogId
{
get
{
return _logId;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionLogId(_nativePtr, value));
_logId = value;
}
}
private string _logId = "";
/// <summary>
/// Log Verbosity Level for the session logs. Default = LogLevel.Verbose
/// </summary>
public LogLevel LogVerbosityLevel
{
get
{
return _logVerbosityLevel;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionLogVerbosityLevel(_nativePtr, value));
_logVerbosityLevel = value;
}
}
private LogLevel _logVerbosityLevel = LogLevel.Verbose;
/// <summary>
/// Threadpool size for the session.Run() calls.
/// Default = 0, meaning threadpool size is aumatically selected from number of available cores.
/// </summary>
public int ThreadPoolSize
{
get
{
return _threadPoolSize;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionThreadPoolSize(_nativePtr, value));
_threadPoolSize = value;
}
}
private int _threadPoolSize = 0; // set to what is set in C++ SessionOptions by default;
/// <summary>
/// Sets the graph optimization level for the session. Default is set to 1.
/// </summary>
/// Available options are : 0, 1, 2
/// 0 -> Disable all optimizations
/// 1 -> Enable basic optimizations
/// 2 -> Enable all optimizations
public uint GraphOptimizationLevel
{
get
{
return _graphOptimizationLevel;
}
set
{
NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionGraphOptimizationLevel(_nativePtr, value));
_graphOptimizationLevel = value;
}
}
private uint _graphOptimizationLevel = 1;
#endregion
#region Private Methods
// Declared, but called only if OS = Windows.
[DllImport("kernel32.dll")]
private static extern IntPtr LoadLibrary(string dllToLoad);
@ -152,32 +311,8 @@ namespace Microsoft.ML.OnnxRuntime
return true;
}
[DllImport("libc", ExactSpelling = true, CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr gnu_get_libc_version();
private static void CheckLibcVersionGreaterThanMinimum()
{
// require libc version 2.23 or higher
var minVersion = new Version(2, 23);
var curVersion = new Version(0, 0);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
try
{
curVersion = Version.Parse(Marshal.PtrToStringAnsi(gnu_get_libc_version()));
if (curVersion >= minVersion)
return;
}
catch (Exception)
{
// trap any obscure exception
}
throw new OnnxRuntimeException(ErrorCode.RuntimeException,
$"libc.so version={curVersion} does not meet the minimun of 2.23 required by OnnxRuntime. " +
"Linux distribution should be similar to Ubuntu 16.04 or higher");
}
}
#endregion
#region destructors disposers
~SessionOptions()

159
csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
View file

@ -17,6 +17,84 @@ namespace Microsoft.ML.OnnxRuntime.Tests
private const string module = "onnxruntime.dll";
private const string propertiesFile = "Properties.txt";
[Fact]
public void TestSessionOptions()
{
using (SessionOptions opt = new SessionOptions())
{
Assert.NotNull(opt);
// check default values of the properties
Assert.True(opt.EnableSequentialExecution);
Assert.True(opt.EnableMemoryPattern);
Assert.False(opt.EnableProfiling);
Assert.Equal("onnxruntime_profile_", opt.ProfileOutputPathPrefix);
Assert.True(opt.EnableCpuMemArena);
Assert.Equal("", opt.LogId);
Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel);
Assert.Equal(0, opt.ThreadPoolSize);
Assert.Equal(1u, opt.GraphOptimizationLevel);
// try setting options
opt.EnableSequentialExecution = false;
Assert.False(opt.EnableSequentialExecution);
opt.EnableMemoryPattern = false;
Assert.False(opt.EnableMemoryPattern);
opt.EnableProfiling = true;
Assert.True(opt.EnableProfiling);
Assert.Equal("onnxruntime_profile_", opt.ProfileOutputPathPrefix);
opt.ProfileOutputPathPrefix = "Ort_P_";
Assert.Equal("Ort_P_", opt.ProfileOutputPathPrefix);
opt.EnableCpuMemArena = false;
Assert.False(opt.EnableCpuMemArena);
opt.LogId = "MyLogId";
Assert.Equal("MyLogId", opt.LogId);
opt.LogVerbosityLevel = LogLevel.Error;
Assert.Equal(LogLevel.Error, opt.LogVerbosityLevel);
opt.ThreadPoolSize = 4;
Assert.Equal(4, opt.ThreadPoolSize);
opt.GraphOptimizationLevel = 3;
Assert.Equal(3u, opt.GraphOptimizationLevel);
Assert.Throws<OnnxRuntimeException>(() => { opt.ThreadPoolSize = -2; });
Assert.Throws<OnnxRuntimeException>(() => { opt.GraphOptimizationLevel = 10; });
}
}
[Fact]
public void TestRunOptions()
{
using (var opt = new RunOptions())
{
Assert.NotNull(opt);
//verify default options
Assert.False(opt.Terminate);
Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel);
Assert.Equal("", opt.LogTag);
// try setting options
opt.Terminate = true;
Assert.True(opt.Terminate);
opt.LogVerbosityLevel = LogLevel.Error;
Assert.Equal(LogLevel.Error, opt.LogVerbosityLevel);
opt.LogTag = "MyLogTag";
Assert.Equal("MyLogTag", opt.LogTag);
}
}
[Fact]
public void CanCreateAndDisposeSessionWithModelPath()
{
@ -61,8 +139,8 @@ namespace Microsoft.ML.OnnxRuntime.Tests
// Set the graph optimization level for this session.
SessionOptions options = new SessionOptions();
options.SetSessionGraphOptimizationLevel(graphOptimizationLevel);
if (disableSequentialExecution) options.DisableSequentialExecution();
options.GraphOptimizationLevel = graphOptimizationLevel;
if (disableSequentialExecution) options.EnableSequentialExecution = false;
using (var session = new InferenceSession(modelPath, options))
{
@ -82,32 +160,51 @@ namespace Microsoft.ML.OnnxRuntime.Tests
// Run the inference
using (var results = session.Run(container)) // results is an IReadOnlyList<NamedOnnxValue> container
{
Assert.Equal(1, results.Count);
validateRunResults(results);
}
float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out");
// validate the results
foreach (var r in results)
// Run Inference with RunOptions
using (var runOptions = new RunOptions())
{
runOptions.LogTag = "CsharpTest";
runOptions.Terminate = true;
runOptions.LogVerbosityLevel = LogLevel.Error;
IReadOnlyCollection<string> outputNames = session.OutputMetadata.Keys.ToList();
using (var results = session.Run(container, outputNames, runOptions)) // results is an IReadOnlyList<NamedOnnxValue> container
{
Assert.Equal("softmaxout_1", r.Name);
var resultTensor = r.AsTensor<float>();
int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data
Assert.Equal(expectedDimensions.Length, resultTensor.Rank);
var resultDimensions = resultTensor.Dimensions;
for (int i = 0; i < expectedDimensions.Length; i++)
{
Assert.Equal(expectedDimensions[i], resultDimensions[i]);
}
var resultArray = r.AsTensor<float>().ToArray();
Assert.Equal(expectedOutput.Length, resultArray.Length);
Assert.Equal(expectedOutput, resultArray, new floatComparer());
validateRunResults(results);
}
}
}
}
private void validateRunResults(IDisposableReadOnlyCollection<DisposableNamedOnnxValue> results)
{
float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out");
// validate the results
foreach (var r in results)
{
Assert.Equal(1, results.Count);
Assert.Equal("softmaxout_1", r.Name);
var resultTensor = r.AsTensor<float>();
int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data
Assert.Equal(expectedDimensions.Length, resultTensor.Rank);
var resultDimensions = resultTensor.Dimensions;
for (int i = 0; i < expectedDimensions.Length; i++)
{
Assert.Equal(expectedDimensions[i], resultDimensions[i]);
}
var resultArray = r.AsTensor<float>().ToArray();
Assert.Equal(expectedOutput.Length, resultArray.Length);
Assert.Equal(expectedOutput, resultArray, new floatComparer());
}
}
[Fact]
private void ThrowWrongInputName()
{
@ -645,8 +742,8 @@ namespace Microsoft.ML.OnnxRuntime.Tests
string modelOutputPath = Path.Combine(Directory.GetCurrentDirectory(), "optimized-squeezenet.onnx");
// Set the optimized model file path to assert that no exception are thrown.
SessionOptions options = new SessionOptions();
options.SetOptimizedModelFilePath(modelOutputPath);
options.SetSessionGraphOptimizationLevel(1);
options.OptimizedModelFilePath = modelOutputPath;
options.GraphOptimizationLevel = 1;
var session = new InferenceSession(modelPath, options);
Assert.NotNull(session);
Assert.True(File.Exists(modelOutputPath));
@ -672,6 +769,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
[DllImport("kernel32", SetLastError = true)]
static extern IntPtr LoadLibrary(string lpFileName);
@ -685,14 +783,19 @@ namespace Microsoft.ML.OnnxRuntime.Tests
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return;
var entryPointNames = new[]{
"OrtCreateEnv","OrtReleaseEnv","OrtGetErrorCode","OrtGetErrorMessage",
"OrtReleaseStatus","OrtCreateSession","OrtRun","OrtSessionGetInputCount",
"OrtSessionGetOutputCount","OrtSessionGetInputName","OrtSessionGetOutputName","OrtSessionGetInputTypeInfo",
"OrtSessionGetOutputTypeInfo","OrtReleaseSession","OrtCreateSessionOptions","OrtCloneSessionOptions",
"OrtCreateEnv","OrtReleaseEnv",
"OrtGetErrorCode","OrtGetErrorMessage", "OrtReleaseStatus",
"OrtCreateSession","OrtRun",
"OrtSessionGetInputCount", "OrtSessionGetOutputCount","OrtSessionGetInputName","OrtSessionGetOutputName",
"OrtSessionGetInputTypeInfo", "OrtSessionGetOutputTypeInfo","OrtReleaseSession",
"OrtCreateSessionOptions","OrtCloneSessionOptions",
"OrtEnableSequentialExecution","OrtDisableSequentialExecution","OrtEnableProfiling","OrtDisableProfiling",
"OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena",
"OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel",
"OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo",
"OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU",
"OrtCreateRunOptions", "OrtReleaseRunOptions", "OrtRunOptionsSetRunLogVerbosityLevel", "OrtRunOptionsSetRunTag",
"OrtRunOptionsGetRunLogVerbosityLevel", "OrtRunOptionsGetRunTag","OrtRunOptionsEnableTerminate", "OrtRunOptionsDisableTerminate",
"OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo",
"OrtCreateDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo",
"OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo",
"OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount",

4
csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs
View file

@ -93,8 +93,8 @@ namespace Microsoft.ML.OnnxRuntime.PerfTool
timestamps[(int)TimingPoint.Start] = DateTime.Now;
SessionOptions options = new SessionOptions();
if (parallelExecution) options.DisableSequentialExecution();
options.SetSessionGraphOptimizationLevel(optLevel);
if (parallelExecution) options.EnableSequentialExecution = false;
options.GraphOptimizationLevel = optLevel;
using (var session = new InferenceSession(modelPath, options))
{
timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;