From 1168f4e85a90815da9b065df05e056c77c25c8d6 Mon Sep 17 00:00:00 2001 From: Hariharan Seshadri Date: Mon, 18 May 2020 19:47:52 -0700 Subject: [PATCH] Support session EndProfiling() in the CSharp API (#3934) --- .../InferenceSession.cs | 27 ++++++++++++ .../Microsoft.ML.OnnxRuntime/NativeMethods.cs | 22 +++++++--- .../InferenceTest.cs | 41 +++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index 21ef4f87b1..e03a70e4d9 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -687,6 +687,33 @@ namespace Microsoft.ML.OnnxRuntime } } + /// + /// Ends profiling for the session. Returns the profile file name. + /// + public string EndProfiling() + { + IntPtr nameHandle = IntPtr.Zero; + string str = null; + + IntPtr status = NativeMethods.OrtSessionEndProfiling(_nativeHandle, + NativeMemoryAllocator.DefaultInstance.Handle, + out nameHandle); + + try + { + NativeApiStatus.VerifySuccess(status); + str = Marshal.PtrToStringAnsi(nameHandle); + } + finally + { + if (nameHandle != IntPtr.Zero) + { + NativeMemoryAllocator.DefaultInstance.FreeMemory(nameHandle); + } + } + + return str; + } //TODO: kept internal until implemented internal ModelMetadata ModelMetadata diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index d87c120563..98e2cd013d 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -129,6 +129,15 @@ namespace Microsoft.ML.OnnxRuntime public IntPtr ReleaseTensorTypeAndShapeInfo; public IntPtr ReleaseSessionOptions; public IntPtr ReleaseCustomOpDomain; + public IntPtr GetDenotationFromTypeInfo; + public IntPtr CastTypeInfoToMapTypeInfo; + public IntPtr CastTypeInfoToSequenceTypeInfo; + public IntPtr GetMapKeyType; + public IntPtr GetMapValueType; + public IntPtr GetSequenceElementType; + public IntPtr ReleaseMapTypeInfo; + public IntPtr ReleaseSequenceTypeInfo; + public IntPtr SessionEndProfiling; } internal static class NativeMethods @@ -162,11 +171,11 @@ namespace Microsoft.ML.OnnxRuntime OrtSessionGetInputName = (DOrtSessionGetInputName)Marshal.GetDelegateForFunctionPointer(api_.SessionGetInputName, typeof(DOrtSessionGetInputName)); OrtSessionGetOutputName = (DOrtSessionGetOutputName)Marshal.GetDelegateForFunctionPointer(api_.SessionGetOutputName, typeof(DOrtSessionGetOutputName)); + OrtSessionEndProfiling = (DOrtSessionEndProfiling)Marshal.GetDelegateForFunctionPointer(api_.SessionEndProfiling, typeof(DOrtSessionEndProfiling)); OrtSessionGetOverridableInitializerName = (DOrtSessionGetOverridableInitializerName)Marshal.GetDelegateForFunctionPointer(api_.SessionGetOverridableInitializerName, typeof(DOrtSessionGetOverridableInitializerName)); OrtSessionGetInputTypeInfo = (DOrtSessionGetInputTypeInfo)Marshal.GetDelegateForFunctionPointer(api_.SessionGetInputTypeInfo, typeof(DOrtSessionGetInputTypeInfo)); OrtSessionGetOutputTypeInfo = (DOrtSessionGetOutputTypeInfo)Marshal.GetDelegateForFunctionPointer(api_.SessionGetOutputTypeInfo, typeof(DOrtSessionGetOutputTypeInfo)); OrtSessionGetOverridableInitializerTypeInfo = (DOrtSessionGetOverridableInitializerTypeInfo)Marshal.GetDelegateForFunctionPointer(api_.SessionGetOverridableInitializerTypeInfo, typeof(DOrtSessionGetOverridableInitializerTypeInfo)); - OrtReleaseTypeInfo = (DOrtReleaseTypeInfo)Marshal.GetDelegateForFunctionPointer(api_.ReleaseTypeInfo, typeof(DOrtReleaseTypeInfo)); OrtReleaseSession = (DOrtReleaseSession)Marshal.GetDelegateForFunctionPointer(api_.ReleaseSession, typeof(DOrtReleaseSession)); @@ -317,6 +326,12 @@ namespace Microsoft.ML.OnnxRuntime out IntPtr /*(char**)*/name); public static DOrtSessionGetOutputName OrtSessionGetOutputName; + public delegate IntPtr /*(OrtStatus*)*/DOrtSessionEndProfiling( + IntPtr /*(const OrtSession*)*/ session, + IntPtr /*(OrtAllocator*)*/ allocator, + out IntPtr /*(char**)*/profile_file); + public static DOrtSessionEndProfiling OrtSessionEndProfiling; + public delegate IntPtr /*(OrtStatus*)*/DOrtSessionGetOverridableInitializerName( IntPtr /*(OrtSession*)*/ session, UIntPtr index, @@ -324,28 +339,25 @@ namespace Microsoft.ML.OnnxRuntime out IntPtr /*(char**)*/name); public static DOrtSessionGetOverridableInitializerName OrtSessionGetOverridableInitializerName; - // release the typeinfo using OrtReleaseTypeInfo public delegate IntPtr /*(OrtStatus*)*/DOrtSessionGetInputTypeInfo( IntPtr /*(const OrtSession*)*/ session, UIntPtr index, out IntPtr /*(struct OrtTypeInfo**)*/ typeInfo); public static DOrtSessionGetInputTypeInfo OrtSessionGetInputTypeInfo; - // release the typeinfo using OrtReleaseTypeInfo public delegate IntPtr /*(OrtStatus*)*/DOrtSessionGetOutputTypeInfo( IntPtr /*(const OrtSession*)*/ session, UIntPtr index, out IntPtr /* (struct OrtTypeInfo**)*/ typeInfo); public static DOrtSessionGetOutputTypeInfo OrtSessionGetOutputTypeInfo; - // release the typeinfo using OrtReleaseTypeInfo public delegate IntPtr /*(OrtStatus*)*/DOrtSessionGetOverridableInitializerTypeInfo( IntPtr /*(const OrtSession*)*/ session, UIntPtr index, out IntPtr /* (struct OrtTypeInfo**)*/ typeInfo); public static DOrtSessionGetOverridableInitializerTypeInfo OrtSessionGetOverridableInitializerTypeInfo; - + // release the typeinfo using OrtReleaseTypeInfo public delegate void DOrtReleaseTypeInfo(IntPtr /*(OrtTypeInfo*)*/session); public static DOrtReleaseTypeInfo OrtReleaseTypeInfo; diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 278db87266..b17baab6c4 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -291,6 +291,47 @@ namespace Microsoft.ML.OnnxRuntime.Tests } } + [Fact] + public void InferenceSessionManualDisposeAfterUse() + { + string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); + + // Set the graph optimization level for this session. + SessionOptions options = new SessionOptions(); + options.ProfileOutputPathPrefix = "Ort_P_"; + options.EnableProfiling = true; + var session = new InferenceSession(modelPath, options); + + + var inputMeta = session.InputMetadata; + var container = new List(); + + float[] inputData = LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model + + foreach (var name in inputMeta.Keys) + { + Assert.Equal(typeof(float), inputMeta[name].ElementType); + Assert.True(inputMeta[name].IsTensor); + var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions); + container.Add(NamedOnnxValue.CreateFromTensor(name, tensor)); + } + + // Run inference with named inputs and outputs created with in Run() + using (var results = session.Run(container)) // results is an IReadOnlyList container + { + validateRunResults(results); + } + + string profile_file = session.EndProfiling(); + + // Profile file should have the output path prefix in it + Assert.Contains("Ort_P_", profile_file); + + // Should be able to dispose the session manually + session.Dispose(); + + } + private void validateRunResults(IReadOnlyCollection results) { // validate the results