From 29f9e89226a2e6bb7ba0287618f4dcf6ffe9f5a3 Mon Sep 17 00:00:00 2001 From: "Shah Asaduzzaman (ASAD)" Date: Tue, 27 Nov 2018 19:01:28 -0800 Subject: [PATCH] merged the latest csharp folder changes from VSTS repo @313681f9 --- csharp/OnnxRuntime.CSharp.proj | 27 +++- csharp/OnnxRuntime.CSharp.sln | 9 ++ .../Microsoft.ML.OnnxRuntime.csproj | 6 +- .../Microsoft.ML.OnnxRuntime.PerfTool.csproj | 38 +++++ .../Program.cs | 139 ++++++++++++++++++ .../SonomaRunner.cs | 58 ++++++++ 6 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj create mode 100644 csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs create mode 100644 csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/SonomaRunner.cs diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj index e5c2e8e171..4a0c999e4a 100644 --- a/csharp/OnnxRuntime.CSharp.proj +++ b/csharp/OnnxRuntime.CSharp.proj @@ -18,28 +18,39 @@ CMake creates a target to this project + + Targets="ObtainPackageVersion;Build" + Properties="Platform=AnyCPU"/> + Targets="Build" + Properties="Platform=AnyCPU" + /> - + Targets="Build" + /> + @@ -47,11 +58,11 @@ CMake creates a target to this project - + diff --git a/csharp/OnnxRuntime.CSharp.sln b/csharp/OnnxRuntime.CSharp.sln index 7e1aeeb529..9aaa5f35cc 100644 --- a/csharp/OnnxRuntime.CSharp.sln +++ b/csharp/OnnxRuntime.CSharp.sln @@ -9,7 +9,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.In EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.Tests", "test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj", "{50173D13-DF29-42E7-A30B-8B12D36C77B1}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.PerfTool", "tools\Microsoft.ML.OnnxRuntime.PerfTool\Microsoft.ML.OnnxRuntime.PerfTool.csproj", "{310506FD-6E78-4D62-989B-25D69A85E8CF}" +EndProject Global + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU @@ -27,6 +32,10 @@ Global {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Debug|Any CPU.Build.0 = Debug|Any CPU {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.ActiveCfg = Release|Any CPU {50173D13-DF29-42E7-A30B-8B12D36C77B1}.Release|Any CPU.Build.0 = Release|Any CPU + {310506FD-6E78-4D62-989B-25D69A85E8CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {310506FD-6E78-4D62-989B-25D69A85E8CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {310506FD-6E78-4D62-989B-25D69A85E8CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {310506FD-6E78-4D62-989B-25D69A85E8CF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index d1f0e38bd9..05ed0811f3 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -30,18 +30,21 @@ @@ -58,7 +61,7 @@ /> - + @@ -69,6 +72,7 @@ $(GitCommitHash) @(MajorVersionNumber) + $(PackageVersion) $(PackageVersion)-dev-$(GitCommitHash) diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj new file mode 100644 index 0000000000..bda86180f6 --- /dev/null +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj @@ -0,0 +1,38 @@ + + + + Exe + netcoreapp2.0 + ..\.. + $(OnnxRuntimeCsharpRoot)\..\build\Windows + $(buildDirectory)\$(Configuration)\$(Configuration) + false + + + + + Always + false + + + Always + false + + + Always + false + + + Always + false + + + + + + + + + + + diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs new file mode 100644 index 0000000000..7524276549 --- /dev/null +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using Microsoft.ML.OnnxRuntime; +using System.Numerics.Tensors; +using System.Diagnostics; + + +namespace Microsoft.ML.OnnxRuntime.PerfTool +{ + public enum TimingPoint + { + Start = 0, + ModelLoaded = 1, + InputLoaded = 2, + RunComplete = 3, + TotalCount = 4 + } + + class Program + { + + public static void Main(string[] args) + { + /* + args[0] = model-file-name + args[1] = input-file-name + args[2] = iteration count + */ + + if (args.Length < 3) + { + PrintUsage(); + Environment.Exit(1); + } + + string modelPath = args[0]; + string inputPath = args[1]; + int iteration = Int32.Parse(args[2]); + Console.WriteLine("Running model {0} in OnnxRuntime with input {1} for {2} times", modelPath, inputPath, iteration); + DateTime[] timestamps = new DateTime[(int)TimingPoint.TotalCount]; + + RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps); + PrintReport(timestamps, iteration); + Console.WriteLine("Done"); + + Console.WriteLine("Running model {0} in Sonoma with input {1} for {2} times", modelPath, inputPath, iteration); + RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps); + PrintReport(timestamps, iteration); + Console.WriteLine("Done"); + + } + + + public static float[] LoadTensorFromFile(string filename) + { + var tensorData = new List(); + + // read data from file + using (var inputFile = new System.IO.StreamReader(filename)) + { + inputFile.ReadLine(); //skip the input name + string[] dataStr = inputFile.ReadLine().Split(new char[] { ',', '[', ']' }, StringSplitOptions.RemoveEmptyEntries); + for (int i = 0; i < dataStr.Length; i++) + { + tensorData.Add(Single.Parse(dataStr[i])); + } + } + + return tensorData.ToArray(); + } + + static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps) + { + if (timestamps.Length != (int)TimingPoint.TotalCount) + { + throw new ArgumentException("Timestamps array must have "+(int)TimingPoint.TotalCount+" size"); + } + + timestamps[(int)TimingPoint.Start] = DateTime.Now; + + using (var session = new InferenceSession(modelPath)) + { + timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now; + var inputMeta = session.InputMetadata; + + var container = new List(); + foreach (var name in inputMeta.Keys) + { + float[] rawData = LoadTensorFromFile(inputPath); + var tensor = new DenseTensor(rawData, inputMeta[name].Dimensions); + container.Add(NamedOnnxValue.CreateFromTensor(name, tensor)); + } + + + + timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now; + + // Run the inference + for (int i=0; i < iteration; i++) + { + var results = session.Run(container); // results is an IReadOnlyList container + Debug.Assert(results != null); + Debug.Assert(results.Count == 1); + //results = null; + //GC.Collect(); + //GC.WaitForPendingFinalizers(); + } + + timestamps[(int)TimingPoint.RunComplete] = DateTime.Now; + } + + } + + + static void PrintUsage() + { + Console.WriteLine("Usage:\n" + +"dotnet Microsoft.ML.OnnxRuntime.PerfTool " + ); + } + + static void PrintReport(DateTime[] timestamps, int iterations) + { + Console.WriteLine("Model Load Time = " + (timestamps[(int)TimingPoint.ModelLoaded] - timestamps[(int)TimingPoint.Start]).TotalMilliseconds); + Console.WriteLine("Input Load Time = " + (timestamps[(int)TimingPoint.InputLoaded] - timestamps[(int)TimingPoint.ModelLoaded]).TotalMilliseconds); + + double totalRuntime = (timestamps[(int)TimingPoint.RunComplete] - timestamps[(int)TimingPoint.InputLoaded]).TotalMilliseconds; + double perIterationTime = totalRuntime / iterations; + + Console.WriteLine("Total Run time for {0} iterations = {1}", iterations, totalRuntime); + Console.WriteLine("Per iteration time = {0}", perIterationTime); + } + } +} diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/SonomaRunner.cs b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/SonomaRunner.cs new file mode 100644 index 0000000000..92b8763f67 --- /dev/null +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/SonomaRunner.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.ML.Scoring; +using System.Diagnostics; + +namespace Microsoft.ML.OnnxRuntime.PerfTool +{ + public class SonomaRunner + { + public static void RunModelSonoma(string modelPath, string inputPath, int iteration, DateTime[] timestamps) + { + if (timestamps.Length != (int)TimingPoint.TotalCount) + { + throw new ArgumentException("Timestamps array must have " + (int)TimingPoint.TotalCount + " size"); + } + + timestamps[(int)TimingPoint.Start] = DateTime.Now; + + var modelName = "lotusrt_squeezenet"; + using (var modelManager = new ModelManager(modelPath, true)) + { + modelManager.InitOnnxModel(modelName, int.MaxValue); + timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now; + + Tensor[] inputs = new Tensor[1]; + var inputShape = new long[] { 1, 3, 224, 224 }; // hardcoded values + + float[] inputData0 = Program.LoadTensorFromFile(inputPath); + inputs[0] = Tensor.Create(inputData0, inputShape); + string[] inputNames = new string[] {"data_0"}; + string[] outputNames = new string[] { "softmaxout_1" }; + + timestamps[(int)TimingPoint.InputLoaded] = DateTime.Now; + + for (int i = 0; i < iteration; i++) + { + var outputs = modelManager.RunModel( + modelName, + int.MaxValue, + inputNames, + inputs, + outputNames + ); + Debug.Assert(outputs != null); + Debug.Assert(outputs.Length == 1); + } + + timestamps[(int)TimingPoint.RunComplete] = DateTime.Now; + } + + + + } + + + } +}