From b0feaef9de0eeab34480bec00f83b90fb3e0b5e1 Mon Sep 17 00:00:00 2001
From: shahasad <43590019+shahasad@users.noreply.github.com>
Date: Mon, 7 Oct 2019 19:14:34 -0700
Subject: [PATCH] Update the C# pretrained model test to include opset9 and 10
models (#2003)
---
csharp/OnnxRuntime.CSharp.proj | 2 +-
.../InferenceTest.cs | 424 ++++++++++++++----
2 files changed, 338 insertions(+), 88 deletions(-)
diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj
index 9651738fa3..64cc865f09 100644
--- a/csharp/OnnxRuntime.CSharp.proj
+++ b/csharp/OnnxRuntime.CSharp.proj
@@ -56,7 +56,7 @@ CMake creates a target to this project
-
+
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
index 6e2b5417ca..47c756c2d7 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs
@@ -9,6 +9,7 @@ using System.Runtime.InteropServices;
using Microsoft.ML.OnnxRuntime.Tensors;
using System.Threading.Tasks;
using Xunit;
+using Xunit.Abstractions;
namespace Microsoft.ML.OnnxRuntime.Tests
{
@@ -16,6 +17,12 @@ namespace Microsoft.ML.OnnxRuntime.Tests
{
private const string module = "onnxruntime.dll";
private const string propertiesFile = "Properties.txt";
+ private readonly ITestOutputHelper output;
+
+ public InferenceTest(ITestOutputHelper o)
+ {
+ this.output = o;
+ }
[Fact]
public void TestSessionOptions()
@@ -309,94 +316,211 @@ namespace Microsoft.ML.OnnxRuntime.Tests
session.Dispose();
}
- [x64Fact]
- private void TestPreTrainedModelsOpset7And8()
+ private static Dictionary GetSkippedModels()
{
- var skipModels = new List() {
- "mxnet_arcface", // Model not supported by CPU execution provider
- "tf_inception_v2", // TODO: Debug failing model, skipping for now
- "fp16_inception_v1", // 16-bit float not supported type in C#.
- "fp16_shufflenet", // 16-bit float not supported type in C#.
- "fp16_tiny_yolov2" }; // 16-bit float not supported type in C#.
+ var skipModels = new Dictionary() {
+ { "mxnet_arcface", "Model not supported by CPU execution provider" } ,
+ { "tf_inception_v2", "TODO: Debug failing model, skipping for now" },
+ { "fp16_inception_v1", "16-bit float not supported type in C#." },
+ { "fp16_shufflenet", "16-bit float not supported type in C#." },
+ { "fp16_tiny_yolov2", "16-bit float not supported type in C#." },
+ { "BERT_Squad", "Could not find an implementation for the node bert / embeddings / one_hot:OneHot(9)" },
+ { "mlperf_ssd_mobilenet_300", "Could not find file output_0.pb" }
+ };
+ // The following models fails on nocontribops win CI
var disableContribOpsEnvVar = Environment.GetEnvironmentVariable("DisableContribOps");
var isContribOpsDisabled = (disableContribOpsEnvVar != null) ? disableContribOpsEnvVar.Equals("ON") : false;
if (isContribOpsDisabled)
{
- skipModels.Add("test_tiny_yolov2");
+ skipModels["test_tiny_yolov2"] = "Fails when ContribOps is disabled";
+ skipModels["mask_rcnn_keras"] = "Pad is not a registered function/op";
}
- var opsets = new[] { "opset7", "opset8" };
- var modelsDir = GetTestModelsDir();
- foreach (var opset in opsets)
+ // This model fails on x86 Win CI
+ if (System.Environment.Is64BitProcess == false)
{
- var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opset));
- foreach (var modelDir in modelRoot.EnumerateDirectories())
+ skipModels["test_vgg19"] = "Get preallocated buffer for initializer conv4_4_b_0 failed";
+ }
+
+ return skipModels;
+ }
+
+ public static IEnumerable GetModelsForTest()
+ {
+ var modelsDir = GetTestModelsDir();
+ var modelsDirInfo = new DirectoryInfo(modelsDir);
+ var skipModels = GetSkippedModels();
+
+ foreach (var opsetDir in modelsDirInfo.EnumerateDirectories())
+ {
+ //var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
+ foreach (var modelDir in opsetDir.EnumerateDirectories())
{
- String onnxModelFileName = null;
-
- if (skipModels.Contains(modelDir.Name))
- continue;
-
- try
+ if (!skipModels.ContainsKey(modelDir.Name))
{
- var onnxModelNames = modelDir.GetFiles("*.onnx");
- if (onnxModelNames.Length > 1)
- {
- // TODO remove file "._resnet34v2.onnx" from test set
- bool validModelFound = false;
- for (int i = 0; i < onnxModelNames.Length; i++)
- {
- if (onnxModelNames[i].Name != "._resnet34v2.onnx")
- {
- onnxModelNames[0] = onnxModelNames[i];
- validModelFound = true;
- }
- }
-
- if (!validModelFound)
- {
- var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
- throw new Exception($"Opset {opset}: Model {modelDir}. Can't determine model file name. Found these :{modelNamesList}");
- }
- }
-
- onnxModelFileName = Path.Combine(modelsDir, opset, modelDir.Name, onnxModelNames[0].Name);
- using (var session = new InferenceSession(onnxModelFileName))
- {
- var inMeta = session.InputMetadata;
- var innodepair = inMeta.First();
- var innodename = innodepair.Key;
- var innodedims = innodepair.Value.Dimensions;
- for (int i = 0; i < innodedims.Length; i++)
- {
- if (innodedims[i] < 0)
- innodedims[i] = -1 * innodedims[i];
- }
-
- var testRoot = new DirectoryInfo(Path.Combine(modelsDir, opset, modelDir.Name));
- var testData = testRoot.EnumerateDirectories("test_data*").First();
- var dataIn = LoadTensorFromFilePb(Path.Combine(modelsDir, opset, modelDir.Name, testData.ToString(), "input_0.pb"));
- var dataOut = LoadTensorFromFilePb(Path.Combine(modelsDir, opset, modelDir.Name, testData.ToString(), "output_0.pb"));
- var tensorIn = new DenseTensor(dataIn, innodedims);
- var nov = new List();
- nov.Add(NamedOnnxValue.CreateFromTensor(innodename, tensorIn));
- using (var resnov = session.Run(nov))
- {
- var res = resnov.ToArray()[0].AsTensor().ToArray();
- Assert.Equal(res, dataOut, new floatComparer());
- }
- }
- }
- catch (Exception ex)
- {
- var msg = $"Opset {opset}: Model {modelDir}: ModelFile = {onnxModelFileName} error = {ex.Message}";
- throw new Exception(msg);
+ yield return new object[] { modelDir.Parent.Name, modelDir.Name };
}
} //model
} //opset
}
+ public static IEnumerable GetSkippedModelForTest()
+ {
+ var modelsDir = GetTestModelsDir();
+ var modelsDirInfo = new DirectoryInfo(modelsDir);
+ var skipModels = GetSkippedModels();
+
+ foreach (var opsetDir in modelsDirInfo.EnumerateDirectories())
+ {
+ var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
+ foreach (var modelDir in modelRoot.EnumerateDirectories())
+ {
+ if (skipModels.ContainsKey(modelDir.Name))
+ {
+ //Console.WriteLine("Model {0} is skipped due to the error: {1}", modelDir.FullName, skipModels[modelDir.Name]);
+ yield return new object[] { modelDir.Parent.Name, modelDir.Name };
+ }
+
+ }
+ }
+ }
+
+
+ [Theory]
+ [MemberData(nameof(GetModelsForTest))]
+ [MemberData(nameof(GetSkippedModelForTest), Skip ="Skipped due to Error, please fix the error and enable the test")]
+ private void TestPreTrainedModels(string opset, string modelName)
+ {
+ var modelsDir = GetTestModelsDir();
+ string onnxModelFileName = null;
+
+ var modelDir = new DirectoryInfo(Path.Combine(modelsDir, opset, modelName));
+
+ try
+ {
+ var onnxModelNames = modelDir.GetFiles("*.onnx");
+ bool validModelFound = false;
+ if (onnxModelNames.Length > 0)
+ {
+ // TODO remove file "._resnet34v2.onnx" from test set
+ for (int i = 0; i < onnxModelNames.Length; i++)
+ {
+ if (onnxModelNames[i].Name != "._resnet34v2.onnx")
+ {
+ onnxModelNames[0] = onnxModelNames[i];
+ validModelFound = true;
+ }
+ }
+ }
+
+ if (validModelFound)
+ {
+ onnxModelFileName = Path.Combine(modelDir.FullName, onnxModelNames[0].Name);
+ }
+ else
+ {
+ var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
+ throw new Exception($"Opset {opset} Model {modelName}. Can't determine model file name. Found these :{modelNamesList}");
+ }
+
+ using (var session = new InferenceSession(onnxModelFileName))
+ {
+ var inMeta = session.InputMetadata;
+ string testDataDirNamePattern = "test_data*";
+ if (opset == "opset9" && modelName == "LSTM_Seq_lens_unpacked")
+ {
+ testDataDirNamePattern = "seq_lens*"; // discrepency in data directory
+ }
+ var testDataDir = modelDir.EnumerateDirectories(testDataDirNamePattern).First();
+ var inputContainer = new List();
+ var outputContainer = new List();
+ foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
+ {
+ inputContainer.Add(LoadTensorFromFilePb(f.FullName, inMeta));
+ }
+ foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
+ {
+ outputContainer.Add(LoadTensorFromFilePb(f.FullName, session.OutputMetadata));
+ }
+
+ using (var resultCollection = session.Run(inputContainer))
+ {
+ foreach (var result in resultCollection)
+ {
+ Assert.True(session.OutputMetadata.ContainsKey(result.Name));
+ var outputMeta = session.OutputMetadata[result.Name];
+ NamedOnnxValue outputValue = null;
+ foreach (var o in outputContainer)
+ {
+ if (o.Name == result.Name)
+ {
+ outputValue = o;
+ break;
+ }
+ }
+ if (outputValue == null)
+ {
+ outputValue = outputContainer.First(); // in case the output data file does not contain the name
+ }
+ if (outputMeta.IsTensor)
+ {
+ if (outputMeta.ElementType == typeof (float))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new floatComparer());
+ }
+ else if (outputMeta.ElementType == typeof(int))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(uint))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(short))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(ushort))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(long))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(ulong))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(byte))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else if (outputMeta.ElementType == typeof(bool))
+ {
+ Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
+ }
+ else
+ {
+ Assert.True(false, "The TestPretrainedModels does not yet support output of type " + nameof(outputMeta.ElementType));
+ }
+ }
+ else
+ {
+ Assert.True(false, "TestPretrainedModel cannot handle non-tensor outputs yet");
+ }
+ }
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ var msg = $"Opset {opset}, Model {modelName}: ModelFile = {onnxModelFileName} error = {ex.Message}";
+ throw new Exception(msg+"\n"+ex.StackTrace);
+ }
+ }
+
[Fact]
private void TestOverridableInitializerMetadata()
{
@@ -934,15 +1058,138 @@ namespace Microsoft.ML.OnnxRuntime.Tests
return tensorData.ToArray();
}
- static float[] LoadTensorFromFilePb(string filename)
+ static NamedOnnxValue LoadTensorFromFilePb(string filename, IReadOnlyDictionary nodeMetaDict)
{
var file = File.OpenRead(filename);
var tensor = Onnx.TensorProto.Parser.ParseFrom(file);
file.Close();
- var raw = tensor.RawData.ToArray();
- var floatArr = new float[raw.Length / sizeof(float)];
- Buffer.BlockCopy(raw, 0, floatArr, 0, raw.Length);
- return floatArr;
+
+ Type tensorElemType = null;
+ int width = 0;
+ TensorElementTypeConverter.GetTypeAndWidth((TensorElementType)tensor.DataType, out tensorElemType, out width);
+ var intDims = new int[tensor.Dims.Count];
+ for (int i = 0; i < tensor.Dims.Count; i++)
+ {
+ intDims[i] = (int)tensor.Dims[i];
+ }
+
+ NodeMetadata nodeMeta = null;
+ string nodeName = "";
+ if (nodeMetaDict.Count == 1)
+ {
+ nodeMeta = nodeMetaDict.Values.First();
+ nodeName = nodeMetaDict.Keys.First(); // valid for single node input
+ }
+ else if (nodeMetaDict.Count > 1)
+ {
+ if (tensor.Name != "")
+ {
+ nodeMeta = nodeMetaDict[tensor.Name];
+ nodeName = tensor.Name;
+ }
+ else
+ {
+ bool matchfound = false;
+ // try to find from matching type and shape
+ foreach (var key in nodeMetaDict.Keys)
+ {
+ var meta = nodeMetaDict[key];
+ if (tensorElemType == meta.ElementType && tensor.Dims.Count == meta.Dimensions.Length)
+ {
+ int i = 0;
+ for (; i < meta.Dimensions.Length; i++)
+ {
+ if (meta.Dimensions[i] != -1 && meta.Dimensions[i] != intDims[i])
+ {
+ break;
+ }
+ }
+ if (i >= meta.Dimensions.Length)
+ {
+ matchfound = true;
+ nodeMeta = meta;
+ nodeName = key;
+ break;
+ }
+ }
+ }
+ if (!matchfound)
+ {
+ // throw error
+ throw new Exception("No Matching Tensor found in InputOutputMetadata corresponding to the serliazed tensor loaded from "+ filename);
+ }
+ }
+ }
+ else
+ {
+ // throw error
+ throw new Exception("While reading the serliazed tensor loaded from " + filename + ", metaDataDict has 0 elements");
+ }
+
+ Assert.True(nodeMeta.IsTensor, "LoadTensorFromFile can load Tensor types only");
+ //TODO: support other types when models are available in Onnx model zoo/ test data
+
+ Assert.Equal(tensorElemType, nodeMeta.ElementType);
+ Assert.Equal(nodeMeta.Dimensions.Length, tensor.Dims.Count);
+ for (int i = 0; i < nodeMeta.Dimensions.Length; i++)
+ {
+ Assert.True((nodeMeta.Dimensions[i] == -1) || (nodeMeta.Dimensions[i] == intDims[i]));
+ }
+
+ if (nodeMeta.ElementType == typeof(float))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(float), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(double))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(double), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(int))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(int), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(uint))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(uint), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(long))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(long), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(ulong))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(ulong), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(short))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(short), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(ushort))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(ushort), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(byte))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(byte), intDims);
+ }
+ else if (nodeMeta.ElementType == typeof(bool))
+ {
+ return CreateNamedOnnxValueFromRawData(nodeName, tensor.RawData.ToArray(), sizeof(bool), intDims);
+ }
+ else
+ {
+ //TODO: Add support for remaining types
+ Assert.True(false, "Tensors of type "+nameof(nodeMeta.ElementType)+" not currently supported in the LoadTensorFromFile");
+ return null;
+ }
+ }
+
+ static NamedOnnxValue CreateNamedOnnxValueFromRawData(string name, byte[] rawData, int elemWidth, int[] dimensions)
+ {
+ T[] floatArr = new T[rawData.Length / elemWidth];
+ Buffer.BlockCopy(rawData, 0, floatArr, 0, rawData.Length);
+ var dt = new DenseTensor(floatArr,dimensions);
+ return NamedOnnxValue.CreateFromTensor(name, dt);
}
static Tuple, float[]> OpenSessionSqueezeNet(int? cudaDeviceId = null)
@@ -980,6 +1227,19 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
+ class ExactComparer : IEqualityComparer
+ {
+ public bool Equals(T x, T y)
+ {
+ return x.Equals(y);
+ }
+ public int GetHashCode(T x)
+ {
+ return 0;
+ }
+ }
+
+
private class GpuFact : FactAttribute
{
public GpuFact()
@@ -992,15 +1252,5 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
}
- private class x64Fact : FactAttribute
- {
- public x64Fact()
- {
- if (System.Environment.Is64BitProcess == false)
- {
- Skip = "Not 64-bit process";
- }
- }
- }
}
}