From 0d672e9112f5cba3feabecc6d9cedaf6f42c810f Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 12 Oct 2022 13:53:58 +0800 Subject: [PATCH] Enable C# test load models with more complex directories. (#13251) ### Description Currently, C# test only load models with the directory structure as `{modelroot}->{opsetXX}->{modelname}->{.onnx}` In this PR, C# test can load models from `{modelroot}->{model-source}->{opsetXX}->{modelname}->{.onnx}` ### Motivation and Context There're multiple sources of testing models. 1. model zoo (Not in official image) 2. 1st party models 3. models with contrib-ops 4. others. It'd better to insert a mid-directory for new sources. **This PR is compatible with current models.** From https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=776643&view=logs&j=6df8fe70-7b8f-505a-8ef0-8bf93da2bac7&t=e7d9f128-b630-5ee6-a99e-2fca70d04619&l=79 the test result is same as master build `Passed: 583, Skipped: 14, Total: 597` **model zoo models (mounted in ..\models\zoo) could be loaded** And from this test workflow, it can load both existing models and models from model zoo. https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=777018&view=logs&j=6df8fe70-7b8f-505a-8ef0-8bf93da2bac7&t=e7d9f128-b630-5ee6-a99e-2fca70d04619 Skipping failed models will be in other PRs --- .../InferenceTest.netcore.cs | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index f538daf8dc..161802084d 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -213,6 +213,11 @@ namespace Microsoft.ML.OnnxRuntime.Tests } #endif + private static Func> getOpsetDirectories = delegate (DirectoryInfo modelsDirInfo) + { + return modelsDirInfo.EnumerateDirectories("opset*", new EnumerationOptions(){RecurseSubdirectories = true}); + }; + private static Dictionary GetSkippedModels(DirectoryInfo modelsDirInfo) { var skipModels = new Dictionary() { @@ -229,7 +234,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests { "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" }, - { "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" }, + { "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" }, { "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" }, { "mask_rcnn_keras", "Model should be edited to remove the extra outputs" }, { "test_strnormalizer_export_monday_casesensintive_lower", "ElementType not currently supported"}, @@ -371,7 +376,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests var isMlOpsDisabled = (disableMlOpsEnvVar != null) ? disableMlOpsEnvVar.Equals("ON") : false; if (isMlOpsDisabled) { - foreach (var opsetDir in modelsDirInfo.EnumerateDirectories()) + foreach (var opsetDir in getOpsetDirectories(modelsDirInfo)) { foreach (var modelDir in opsetDir.EnumerateDirectories()) { @@ -413,14 +418,14 @@ namespace Microsoft.ML.OnnxRuntime.Tests var modelsDirInfo = new DirectoryInfo(modelsDir); var skipModels = GetSkippedModels(modelsDirInfo); - foreach (var opsetDir in modelsDirInfo.EnumerateDirectories()) + foreach (var opsetDir in getOpsetDirectories(modelsDirInfo)) { //var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name)); foreach (var modelDir in opsetDir.EnumerateDirectories()) { if (!skipModels.ContainsKey(modelDir.Name)) { - yield return new object[] { modelDir.Parent.Name, modelDir.Name }; + yield return new object[] { modelDir.Parent.FullName, modelDir.Name }; } } //model } //opset @@ -432,15 +437,14 @@ namespace Microsoft.ML.OnnxRuntime.Tests var modelsDirInfo = new DirectoryInfo(modelsDir); var skipModels = GetSkippedModels(modelsDirInfo); - foreach (var opsetDir in modelsDirInfo.EnumerateDirectories()) + foreach (var opsetDir in getOpsetDirectories(modelsDirInfo)) { - var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name)); - foreach (var modelDir in modelRoot.EnumerateDirectories()) + foreach (var modelDir in opsetDir.EnumerateDirectories()) { if (skipModels.ContainsKey(modelDir.Name)) { //Console.WriteLine("Model {0} is skipped due to the error: {1}", modelDir.FullName, skipModels[modelDir.Name]); - yield return new object[] { modelDir.Parent.Name, modelDir.Name }; + yield return new object[] { modelDir.Parent.FullName, modelDir.Name }; } } @@ -450,12 +454,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests [Theory(DisplayName = "TestPreTrainedModels")] [MemberData(nameof(GetModelsForTest))] [MemberData(nameof(GetSkippedModelForTest), Skip = "Skipped due to Error, please fix the error and enable the test")] - private void TestPreTrainedModels(string opset, string modelName) + private void TestPreTrainedModels(string opsetDir, string modelName) { - var modelsDir = GetTestModelsDir(); + var opsetDirInfo = new DirectoryInfo(opsetDir); + var opset = opsetDirInfo.Name; string onnxModelFileName = null; - var modelDir = new DirectoryInfo(Path.Combine(modelsDir, opset, modelName)); + var modelDir = new DirectoryInfo(Path.Combine(opsetDir, modelName)); try {