saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-14 20:48:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable C# test load models with more complex directories. (#13251)

Browse source 
### Description
Currently, C# test only load models with the directory structure as
`{modelroot}->{opsetXX}->{modelname}->{.onnx}`
In this PR, C# test can load models from
`{modelroot}->{model-source}->{opsetXX}->{modelname}->{.onnx}`

### Motivation and Context
There're multiple sources of testing models.
1. model zoo (Not in official image)
2. 1st party models
3.  models with contrib-ops
4.  others.

It'd better to insert a mid-directory for new sources.

**This PR is compatible with current models.**
From
https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=776643&view=logs&j=6df8fe70-7b8f-505a-8ef0-8bf93da2bac7&t=e7d9f128-b630-5ee6-a99e-2fca70d04619&l=79
the test result is same as master build `Passed: 583, Skipped: 14,
Total: 597`

**model zoo models (mounted in ..\models\zoo) could be loaded**
And from this test workflow, it can load both existing models and models
from model zoo.

https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=777018&view=logs&j=6df8fe70-7b8f-505a-8ef0-8bf93da2bac7&t=e7d9f128-b630-5ee6-a99e-2fca70d04619
Skipping failed models will be in other PRs
This commit is contained in:
Yi Zhang 2022-10-12 13:53:58 +08:00 committed by GitHub
parent 8a3407d54f
commit 0d672e9112
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs
View file

@ -213,6 +213,11 @@ namespace Microsoft.ML.OnnxRuntime.Tests
}
#endif
private static Func<DirectoryInfo, IEnumerable<DirectoryInfo>> getOpsetDirectories = delegate (DirectoryInfo modelsDirInfo)
{
return modelsDirInfo.EnumerateDirectories("opset*", new EnumerationOptions(){RecurseSubdirectories = true});
};
private static Dictionary<string, string> GetSkippedModels(DirectoryInfo modelsDirInfo)
{
var skipModels = new Dictionary<string, string>() {
@ -229,7 +234,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
{ "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" },
{ "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" },
{ "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" },
{ "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" },
{ "mask_rcnn_keras", "Model should be edited to remove the extra outputs" },
{ "test_strnormalizer_export_monday_casesensintive_lower", "ElementType not currently supported"},
@ -371,7 +376,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
var isMlOpsDisabled = (disableMlOpsEnvVar != null) ? disableMlOpsEnvVar.Equals("ON") : false;
if (isMlOpsDisabled)
{
foreach (var opsetDir in modelsDirInfo.EnumerateDirectories())
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
@ -413,14 +418,14 @@ namespace Microsoft.ML.OnnxRuntime.Tests
var modelsDirInfo = new DirectoryInfo(modelsDir);
var skipModels = GetSkippedModels(modelsDirInfo);
foreach (var opsetDir in modelsDirInfo.EnumerateDirectories())
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
//var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
if (!skipModels.ContainsKey(modelDir.Name))
{
yield return new object[] { modelDir.Parent.Name, modelDir.Name };
yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
}
} //model
} //opset
@ -432,15 +437,14 @@ namespace Microsoft.ML.OnnxRuntime.Tests
var modelsDirInfo = new DirectoryInfo(modelsDir);
var skipModels = GetSkippedModels(modelsDirInfo);
foreach (var opsetDir in modelsDirInfo.EnumerateDirectories())
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
foreach (var modelDir in modelRoot.EnumerateDirectories())
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
if (skipModels.ContainsKey(modelDir.Name))
{
//Console.WriteLine("Model {0} is skipped due to the error: {1}", modelDir.FullName, skipModels[modelDir.Name]);
yield return new object[] { modelDir.Parent.Name, modelDir.Name };
yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
}
}
@ -450,12 +454,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests
[Theory(DisplayName = "TestPreTrainedModels")]
[MemberData(nameof(GetModelsForTest))]
[MemberData(nameof(GetSkippedModelForTest), Skip = "Skipped due to Error, please fix the error and enable the test")]
private void TestPreTrainedModels(string opset, string modelName)
private void TestPreTrainedModels(string opsetDir, string modelName)
{
var modelsDir = GetTestModelsDir();
var opsetDirInfo = new DirectoryInfo(opsetDir);
var opset = opsetDirInfo.Name;
string onnxModelFileName = null;
var modelDir = new DirectoryInfo(Path.Combine(modelsDir, opset, modelName));
var modelDir = new DirectoryInfo(Path.Combine(opsetDir, modelName));
try
{