onnxruntime/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs

using Microsoft.ML.OnnxRuntime.Tensors;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Xunit;

#if NET8_0_OR_GREATER
using SystemNumericsTensors = System.Numerics.Tensors;
#endif

namespace Microsoft.ML.OnnxRuntime.Tests
{
    /// <summary>
    /// This is compensate for the absence of string.Contains() in .NET Standard 2.0
    /// Contains(String, StringComparison)
    /// </summary>
    public static class StringExtensions
    {
        public static bool Contains(this String str, String substring,
                                    StringComparison comp)
        {
            if (substring == null)
                throw new ArgumentNullException("substring",
                                             "substring cannot be null.");
            else if (!Enum.IsDefined(typeof(StringComparison), comp))
                throw new ArgumentException("comp is not a member of StringComparison",
                                         "comp");

            return str.IndexOf(substring, comp) >= 0;
        }
    }

    public partial class InferenceTest
    {
        private const string module = "onnxruntime.dll";
        private const string propertiesFile = "Properties.txt";

        [Fact(DisplayName = "CanCreateAndDisposeSessionWithModelPath")]
        public void CanCreateAndDisposeSessionWithModelPath()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
            using (var session = new InferenceSession(modelPath))
            {
                Assert.NotNull(session);
                Assert.NotNull(session.InputMetadata);
                Assert.Single(session.InputMetadata); // 1 input nodeMeta
                Assert.True(session.InputMetadata.ContainsKey("data_0")); // input nodeMeta name
                Assert.Equal(typeof(float), session.InputMetadata["data_0"].ElementType);
                Assert.True(session.InputMetadata["data_0"].IsTensor);
                var expectedInputDimensions = new int[] { 1, 3, 224, 224 };
                Assert.Equal(expectedInputDimensions.Length, session.InputMetadata["data_0"].Dimensions.Length);
                for (int i = 0; i < expectedInputDimensions.Length; i++)
                {
                    Assert.Equal(expectedInputDimensions[i], session.InputMetadata["data_0"].Dimensions[i]);
                }

                Assert.NotNull(session.OutputMetadata);
                Assert.Single(session.OutputMetadata); // 1 output nodeMeta
                Assert.True(session.OutputMetadata.ContainsKey("softmaxout_1")); // output nodeMeta name
                Assert.Equal(typeof(float), session.OutputMetadata["softmaxout_1"].ElementType);
                Assert.True(session.OutputMetadata["softmaxout_1"].IsTensor);
                var expectedOutputDimensions = new int[] { 1, 1000, 1, 1 };
                Assert.Equal(expectedOutputDimensions.Length, session.OutputMetadata["softmaxout_1"].Dimensions.Length);
                for (int i = 0; i < expectedOutputDimensions.Length; i++)
                {
                    Assert.Equal(expectedOutputDimensions[i], session.OutputMetadata["softmaxout_1"].Dimensions[i]);
                }
            }
        }

#if NET8_0_OR_GREATER
#pragma warning disable SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback
        [Theory]
        [InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, true)]
        [InlineData(GraphOptimizationLevel.ORT_DISABLE_ALL, false)]
        [InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, true)]
        [InlineData(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, false)]
        private void CanRunInferenceOnAModelDotnetTensors(GraphOptimizationLevel graphOptimizationLevel, bool enableParallelExecution)
        {
            var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx");

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                // Set the graph optimization level for this session.
                SessionOptions options = new SessionOptions();
                cleanUp.Add(options);
                options.GraphOptimizationLevel = graphOptimizationLevel;

                var session = new InferenceSession(model, options);
                cleanUp.Add(session);

                using var runOptions = new RunOptions();
                var inputMeta = session.InputMetadata;
                var outputMeta = session.OutputMetadata;

                float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out");
                long[] expectedDimensions = { 1, 1000, 1, 1 };  // hardcoded for now for the test data
                ReadOnlySpan<long> expectedOutputDimensions = expectedDimensions;

                float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model

                using var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count);

                foreach (var name in inputMeta.Keys)
                {
                    Assert.Equal(typeof(float), inputMeta[name].ElementType);
                    Assert.True(inputMeta[name].IsTensor);
                    var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, inputMeta[name].Dimensions.Select(x => (nint)x).ToArray());
                    inputOrtValues.Add(new DisposableTestPair<OrtValue>(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));

                }

                runOptions.LogId = "CsharpTest";
                runOptions.Terminate = false;  // TODO: Test terminate = true, it currently crashes
                runOptions.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR;
                // Run inference with named inputs and outputs created with in Run()
                using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List<string>(["softmaxout_1"])))  // results is an IDisposableReadOnlyCollection<OrtValue> container
                {
                    // validate the results
                    foreach (var r in results)
                    {
                        Assert.Single(results);

                        ValidateRunResult(r, expectedOutput, expectedDimensions);
                    }
                }
            }
        }

        [Fact]
        public void InferenceSessionDisposedDotnetTensors()
        {
            var model = TestDataLoader.LoadModelFromEmbeddedResource("squeezenet.onnx");

            // Set the graph optimization level for this session.
            using (SessionOptions options = new SessionOptions())
            {
                options.ProfileOutputPathPrefix = "Ort_P_";
                options.EnableProfiling = true;
                using (var session = new InferenceSession(model, options))
                {
                    var inputMeta = session.InputMetadata;
                    var container = new List<NamedOnnxValue>();

                    float[] inputData = TestDataLoader.LoadTensorFromEmbeddedResource("bench.in"); // this is the data for only one input tensor for this model

                    using (var runOptions = new RunOptions())
                    using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
                    using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
                    {
                        
                        foreach (var name in inputMeta.Keys)
                        {
                            Assert.Equal(typeof(float), inputMeta[name].ElementType);
                            Assert.True(inputMeta[name].IsTensor);
                            var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, inputMeta[name].Dimensions.Select(x => (nint) x).ToArray());
                            inputOrtValues.Add(new DisposableTestPair<OrtValue>(name, OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
                        }
                        
                        // Run inference with named inputs and outputs created with in Run()
                        using (var results = session.Run(runOptions, inputOrtValues.Select(x => x.Key).ToList(), inputOrtValues.Select(x => x.Value).ToList(), new List<string>(["softmaxout_1"])))  // results is an IDisposableReadOnlyCollection<OrtValue> container
                        {
                            // validate the results
                            foreach (var r in results)
                            {
                                Assert.Single(results);

                                float[] expectedOutput = TestDataLoader.LoadTensorFromEmbeddedResource("bench.expected_out");
                                long[] expectedDimensions = { 1, 1000, 1, 1 };  // hardcoded for now for the test data
                                ValidateRunResult(r, expectedOutput, expectedDimensions);
                            }
                        }
                    }

                    string profile_file = session.EndProfiling();

                    // Profile file should have the output path prefix in it
                    Assert.Contains("Ort_P_", profile_file);
                }
            }
        }

        [Fact]
        private void ThrowWrongOutputNameDotnetTensors()
        {
            var tuple = OpenSessionSqueezeNet();
            var session = tuple.Item1;
            var inputData = tuple.Item2;
            var inputTensor = tuple.Item3;

            using (var runOptions = new RunOptions())
            using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
            using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
            {
                var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));

                inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
                outputOrtValues.Add(new DisposableTestPair<OrtValue>("bad_output_name", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(tensor)));

                var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["bad_output_name"], [outputOrtValues[0].Value]));
                Assert.Contains("Output name: 'bad_output_name' is not in the metadata", ex.Message);
            }

            session.Dispose();
        }

        [Fact]
        private void ThrowWrongOutputDimensionDotnetTensors()
        {
            var tuple = OpenSessionSqueezeNet();
            var session = tuple.Item1;
            var inputData = tuple.Item2;
            var inputTensor = tuple.Item3;
            var outputTensor = SystemNumericsTensors.Tensor.Create<float>([1, 1001, 1, 1]);

            using (var runOptions = new RunOptions())
            using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
            using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
            {
                var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));

                inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
                outputOrtValues.Add(new DisposableTestPair<OrtValue>("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor)));
                
                var ex = Assert.Throws<OnnxRuntimeException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], [outputOrtValues[0].Value]));
            }

            session.Dispose();
        }

        [Fact]
        private void ThrowInconsistentPinnedOutputsDotnetTensors()
        {
            var tuple = OpenSessionSqueezeNet();
            using var cleanUp = new DisposableListTest<IDisposable>();
            cleanUp.Add(tuple.Item1);
            var session = tuple.Item1;
            var inputData = tuple.Item2;
            var inputTensor = tuple.Item3;
            var outputTensor = SystemNumericsTensors.Tensor.Create([1, 1001, 1, 1], [4]);

            using (var runOptions = new RunOptions())
            using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
            using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
            {
                var tensor = SystemNumericsTensors.Tensor.Create<float>(inputData, Array.ConvertAll<int, nint>(inputTensor.Dimensions.ToArray(), x => (nint)x));

                inputOrtValues.Add(new DisposableTestPair<OrtValue>("data_0", OrtValue.CreateTensorValueFromSystemNumericsTensorObject<float>(tensor)));
                outputOrtValues.Add(new DisposableTestPair<OrtValue>("softmaxout_1", OrtValue.CreateTensorValueFromSystemNumericsTensorObject(outputTensor)));
                OrtValue[] outputs = [];
                var ex = Assert.Throws<ArgumentException>(() => session.Run(runOptions, ["data_0"], [inputOrtValues[0].Value], ["softmaxout_1"], outputs));
                Assert.StartsWith("Length of outputNames (1) must match that of outputValues (0).", ex.Message);
            }
        }
#pragma warning restore SYSLIB5001 // System.Numerics.Tensors is only in preview so we can continue receiving API feedback
#endif


#if USE_CUDA
        [Fact(DisplayName = "TestCUDAProviderOptions")]
        private void TestCUDAProviderOptions()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");

            string defaultDeviceId = "0";
            string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId");
            if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0)
            {
                defaultDeviceId = deviceIdFromEnv;
                output.WriteLine($"Parsed ID: {deviceIdFromEnv}");
            }

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                var cudaProviderOptions = new OrtCUDAProviderOptions();
                cleanUp.Add(cudaProviderOptions);

                var providerOptionsDict = new Dictionary<string, string>();
                providerOptionsDict["device_id"] = defaultDeviceId;
                // 256MB
                providerOptionsDict["gpu_mem_limit"] = "268435456";
                providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested";
                providerOptionsDict["cudnn_conv_algo_search"] = "DEFAULT";
                providerOptionsDict["do_copy_in_default_stream"] = "1";
                providerOptionsDict["cudnn_conv_use_max_workspace"] = "1";
                providerOptionsDict["cudnn_conv1d_pad_to_nc1d"] = "1";
                cudaProviderOptions.UpdateOptions(providerOptionsDict);

                var resultProviderOptionsDict = new Dictionary<string, string>();
                ProviderOptionsValueHelper.StringToDict(cudaProviderOptions.GetOptions(), resultProviderOptionsDict);

                // test provider options configuration
                string value;
                value = resultProviderOptionsDict["device_id"];
                Assert.Equal("0", value);
                value = resultProviderOptionsDict["gpu_mem_limit"];
                Assert.Equal("268435456", value);
                value = resultProviderOptionsDict["arena_extend_strategy"];
                Assert.Equal("kSameAsRequested", value);
                value = resultProviderOptionsDict["cudnn_conv_algo_search"];
                Assert.Equal("DEFAULT", value);
                value = resultProviderOptionsDict["do_copy_in_default_stream"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["cudnn_conv_use_max_workspace"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["cudnn_conv1d_pad_to_nc1d"];
                Assert.Equal("1", value);

                // test correctness of provider options
                SessionOptions options = SessionOptions.MakeSessionOptionWithCudaProvider(cudaProviderOptions);
                cleanUp.Add(options);

                var session = new InferenceSession(modelPath, options);
                cleanUp.Add(session);

                var inputMeta = session.InputMetadata;
                var container = new List<NamedOnnxValue>();
                float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
                foreach (var name in inputMeta.Keys)
                {
                    Assert.Equal(typeof(float), inputMeta[name].ElementType);
                    Assert.True(inputMeta[name].IsTensor);
                    var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
                    container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
                }

                session.Run(container);
            }
        }
#endif

#if USE_TENSORRT
        [Fact(DisplayName = "CanRunInferenceOnAModelWithTensorRT")]
        private void CanRunInferenceOnAModelWithTensorRT()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");

            int deviceId = 0;
            string deviceIdStr = System.Environment.GetEnvironmentVariable("ONNXRUNTIME_TEST_GPU_DEVICE_ID");
            if (!string.IsNullOrEmpty(deviceIdStr) && int.TryParse(deviceIdStr, out int parsedValue) && parsedValue >= 0)
            {
                deviceId = parsedValue;
                output.WriteLine($"Parsed ID: {parsedValue}");
            }

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(deviceId);
                cleanUp.Add(options);

                var session = new InferenceSession(modelPath, options);
                cleanUp.Add(session);

                var inputMeta = session.InputMetadata;
                var container = new List<NamedOnnxValue>();
                float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
                foreach (var name in inputMeta.Keys)
                {
                    Assert.Equal(typeof(float), inputMeta[name].ElementType);
                    Assert.True(inputMeta[name].IsTensor);
                    var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
                    container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
                }


                using (var results = session.Run(container))
                {
                    ValidateRunResults(results);
                }
            }
        }

        [Fact(DisplayName = "TestTensorRTProviderOptions")]
        private void TestTensorRTProviderOptions()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
            string calTablePath = "squeezenet_calibration.flatbuffers";
            string enginePath = "./";
            string engineDecrptLibPath = "engine_decryp";
            string defaultDeviceId = "0";
            string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId");
            if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0)
            {
                defaultDeviceId = deviceIdFromEnv;
                output.WriteLine($"Parsed ID: {deviceIdFromEnv}");
            }

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                var trtProviderOptions = new OrtTensorRTProviderOptions();
                cleanUp.Add(trtProviderOptions);

                var providerOptionsDict = new Dictionary<string, string>();
                providerOptionsDict["device_id"] = defaultDeviceId;
                providerOptionsDict["trt_fp16_enable"] = "1";
                providerOptionsDict["trt_int8_enable"] = "1";
                providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath;
                providerOptionsDict["trt_engine_cache_enable"] = "1";
                providerOptionsDict["trt_engine_cache_path"] = enginePath;
                providerOptionsDict["trt_engine_decryption_enable"] = "0";
                providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath;
                trtProviderOptions.UpdateOptions(providerOptionsDict);

                var resultProviderOptionsDict = new Dictionary<string, string>();
                ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict);

                // test provider options configuration
                string value;
                value = resultProviderOptionsDict["device_id"];
                Assert.Equal(defaultDeviceId, value);
                value = resultProviderOptionsDict["trt_fp16_enable"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["trt_int8_enable"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["trt_int8_calibration_table_name"];
                Assert.Equal(calTablePath, value);
                value = resultProviderOptionsDict["trt_engine_cache_enable"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["trt_engine_cache_path"];
                Assert.Equal(enginePath, value);
                value = resultProviderOptionsDict["trt_engine_decryption_enable"];
                Assert.Equal("0", value);
                value = resultProviderOptionsDict["trt_engine_decryption_lib_path"];
                Assert.Equal(engineDecrptLibPath, value);

                // test correctness of provider options
                SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions);
                cleanUp.Add(options);

                var session = new InferenceSession(modelPath, options);
                cleanUp.Add(session);

                var inputMeta = session.InputMetadata;
                var container = new List<NamedOnnxValue>();
                float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
                foreach (var name in inputMeta.Keys)
                {
                    Assert.Equal(typeof(float), inputMeta[name].ElementType);
                    Assert.True(inputMeta[name].IsTensor);
                    var tensor = new DenseTensor<float>(inputData, inputMeta[name].Dimensions);
                    container.Add(NamedOnnxValue.CreateFromTensor<float>(name, tensor));
                }

                session.Run(container);
            }
        }
#endif

        private static Func<DirectoryInfo, IEnumerable<DirectoryInfo>> getOpsetDirectories = delegate (DirectoryInfo modelsDirInfo)
        {
            return modelsDirInfo.EnumerateDirectories("opset*", SearchOption.AllDirectories);
        };

        private static Dictionary<string, string> GetSkippedModels(DirectoryInfo modelsDirInfo)
        {
            var skipModels = new Dictionary<string, string>() {
                { "mxnet_arcface", "Model is an invalid ONNX model"},
                { "tf_inception_v2", "TODO: Debug failing model, skipping for now" },
                { "fp16_tiny_yolov2", "Tolerance level for float16 is not known. We now support fp16." },
                { "fp16_test_tiny_yolov2", "ImageScaler is not a registered function/op"},
                { "fp16_coreml_FNS-Candy", "ImageScaler is not a registered function/op" },
                { "fp16_coreml_LinearRegression_NYCTaxi", "Error in Node:featureVectorizer : No Op registered for FeatureVectorizer with domain_version of 1"},
                { "test_mnist", "Does not run in opset9, runs in other opsets. The model runs but I don't have a data set to debug output locally. Tensors of type ElementType not currently supported in the LoadTensorFromFile" },
                { "BERT_Squad", "Could not find an implementation for the nodeMeta bert / embeddings / one_hot:OneHot(9)" },

                { "mlperf_ssd_mobilenet_300", "Could not find file output_0.pb" },
                { "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" },
                { "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" },
                { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" },
                { "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" },
                { "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" },
                { "mask_rcnn_keras", "Model should be edited to remove the extra outputs" },

                { "test_maxunpool_export_with_output_shape", "results mismatch"},

                { "test_min_int8", "Could not find an implementation for Min(13) node with name"},
                { "test_min_uint8", "Could not find an implementation for Min(13) node with name"},
                { "test_min_int16", "Could not find an implementation for Min(13) node with name"},
                { "test_min_uint16", "Could not find an implementation for Min(13) node with name"},

                { "test_max_int8", "Could not find an implementation for Max(13) node with name"},
                { "test_max_uint8", "Could not find an implementation for Max(13) node with name"},
                { "test_max_int16", "Could not find an implementation for Max(13) node with name"},
                { "test_max_uint16", "Could not find an implementation for Max(13) nodeMeta with name '"},

                { "test_mul_uint8", "Could not find an implementation for Mul(14) node with name" },

                { "test_bitshift_right_uint16", "Could not find an implementation for BitShift(11) nodeMeta with name ''"},
                { "test_bitshift_left_uint16", "Could not find an implementation for BitShift(11)"},

                { "test_pow_types_float32_uint64", "Could not find an implementation for Pow(15) node with name ''"},
                { "test_pow_types_float32_uint32", "Could not find an implementation for Pow(15) node with name ''"},

                { "test_resize_downsample_scales_cubic_align_corners", "Results mismatch"},
                { "test_resize_downsample_scales_linear_align_corners", "Results mismatch"},

                { "test_gru_batchwise", "batchwise operations not supported"},
                { "test_lstm_batchwise", "Batchwise recurrent operations(layout == 1) are not supported.If you need support create a github issue with justification."},
                { "test_simple_rnn_batchwise", "batchwise operations not supported"},
                { "test_batchnorm_example_training_mode", "opset14 version not implemented yet"},

                { "test_bernoulli", "random generator, results mismatch"},
                { "test_bernoulli_seed", "random generator, results mismatch"},
                { "test_bernoulli_double", "random generator, results mismatch"},
                { "test_bernoulli_expanded", "random generator, results mismatch"},
                { "test_bernoulli_seed_expanded", "random generator, results mismatch"},
                { "test_bernoulli_double_expanded", "random generator, results mismatch"},

                // the expansion of Softplus uses Exp(1). ORT has a Softplus kernel, so testing the expansion is
                // unnecessary and fails as ORT support for Exp started at opset 6 (as ORT didn't exist until opset 7).

                { "test_clip_default_int8_max_expanded", "Could not find an implementation for Less(13) nodeMeta with name ''" },
                { "test_softplus_expanded", "Could not find an implementation for Exp(1) node with name ''"},
                { "test_softplus_example_expanded", "Could not find an implementation for Exp(1) node with name ''"},
                { "test_div_uint8", "Could not find an implementation for Div(14) nodeMeta with name ''"},
                { "test_add_uint8", "Opset18 Could not find an implementation for Add(14) nodeMeta with name ''"},
                { "test_col2im_pads", "Results mismatch due to a typo in test data"},

                { "test_optional_has_element_empty_optional_input", "OptionalProto test metadata. Unable to load 'optional_input' optional element type of: Undefined type"},
                { "test_loop13_seq", "3rd input is an empty sequence. Ort API does not tolerate empty seq: Number of values should be at least 1" },

                // Training tests
                { "BERT-Squad-int8", "training domain"},
                { "YOLOv3-12-int8", "training_domain"},

                { "test_training_dropout_default", "results mismatch"},
                { "test_training_dropout_default_mask", "Results mismatch"},
                { "test_training_dropout", "results mismatch"},
                { "test_training_dropout_mask", "results mismatch."},

                { "test_momentum", "ai.onnx.preview.training:Momentum(-1) is not a registered function/op"},
                { "test_momentum_multiple", "ai.onnx.preview.training:Momentum(-1) is not a registered function/op"},
                { "test_nesterov_momentum", "ai.onnx.preview.training:Momentum(-1) is not a registered function/op"},

                { "test_adam", "ai.onnx.preview.training:Adam(-1) is not a registered function/op"},
                { "test_adam_multiple", "ai.onnx.preview.training:Adam(-1) is not a registered function/op"},

                { "test_adagrad", "ai.onnx.preview.training:Adagrad(-1) is not a registered function/op"},
                { "test_adagrad_multiple", "ai.onnx.preview.training:Adagrad(-1) is not a registered function/op"},

                { "test_zfnet512", "skip it as ZFNET-512"},
            };

            // The following models fails on nocontribops win CI
            var disableContribOpsEnvVar = Environment.GetEnvironmentVariable("DisableContribOps");
            var isContribOpsDisabled = (disableContribOpsEnvVar != null) ? disableContribOpsEnvVar.Equals("ON") : false;
            if (isContribOpsDisabled)
            {
                skipModels["test_tiny_yolov2"] = "Fails when ContribOps is disabled";
                skipModels["mask_rcnn_keras"] = "Pad is not a registered function/op";
            }

            // Skip traditional ML models
            var disableMlOpsEnvVar = Environment.GetEnvironmentVariable("DisableMlOps");
            var isMlOpsDisabled = (disableMlOpsEnvVar != null) ? disableMlOpsEnvVar.Equals("ON") : false;
            if (isMlOpsDisabled)
            {
                foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
                {
                    foreach (var modelDir in opsetDir.EnumerateDirectories())
                    {
                        var modelDirName = modelDir.Name;
                        if (modelDirName.StartsWith("scikit_") ||
                        modelDirName.StartsWith("libsvm_") ||
                        modelDirName.StartsWith("coreml_") ||
                        modelDirName.StartsWith("keras2coreml_") ||
                        modelDirName.StartsWith("XGBoost_"))
                        {
                            skipModels[modelDirName] = "Fails when ML ops are disabled";
                        }
                    } //model
                } //opset
            }

            // This model fails on x86 Win CI
            if (System.Environment.Is64BitProcess == false)
            {
                skipModels["test_vgg19"] = "Get preallocated buffer for initializer conv4_4_b_0 failed";
                skipModels["GPT2_LM_HEAD"] = "System out of memory";
                skipModels["GPT2"] = "System out of memory";
                skipModels["test_GPT2"] = "System out of memory";
                skipModels["tf_pnasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_5/comb_iter_1/left/bn_sep_7x7_1/beta:0_203 failed";
                skipModels["tf_nasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_11/beginning_bn/beta:0_331 failed";
                skipModels["ZFNet-512"] = "System out of memory";
                skipModels["test_bvlc_reference_caffenet"] = "System out of memory";
                skipModels["coreml_VGG16_ImageNet"] = "System out of memory";
                skipModels["test_ssd"] = "System out of memory";
                skipModels["roberta_sequence_classification"] = "System out of memory";
                // models from model zoo
                skipModels["VGG 19"] = "bad allocation";
                skipModels["VGG 19-caffe2"] = "bad allocation";
                skipModels["VGG 19-bn"] = "bad allocation";
                skipModels["VGG 16"] = "bad allocation";
                skipModels["VGG 16-bn"] = "bad allocation";
                skipModels["VGG 16-fp32"] = "bad allocation";
            }

            return skipModels;
        }

        public static IEnumerable<object[]> GetModelsForTest()
        {
            var modelsDir = GetTestModelsDir();
            var modelsDirInfo = new DirectoryInfo(modelsDir);
            var skipModels = GetSkippedModels(modelsDirInfo);

            foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
            {
                //var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
                foreach (var modelDir in opsetDir.EnumerateDirectories())
                {
                    if (!(skipModels.ContainsKey(modelDir.Name) ||
                          modelDir.Name.Contains("int8", StringComparison.OrdinalIgnoreCase) ||
                          modelDir.Name.Contains("qdq", StringComparison.OrdinalIgnoreCase)))
                    {
                        yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
                    }
                } //model
            } //opset
        }

        public static IEnumerable<object[]> GetSkippedModelForTest()
        {
            var modelsDir = GetTestModelsDir();
            var modelsDirInfo = new DirectoryInfo(modelsDir);
            var skipModels = GetSkippedModels(modelsDirInfo);

            foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
            {
                foreach (var modelDir in opsetDir.EnumerateDirectories())
                {
                    if (skipModels.ContainsKey(modelDir.Name) ||
                        modelDir.Name.Contains("int8", StringComparison.OrdinalIgnoreCase) ||
                        modelDir.Name.Contains("qdq", StringComparison.OrdinalIgnoreCase))
                    {
                        //Console.WriteLine("Model {0} is skipped due to the error: {1}", modelDir.FullName, skipModels[modelDir.Name]);
                        yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
                    }

                }
            }
        }

        private string MatchInputOutputWithFile(string fileName, InferenceSession session, bool input, out NodeMetadata result)
        {
            string nodeName = string.Empty;
            result = null;
            var names = (input) ? session.InputNames : session.OutputNames;
            var metadata = (input) ? session.InputMetadata : session.OutputMetadata;
            string regEx = (input) ? @"input_(\d{1,}).pb" : @"output_(\d{1,}).pb";
            var inpOut = (input) ? "input" : "output";

            // Extract the number from the file name, if not try to match the input/output name with the name of the file.
            try
            {
                // captures start at index 1
                var group = Regex.Matches(fileName, regEx).Single().Groups[1];
                var num = int.Parse(group.Value);
                if (num >= 0 && num < names.Count)
                {
                    nodeName = names[num];
                    result = metadata[nodeName];
                }
                else
                {
                    throw new InvalidDataException($"Filename '{fileName}' {inpOut} number '{num}' is out of range for '{names.Count}' {inpOut}(s)");
                }
            }
            catch (Exception)
            {
                // Either does not match or can not parse the number
            }

            if (result is null)
            {
                throw new InvalidDataException($"Unable to match file: {fileName} to input/output metadata");
            }
            return nodeName;
        }

        // The numbering of the input files does not match the order of outputs
        // listed in the metadata of test_BERT_Squad. Model metadata order:
        //                 "unique_ids_raw_output___9:0", "segment_ids:0", "input_mask:0", "input_ids:0"
        // The corr input files are: input_0.pb, input_3.pb, input_2.pb, input_1.pb
        // Everything in reverse, but the 0.

        // Previously, it worked because our test data has matching
        // tensor names that we could match to metadata after we load the tensor.
        // But now, we need to know ahead of time what Onnx type we load, and thus match
        // metadata with the test data file before loading. Protobuf can happily load whatever
        // and give you garbage.

        private string MatchBertSquadInputs(string fileName)
        {
            string nodeName = string.Empty;
            switch (fileName)
            {
                case "input_0.pb":
                    nodeName = "unique_ids_raw_output___9:0";
                    break;
                case "input_1.pb":
                    nodeName = "input_ids:0";
                    break;
                case "input_2.pb":
                    nodeName = "input_mask:0";
                    break;
                case "input_3.pb":
                    nodeName = "segment_ids:0";
                    break;
                default:
                    throw new InvalidDataException($"Unhandled input file name: '{fileName}' for test_BERT_Squad");
            }
            return nodeName;
        }

        // The model actually has only 3 outputs, but the Zoo version has 4 files are supplied.
        // The numbering of the output files does not match the order of outputs
        // listed in the metadata.

        // Previously, it worked because our CI test data version has matching
        // tensor names that we could match to metadata after we load the tensor.
        // But now, we need to know ahead of time what Onnx type we load, and thus match
        // metadata with the test data file before loading. Protobuf can happily load whatever
        // and give you garbage.

        // Order in the metadata: unstack:1, unstack:0, unique_ids:0
        // The files are in reverse order
        private string MatchBertSquadOutputs(string fileName)
        {
            string nodeName = string.Empty;
            switch (fileName)
            {
                case "output_0.pb": // Int64
                    nodeName = "unique_ids:0";
                    break;
                case "output_1.pb":
                    nodeName = "unstack:0";
                    break;
                case "output_2.pb":
                    nodeName = "unstack:1";
                    break;
                default:
                    throw new InvalidDataException($"Unhandled output file name: '{fileName}' for test_BERT_Squad");
            }
            return nodeName;
        }

        private const string keras_prelu_ImageNet_small_nodeName_Input = "p_re_lu_3_input";
        private const string keras_prelu_ImageNet_small_nodeName_Output = "p_re_lu_3/add:0";

        private void LoadInputData<T>(string opset, string modelName,
            DirectoryInfo testDataDir,
            InferenceSession session,
            IList<T> inputContainer,
            Func<string, string, NodeMetadata, T> loader)
        {
            var inMeta = session.InputMetadata;
            foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
            {
                if (modelName == "keras_prelu_ImageNet_small" && opset == "opset9")
                {
                    // The model has 1 input, match all file names (they are different in each data set)
                    // to the same input
                    var nodeName = keras_prelu_ImageNet_small_nodeName_Input;
                    var nodeMeta = inMeta[nodeName];
                    inputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
                else if (modelName == "test_BERT_Squad" && opset == "opset8")
                {
                    string nodeName = MatchBertSquadInputs(f.Name);
                    var nodeMeta = inMeta[nodeName];
                    inputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
                else
                {
                    var nodeName = MatchInputOutputWithFile(f.Name, session, true, out NodeMetadata nodeMeta);
                    inputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
            }
        }

        private void LoadOutputData<T>(string opset, string modelName,
                                                DirectoryInfo testDataDir,
                                                InferenceSession session,
                                                IList<T> outputContainer,
                                                Func<string, string, NodeMetadata, T> loader)
        {
            var outMeta = session.OutputMetadata;
            foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
            {
                if (modelName == "keras_prelu_ImageNet_small" && opset == "opset9")
                {
                    // The model has 1 output, match all file names (they are different in each data set)
                    // to the same output
                    var nodeName = keras_prelu_ImageNet_small_nodeName_Output;
                    var nodeMeta = outMeta[nodeName];
                    outputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
                else if (modelName == "test_BERT_Squad" && opset == "opset8")
                {
                    string nodeName = MatchBertSquadOutputs(f.Name);
                    var nodeMeta = outMeta[nodeName];
                    outputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
                else
                {
                    // Otherwise, just match trailing filename number to the input name -> metadata
                    var nodeName = MatchInputOutputWithFile(f.Name, session, false, out NodeMetadata nodeMeta);
                    outputContainer.Add(loader(f.FullName, nodeName, nodeMeta));
                }
            }
        }

        private void RunPretrainedModel(InferenceSession session,
                     IReadOnlyList<NamedOnnxValue> inputContainer, IReadOnlyList<NamedOnnxValue> outputContainer)
        {
            var outMeta = session.OutputMetadata;

            var orderedOutputNames = new List<string>(outputContainer.Count);
            foreach (var output in outputContainer)
            {
                orderedOutputNames.Add(output.Name);
            }

            using (var resultCollection = session.Run(inputContainer, orderedOutputNames))
            {
                Assert.Equal(outputContainer.Count, resultCollection.Count);
                for (int i = 0; i < resultCollection.Count; ++i)
                {
                    var result = resultCollection[i];
                    var outputValue = outputContainer[i];

                    Assert.NotNull(outputValue);
                    Assert.Equal(result.Name, outputValue.Name);

                    var outputMeta = outMeta[outputValue.Name];
                    if (outputMeta.OnnxValueType == OnnxValueType.ONNX_TYPE_OPTIONAL)
                    {
                        outputMeta = outputMeta.AsOptionalMetadata().ElementMeta;
                    }

                    Assert.Equal(outputValue.ValueType, outputMeta.OnnxValueType);

                    switch (outputValue.ValueType)
                    {
                        case OnnxValueType.ONNX_TYPE_TENSOR:  // Only Dense tensors now
                            {
                                VerifyTensorResults(outputMeta.ElementDataType, result, outputValue);
                            }
                            break;
                        case OnnxValueType.ONNX_TYPE_SEQUENCE:
                            {
                                VerifySequenceResults(result, outputValue, outputMeta);
                            }
                            break;
                        default:
                            Assert.Fail($"TestPreTrainedModels cannot handle Onnxtype: {outputValue.ValueType}");
                            break;
                    }
                }
            }
        }

        private void RunPretrainedModel(InferenceSession session, RunOptions runOptions,
                     IReadOnlyList<DisposableTestPair<OrtValue>> inputContainer,
                     IReadOnlyList<DisposableTestPair<OrtValue>> outputContainer)
        {
            var outMeta = session.OutputMetadata;

            var orderedInputNames = new List<string>(inputContainer.Count);
            var orderdedInputs = new List<OrtValue>(inputContainer.Count);
            foreach (var pair in inputContainer)
            {
                orderedInputNames.Add(pair.Key);
                orderdedInputs.Add(pair.Value);
            }

            var orderedOutputNames = new List<string>(outputContainer.Count);
            var orderedOutputs = new List<OrtValue>(outputContainer.Count);
            foreach (var pair in outputContainer)
            {
                orderedOutputNames.Add(pair.Key);
                orderedOutputs.Add(pair.Value);
            }

            using (var results = session.Run(runOptions, orderedInputNames, orderdedInputs, orderedOutputNames))
            {
                Assert.Equal(outMeta.Count, results.Count);
                Assert.Equal(outputContainer.Count, results.Count);

                for (int i = 0; i < outputContainer.Count; ++i)
                {
                    var resultValue = results[i];
                    var expectedValue = outputContainer[i].Value;

                    var outputMeta = outMeta[orderedOutputNames[i]];
                    if (outputMeta.OnnxValueType == OnnxValueType.ONNX_TYPE_OPTIONAL)
                    {
                        outputMeta = outputMeta.AsOptionalMetadata().ElementMeta;
                    }

                    if (outputMeta.OnnxValueType == OnnxValueType.ONNX_TYPE_TENSOR)
                    {
                        VerifyTensorResults(outputMeta.ElementDataType, resultValue, expectedValue);
                    }
                    else if (outputMeta.OnnxValueType == OnnxValueType.ONNX_TYPE_SEQUENCE)
                    {
                        VerifySequenceResults(resultValue, expectedValue, outputMeta);
                    }
                    else
                    {
                        Assert.Fail($"TestPreTrainedModels cannot handle Onnxtype: {outputMeta.OnnxValueType}");
                    }
                }
            }
        }

        [Theory(DisplayName = "TestPretrainedModelsWithOrtValue")]
        [MemberData(nameof(GetModelsForTest))]
        [MemberData(nameof(GetSkippedModelForTest), Skip = "Skipped due to Error, please fix the error and enable the test")]
        public void TestPretrainedModelsWithOrtValue(string opsetDir, string modelName)
        {
            TestPreTrainedModels(opsetDir, modelName, true);
        }

        [Theory(DisplayName = "TestPreTrainedModels")]
        [MemberData(nameof(GetModelsForTest))]
        [MemberData(nameof(GetSkippedModelForTest), Skip = "Skipped due to Error, please fix the error and enable the test")]
        private void TestPreTrainedModels(string opsetDir, string modelName, bool useOrtValueAPIs = false)
        {
            var opsetDirInfo = new DirectoryInfo(opsetDir);
            var opset = opsetDirInfo.Name;
            string onnxModelFileName = null;

            var modelDir = new DirectoryInfo(Path.Combine(opsetDir, modelName));

            try
            {
                var onnxModelNames = modelDir.GetFiles("*.onnx");
                bool validModelFound = false;
                if (onnxModelNames.Length > 0)
                {
                    // TODO remove file "._resnet34v2.onnx" from test set
                    for (int i = 0; i < onnxModelNames.Length; i++)
                    {
                        if (onnxModelNames[i].Name != "._resnet34v2.onnx")
                        {
                            onnxModelNames[0] = onnxModelNames[i];
                            validModelFound = true;
                        }
                    }
                }

                if (validModelFound)
                {
                    onnxModelFileName = Path.Combine(modelDir.FullName, onnxModelNames[0].Name);
                }
                else
                {
                    var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
                    throw new Exception($"Opset {opset} Model {modelName}. Can't determine model file name. Found these :{modelNamesList}");
                }

                using (var runOptions = new RunOptions())
                using (var session = new InferenceSession(onnxModelFileName))
                {
                    string testDataDirNamePattern = "test_data*";
                    if (opset == "opset9" && modelName == "LSTM_Seq_lens_unpacked")
                    {
                        testDataDirNamePattern = "seq_lens*"; // discrepancy in data directory
                    }
                    foreach (var testDataDir in modelDir.EnumerateDirectories(testDataDirNamePattern))
                    {
                        if (useOrtValueAPIs)
                        {
                            using (var inputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.InputMetadata.Count))
                            using (var outputOrtValues = new DisposableListTest<DisposableTestPair<OrtValue>>(session.OutputMetadata.Count))
                            {
                                LoadInputData(opset, modelName, testDataDir, session, inputOrtValues, TestDataLoader.LoadOrtValueFromFilePb);
                                LoadOutputData(opset, modelName, testDataDir, session, outputOrtValues, TestDataLoader.LoadOrtValueFromFilePb);
                                RunPretrainedModel(session, runOptions, inputOrtValues, outputOrtValues);
                            }
                        }
                        else
                        {
                            var inputContainer = new List<NamedOnnxValue>(session.InputMetadata.Count);
                            LoadInputData(opset, modelName, testDataDir, session, inputContainer, TestDataLoader.LoadOnnxValueFromFilePb);
                            var outputContainer = new List<NamedOnnxValue>(session.OutputMetadata.Count);
                            LoadOutputData(opset, modelName, testDataDir, session, outputContainer, TestDataLoader.LoadOnnxValueFromFilePb);
                            RunPretrainedModel(session, inputContainer, outputContainer);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                var msg = $"Opset {opset}, Model {modelName}: ModelFile = {onnxModelFileName} error = {ex.Message}";
                if (ex.Message.Contains("ONNX Runtime only *guarantees* support for models stamped with official released onnx opset versions"))
                {
                    // If the exception is thrown because the opset version of the test model is
                    // not supported by ONNXRuntime yet, then ignore the test and proceed.
                    // ORT allows commits from ONNX master and in such cases we do come across new opsets which are
                    // not supported in ORT yet. In order to force these tests to run set env var ALLOW_RELEASED_ONNX_OPSET_ONLY=0
                    output.WriteLine("Skipping the model test as the latest ONNX opset is not supported yet. Error Message: " + msg);
                }
                else
                {
                    throw new Exception(msg + "\n" + ex.StackTrace);
                }
            }
        }

        private static void VerifySequenceResults(NamedOnnxValue result, NamedOnnxValue expectedValue, NodeMetadata metaData)
        {
            var meta = metaData.AsSequenceMetadata();
            var resultSequence = result.AsEnumerable<NamedOnnxValue>();
            var expectedSequence = expectedValue.AsEnumerable<NamedOnnxValue>();
            Assert.Equal(resultSequence.Count(), expectedSequence.Count());

            foreach (var (resultItem, expectedItem) in resultSequence.Zip(expectedSequence, (r, e) => (r, e)))
            {
                Assert.Equal(resultItem.ValueType, expectedItem.ValueType);
                Assert.Equal(resultItem.ValueType, meta.ElementMeta.OnnxValueType);
                switch (resultItem.ValueType)
                {
                    case OnnxValueType.ONNX_TYPE_TENSOR:
                        VerifyTensorResults(meta.ElementMeta.ElementDataType, resultItem, expectedItem);
                        break;
                    case OnnxValueType.ONNX_TYPE_SEQUENCE:
                        {
                            VerifySequenceResults(resultItem, expectedItem, meta.ElementMeta);
                        }
                        break;
                    default:
                        Assert.Fail("VerifySequenceResults cannot handle Onnxtype: " + resultItem.ValueType.ToString());
                        break;
                }
                Assert.Equal(resultItem.AsTensor<float>(), expectedItem.AsTensor<float>(), new FloatComparer());
            }
        }

        private static void VerifyTensorResults(TensorElementType elementType, NamedOnnxValue result, NamedOnnxValue expectedValue)
        {
            switch (elementType)
            {
                case TensorElementType.Float:
                    Assert.Equal(expectedValue.AsTensor<float>(), result.AsTensor<float>(), new FloatComparer());
                    break;
                case TensorElementType.Double:
                    Assert.Equal(expectedValue.AsTensor<double>(), result.AsTensor<double>(), new DoubleComparer());
                    break;
                case TensorElementType.Int32:
                    Assert.Equal(expectedValue.AsTensor<int>(), result.AsTensor<int>(), new ExactComparer<int>());
                    break;
                case TensorElementType.UInt32:
                    Assert.Equal(expectedValue.AsTensor<uint>(), result.AsTensor<uint>(), new ExactComparer<uint>());
                    break;
                case TensorElementType.Int16:
                    Assert.Equal(expectedValue.AsTensor<short>(), result.AsTensor<short>(), new ExactComparer<short>());
                    break;
                case TensorElementType.UInt16:
                    Assert.Equal(expectedValue.AsTensor<ushort>(), result.AsTensor<ushort>(), new ExactComparer<ushort>());
                    break;
                case TensorElementType.Int64:
                    Assert.Equal(expectedValue.AsTensor<long>(), result.AsTensor<long>(), new ExactComparer<long>());
                    break;
                case TensorElementType.UInt64:
                    Assert.Equal(expectedValue.AsTensor<ulong>(), result.AsTensor<ulong>(), new ExactComparer<ulong>());
                    break;
                case TensorElementType.UInt8:
                    Assert.Equal(expectedValue.AsTensor<byte>(), result.AsTensor<byte>(), new ExactComparer<byte>());
                    break;
                case TensorElementType.Int8:
                    Assert.Equal(result.AsTensor<sbyte>(), result.AsTensor<sbyte>(), new ExactComparer<sbyte>());
                    break;
                case TensorElementType.Bool:
                    Assert.Equal(expectedValue.AsTensor<bool>(), result.AsTensor<bool>(), new ExactComparer<bool>());
                    break;
                case TensorElementType.Float16:
                    Assert.Equal(expectedValue.AsTensor<Float16>(), result.AsTensor<Float16>(), new Float16Comparer { tolerance = 2 });
                    break;
                case TensorElementType.BFloat16:
                    Assert.Equal(expectedValue.AsTensor<BFloat16>(), result.AsTensor<BFloat16>(), new BFloat16Comparer { tolerance = 2 });
                    break;
                case TensorElementType.String:
                    Assert.Equal(expectedValue.AsTensor<string>(), result.AsTensor<string>(), new ExactComparer<string>());
                    break;
                default:
                    Assert.Fail("TestPreTrainedModels does not yet support output of type: " + elementType.ToString());
                    break;
            }
        }

        private static void VerifySequenceResults(OrtValue resultSequence, OrtValue expectedSequence, NodeMetadata metaData)
        {
            var allocator = OrtAllocator.DefaultInstance;
            Assert.Equal(OnnxValueType.ONNX_TYPE_SEQUENCE, resultSequence.OnnxType);
            Assert.Equal(OnnxValueType.ONNX_TYPE_SEQUENCE, expectedSequence.OnnxType);

            var elementMeta = metaData.AsSequenceMetadata().ElementMeta;

            var resultCount = resultSequence.GetValueCount();
            Assert.Equal(expectedSequence.GetValueCount(), resultCount);

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                for (int i = 0; i < resultCount; ++i)
                {
                    var resultItem = resultSequence.GetValue(i, allocator);
                    cleanUp.Add(resultItem);

                    var expectedItem = expectedSequence.GetValue(i, allocator);
                    cleanUp.Add(expectedItem);

                    Assert.Equal(elementMeta.OnnxValueType, expectedItem.OnnxType);
                    Assert.Equal(elementMeta.OnnxValueType, resultItem.OnnxType);

                    switch (elementMeta.OnnxValueType)
                    {
                        case OnnxValueType.ONNX_TYPE_TENSOR:
                            VerifyTensorResults(elementMeta.ElementDataType, resultItem, expectedItem);
                            break;
                        case OnnxValueType.ONNX_TYPE_SEQUENCE:
                            {
                                VerifySequenceResults(resultItem, expectedItem, elementMeta);
                            }
                            break;
                        default:
                            Assert.Fail($"VerifySequenceResults cannot handle Onnxtype: {elementMeta.OnnxValueType}");
                            break;
                    }
                }
            }
        }

        private static void VerifyTensorResults(TensorElementType expectedElementType, OrtValue result, OrtValue expectedValue)
        {
            Assert.True(result.IsTensor);
            Assert.True(expectedValue.IsTensor);

            var resultTypeShape = result.GetTensorTypeAndShape();
            var expectedTypeShape = expectedValue.GetTensorTypeAndShape();
            Assert.Equal(expectedElementType, resultTypeShape.ElementDataType);
            Assert.Equal(expectedElementType, expectedTypeShape.ElementDataType);
            Assert.Equal(expectedTypeShape.Shape, resultTypeShape.Shape);

            if (expectedElementType == TensorElementType.String)
            {
                var resStrings = result.GetStringTensorAsArray();
                var expStrings = expectedValue.GetStringTensorAsArray();
                Assert.Equal(expStrings, resStrings);
                return;
            }

            switch (expectedElementType)
            {
                case TensorElementType.Float:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<float>().ToArray(), result.GetTensorDataAsSpan<float>().ToArray(),
                        new FloatComparer());
                    break;
                case TensorElementType.Double:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<double>().ToArray(), result.GetTensorDataAsSpan<double>().ToArray(),
                        new DoubleComparer());
                    break;
                case TensorElementType.Int32:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<int>().ToArray(), result.GetTensorDataAsSpan<int>().ToArray(), new ExactComparer<int>());
                    break;
                case TensorElementType.UInt32:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<uint>().ToArray(), result.GetTensorDataAsSpan<uint>().ToArray(), new ExactComparer<uint>());
                    break;
                case TensorElementType.Int16:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<short>().ToArray(), result.GetTensorDataAsSpan<short>().ToArray(), new ExactComparer<short>());
                    break;
                case TensorElementType.UInt16:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<ushort>().ToArray(), result.GetTensorDataAsSpan<ushort>().ToArray(), new ExactComparer<ushort>());
                    break;
                case TensorElementType.Int64:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<long>().ToArray(), result.GetTensorDataAsSpan<long>().ToArray(), new ExactComparer<long>());
                    break;
                case TensorElementType.UInt64:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<ulong>().ToArray(), result.GetTensorDataAsSpan<ulong>().ToArray(), new ExactComparer<ulong>());
                    break;
                case TensorElementType.UInt8:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<byte>().ToArray(), result.GetTensorDataAsSpan<byte>().ToArray(), new ExactComparer<byte>());
                    break;
                case TensorElementType.Int8:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<sbyte>().ToArray(), result.GetTensorDataAsSpan<sbyte>().ToArray(), new ExactComparer<sbyte>());
                    break;
                case TensorElementType.Bool:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<bool>().ToArray(), result.GetTensorDataAsSpan<bool>().ToArray(), new ExactComparer<bool>());
                    break;
                case TensorElementType.Float16:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<Float16>().ToArray(), result.GetTensorDataAsSpan<Float16>().ToArray(),
                        new Float16Comparer { tolerance = 2 });
                    break;
                case TensorElementType.BFloat16:
                    Assert.Equal(expectedValue.GetTensorDataAsSpan<BFloat16>().ToArray(), result.GetTensorDataAsSpan<BFloat16>().ToArray(),
                                               new BFloat16Comparer { tolerance = 2 });
                    break;
                default:
                    Assert.Fail("VerifyTensorResults cannot handle ElementType: " + expectedElementType.ToString());
                    break;
            }
        }

        private static void VerifyContainerContent(IReadOnlyList<OrtValue> results,
            IReadOnlyList<NamedOnnxValue> expectedValues)
        {
            Assert.Equal(results.Count, expectedValues.Count);

            for (int i = 0; i < expectedValues.Count; ++i)
            {
                var result = results[i];

                var resultTypeShape = result.GetTensorTypeAndShape();

                var expectedValue = expectedValues[i];
                Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, expectedValue.ValueType);

                switch (resultTypeShape.ElementDataType)
                {
                    case TensorElementType.Float:
                        Assert.Equal(result.GetTensorDataAsSpan<float>().ToArray(), expectedValue.AsTensor<float>().ToArray(),
                            new ExactComparer<float>());
                        break;
                    case TensorElementType.Double:
                        Assert.Equal(result.GetTensorDataAsSpan<double>().ToArray(), expectedValue.AsTensor<double>().ToArray(),
                            new DoubleComparer());
                        break;
                    case TensorElementType.Int32:
                        Assert.Equal(result.GetTensorDataAsSpan<int>().ToArray(), expectedValue.AsTensor<int>().ToArray(), new ExactComparer<int>());
                        break;
                    case TensorElementType.UInt32:
                        Assert.Equal(result.GetTensorDataAsSpan<uint>().ToArray(), expectedValue.AsTensor<uint>().ToArray(), new ExactComparer<uint>());
                        break;
                    case TensorElementType.Int16:
                        Assert.Equal(result.GetTensorDataAsSpan<short>().ToArray(), expectedValue.AsTensor<short>().ToArray(), new ExactComparer<short>());
                        break;
                    case TensorElementType.UInt16:
                        Assert.Equal(result.GetTensorDataAsSpan<ushort>().ToArray(), expectedValue.AsTensor<ushort>().ToArray(), new ExactComparer<ushort>());
                        break;
                    case TensorElementType.Int64:
                        Assert.Equal(result.GetTensorDataAsSpan<long>().ToArray(), expectedValue.AsTensor<long>().ToArray(), new ExactComparer<long>());
                        break;
                    case TensorElementType.UInt64:
                        Assert.Equal(result.GetTensorDataAsSpan<ulong>().ToArray(), expectedValue.AsTensor<ulong>().ToArray(), new ExactComparer<ulong>());
                        break;
                    case TensorElementType.UInt8:
                        Assert.Equal(result.GetTensorDataAsSpan<byte>().ToArray(), expectedValue.AsTensor<byte>().ToArray(), new ExactComparer<byte>());
                        break;
                    case TensorElementType.Int8:
                        Assert.Equal(result.GetTensorDataAsSpan<sbyte>().ToArray(), expectedValue.AsTensor<sbyte>().ToArray(), new ExactComparer<sbyte>());
                        break;
                    case TensorElementType.Bool:
                        Assert.Equal(result.GetTensorDataAsSpan<bool>().ToArray(), expectedValue.AsTensor<bool>().ToArray(), new ExactComparer<bool>());
                        break;
                    case TensorElementType.Float16:
                        Assert.Equal(result.GetTensorDataAsSpan<Float16>().ToArray(), expectedValue.AsTensor<Float16>().ToArray(),
                            new Float16Comparer { tolerance = 2 });
                        break;
                    case TensorElementType.BFloat16:
                        Assert.Equal(result.GetTensorDataAsSpan<BFloat16>().ToArray(), expectedValue.AsTensor<BFloat16>().ToArray(),
                                                   new BFloat16Comparer { tolerance = 2 });
                        break;
                    case TensorElementType.String:
                        Assert.Equal(result.GetStringTensorAsArray(), expectedValue.AsTensor<string>().ToArray(), new ExactComparer<string>());
                        break;
                    default:
                        Assert.Fail($"VerifyTensorResults cannot handle ElementType: {resultTypeShape.ElementDataType}");
                        break;
                }
            }
        }

        // Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
        private void UnloadLibrary(IntPtr libraryHandle)
        {
            if (libraryHandle != IntPtr.Zero)
            {
                if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                {
                    if (!FreeLibrary(libraryHandle))
                    {
                        throw new Exception("Could not unload the provided shared library using its handle");
                    }
                }

                else
                {
                    // TODO: Deal with non-Windows platforms for the .NET Core use-case
                }
            }
        }

        private string GetCustomOpLibFullPath()
        {
            string libName = "custom_op_library.dll";
            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
            {
                libName = "custom_op_library.dll";
            }
            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                libName = "libcustom_op_library.so";
            }
            else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
            {
                libName = "libcustom_op_library.dylib";
            }

            string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName);
            Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist.");

            return libFullPath;
        }

        private void ValidateModelWithCustomOps(SessionOptions options)
        {
            string modelPath = "custom_op_test.onnx";

            using (var session = new InferenceSession(modelPath, options))
            {
                var inputContainer = new List<NamedOnnxValue>();
                inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_1",
                    new DenseTensor<float>(
                        new float[]
                        {
                                1.1f,   2.2f,   3.3f,   4.4f,   5.5f,
                                6.6f,   7.7f,   8.8f,   9.9f,   10.0f,
                                11.1f,  12.2f,  13.3f,  14.4f,  15.5f
                        },
                        new int[] { 3, 5 }
                        )));

                inputContainer.Add(NamedOnnxValue.CreateFromTensor<float>("input_2",
                    new DenseTensor<float>(
                        new float[]
                        {
                                15.5f,   14.4f,   13.3f,   12.2f,   11.1f,
                                10.0f,   9.9f,    8.8f,    7.7f,    6.6f,
                                5.5f,    4.4f,    3.3f,    2.2f,    1.1f
                        },
                        new int[] { 3, 5 }
                        )));

                using (var result = session.Run(inputContainer))
                {
                    Assert.Equal("output", result.First().Name);
                    var tensorOut = result.First().AsTensor<int>();

                    var expectedOut = new DenseTensor<int>(
                        new int[]
                        {
                                17, 17, 17, 17, 17,
                                17, 18, 18, 18, 17,
                                17, 17, 17, 17, 17
                        },
                        new int[] { 3, 5 }
                        );
                    Assert.True(tensorOut.SequenceEqual(expectedOut));
                }
            }
        }

        [SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibrary")]
        private void TestRegisterCustomOpLibrary()
        {
            using (var option = new SessionOptions())
            {
                string libFullPath = GetCustomOpLibFullPath();

                try
                {
                    option.RegisterCustomOpLibrary(libFullPath);
                }
                catch (Exception ex)
                {
                    var msg = $"Failed to load custom op library {libFullPath}, error = {ex.Message}";
                    throw new Exception(msg + "\n" + ex.StackTrace);
                }

                var ortEnvInstance = OrtEnv.Instance();
                string[] providers = ortEnvInstance.GetAvailableProviders();
                if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider"))
                {
                    option.AppendExecutionProvider_CUDA(0);
                }

                ValidateModelWithCustomOps(option);
            }
        }

        [SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibraryV2")]
        private void TestRegisterCustomOpLibraryV2()
        {
            using (var option = new SessionOptions())
            {
                string libFullPath = GetCustomOpLibFullPath();

                var ortEnvInstance = OrtEnv.Instance();
                string[] providers = ortEnvInstance.GetAvailableProviders();
                if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider"))
                {
                    option.AppendExecutionProvider_CUDA(0);
                }

                IntPtr libraryHandle = IntPtr.Zero;
                try
                {
                    option.RegisterCustomOpLibraryV2(libFullPath, out libraryHandle);
                }
                catch (Exception ex)
                {
                    var msg = $"Failed to load custom op library {libFullPath}, error = {ex.Message}";
                    throw new Exception(msg + "\n" + ex.StackTrace);
                }

                ValidateModelWithCustomOps(option);

                // Safe to unload the custom op shared library now
                UnloadLibrary(libraryHandle);
            }
        }

        [Fact(DisplayName = "TestModelSerialization")]
        private void TestModelSerialization()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
            string modelOutputPath = Path.Combine(Directory.GetCurrentDirectory(), "optimized-squeezenet.onnx");
            // Set the optimized model file path to assert that no exception are thrown.
            using (SessionOptions options = new SessionOptions())
            {
                options.OptimizedModelFilePath = modelOutputPath;
                options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_BASIC;
                using (var session = new InferenceSession(modelPath, options))
                {
                    Assert.NotNull(session);
                    Assert.True(File.Exists(modelOutputPath));
                }
            }
        }

        private static OrtLoraAdapter CreateLoraAdapterFromFile()
        {
            var adapterPath = Path.Combine(Directory.GetCurrentDirectory(), "two_params_lora_model.onnx_adapter");
            return OrtLoraAdapter.Create(adapterPath, null);
        }

        private static OrtLoraAdapter CreateLoraAdapterFromArray()
        {
            var adapterPath = Path.Combine(Directory.GetCurrentDirectory(), "two_params_lora_model.onnx_adapter");
            var adapterBytes = File.ReadAllBytes(adapterPath);
            return OrtLoraAdapter.Create(adapterBytes, null);
        }

        // See tests below for running with Lora Adapters
        [Fact(DisplayName = "TestInferenceWithBaseLoraModel")]
        private void TestInferenceWithBaseLoraModel()
        {
            var modelPath = Path.Combine(Directory.GetCurrentDirectory(), "two_params_lora_model.onnx");

            var inputShape = new long[] { 4, 4 };
            var inputData = new float[16];
            Array.Fill(inputData, 1);
            using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(inputData, inputShape);

            var expectedOutput = new float[] {
                    28, 32, 36, 40,
                    28, 32, 36, 40,
                    28, 32, 36, 40,
                    28, 32, 36, 40 };

            using var session = new InferenceSession(modelPath);
            using var runOptions = new RunOptions();

            using var outputs = session.Run(runOptions, ["input_x"], [inputOrtValue], ["output"]);
            Assert.Single(outputs);
            var output = outputs[0].GetTensorDataAsSpan<float>();
            Assert.Equal(expectedOutput.Length, output.Length);
            Assert.Equal(expectedOutput, output.ToArray(), new FloatComparer());
        }


        private static void TestInferenceWithLoraAdapter(OrtLoraAdapter ortLoraAdapter)
        {
            var modelPath = Path.Combine(Directory.GetCurrentDirectory(), "two_params_lora_model.onnx");
            var adapterPath = Path.Combine(Directory.GetCurrentDirectory(), "two_params_lora_model.onnx_adapter");

            var inputShape = new long[] { 4, 4 };
            var inputData = new float[16];
            Array.Fill(inputData, 1);
            using var inputOrtValue = OrtValue.CreateTensorValueFromMemory(inputData, inputShape);

            var expectedOutput = new float[] {
                  154, 176, 198, 220,
                  154, 176, 198, 220,
                  154, 176, 198, 220,
                  154, 176, 198, 220 };

            using var session = new InferenceSession(modelPath);
            using var runOptions = new RunOptions();
            runOptions.AddActiveLoraAdapter(ortLoraAdapter);

            using var outputs = session.Run(runOptions, ["input_x"], [inputOrtValue], ["output"]);
            Assert.Single(outputs);
            var output = outputs[0].GetTensorDataAsSpan<float>();
            Assert.Equal(expectedOutput.Length, output.Length);
            Assert.Equal(expectedOutput, output.ToArray(), new FloatComparer());
        }

        [Fact(DisplayName = "TestInferenceWithLoraAdapterFromFile")]
        private void TestInferenceWithLoraAdapterFromFile()
        {
            using var ortAdapter = CreateLoraAdapterFromFile();
            TestInferenceWithLoraAdapter(ortAdapter);
        }

        [Fact(DisplayName = "TestInferenceWithLoraAdapterFromArray")]
        private void TestInferenceWithLoraAdapterFromArray()
        {
            using var ortAdapter = CreateLoraAdapterFromArray();
            TestInferenceWithLoraAdapter(ortAdapter);
        }

        // TestGpu() will test
        //  - the CUDA EP on CUDA enabled builds
        //  - the DML EP on DML enabled builds
        //  - the ROCm EP on ROCm enabled builds
        [GpuFact(DisplayName = "TestGpu")]
        private void TestGpu()
        {
            var tuple = OpenSessionSqueezeNet(0); // run on deviceID 0
            float[] expectedOutput = TestDataLoader.LoadTensorFromFile(@"bench.expected_out");

            using (var session = tuple.Item1)
            {
                var inputData = tuple.Item2;
                var tensor = tuple.Item3;
                var inputMeta = session.InputMetadata;
                var container = new List<NamedOnnxValue>();
                container.Add(NamedOnnxValue.CreateFromTensor<float>("data_0", tensor));
                var res = session.Run(container);
                var resultArray = res.First().AsTensor<float>().ToArray();
                Assert.Equal(expectedOutput, resultArray, new FloatComparer());
            }
        }

        [DllImport("kernel32", SetLastError = true)]
        static extern IntPtr LoadLibrary(string lpFileName);

        [DllImport("kernel32", CharSet = CharSet.Ansi)]
        static extern UIntPtr GetProcAddress(IntPtr hModule, string procName);

        [DllImport("kernel32.dll", CharSet = CharSet.Ansi)]
        private static extern bool FreeLibrary(IntPtr hModule);

        [Fact(DisplayName = "VerifyNativeMethodsExist")]
        private void VerifyNativeMethodsExist()
        {
            // Check for  external API changes
            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                return;
            var entryPointNames = new[]{
            "OrtGetApiBase",
            "OrtSessionOptionsAppendExecutionProvider_CPU"
#if USE_DNNL
            ,"OrtSessionOptionsAppendExecutionProvider_Dnnl"
#endif
#if USE_CUDA
            ,"OrtSessionOptionsAppendExecutionProvider_CUDA"
#endif
#if USE_ROCM
            ,"OrtSessionOptionsAppendExecutionProvider_ROCM"
#endif
#if USE_DML
            ,"OrtSessionOptionsAppendExecutionProvider_DML"
#endif
#if USE_OPENVINO
            ,"OrtSessionOptionsAppendExecutionProvider_OpenVINO"
#endif
#if USE_TENSORRT
            ,"OrtSessionOptionsAppendExecutionProvider_Tensorrt"
#endif
#if USE_MIGRAPHX
            ,"OrtSessionOptionsAppendExecutionProvider_MIGraphX"
#endif
#if USE_NNAPI
            ,"OrtSessionOptionsAppendExecutionProvider_Nnapi"
#endif
    };
            IntPtr libraryHandle = IntPtr.Zero;
            try
            {
                libraryHandle = LoadLibrary(module);
                foreach (var ep in entryPointNames)
                {
                    var x = GetProcAddress(libraryHandle, ep);
                    Assert.False(x == UIntPtr.Zero, $"Entrypoint {ep} not found in module {module}");
                }
            }

            finally
            {
                UnloadLibrary(libraryHandle);
            }
        }

#if NET8_0_OR_GREATER
#pragma warning disable SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
        private void ValidateRunResultData(SystemNumericsTensors.Tensor<float> resultTensor, float[] expectedOutput, int[] expectedDimensions)
        {
            Assert.Equal(expectedDimensions.Length, resultTensor.Rank);

            var resultDimensions = resultTensor.Lengths;
            for (int i = 0; i < expectedDimensions.Length; i++)
            {
                Assert.Equal(expectedDimensions[i], resultDimensions[i]);
            }

            var resultArray = resultTensor.ToArray();
            Assert.Equal(expectedOutput.Length, resultArray.Length);
            Assert.Equal(expectedOutput, resultArray, new FloatComparer());
        }
#pragma warning restore SYSLIB5001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
#endif

        static string GetTestModelsDir()
        {
            // get build directory, append downloaded models location
            var cwd = Directory.GetCurrentDirectory();
            var props = File.ReadAllLines(Path.Combine(cwd, propertiesFile));
            var modelsRelDir = Path.Combine(props[0].Split('=')[1].Trim());
            var modelsDir = Path.Combine(cwd, @"../../..", modelsRelDir, "models");
            return modelsDir;
        }
    }
}