using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Microsoft.ML.OnnxRuntime.Tensors;
using Xunit;
namespace Microsoft.ML.OnnxRuntime.Tests
{
///
/// This is compensate for the absence of string.Contains() in .NET Standard 2.0
/// Contains(String, StringComparison)
///
public static class StringExtensions
{
public static bool Contains(this String str, String substring,
StringComparison comp)
{
if (substring == null)
throw new ArgumentNullException("substring",
"substring cannot be null.");
else if (!Enum.IsDefined(typeof(StringComparison), comp))
throw new ArgumentException("comp is not a member of StringComparison",
"comp");
return str.IndexOf(substring, comp) >= 0;
}
}
public partial class InferenceTest
{
private const string module = "onnxruntime.dll";
private const string propertiesFile = "Properties.txt";
[Fact(DisplayName = "CanCreateAndDisposeSessionWithModelPath")]
public void CanCreateAndDisposeSessionWithModelPath()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
using (var session = new InferenceSession(modelPath))
{
Assert.NotNull(session);
Assert.NotNull(session.InputMetadata);
Assert.Equal(1, session.InputMetadata.Count); // 1 input node
Assert.True(session.InputMetadata.ContainsKey("data_0")); // input node name
Assert.Equal(typeof(float), session.InputMetadata["data_0"].ElementType);
Assert.True(session.InputMetadata["data_0"].IsTensor);
var expectedInputDimensions = new int[] { 1, 3, 224, 224 };
Assert.Equal(expectedInputDimensions.Length, session.InputMetadata["data_0"].Dimensions.Length);
for (int i = 0; i < expectedInputDimensions.Length; i++)
{
Assert.Equal(expectedInputDimensions[i], session.InputMetadata["data_0"].Dimensions[i]);
}
Assert.NotNull(session.OutputMetadata);
Assert.Equal(1, session.OutputMetadata.Count); // 1 output node
Assert.True(session.OutputMetadata.ContainsKey("softmaxout_1")); // output node name
Assert.Equal(typeof(float), session.OutputMetadata["softmaxout_1"].ElementType);
Assert.True(session.OutputMetadata["softmaxout_1"].IsTensor);
var expectedOutputDimensions = new int[] { 1, 1000, 1, 1 };
Assert.Equal(expectedOutputDimensions.Length, session.OutputMetadata["softmaxout_1"].Dimensions.Length);
for (int i = 0; i < expectedOutputDimensions.Length; i++)
{
Assert.Equal(expectedOutputDimensions[i], session.OutputMetadata["softmaxout_1"].Dimensions[i]);
}
}
}
#if USE_CUDA
[Fact(DisplayName = "TestCUDAProviderOptions")]
private void TestCUDAProviderOptions()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
using (var cleanUp = new DisposableListTest())
{
var cudaProviderOptions = new OrtCUDAProviderOptions();
cleanUp.Add(cudaProviderOptions);
var providerOptionsDict = new Dictionary();
providerOptionsDict["device_id"] = "0";
providerOptionsDict["gpu_mem_limit"] = "20971520";
providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested";
providerOptionsDict["cudnn_conv_algo_search"] = "DEFAULT";
providerOptionsDict["do_copy_in_default_stream"] = "1";
providerOptionsDict["cudnn_conv_use_max_workspace"] = "1";
providerOptionsDict["cudnn_conv1d_pad_to_nc1d"] = "1";
cudaProviderOptions.UpdateOptions(providerOptionsDict);
var resultProviderOptionsDict = new Dictionary();
ProviderOptionsValueHelper.StringToDict(cudaProviderOptions.GetOptions(), resultProviderOptionsDict);
// test provider options configuration
string value;
value = resultProviderOptionsDict["device_id"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["gpu_mem_limit"];
Assert.Equal("20971520", value);
value = resultProviderOptionsDict["arena_extend_strategy"];
Assert.Equal("kSameAsRequested", value);
value = resultProviderOptionsDict["cudnn_conv_algo_search"];
Assert.Equal("DEFAULT", value);
value = resultProviderOptionsDict["do_copy_in_default_stream"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["cudnn_conv_use_max_workspace"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["cudnn_conv1d_pad_to_nc1d"];
Assert.Equal("1", value);
// test correctness of provider options
SessionOptions options = SessionOptions.MakeSessionOptionWithCudaProvider(cudaProviderOptions);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor(name, tensor));
}
session.Run(container);
}
}
#endif
#if USE_TENSORRT
[Fact(DisplayName = "CanRunInferenceOnAModelWithTensorRT")]
private void CanRunInferenceOnAModelWithTensorRT()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
using (var cleanUp = new DisposableListTest())
{
SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor(name, tensor));
}
using (var results = session.Run(container))
{
ValidateRunResults(results);
}
}
}
[Fact(DisplayName = "TestTensorRTProviderOptions")]
private void TestTensorRTProviderOptions()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
string calTablePath = "squeezenet_calibration.flatbuffers";
string enginePath = "./";
string engineDecrptLibPath = "engine_decryp";
using (var cleanUp = new DisposableListTest())
{
var trtProviderOptions = new OrtTensorRTProviderOptions();
cleanUp.Add(trtProviderOptions);
var providerOptionsDict = new Dictionary();
providerOptionsDict["device_id"] = "0";
providerOptionsDict["trt_fp16_enable"] = "1";
providerOptionsDict["trt_int8_enable"] = "1";
providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath;
providerOptionsDict["trt_engine_cache_enable"] = "1";
providerOptionsDict["trt_engine_cache_path"] = enginePath;
providerOptionsDict["trt_engine_decryption_enable"] = "0";
providerOptionsDict["trt_engine_decryption_lib_path"] = engineDecrptLibPath;
trtProviderOptions.UpdateOptions(providerOptionsDict);
var resultProviderOptionsDict = new Dictionary();
ProviderOptionsValueHelper.StringToDict(trtProviderOptions.GetOptions(), resultProviderOptionsDict);
// test provider options configuration
string value;
value = resultProviderOptionsDict["device_id"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["trt_fp16_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_int8_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_int8_calibration_table_name"];
Assert.Equal(calTablePath, value);
value = resultProviderOptionsDict["trt_engine_cache_enable"];
Assert.Equal("1", value);
value = resultProviderOptionsDict["trt_engine_cache_path"];
Assert.Equal(enginePath, value);
value = resultProviderOptionsDict["trt_engine_decryption_enable"];
Assert.Equal("0", value);
value = resultProviderOptionsDict["trt_engine_decryption_lib_path"];
Assert.Equal(engineDecrptLibPath, value);
// test correctness of provider options
SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(trtProviderOptions);
cleanUp.Add(options);
var session = new InferenceSession(modelPath, options);
cleanUp.Add(session);
var inputMeta = session.InputMetadata;
var container = new List();
float[] inputData = TestDataLoader.LoadTensorFromFile(@"bench.in"); // this is the data for only one input tensor for this model
foreach (var name in inputMeta.Keys)
{
Assert.Equal(typeof(float), inputMeta[name].ElementType);
Assert.True(inputMeta[name].IsTensor);
var tensor = new DenseTensor(inputData, inputMeta[name].Dimensions);
container.Add(NamedOnnxValue.CreateFromTensor(name, tensor));
}
session.Run(container);
}
}
#endif
private static Func> getOpsetDirectories = delegate (DirectoryInfo modelsDirInfo)
{
return modelsDirInfo.EnumerateDirectories("opset*", SearchOption.AllDirectories);
};
private static Dictionary GetSkippedModels(DirectoryInfo modelsDirInfo)
{
var skipModels = new Dictionary() {
{ "mxnet_arcface", "Model is an invalid ONNX model"},
{ "tf_inception_v2", "TODO: Debug failing model, skipping for now" },
{ "fp16_tiny_yolov2", "Tolerance level for float16 is not known. We now support fp16." },
{ "fp16_test_tiny_yolov2", "ImageScaler is not a registered function/op"},
{ "fp16_coreml_FNS-Candy", "ImageScaler is not a registered function/op" },
{ "fp16_coreml_LinearRegression_NYCTaxi", "Error in Node:featureVectorizer : No Op registered for FeatureVectorizer with domain_version of 1"},
{ "test_bidaf", "Does not run in opset9, runs in other opsets. The model runs but I don't have a data set to debug output locally. Tensors of type ElementType not currently supported in the LoadTensorFromFile." },
{ "test_mnist", "Does not run in opset9, runs in other opsets. The model runs but I don't have a data set to debug output locally. Tensors of type ElementType not currently supported in the LoadTensorFromFile" },
{ "BERT_Squad", "Could not find an implementation for the node bert / embeddings / one_hot:OneHot(9)" },
{ "mlperf_ssd_mobilenet_300", "Could not find file output_0.pb" },
{ "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" },
{ "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" },
{ "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" },
{ "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" },
{ "mask_rcnn_keras", "Model should be edited to remove the extra outputs" },
{ "test_strnormalizer_export_monday_casesensintive_lower", "ElementType not currently supported"},
{ "test_max_float64", "node test error"},
{ "test_min_uint8", "node test error"},
{ "test_mod_mixed_sign_float64", "node test error"},
{ "test_einsum_transpose", "node test error"},
{ "test_momentum", "node test error"},
{ "test_max_uint16", "node test error"},
{ "test_resize_downsample_scales_linear_align_corners", "node test error"},
{ "test_strnormalizer_nostopwords_nochangecase", "node test error"},
{ "test_cumsum_2d_negative_axis", "node test error"},
{ "test_adagrad_multiple", "node test error"},
{ "test_einsum_inner_prod", "node test error"},
{ "test_clip_default_int8_min", "node test error"},
{ "test_max_int8", "node test error"},
{ "test_sequence_insert_at_back", "node test error"},
{ "test_mod_mixed_sign_int8", "node test error"},
{ "test_maxunpool_export_with_output_shape", "node test error"},
{ "test_strnormalizer_export_monday_empty_output", "node test error"},
{ "test_strnormalizer_export_monday_insensintive_upper_twodim", "ElementType not currently supported"},
{ "test_clip_default_int8_max", "node test error"},
{ "test_einsum_sum", "node test error"},
{ "test_min_int16", "node test error"},
{ "test_adagrad", "node test error"},
{ "test_min_float64", "node test error"},
{ "test_max_int16", "node test error"},
{ "test_einsum_batch_diagonal", "node test error"},
{ "test_sequence_insert_at_front", "node test error"},
{ "test_cumsum_1d_exclusive", "node test error"},
{ "test_training_dropout_default", "node test error"},
{ "test_training_dropout", "node test error"},
{ "test_adam", "node test error"},
{ "test_training_dropout_mask", "node test error"},
{ "test_clip_default_int8_inbounds", "node test error"},
{ "test_eyelike_with_dtype", "node test error"},
{ "test_cumsum_1d", "node test error"},
{ "test_conv_with_autopad_same", "node test error"},
{ "test_cumsum_1d_reverse_exclusive", "node test error"},
{ "test_cast_STRING_to_FLOAT", "node test error"},
{ "test_cast_FLOAT16_to_DOUBLE", "node test error"},
{ "test_cast_FLOAT_to_DOUBLE", "node test error"},
{ "test_cast_BFLOAT16_to_FLOAT", "node test error"},
{ "test_cast_FLOAT_to_BFLOAT16", "node test error"},
{ "test_cast_FLOAT_to_STRING", "node test error"},
{ "test_castlike_STRING_to_FLOAT", "node test error"},
{ "test_castlike_STRING_to_FLOAT_expanded", "node test error"},
{ "test_castlike_FLOAT16_to_DOUBLE", "node test error"},
{ "test_castlike_FLOAT16_to_DOUBLE_expanded", "node test error"},
{ "test_castlike_FLOAT_to_DOUBLE", "node test error"},
{ "test_castlike_FLOAT_to_DOUBLE_expanded", "node test error"},
{ "test_castlike_BFLOAT16_to_FLOAT", "node test error"},
{ "test_castlike_BFLOAT16_to_FLOAT_expanded", "node test error"},
{ "test_castlike_FLOAT_to_BFLOAT16", "node test error"},
{ "test_castlike_FLOAT_to_BFLOAT16_expanded", "node test error"},
{ "test_castlike_FLOAT_to_STRING", "node test error"},
{ "test_castlike_FLOAT_to_STRING_expanded", "node test error"},
{ "test_bitshift_right_uint16", "node test error"},
{ "test_bitshift_left_uint16", "node test error"},
{ "test_pow_types_float32_uint64", "node test error"},
{ "test_cumsum_2d_axis_0", "node test error"},
{ "test_max_uint8", "node test error"},
{ "test_strnormalizer_export_monday_casesensintive_nochangecase", "ElementType not currently supported"},
{ "test_momentum_multiple", "node test error"},
{ "test_cumsum_1d_reverse", "node test error"},
{ "test_pow_types_float32_uint32", "node test error"},
{ "test_if_seq", "node test error"},
{ "test_resize_downsample_scales_cubic_align_corners", "node test error"},
{ "test_einsum_batch_matmul", "node test error"},
{ "test_nesterov_momentum", "node test error"},
{ "test_cumsum_2d_axis_1", "node test error"},
{ "test_strnormalizer_export_monday_casesensintive_upper", "node test error"},
{ "test_min_uint16", "node test error"},
{ "test_adam_multiple", "node test error"},
{ "test_loop13_seq", "node test error"},
{ "test_training_dropout_default_mask", "node test error"},
{ "test_min_int8", "node test error"},
{ "test_identity_sequence", "data type not supported"},
{ "test_gru_batchwise", "batchwise operations not supported"},
{ "test_lstm_batchwise", "batchwise operations not supported"},
{ "test_simple_rnn_batchwise", "batchwise operations not supported"},
{ "test_sub_uint8", "data type not supported"},
{ "test_mul_uint8", "data type not supported"},
{ "test_add_uint8", "data type not supported"},
{ "test_div_uint8", "data type not supported"},
{ "test_batchnorm_epsilon", "opset14 version not implemented yet"},
{ "test_batchnorm_epsilon_training_mode", "opset14 version not implemented yet"},
{ "test_batchnorm_example", "opset14 version not implemented yet"},
{ "test_batchnorm_example_training_mode", "opset14 version not implemented yet"},
{ "test_bernoulli", "random generator"},
{ "test_bernoulli_seed", "random generator"},
{ "test_bernoulli_double", "random generator"},
{ "test_bernoulli_expanded", "random generator"},
{ "test_bernoulli_seed_expanded", "random generator"},
{ "test_bernoulli_double_expanded", "random generator"},
{ "test_shape", "opset15 version not implemented yet"},
{ "test_shape_clip_end", "opset15 version not implemented yet"},
{ "test_shape_clip_start", "opset15 version not implemented yet"},
{ "test_shape_end_1", "opset15 version not implemented yet"},
{ "test_shape_end_negative", "opset15 version not implemented yet"},
{ "test_shape_example", "opset15 version not implemented yet"},
{ "test_shape_start_1", "opset15 version not implemented yet"},
{ "test_shape_start_negative_1", "opset15 version not implemented yet"},
{ "test_shape_start_1_end_2", "opset15 version not implemented yet"},
{ "test_shape_start_1_end_negative_1", "opset15 version not implemented yet"},
{ "test_shape_end_negative_1", "opset15 version not implemented yet"},
{ "test_optional_get_element", "not implemented yet"},
{ "test_optional_get_element_sequence", "not implemented yet"},
{ "test_optional_has_element", "not implemented yet"},
{ "test_optional_has_element_empty", "not implemented yet"},
{ "test_identity_opt", "opset16 version not implemented yet"},
{ "test_if_opt", "opset16 version not implemented yet"},
{ "test_loop16_seq_none", "opset16 version not implemented yet"},
{ "test_sequence_map_extract_shapes", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_1_sequence_1_tensor", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_1_sequence_1_tensor_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_add_1_sequence_1_tensor", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_1_sequence_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_2_sequences", "sequence type is not supported in test infra." },
{ "test_sequence_map_add_2_sequences_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_2_sequences_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_extract_shapes_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_add_1_sequence_1_tensor_expanded", "sequence type is not supported in test infra." },
{ "test_sequence_map_add_2_sequences", "sequence type is not supported in test infra." },
{ "test_sequence_map_identity_1_sequence", "sequence type is not supported in test infra." },
{ "BERT-Squad-int8", "training domain"},
{ "YOLOv3-12-int8", "training_domain"},
// opset 18 models. these should be supported by ORT 1.14 when released
{ "test_bitwise_and_i16_3d", "pending opset 18 support"},
{ "test_bitwise_and_i32_2d", "pending opset 18 support"},
{ "test_bitwise_and_ui64_bcast_3v1d", "pending opset 18 support"},
{ "test_bitwise_and_ui8_bcast_4v3d", "pending opset 18 support"},
{ "test_bitwise_not_2d", "pending opset 18 support"},
{ "test_bitwise_not_3d", "pending opset 18 support"},
{ "test_bitwise_not_4d", "pending opset 18 support"},
{ "test_bitwise_or_i16_4d", "pending opset 18 support"},
{ "test_bitwise_or_i32_2d", "pending opset 18 support"},
{ "test_bitwise_or_ui64_bcast_3v1d", "pending opset 18 support"},
{ "test_bitwise_or_ui8_bcast_4v3d", "pending opset 18 support"},
{ "test_bitwise_xor_i16_3d", "pending opset 18 support"},
{ "test_bitwise_xor_i32_2d", "pending opset 18 support"},
{ "test_bitwise_xor_ui8_bcast_4v3d", "pending opset 18 support"},
{ "test_bitwise_xor_ui64_bcast_3v1d", "pending opset 18 support"},
{ "test_center_crop_pad_crop", "pending opset 18 support"},
{ "test_center_crop_pad_crop_and_pad", "pending opset 18 support"},
{ "test_center_crop_pad_crop_and_pad_expanded", "pending opset 18 support"},
{ "test_center_crop_pad_crop_axes_chw", "pending opset 18 support"},
{ "test_center_crop_pad_crop_axes_chw_expanded", "pending opset 18 support"},
{ "test_center_crop_pad_crop_axes_hwc", "pending opset 18 support"},
{ "test_center_crop_pad_crop_axes_hwc_expanded", "pending opset 18 support"},
{ "test_center_crop_pad_crop_expanded", "pending opset 18 support"},
{ "test_center_crop_pad_pad", "pending opset 18 support"},
{ "test_center_crop_pad_pad_expanded", "pending opset 18 support"},
{ "test_col2im", "pending opset 18 support"},
{ "test_col2im_5d", "pending opset 18 support"},
{ "test_col2im_dilations", "pending opset 18 support"},
{ "test_col2im_pads", "pending opset 18 support"},
{ "test_col2im_strides", "pending opset 18 support"},
{ "test_constant_pad", "pending opset 18 support"},
{ "test_constant_pad_axes", "pending opset 18 support"},
{ "test_edge_pad", "pending opset 18 support"},
{ "test_reflect_pad", "pending opset 18 support"},
{ "test_scatter_elements_with_axis", "pending opset 18 support"},
{ "test_scatter_elements_without_axis", "pending opset 18 support"},
{ "test_scatter_elements_with_duplicate_indices", "pending opset 18 support"},
{ "test_scatter_elements_with_negative_indices", "pending opset 18 support"},
{ "test_scatter_elements_with_reduction_max", "pending opset 18 support"},
{ "test_scatter_elements_with_reduction_min", "pending opset 18 support"},
{ "test_scatternd", "pending opset 18 support"},
{ "test_scatternd_add", "pending opset 18 support"},
{ "test_scatternd_max", "pending opset 18 support"},
{ "test_scatternd_min", "pending opset 18 support"},
{ "test_scatternd_multiply", "pending opset 18 support"},
{ "test_softplus_example_expanded", "pending opset 18 support"},
{ "test_softplus_expanded", "pending opset 18 support"},
{ "test_optional_get_element_optional_sequence", "pending opset 18 support"},
{ "test_optional_get_element_optional_tensor", "pending opset 18 support"},
{ "test_optional_has_element_empty_optional_input", "pending opset 18 support"},
{ "test_optional_has_element_optional_input", "pending opset 18 support"},
{ "test_optional_has_element_tensor_input", "pending opset 18 support"},
};
// The following models fails on nocontribops win CI
var disableContribOpsEnvVar = Environment.GetEnvironmentVariable("DisableContribOps");
var isContribOpsDisabled = (disableContribOpsEnvVar != null) ? disableContribOpsEnvVar.Equals("ON") : false;
if (isContribOpsDisabled)
{
skipModels["test_tiny_yolov2"] = "Fails when ContribOps is disabled";
skipModels["mask_rcnn_keras"] = "Pad is not a registered function/op";
}
// Skip traditional ML models
var disableMlOpsEnvVar = Environment.GetEnvironmentVariable("DisableMlOps");
var isMlOpsDisabled = (disableMlOpsEnvVar != null) ? disableMlOpsEnvVar.Equals("ON") : false;
if (isMlOpsDisabled)
{
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
var modelDirName = modelDir.Name;
if (modelDirName.StartsWith("scikit_") ||
modelDirName.StartsWith("libsvm_") ||
modelDirName.StartsWith("coreml_") ||
modelDirName.StartsWith("keras2coreml_") ||
modelDirName.StartsWith("XGBoost_"))
{
skipModels[modelDirName] = "Fails when ML ops are disabled";
}
} //model
} //opset
}
// This model fails on x86 Win CI
if (System.Environment.Is64BitProcess == false)
{
skipModels["test_vgg19"] = "Get preallocated buffer for initializer conv4_4_b_0 failed";
skipModels["GPT2_LM_HEAD"] = "System out of memory";
skipModels["GPT2"] = "System out of memory";
skipModels["test_GPT2"] = "System out of memory";
skipModels["tf_pnasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_5/comb_iter_1/left/bn_sep_7x7_1/beta:0_203 failed";
skipModels["tf_nasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_11/beginning_bn/beta:0_331 failed";
skipModels["test_zfnet512"] = "System out of memory";
skipModels["test_bvlc_reference_caffenet"] = "System out of memory";
skipModels["coreml_VGG16_ImageNet"] = "System out of memory";
skipModels["test_ssd"] = "System out of memory";
skipModels["roberta_sequence_classification"] = "System out of memory";
// models from model zoo
skipModels["VGG 19"] = "bad allocation";
skipModels["VGG 19-caffe2"] = "bad allocation";
skipModels["VGG 19-bn"] = "bad allocation";
skipModels["VGG 16"] = "bad allocation";
skipModels["VGG 16-bn"] = "bad allocation";
skipModels["VGG 16-fp32"] = "bad allocation";
}
return skipModels;
}
public static IEnumerable GetModelsForTest()
{
var modelsDir = GetTestModelsDir();
var modelsDirInfo = new DirectoryInfo(modelsDir);
var skipModels = GetSkippedModels(modelsDirInfo);
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
//var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opsetDir.Name));
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
if (!(skipModels.ContainsKey(modelDir.Name) ||
modelDir.Name.Contains("int8", StringComparison.OrdinalIgnoreCase) ||
modelDir.Name.Contains("qdq", StringComparison.OrdinalIgnoreCase)))
{
yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
}
} //model
} //opset
}
public static IEnumerable GetSkippedModelForTest()
{
var modelsDir = GetTestModelsDir();
var modelsDirInfo = new DirectoryInfo(modelsDir);
var skipModels = GetSkippedModels(modelsDirInfo);
foreach (var opsetDir in getOpsetDirectories(modelsDirInfo))
{
foreach (var modelDir in opsetDir.EnumerateDirectories())
{
if (skipModels.ContainsKey(modelDir.Name) ||
modelDir.Name.Contains("int8", StringComparison.OrdinalIgnoreCase) ||
modelDir.Name.Contains("qdq", StringComparison.OrdinalIgnoreCase))
{
//Console.WriteLine("Model {0} is skipped due to the error: {1}", modelDir.FullName, skipModels[modelDir.Name]);
yield return new object[] { modelDir.Parent.FullName, modelDir.Name };
}
}
}
}
[Theory(DisplayName = "TestPreTrainedModels")]
[MemberData(nameof(GetModelsForTest))]
[MemberData(nameof(GetSkippedModelForTest), Skip = "Skipped due to Error, please fix the error and enable the test")]
private void TestPreTrainedModels(string opsetDir, string modelName)
{
var opsetDirInfo = new DirectoryInfo(opsetDir);
var opset = opsetDirInfo.Name;
string onnxModelFileName = null;
var modelDir = new DirectoryInfo(Path.Combine(opsetDir, modelName));
try
{
var onnxModelNames = modelDir.GetFiles("*.onnx");
bool validModelFound = false;
if (onnxModelNames.Length > 0)
{
// TODO remove file "._resnet34v2.onnx" from test set
for (int i = 0; i < onnxModelNames.Length; i++)
{
if (onnxModelNames[i].Name != "._resnet34v2.onnx")
{
onnxModelNames[0] = onnxModelNames[i];
validModelFound = true;
}
}
}
if (validModelFound)
{
onnxModelFileName = Path.Combine(modelDir.FullName, onnxModelNames[0].Name);
}
else
{
var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString()));
throw new Exception($"Opset {opset} Model {modelName}. Can't determine model file name. Found these :{modelNamesList}");
}
using (var session = new InferenceSession(onnxModelFileName))
{
var inMeta = session.InputMetadata;
string testDataDirNamePattern = "test_data*";
if (opset == "opset9" && modelName == "LSTM_Seq_lens_unpacked")
{
testDataDirNamePattern = "seq_lens*"; // discrepancy in data directory
}
foreach (var testDataDir in modelDir.EnumerateDirectories(testDataDirNamePattern))
{
var inputContainer = new List();
var outputContainer = new List();
foreach (var f in testDataDir.EnumerateFiles("input_*.pb"))
{
inputContainer.Add(TestDataLoader.LoadTensorFromFilePb(f.FullName, inMeta));
}
foreach (var f in testDataDir.EnumerateFiles("output_*.pb"))
{
outputContainer.Add(TestDataLoader.LoadTensorFromFilePb(f.FullName, session.OutputMetadata));
}
using (var resultCollection = session.Run(inputContainer))
{
foreach (var result in resultCollection)
{
Assert.True(session.OutputMetadata.ContainsKey(result.Name));
var outputMeta = session.OutputMetadata[result.Name];
NamedOnnxValue outputValue = null;
foreach (var o in outputContainer)
{
if (o.Name == result.Name)
{
outputValue = o;
break;
}
}
if (outputValue == null)
{
outputValue = outputContainer.First(); // in case the output data file does not contain the name
}
if (outputMeta.IsTensor)
{
if (outputMeta.ElementType == typeof(float))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new FloatComparer());
}
else if (outputMeta.ElementType == typeof(double))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new DoubleComparer());
}
else if (outputMeta.ElementType == typeof(int))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(uint))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(short))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(ushort))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(long))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(ulong))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(byte))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(bool))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new ExactComparer());
}
else if (outputMeta.ElementType == typeof(Float16))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new Float16Comparer { tolerance = 2 });
}
else if (outputMeta.ElementType == typeof(BFloat16))
{
Assert.Equal(result.AsTensor(), outputValue.AsTensor(), new BFloat16Comparer { tolerance = 2 });
}
else
{
Assert.True(false, $"{nameof(TestPreTrainedModels)} does not yet support output of type {outputMeta.ElementType}");
}
}
else
{
Assert.True(false, $"{nameof(TestPreTrainedModels)} cannot handle non-tensor outputs yet");
}
}
}
}
}
}
catch (Exception ex)
{
var msg = $"Opset {opset}, Model {modelName}: ModelFile = {onnxModelFileName} error = {ex.Message}";
if (ex.Message.Contains("ONNX Runtime only *guarantees* support for models stamped with official released onnx opset versions"))
{
// If the exception is thrown because the opset version of the test model is
// not supported by ONNXRuntime yet, then ignore the test and proceed.
// ORT allows commits from ONNX master and in such cases we do come across new opsets which are
// not supported in ORT yet. In order to force these tests to run set env var ALLOW_RELEASED_ONNX_OPSET_ONLY=0
output.WriteLine("Skipping the model test as the latest ONNX opset is not supported yet. Error Message: " + msg);
}
else
{
throw new Exception(msg + "\n" + ex.StackTrace);
}
}
}
// Hint: .NET Core 3.1 has a 'NativeLibrary' class that can be used to free the library handle
private void UnloadLibrary(IntPtr libraryHandle)
{
if (libraryHandle != IntPtr.Zero)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
if (!FreeLibrary(libraryHandle))
{
throw new Exception("Could not unload the provided shared library using its handle");
}
}
else
{
// TODO: Deal with non-Windows platforms for the .NET Core use-case
}
}
}
[SkipNonPackageTests(DisplayName = "TestRegisterCustomOpLibrary")]
private void TestRegisterCustomOpLibrary()
{
using (var option = new SessionOptions())
{
string libName = "custom_op_library.dll";
string modelPath = "custom_op_test.onnx";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
libName = "custom_op_library.dll";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
libName = "libcustom_op_library.so";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
libName = "libcustom_op_library.dylib";
}
string libFullPath = Path.Combine(Directory.GetCurrentDirectory(), libName);
Assert.True(File.Exists(libFullPath), $"Expected lib {libFullPath} does not exist.");
var ortEnvInstance = OrtEnv.Instance();
string[] providers = ortEnvInstance.GetAvailableProviders();
if (Array.Exists(providers, provider => provider == "CUDAExecutionProvider")) {
option.AppendExecutionProvider_CUDA(0);
}
IntPtr libraryHandle = IntPtr.Zero;
try
{
option.RegisterCustomOpLibraryV2(libFullPath, out libraryHandle);
}
catch (Exception ex)
{
var msg = $"Failed to load custom op library {libFullPath}, error = {ex.Message}";
throw new Exception(msg + "\n" + ex.StackTrace);
}
using (var session = new InferenceSession(modelPath, option))
{
var inputContainer = new List();
inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_1",
new DenseTensor(
new float[]
{
1.1f, 2.2f, 3.3f, 4.4f, 5.5f,
6.6f, 7.7f, 8.8f, 9.9f, 10.0f,
11.1f, 12.2f, 13.3f, 14.4f, 15.5f
},
new int[] { 3, 5 }
)));
inputContainer.Add(NamedOnnxValue.CreateFromTensor("input_2",
new DenseTensor(
new float[]
{
15.5f, 14.4f, 13.3f, 12.2f, 11.1f,
10.0f, 9.9f, 8.8f, 7.7f, 6.6f,
5.5f, 4.4f, 3.3f, 2.2f, 1.1f
},
new int[] { 3, 5 }
)));
using (var result = session.Run(inputContainer))
{
Assert.Equal("output", result.First().Name);
var tensorOut = result.First().AsTensor();
var expectedOut = new DenseTensor(
new int[]
{
17, 17, 17, 17, 17,
17, 18, 18, 18, 17,
17, 17, 17, 17, 17
},
new int[] { 3, 5 }
);
Assert.True(tensorOut.SequenceEqual(expectedOut));
}
}
// Safe to unload the custom op shared library now
UnloadLibrary(libraryHandle);
}
}
[Fact(DisplayName = "TestModelSerialization")]
private void TestModelSerialization()
{
string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
string modelOutputPath = Path.Combine(Directory.GetCurrentDirectory(), "optimized-squeezenet.onnx");
// Set the optimized model file path to assert that no exception are thrown.
using (SessionOptions options = new SessionOptions())
{
options.OptimizedModelFilePath = modelOutputPath;
options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_BASIC;
using (var session = new InferenceSession(modelPath, options))
{
Assert.NotNull(session);
Assert.True(File.Exists(modelOutputPath));
}
}
}
// TestGpu() will test the CUDA EP on CUDA enabled builds and
// the DML EP on DML enabled builds
[GpuFact(DisplayName = "TestGpu")]
private void TestGpu()
{
var tuple = OpenSessionSqueezeNet(0); // run on deviceID 0
float[] expectedOutput = TestDataLoader.LoadTensorFromFile(@"bench.expected_out");
using (var session = tuple.Item1)
{
var inputData = tuple.Item2;
var tensor = tuple.Item3;
var inputMeta = session.InputMetadata;
var container = new List();
container.Add(NamedOnnxValue.CreateFromTensor("data_0", tensor));
var res = session.Run(container);
var resultArray = res.First().AsTensor().ToArray();
Assert.Equal(expectedOutput, resultArray, new FloatComparer());
}
}
[DllImport("kernel32", SetLastError = true)]
static extern IntPtr LoadLibrary(string lpFileName);
[DllImport("kernel32", CharSet = CharSet.Ansi)]
static extern UIntPtr GetProcAddress(IntPtr hModule, string procName);
[DllImport("kernel32.dll", CharSet = CharSet.Ansi)]
private static extern bool FreeLibrary(IntPtr hModule);
[Fact(DisplayName = "VerifyNativeMethodsExist")]
private void VerifyNativeMethodsExist()
{
// Check for external API changes
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return;
var entryPointNames = new[]{
"OrtGetApiBase",
"OrtSessionOptionsAppendExecutionProvider_CPU"
#if USE_DNNL
,"OrtSessionOptionsAppendExecutionProvider_Dnnl"
#endif
#if USE_CUDA
,"OrtSessionOptionsAppendExecutionProvider_CUDA"
#endif
#if USE_ROCM
,"OrtSessionOptionsAppendExecutionProvider_ROCM"
#endif
#if USE_DML
,"OrtSessionOptionsAppendExecutionProvider_DML"
#endif
#if USE_OPENVINO
,"OrtSessionOptionsAppendExecutionProvider_OpenVINO"
#endif
#if USE_TENSORRT
,"OrtSessionOptionsAppendExecutionProvider_Tensorrt"
#endif
#if USE_MIGRAPHX
,"OrtSessionOptionsAppendExecutionProvider_MIGraphX"
#endif
#if USE_NNAPI
,"OrtSessionOptionsAppendExecutionProvider_Nnapi"
#endif
};
IntPtr libraryHandle = IntPtr.Zero;
try
{
libraryHandle = LoadLibrary(module);
foreach (var ep in entryPointNames)
{
var x = GetProcAddress(libraryHandle, ep);
Assert.False(x == UIntPtr.Zero, $"Entrypoint {ep} not found in module {module}");
}
}
finally
{
UnloadLibrary(libraryHandle);
}
}
static string GetTestModelsDir()
{
// get build directory, append downloaded models location
var cwd = Directory.GetCurrentDirectory();
var props = File.ReadAllLines(Path.Combine(cwd, propertiesFile));
var modelsRelDir = Path.Combine(props[0].Split('=')[1].Trim());
var modelsDir = Path.Combine(cwd, @"../../..", modelsRelDir, "models");
return modelsDir;
}
}
}