From ea12b674c0d76bcc3dc004a8d31ccf279a0c8a2b Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Jan 2023 10:25:52 +1030 Subject: [PATCH] Disable the failing opset 18 model tests that are breaking the packaging pipeline (#14259) ### Description Skip tests for opset18 models that we haven't implemented kernels for yet. Slice was checked in today so those failures should go away. Resize: #13890 (all resize failures are fixed by this PR as confirmed in output [here](https://dev.azure.com/aiinfra/530acbc4-21bc-487d-8cd8-348ff451d2ff/_apis/build/builds/264725/logs/729)) Col2Im: #12311 ScatterND and ScatterElement: #14224 Pad (should also fix CenterCropPad failures): #14219 Bitwise ops: #14197 Optional: Unknown if we're intending to support this in 1.14 Not sure about SoftPlus as that is failing due to `Could not find an implementation for Exp(1)`. ORT supports Exp from opset 6 and on, and it seems incorrect for the test model created for opset 18 to be using a version of Exp that is so old. Would have expected it to use the latest - Exp(13). @liqunfu is this something that requires a fix to the ONNX model? ### Motivation and Context Fix pipeline --- .../InferenceTest.netcore.cs | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index 57d869b83e..c520e4bef6 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -380,7 +380,73 @@ namespace Microsoft.ML.OnnxRuntime.Tests { "test_sequence_map_add_2_sequences", "sequence type is not supported in test infra." }, { "test_sequence_map_identity_1_sequence", "sequence type is not supported in test infra." }, { "BERT-Squad-int8", "training domain"}, - { "YOLOv3-12-int8", "training_domain"} + { "YOLOv3-12-int8", "training_domain"}, + // opset 18 models. these should be supported by ORT 1.14 when released + { "test_bitwise_and_i16_3d", "pending opset 18 support"}, + { "test_bitwise_and_i32_2d", "pending opset 18 support"}, + { "test_bitwise_and_ui64_bcast_3v1d", "pending opset 18 support"}, + { "test_bitwise_and_ui8_bcast_4v3d", "pending opset 18 support"}, + { "test_bitwise_not_2d", "pending opset 18 support"}, + { "test_bitwise_not_3d", "pending opset 18 support"}, + { "test_bitwise_not_4d", "pending opset 18 support"}, + { "test_bitwise_or_i16_4d", "pending opset 18 support"}, + { "test_bitwise_or_i32_2d", "pending opset 18 support"}, + { "test_bitwise_or_ui64_bcast_3v1d", "pending opset 18 support"}, + { "test_bitwise_or_ui8_bcast_4v3d", "pending opset 18 support"}, + { "test_bitwise_xor_i16_3d", "pending opset 18 support"}, + { "test_bitwise_xor_i32_2d", "pending opset 18 support"}, + { "test_bitwise_xor_ui8_bcast_4v3d", "pending opset 18 support"}, + { "test_bitwise_xor_ui64_bcast_3v1d", "pending opset 18 support"}, + { "test_center_crop_pad_crop", "pending opset 18 support"}, + { "test_center_crop_pad_crop_and_pad", "pending opset 18 support"}, + { "test_center_crop_pad_crop_and_pad_expanded", "pending opset 18 support"}, + { "test_center_crop_pad_crop_axes_chw", "pending opset 18 support"}, + { "test_center_crop_pad_crop_axes_chw_expanded", "pending opset 18 support"}, + { "test_center_crop_pad_crop_axes_hwc", "pending opset 18 support"}, + { "test_center_crop_pad_crop_axes_hwc_expanded", "pending opset 18 support"}, + { "test_center_crop_pad_crop_expanded", "pending opset 18 support"}, + { "test_center_crop_pad_pad", "pending opset 18 support"}, + { "test_center_crop_pad_pad_expanded", "pending opset 18 support"}, + { "test_col2im", "pending opset 18 support"}, + { "test_col2im_5d", "pending opset 18 support"}, + { "test_col2im_dilations", "pending opset 18 support"}, + { "test_col2im_pads", "pending opset 18 support"}, + { "test_col2im_strides", "pending opset 18 support"}, + { "test_constant_pad", "pending opset 18 support"}, + { "test_constant_pad_axes", "pending opset 18 support"}, + { "test_edge_pad", "pending opset 18 support"}, + { "test_reflect_pad", "pending opset 18 support"}, + { "test_resize_downsample_scales_cubic_antialias", "pending opset 18 support"}, + { "test_resize_downsample_scales_linear_antialias", "pending opset 18 support"}, + { "test_resize_downsample_sizes_cubic_antialias", "pending opset 18 support"}, + { "test_resize_downsample_sizes_linear_antialias", "pending opset 18 support"}, + { "test_resize_downsample_sizes_nearest_not_larger", "pending opset 18 support"}, + { "test_resize_downsample_sizes_nearest_not_smaller", "pending opset 18 support"}, + { "test_resize_tf_crop_and_resize_axes_2_3", "pending opset 18 support"}, + { "test_resize_tf_crop_and_resize_axes_3_2", "pending opset 18 support"}, + { "test_resize_upsample_scales_nearest_axes_2_3", "pending opset 18 support"}, + { "test_resize_upsample_scales_nearest_axes_3_2", "pending opset 18 support"}, + { "test_resize_upsample_sizes_nearest_axes_2_3", "pending opset 18 support"}, + { "test_resize_upsample_sizes_nearest_axes_3_2", "pending opset 18 support"}, + { "test_resize_upsample_sizes_nearest_not_larger", "pending opset 18 support"}, + { "test_scatter_elements_with_axis", "pending opset 18 support"}, + { "test_scatter_elements_without_axis", "pending opset 18 support"}, + { "test_scatter_elements_with_duplicate_indices", "pending opset 18 support"}, + { "test_scatter_elements_with_negative_indices", "pending opset 18 support"}, + { "test_scatter_elements_with_reduction_max", "pending opset 18 support"}, + { "test_scatter_elements_with_reduction_min", "pending opset 18 support"}, + { "test_scatternd", "pending opset 18 support"}, + { "test_scatternd_add", "pending opset 18 support"}, + { "test_scatternd_max", "pending opset 18 support"}, + { "test_scatternd_min", "pending opset 18 support"}, + { "test_scatternd_multiply", "pending opset 18 support"}, + { "test_softplus_example_expanded", "pending opset 18 support"}, + { "test_softplus_expanded", "pending opset 18 support"}, + { "test_optional_get_element_optional_sequence", "pending opset 18 support"}, + { "test_optional_get_element_optional_tensor", "pending opset 18 support"}, + { "test_optional_has_element_empty_optional_input", "pending opset 18 support"}, + { "test_optional_has_element_optional_input", "pending opset 18 support"}, + { "test_optional_has_element_tensor_input", "pending opset 18 support"}, }; // The following models fails on nocontribops win CI