diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index bec0b65ed2..720ddbb29d 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -427,29 +427,18 @@ int real_main(int argc, char* argv[], Ort::Env& env) { {"mxnet_arcface", "Model is an invalid ONNX model"}, {"unique_not_sorted_without_axis", "Expected data for 'Y' is incorrect and in sorted order."}, {"cumsum_1d_reverse_exclusive", "only failing linux GPU CI. Likely build error."}, - {"det_2d", "not implemented yet"}, - {"det_nd", "not implemented yet"}, {"resize_downsample_scales_cubic_align_corners", "results mismatch with onnx tests"}, {"resize_downsample_scales_linear_align_corners", "results mismatch with onnx tests"}, {"resize_tf_crop_and_resize", "Bad onnx test output. Needs test fix."}, {"resize_upsample_sizes_nearest_ceil_half_pixel", "Bad onnx test output. Needs test fix."}, {"resize_upsample_sizes_nearest_floor_align_corners", "Bad onnx test output. Needs test fix."}, {"resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", "Bad onnx test output. Needs test fix."}, - {"scatternd", "not implemented yet"}, - {"sequence_model7", "SplitToSequence not implemented yet"}, - {"sequence_model6", "SplitToSequence not implemented yet"}, - {"sequence_model5", "SequenceConstruct not implemented yet"}, - {"sequence_model4", "SequenceConstruct not implemented yet"}, - {"sequence_model3", "SequenceConstruct not implemented yet"}, - {"sequence_model2", "SequenceConstruct not implemented yet"}, - {"sequence_model1", "Sequence* not implemented yet"}, - {"scatter_elements_with_negative_indices", "ScatterElements(11) not implemented yet"}, {"bitshift_right_uint8", "BitShift(11) uint8 support not enabled currently"}, {"bitshift_right_uint16", "BitShift(11) uint16 support not enabled currently"}, {"bitshift_left_uint8", "BitShift(11) uint8 support not enabled currently"}, {"bitshift_left_uint16", "BitShift(11) uint16 support not enabled currently"}, - {"reflect_pad", "test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests"}, - {"edge_pad", "test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests"}, + {"reflect_pad", "Pad(11) int32 support not enabled currently"}, + {"edge_pad", "Pad(11) int32 support not enabled currently"}, {"maxunpool_export_with_output_shape", "Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398" }, }; diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index e0d3be55c0..7a4ef79d83 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -95,7 +95,9 @@ def test_with_types_disabled_due_to_binary_size_concerns_filters(): filters = ['^test_bitshift_right_uint16_cpu', '^test_bitshift_right_uint8_cpu', '^test_bitshift_left_uint16_cpu', - '^test_bitshift_left_uint8_cpu'] + '^test_bitshift_left_uint8_cpu', + '^test_edge_pad_cpu', + '^test_reflect_pad_cpu'] return filters @@ -111,56 +113,23 @@ def create_backend_test(testname=None): else: # Tests that are failing temporarily and should be fixed current_failing_tests = [#'^test_cast_STRING_to_FLOAT_cpu', # old test data that is bad on Linux CI builds - '^test_qlinearconv_cpu', - '^test_gru_seq_length_cpu', - '^test_dynamicquantizelinear_expanded.*', - '^test_dynamicquantizelinear_max_adjusted_expanded.*', - '^test_dynamicquantizelinear_min_adjusted_expanded.*', - '^test_top_k.*', '^test_unique_not_sorted_without_axis_cpu', # bad expected data. enable after https://github.com/onnx/onnx/pull/2381 is picked up '^test_mod_float_mixed_sign_example_cpu', #onnxruntime::Mod::Compute fmod_ was false. fmod attribute must be true for float, float16 and double types - '^test_shrink_cpu', #Invalid rank for input: x Got: 1 Expected: 2 Please fix either the inputs or the model. - '^test_range_float_type_positive_delta_cpu', - '^test_range_float_type_positive_delta_expanded_cpu', - '^test_range_int32_type_negative_delta_cpu', - '^test_range_int32_type_negative_delta_expanded_cpu', - '^test_det_2d_cpu', - '^test_det_nd_cpu', - '^test_resize_downsample_scales_cubic_A_n0p5_exclude_outside_cpu', - '^test_resize_downsample_scales_cubic_align_corners_cpu', - '^test_resize_downsample_scales_cubic_cpu', - '^test_resize_downsample_scales_linear_align_corners_cpu', - '^test_resize_downsample_scales_linear_cpu', - '^test_resize_downsample_scales_nearest_cpu', - '^test_resize_downsample_sizes_cubic_cpu', - '^test_resize_downsample_sizes_linear_pytorch_half_pixel_cpu', - '^test_resize_downsample_sizes_nearest_cpu', - '^test_resize_downsample_sizes_nearest_tf_half_pixel_for_nn_cpu', - '^test_resize_tf_crop_and_resize_cpu', - '^test_resize_upsample_scales_cubic_A_n0p5_exclude_outside_cpu', - '^test_resize_upsample_scales_cubic_align_corners_cpu', - '^test_resize_upsample_scales_cubic_asymmetric_cpu', - '^test_resize_upsample_scales_cubic_cpu', - '^test_resize_upsample_scales_linear_align_corners_cpu', - '^test_resize_upsample_scales_linear_cpu', - '^test_resize_upsample_scales_nearest_cpu', - '^test_resize_upsample_sizes_cubic_cpu', - '^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu', - '^test_resize_upsample_sizes_nearest_cpu', - '^test_resize_upsample_sizes_nearest_floor_align_corners_cpu', - '^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu', - '^test_sequence_*', - '^test_scatter_.*', - '^test_edge_pad_cpu.*', # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests - '^test_reflect_pad_cpu.*', # test data type `int32_t` not supported yet, the `float` equivalent is covered via unit tests + '^test_resize_downsample_scales_cubic_align_corners_cpu', # results mismatch with onnx tests + '^test_resize_downsample_scales_linear_align_corners_cpu', # results mismatch with onnx tests + '^test_resize_tf_crop_and_resize_cpu', # bad expected data, needs test fix + '^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu', # bad expected data, needs test fix + '^test_resize_upsample_sizes_nearest_floor_align_corners_cpu', # bad expected data, needs test fix + '^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu', # bad expected data, needs test fix '^test_maxunpool_export_with_output_shape_cpu', # Invalid output in ONNX test. See https://github.com/onnx/onnx/issues/2398' - ] + ] # Example of how to disable tests for a specific provider. # if c2.supports_device('NGRAPH'): # current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu') if c2.supports_device('NGRAPH'): current_failing_tests += ['^test_clip.*', + '^test_qlinearconv_cpu', '^test_depthtospace_crd.*', '^test_argmax_negative_axis.*', '^test_argmin_negative_axis.*', @@ -172,6 +141,10 @@ def create_backend_test(testname=None): 'test_unsqueeze_negative_axes_cpu', 'test_constant_pad_cpu'] + if c2.supports_device('MKL-DNN'): + current_failing_tests += ['^test_range_float_type_positive_delta_expanded_cpu', + '^test_range_int32_type_negative_delta_expanded_cpu'] + if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device('OPENVINO_GPU_FP16'): current_failing_tests.append('^test_div_cpu*')