From c8269e4b89813e9165d30e9507739ef826f2e612 Mon Sep 17 00:00:00 2001 From: Yulong Wang Date: Sat, 2 May 2020 19:05:58 -0700 Subject: [PATCH] move backend test filters into data file (#3798) * move backend test filters into data file * update data * update data * update document * fix list for current_failing_tests_OPENVINO_CPU_FP32 --- docs/How_To_Update_ONNX_Dev_Notes.md | 11 +- .../test/python/onnx_backend_test_series.py | 173 +++--------------- .../onnx_backend_test_series_filters.jsonc | 173 ++++++++++++++++++ 3 files changed, 198 insertions(+), 159 deletions(-) create mode 100644 onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc diff --git a/docs/How_To_Update_ONNX_Dev_Notes.md b/docs/How_To_Update_ONNX_Dev_Notes.md index e2da338bc0..25162530b1 100644 --- a/docs/How_To_Update_ONNX_Dev_Notes.md +++ b/docs/How_To_Update_ONNX_Dev_Notes.md @@ -29,13 +29,4 @@ Search 'for version2tag', update the commit hashes. The list should contain ever 6. If there is any unitest failure, caught by onnx_test_runner. Please also update - [onnxruntime/test/onnx/main.cc](/onnxruntime/test/onnx/main.cc) -- [onnxruntime/test/python/onnx_backend_test_series.py](/onnxruntime/test/python/onnx_backend_test_series.py) - - - - - - - - - +- [onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc) diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 62fd6370c3..f968c4bf3b 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import argparse +import json import sys import os import platform @@ -31,49 +32,6 @@ class OrtBackendTest(onnx.backend.test.BackendTest): np.testing.assert_allclose(ref_outputs[i], outputs[i], rtol=1e-3, atol=1e-5) -# ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported -def tests_with_pre_opset7_dependencies_filters(): - filters = [ - '^test_AvgPool1d_cpu', '^test_AvgPool1d_stride_cpu', '^test_AvgPool2d_cpu', '^test_AvgPool2d_stride_cpu', - '^test_AvgPool3d_cpu', '^test_AvgPool3d_stride1_pad0_gpu_input_cpu', '^test_AvgPool3d_stride_cpu', - '^test_BatchNorm1d_3d_input_eval_cpu', '^test_BatchNorm2d_eval_cpu', '^test_BatchNorm2d_momentum_eval_cpu', - '^test_BatchNorm3d_eval_cpu', '^test_BatchNorm3d_momentum_eval_cpu', '^test_GLU_cpu', '^test_GLU_dim_cpu', - '^test_Linear_cpu', '^test_PReLU_1d_cpu', '^test_PReLU_1d_multiparam_cpu', '^test_PReLU_2d_cpu', - '^test_PReLU_2d_multiparam_cpu', '^test_PReLU_3d_cpu', '^test_PReLU_3d_multiparam_cpu', - '^test_PoissonNLLLLoss_no_reduce_cpu', '^test_Softsign_cpu', '^test_operator_add_broadcast_cpu', - '^test_operator_add_size1_broadcast_cpu', '^test_operator_add_size1_right_broadcast_cpu', - '^test_operator_add_size1_singleton_broadcast_cpu', '^test_operator_addconstant_cpu', - '^test_operator_addmm_cpu', '^test_operator_basic_cpu', '^test_operator_mm_cpu', - '^test_operator_non_float_params_cpu', '^test_operator_params_cpu', '^test_operator_pow_cpu' - ] - - return filters - - -def unsupported_usages_filters(): - filters = [ - '^test_convtranspose_1d_cpu', # ConvTransponse supports 4-D only - '^test_convtranspose_3d_cpu' - ] - - return filters - - -def other_tests_failing_permanently_filters(): - # Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. - # e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8, which - # doesn't match the expected output that was generated with numpy. - filters = ['^test_cast_FLOAT_to_STRING_cpu'] - - return filters - - -def test_with_types_disabled_due_to_binary_size_concerns_filters(): - filters = ['^test_bitshift_right_uint16_cpu', '^test_bitshift_left_uint16_cpu'] - - return filters - - def create_backend_test(testname=None): backend_test = OrtBackendTest(c2, __name__) @@ -83,123 +41,40 @@ def create_backend_test(testname=None): if testname: backend_test.include(testname + '.*') else: - # Tests that are failing temporarily and should be fixed - current_failing_tests = [ - '^test_adagrad_cpu', - '^test_adagrad_multiple_cpu', - '^test_batchnorm_epsilon_old_cpu', - '^test_batchnorm_epsilon_training_mode_cpu', - '^test_batchnorm_example_old_cpu', - '^test_batchnorm_example_training_mode_cpu', - '^test_einsum_batch_diagonal_cpu', - '^test_einsum_batch_matmul_cpu', - '^test_einsum_inner_prod_cpu', - '^test_einsum_sum_cpu', - '^test_einsum_transpose_cpu', - '^test_gathernd_example_int32_batch_dim1_cpu', - '^test_max_int16_cpu', - '^test_max_int8_cpu', - '^test_max_uint16_cpu', - '^test_max_uint8_cpu', - '^test_min_int16_cpu', - '^test_min_int8_cpu', - '^test_min_uint16_cpu', - '^test_min_uint8_cpu', - '^test_momentum_cpu', - '^test_momentum_multiple_cpu', - '^test_nesterov_momentum_cpu', - '^test_pow_types_float32_uint32_cpu', - '^test_pow_types_float32_uint64_cpu', - '^test_gradient_of_add_and_mul_cpu', - '^test_gradient_of_add_cpu', - '^test_batchnorm_example_training_mode_cpu', - '^test_batchnorm_epsilon_training_mode_cpu', - '^test_maxunpool_export_with_output_shape_cpu', #result mismatch - '^test_resize_downsample_scales_cubic_align_corners_cpu', # results mismatch with onnx tests - '^test_resize_downsample_scales_linear_align_corners_cpu', # results mismatch with onnx tests - '^test_adam_cpu', # NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) - '^test_adam_multiple_cpu', # NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) - '^test_dropout.*', # NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) - '^test_training_dropout.*' # NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) - ] + # read filters data + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'testdata', 'onnx_backend_test_series_filters.jsonc')) as f: + filters_lines = f.readlines() + filters_lines = [x.split('//')[0] for x in filters_lines] + filters = json.loads('\n'.join(filters_lines)) + + current_failing_tests = filters['current_failing_tests'] + if platform.architecture()[0] == '32bit': - current_failing_tests += ['^test_vgg19', '^test_zfnet512', '^test_bvlc_alexnet_cpu'] - # Example of how to disable tests for a specific provider. - # if c2.supports_device('NGRAPH'): - # current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu') + current_failing_tests += filters['current_failing_tests_x86'] + if c2.supports_device('NGRAPH'): - current_failing_tests += [ - '^test_clip.*', '^test_qlinearconv_cpu', '^test_depthtospace_crd.*', '^test_argmax_negative_axis.*', - '^test_argmin_negative_axis.*', '^test_hardmax_negative_axis.*', '^test_gemm_default_no_bias_cpu', - '^test_flatten_negative_axis.*', '^test_reduce_[a-z1-9_]*_negative_axes_.*', - 'test_squeeze_negative_axes_cpu', 'test_unsqueeze_negative_axes_cpu', 'test_constant_pad_cpu', - 'test_edge_pad_cpu', 'test_reflect_pad_cpu', '^test_split_zero_size_splits_.*', - '^test_argmax_keepdims_example_select_last_index_cpu', '^test_argmax_no_keepdims_example_select_last_index_cpu', - '^test_argmin_no_keepdims_example_select_last_index_cpu','^test_argmin_keepdims_example_select_last_index_cpu', - '^test_pow_types_float32_int32_*', '^test_pow_types_float32_int64_*', '^test_pow_types_float_*', - '^test_pow_types_int32_float32_*', '^test_pow_types_int_*', '^test_pow_types_int64_float32_*' - ] + current_failing_tests += filters['current_failing_tests_NGRAPH'] if c2.supports_device('DNNL'): - current_failing_tests += [ - '^test_range_float_type_positive_delta_expanded_cpu', - '^test_range_int32_type_negative_delta_expanded_cpu', '^test_averagepool_2d_ceil_cpu', - '^test_maxpool_2d_ceil_cpu', '^test_maxpool_2d_dilations_cpu', - '^test_maxpool_2d_uint8', - '^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op. - '^test_softmax_cross_entropy.*', # Does not support 5-D or above tensors for SUB op. - ] + current_failing_tests += filters['current_failing_tests_DNNL'] if c2.supports_device('NNAPI'): - current_failing_tests += [ - '^test_maxpool_2d_uint8', - '^test_negative_log_likelihood.*', - '^test_softmax_cross_entropy.*' - ] + current_failing_tests += filters['current_failing_tests_NNAPI'] if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device('OPENVINO_GPU_FP16'): - current_failing_tests.append('^test_div_cpu') - # temporarily exclude vgg19 test which comsumes too much memory, run out of memory on Upsquared device. - # single test pass for vgg19, need furture investigation - current_failing_tests.append('^test_vgg19_cpu') - current_failing_tests.append('^test_negative_log_likelihood.*') # Does not support 5-D or above tensors for SUB op. - current_failing_tests.append('^test_softmax_cross_entropy.*') # Does not support 5-D or above tensors for SUB op. - - if c2.supports_device('OPENVINO_CPU_FP32'): - current_failing_tests += [ - '^test_operator_permute2_cpu', - '^test_operator_repeat_cpu', - '^test_operator_repeat_dim_overflow_cpu', - '^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op. - '^test_softmax_cross_entropy.*' # Does not support 5-D or above tensors for SUB op. - ] - if c2.supports_device('OPENVINO_GPU_FP32'): - current_failing_tests += [ - '^test_operator_permute2_cpu', - '^test_operator_repeat_cpu', - '^test_operator_repeat_dim_overflow_cpu', - '^test_add_bcast_cpu', - '^test_batchnorm_epsilon_cpu', - '^test_div_bcast_cpu', - '^test_mul_bcast_cpu', - '^test_pow_bcast_array_cpu', - '^test_sub_bcast_cpu', - '^test_batchnorm_example_cpu', - '^test_clip_default_inbounds_cpu', - '^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu', - '^test_resize_upsample_sizes_nearest_floor_align_corners_cpu', - '^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu', - '^test_unique_not_sorted_without_axis_cpu', - '^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op. - '^test_softmax_cross_entropy.*' # Does not support 5-D or above tensors for SUB op. - ] + current_failing_tests += filters['current_failing_tests_OPENVINO_GPU'] + if c2.supports_device('OPENVINO_GPU_FP32'): + current_failing_tests += filters['current_failing_tests_OPENVINO_GPU_FP32'] + + if c2.supports_device('OPENVINO_CPU_FP32'): + current_failing_tests += filters['current_failing_tests_OPENVINO_CPU_FP32'] filters = current_failing_tests + \ - tests_with_pre_opset7_dependencies_filters() + \ - unsupported_usages_filters() + \ - other_tests_failing_permanently_filters() + \ - test_with_types_disabled_due_to_binary_size_concerns_filters() + filters['tests_with_pre_opset7_dependencies'] + \ + filters['unsupported_usages'] + \ + filters['failing_permanently'] + \ + filters['test_with_types_disabled_due_to_binary_size_concerns'] backend_test.exclude('(' + '|'.join(filters) + ')') print('excluded tests:', filters) diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc new file mode 100644 index 0000000000..3449828985 --- /dev/null +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -0,0 +1,173 @@ +{ + // Tests that are failing temporarily and should be fixed + "current_failing_tests": [ + "^test_adagrad", + "^test_adagrad_multiple", + "^test_batchnorm_epsilon_old", + "^test_batchnorm_epsilon_training_mode", + "^test_batchnorm_example_old", + "^test_batchnorm_example_training_mode", + "^test_einsum_batch_diagonal", + "^test_einsum_batch_matmul", + "^test_einsum_inner_prod", + "^test_einsum_sum", + "^test_einsum_transpose", + "^test_gathernd_example_int32_batch_dim1", + "^test_max_int16", + "^test_max_int8", + "^test_max_uint16", + "^test_max_uint8", + "^test_min_int16", + "^test_min_int8", + "^test_min_uint16", + "^test_min_uint8", + "^test_momentum", + "^test_momentum_multiple", + "^test_nesterov_momentum", + "^test_pow_types_float32_uint32", + "^test_pow_types_float32_uint64", + "^test_gradient_of_add_and_mul", + "^test_gradient_of_add", + "^test_batchnorm_example_training_mode", + "^test_batchnorm_epsilon_training_mode", + "^test_maxunpool_export_with_output_shape", // result mismatch + "^test_resize_downsample_scales_cubic_align_corners", // results mismatch with onnx tests + "^test_resize_downsample_scales_linear_align_corners", // results mismatch with onnx tests + "^test_adam", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) + "^test_adam_multiple", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1) + "^test_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) + "^test_training_dropout.*" // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel) + ], + "current_failing_tests_x86": [ + "^test_vgg19", + "^test_zfnet512", + "^test_bvlc_alexnet" + ], + "current_failing_tests_NGRAPH": [ + "^test_clip.*", + "^test_qlinearconv", + "^test_depthtospace_crd.*", + "^test_argmax_negative_axis.*", + "^test_argmin_negative_axis.*", + "^test_hardmax_negative_axis.*", + "^test_gemm_default_no_bias", + "^test_flatten_negative_axis.*", + "^test_reduce_[a-z1-9_]*_negative_axes_.*", + "test_squeeze_negative_axes", + "test_unsqueeze_negative_axes", + "test_constant_pad", + "test_edge_pad", + "test_reflect_pad", + "^test_split_zero_size_splits_.*", + "^test_argmax_keepdims_example_select_last_index", + "^test_argmax_no_keepdims_example_select_last_index", + "^test_argmin_no_keepdims_example_select_last_index", + "^test_argmin_keepdims_example_select_last_index", + "^test_pow_types_float32_int32_*", + "^test_pow_types_float32_int64_*", + "^test_pow_types_float_*", + "^test_pow_types_int32_float32_*", + "^test_pow_types_int_*", + "^test_pow_types_int64_float32_*" + ], + "current_failing_tests_DNNL": [ + "^test_range_float_type_positive_delta_expanded", + "^test_range_int32_type_negative_delta_expanded", + "^test_averagepool_2d_ceil", + "^test_maxpool_2d_ceil", + "^test_maxpool_2d_dilations", + "^test_maxpool_2d_uint8", + "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. + "^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op. + ], + "current_failing_tests_NNAPI": [ + "^test_maxpool_2d_uint8", + "^test_negative_log_likelihood.*", + "^test_softmax_cross_entropy.*" + ], + "current_failing_tests_OPENVINO_GPU": [ + "^test_div", + // temporarily exclude vgg19 test which comsumes too much memory, run out of memory on Upsquared device. + // single test pass for vgg19, need furture investigation + "^test_vgg19", + "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. + "^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op. + ], + "current_failing_tests_OPENVINO_GPU_FP32": [ + "^test_operator_permute2", + "^test_operator_repeat", + "^test_operator_repeat_dim_overflow", + "^test_add_bcast", + "^test_batchnorm_epsilon", + "^test_div_bcast", + "^test_mul_bcast", + "^test_pow_bcast_array", + "^test_sub_bcast", + "^test_batchnorm_example", + "^test_clip_default_inbounds", + "^test_resize_upsample_sizes_nearest_ceil_half_pixel", + "^test_resize_upsample_sizes_nearest_floor_align_corners", + "^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", + "^test_unique_not_sorted_without_axis", + "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. + "^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op. + ], + "current_failing_tests_OPENVINO_CPU_FP32": [ + "^test_operator_permute2", + "^test_operator_repeat", + "^test_operator_repeat_dim_overflow", + "^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op. + "^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op. + ], + // ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported + "tests_with_pre_opset7_dependencies": [ + "^test_AvgPool1d", + "^test_AvgPool1d_stride", + "^test_AvgPool2d", + "^test_AvgPool2d_stride", + "^test_AvgPool3d", + "^test_AvgPool3d_stride1_pad0_gpu_input", + "^test_AvgPool3d_stride", + "^test_BatchNorm1d_3d_input_eval", + "^test_BatchNorm2d_eval", + "^test_BatchNorm2d_momentum_eval", + "^test_BatchNorm3d_eval", + "^test_BatchNorm3d_momentum_eval", + "^test_GLU", + "^test_GLU_dim", + "^test_Linear", + "^test_PReLU_1d", + "^test_PReLU_1d_multiparam", + "^test_PReLU_2d", + "^test_PReLU_2d_multiparam", + "^test_PReLU_3d", + "^test_PReLU_3d_multiparam", + "^test_PoissonNLLLLoss_no_reduce", + "^test_Softsign", + "^test_operator_add_broadcast", + "^test_operator_add_size1_broadcast", + "^test_operator_add_size1_right_broadcast", + "^test_operator_add_size1_singleton_broadcast", + "^test_operator_addconstant", + "^test_operator_addmm", + "^test_operator_basic", + "^test_operator_mm", + "^test_operator_non_float_params", + "^test_operator_params", + "^test_operator_pow" + ], + "unsupported_usages": [ + "^test_convtranspose_1d", // ConvTransponse supports 4-D only + "^test_convtranspose_3d" + ], + "failing_permanently": [ + // Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8. + // e.g. 0.296140194 -> "0.2961402" not "0.29614019". ORT produces the latter with precision set to 8, which + // doesn"t match the expected output that was generated with numpy. + "^test_cast_FLOAT_to_STRING" + ], + "test_with_types_disabled_due_to_binary_size_concerns": [ + "^test_bitshift_right_uint16", + "^test_bitshift_left_uint16" + ] +} \ No newline at end of file