mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-09 17:28:58 +00:00
move backend test filters into data file (#3798)
* move backend test filters into data file * update data * update data * update document * fix list for current_failing_tests_OPENVINO_CPU_FP32
This commit is contained in:
Yulong Wang
GitHub
parent
2684d47fc5
commit
c8269e4b89
3 changed files with 198 additions and 159 deletions
|
|
@ -29,13 +29,4 @@ Search 'for version2tag', update the commit hashes. The list should contain ever
|
|||
|
||||
6. If there is any unitest failure, caught by onnx_test_runner. Please also update
|
||||
- [onnxruntime/test/onnx/main.cc](/onnxruntime/test/onnx/main.cc)
|
||||
- [onnxruntime/test/python/onnx_backend_test_series.py](/onnxruntime/test/python/onnx_backend_test_series.py)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
- [onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc](/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import platform
|
||||
|
|
@ -31,49 +32,6 @@ class OrtBackendTest(onnx.backend.test.BackendTest):
|
|||
np.testing.assert_allclose(ref_outputs[i], outputs[i], rtol=1e-3, atol=1e-5)
|
||||
|
||||
|
||||
# ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported
|
||||
def tests_with_pre_opset7_dependencies_filters():
|
||||
filters = [
|
||||
'^test_AvgPool1d_cpu', '^test_AvgPool1d_stride_cpu', '^test_AvgPool2d_cpu', '^test_AvgPool2d_stride_cpu',
|
||||
'^test_AvgPool3d_cpu', '^test_AvgPool3d_stride1_pad0_gpu_input_cpu', '^test_AvgPool3d_stride_cpu',
|
||||
'^test_BatchNorm1d_3d_input_eval_cpu', '^test_BatchNorm2d_eval_cpu', '^test_BatchNorm2d_momentum_eval_cpu',
|
||||
'^test_BatchNorm3d_eval_cpu', '^test_BatchNorm3d_momentum_eval_cpu', '^test_GLU_cpu', '^test_GLU_dim_cpu',
|
||||
'^test_Linear_cpu', '^test_PReLU_1d_cpu', '^test_PReLU_1d_multiparam_cpu', '^test_PReLU_2d_cpu',
|
||||
'^test_PReLU_2d_multiparam_cpu', '^test_PReLU_3d_cpu', '^test_PReLU_3d_multiparam_cpu',
|
||||
'^test_PoissonNLLLLoss_no_reduce_cpu', '^test_Softsign_cpu', '^test_operator_add_broadcast_cpu',
|
||||
'^test_operator_add_size1_broadcast_cpu', '^test_operator_add_size1_right_broadcast_cpu',
|
||||
'^test_operator_add_size1_singleton_broadcast_cpu', '^test_operator_addconstant_cpu',
|
||||
'^test_operator_addmm_cpu', '^test_operator_basic_cpu', '^test_operator_mm_cpu',
|
||||
'^test_operator_non_float_params_cpu', '^test_operator_params_cpu', '^test_operator_pow_cpu'
|
||||
]
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def unsupported_usages_filters():
|
||||
filters = [
|
||||
'^test_convtranspose_1d_cpu', # ConvTransponse supports 4-D only
|
||||
'^test_convtranspose_3d_cpu'
|
||||
]
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def other_tests_failing_permanently_filters():
|
||||
# Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8.
|
||||
# e.g. 0.296140194 -> '0.2961402' not '0.29614019'. ORT produces the latter with precision set to 8, which
|
||||
# doesn't match the expected output that was generated with numpy.
|
||||
filters = ['^test_cast_FLOAT_to_STRING_cpu']
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def test_with_types_disabled_due_to_binary_size_concerns_filters():
|
||||
filters = ['^test_bitshift_right_uint16_cpu', '^test_bitshift_left_uint16_cpu']
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def create_backend_test(testname=None):
|
||||
backend_test = OrtBackendTest(c2, __name__)
|
||||
|
||||
|
|
@ -83,123 +41,40 @@ def create_backend_test(testname=None):
|
|||
if testname:
|
||||
backend_test.include(testname + '.*')
|
||||
else:
|
||||
# Tests that are failing temporarily and should be fixed
|
||||
current_failing_tests = [
|
||||
'^test_adagrad_cpu',
|
||||
'^test_adagrad_multiple_cpu',
|
||||
'^test_batchnorm_epsilon_old_cpu',
|
||||
'^test_batchnorm_epsilon_training_mode_cpu',
|
||||
'^test_batchnorm_example_old_cpu',
|
||||
'^test_batchnorm_example_training_mode_cpu',
|
||||
'^test_einsum_batch_diagonal_cpu',
|
||||
'^test_einsum_batch_matmul_cpu',
|
||||
'^test_einsum_inner_prod_cpu',
|
||||
'^test_einsum_sum_cpu',
|
||||
'^test_einsum_transpose_cpu',
|
||||
'^test_gathernd_example_int32_batch_dim1_cpu',
|
||||
'^test_max_int16_cpu',
|
||||
'^test_max_int8_cpu',
|
||||
'^test_max_uint16_cpu',
|
||||
'^test_max_uint8_cpu',
|
||||
'^test_min_int16_cpu',
|
||||
'^test_min_int8_cpu',
|
||||
'^test_min_uint16_cpu',
|
||||
'^test_min_uint8_cpu',
|
||||
'^test_momentum_cpu',
|
||||
'^test_momentum_multiple_cpu',
|
||||
'^test_nesterov_momentum_cpu',
|
||||
'^test_pow_types_float32_uint32_cpu',
|
||||
'^test_pow_types_float32_uint64_cpu',
|
||||
'^test_gradient_of_add_and_mul_cpu',
|
||||
'^test_gradient_of_add_cpu',
|
||||
'^test_batchnorm_example_training_mode_cpu',
|
||||
'^test_batchnorm_epsilon_training_mode_cpu',
|
||||
'^test_maxunpool_export_with_output_shape_cpu', #result mismatch
|
||||
'^test_resize_downsample_scales_cubic_align_corners_cpu', # results mismatch with onnx tests
|
||||
'^test_resize_downsample_scales_linear_align_corners_cpu', # results mismatch with onnx tests
|
||||
'^test_adam_cpu', # NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
|
||||
'^test_adam_multiple_cpu', # NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
|
||||
'^test_dropout.*', # NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
|
||||
'^test_training_dropout.*' # NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
|
||||
]
|
||||
# read filters data
|
||||
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'testdata', 'onnx_backend_test_series_filters.jsonc')) as f:
|
||||
filters_lines = f.readlines()
|
||||
filters_lines = [x.split('//')[0] for x in filters_lines]
|
||||
filters = json.loads('\n'.join(filters_lines))
|
||||
|
||||
current_failing_tests = filters['current_failing_tests']
|
||||
|
||||
if platform.architecture()[0] == '32bit':
|
||||
current_failing_tests += ['^test_vgg19', '^test_zfnet512', '^test_bvlc_alexnet_cpu']
|
||||
# Example of how to disable tests for a specific provider.
|
||||
# if c2.supports_device('NGRAPH'):
|
||||
# current_failing_tests.append('^test_operator_repeat_dim_overflow_cpu')
|
||||
current_failing_tests += filters['current_failing_tests_x86']
|
||||
|
||||
if c2.supports_device('NGRAPH'):
|
||||
current_failing_tests += [
|
||||
'^test_clip.*', '^test_qlinearconv_cpu', '^test_depthtospace_crd.*', '^test_argmax_negative_axis.*',
|
||||
'^test_argmin_negative_axis.*', '^test_hardmax_negative_axis.*', '^test_gemm_default_no_bias_cpu',
|
||||
'^test_flatten_negative_axis.*', '^test_reduce_[a-z1-9_]*_negative_axes_.*',
|
||||
'test_squeeze_negative_axes_cpu', 'test_unsqueeze_negative_axes_cpu', 'test_constant_pad_cpu',
|
||||
'test_edge_pad_cpu', 'test_reflect_pad_cpu', '^test_split_zero_size_splits_.*',
|
||||
'^test_argmax_keepdims_example_select_last_index_cpu', '^test_argmax_no_keepdims_example_select_last_index_cpu',
|
||||
'^test_argmin_no_keepdims_example_select_last_index_cpu','^test_argmin_keepdims_example_select_last_index_cpu',
|
||||
'^test_pow_types_float32_int32_*', '^test_pow_types_float32_int64_*', '^test_pow_types_float_*',
|
||||
'^test_pow_types_int32_float32_*', '^test_pow_types_int_*', '^test_pow_types_int64_float32_*'
|
||||
]
|
||||
current_failing_tests += filters['current_failing_tests_NGRAPH']
|
||||
|
||||
if c2.supports_device('DNNL'):
|
||||
current_failing_tests += [
|
||||
'^test_range_float_type_positive_delta_expanded_cpu',
|
||||
'^test_range_int32_type_negative_delta_expanded_cpu', '^test_averagepool_2d_ceil_cpu',
|
||||
'^test_maxpool_2d_ceil_cpu', '^test_maxpool_2d_dilations_cpu',
|
||||
'^test_maxpool_2d_uint8',
|
||||
'^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op.
|
||||
'^test_softmax_cross_entropy.*', # Does not support 5-D or above tensors for SUB op.
|
||||
]
|
||||
current_failing_tests += filters['current_failing_tests_DNNL']
|
||||
|
||||
if c2.supports_device('NNAPI'):
|
||||
current_failing_tests += [
|
||||
'^test_maxpool_2d_uint8',
|
||||
'^test_negative_log_likelihood.*',
|
||||
'^test_softmax_cross_entropy.*'
|
||||
]
|
||||
current_failing_tests += filters['current_failing_tests_NNAPI']
|
||||
|
||||
if c2.supports_device('OPENVINO_GPU_FP32') or c2.supports_device('OPENVINO_GPU_FP16'):
|
||||
current_failing_tests.append('^test_div_cpu')
|
||||
# temporarily exclude vgg19 test which comsumes too much memory, run out of memory on Upsquared device.
|
||||
# single test pass for vgg19, need furture investigation
|
||||
current_failing_tests.append('^test_vgg19_cpu')
|
||||
current_failing_tests.append('^test_negative_log_likelihood.*') # Does not support 5-D or above tensors for SUB op.
|
||||
current_failing_tests.append('^test_softmax_cross_entropy.*') # Does not support 5-D or above tensors for SUB op.
|
||||
|
||||
if c2.supports_device('OPENVINO_CPU_FP32'):
|
||||
current_failing_tests += [
|
||||
'^test_operator_permute2_cpu',
|
||||
'^test_operator_repeat_cpu',
|
||||
'^test_operator_repeat_dim_overflow_cpu',
|
||||
'^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op.
|
||||
'^test_softmax_cross_entropy.*' # Does not support 5-D or above tensors for SUB op.
|
||||
]
|
||||
if c2.supports_device('OPENVINO_GPU_FP32'):
|
||||
current_failing_tests += [
|
||||
'^test_operator_permute2_cpu',
|
||||
'^test_operator_repeat_cpu',
|
||||
'^test_operator_repeat_dim_overflow_cpu',
|
||||
'^test_add_bcast_cpu',
|
||||
'^test_batchnorm_epsilon_cpu',
|
||||
'^test_div_bcast_cpu',
|
||||
'^test_mul_bcast_cpu',
|
||||
'^test_pow_bcast_array_cpu',
|
||||
'^test_sub_bcast_cpu',
|
||||
'^test_batchnorm_example_cpu',
|
||||
'^test_clip_default_inbounds_cpu',
|
||||
'^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu',
|
||||
'^test_resize_upsample_sizes_nearest_floor_align_corners_cpu',
|
||||
'^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu',
|
||||
'^test_unique_not_sorted_without_axis_cpu',
|
||||
'^test_negative_log_likelihood.*', # Does not support 5-D or above tensors for SUB op.
|
||||
'^test_softmax_cross_entropy.*' # Does not support 5-D or above tensors for SUB op.
|
||||
]
|
||||
current_failing_tests += filters['current_failing_tests_OPENVINO_GPU']
|
||||
|
||||
if c2.supports_device('OPENVINO_GPU_FP32'):
|
||||
current_failing_tests += filters['current_failing_tests_OPENVINO_GPU_FP32']
|
||||
|
||||
if c2.supports_device('OPENVINO_CPU_FP32'):
|
||||
current_failing_tests += filters['current_failing_tests_OPENVINO_CPU_FP32']
|
||||
|
||||
filters = current_failing_tests + \
|
||||
tests_with_pre_opset7_dependencies_filters() + \
|
||||
unsupported_usages_filters() + \
|
||||
other_tests_failing_permanently_filters() + \
|
||||
test_with_types_disabled_due_to_binary_size_concerns_filters()
|
||||
filters['tests_with_pre_opset7_dependencies'] + \
|
||||
filters['unsupported_usages'] + \
|
||||
filters['failing_permanently'] + \
|
||||
filters['test_with_types_disabled_due_to_binary_size_concerns']
|
||||
|
||||
backend_test.exclude('(' + '|'.join(filters) + ')')
|
||||
print('excluded tests:', filters)
|
||||
|
|
|
|||
173
onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc
vendored
Normal file
173
onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc
vendored
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
{
|
||||
// Tests that are failing temporarily and should be fixed
|
||||
"current_failing_tests": [
|
||||
"^test_adagrad",
|
||||
"^test_adagrad_multiple",
|
||||
"^test_batchnorm_epsilon_old",
|
||||
"^test_batchnorm_epsilon_training_mode",
|
||||
"^test_batchnorm_example_old",
|
||||
"^test_batchnorm_example_training_mode",
|
||||
"^test_einsum_batch_diagonal",
|
||||
"^test_einsum_batch_matmul",
|
||||
"^test_einsum_inner_prod",
|
||||
"^test_einsum_sum",
|
||||
"^test_einsum_transpose",
|
||||
"^test_gathernd_example_int32_batch_dim1",
|
||||
"^test_max_int16",
|
||||
"^test_max_int8",
|
||||
"^test_max_uint16",
|
||||
"^test_max_uint8",
|
||||
"^test_min_int16",
|
||||
"^test_min_int8",
|
||||
"^test_min_uint16",
|
||||
"^test_min_uint8",
|
||||
"^test_momentum",
|
||||
"^test_momentum_multiple",
|
||||
"^test_nesterov_momentum",
|
||||
"^test_pow_types_float32_uint32",
|
||||
"^test_pow_types_float32_uint64",
|
||||
"^test_gradient_of_add_and_mul",
|
||||
"^test_gradient_of_add",
|
||||
"^test_batchnorm_example_training_mode",
|
||||
"^test_batchnorm_epsilon_training_mode",
|
||||
"^test_maxunpool_export_with_output_shape", // result mismatch
|
||||
"^test_resize_downsample_scales_cubic_align_corners", // results mismatch with onnx tests
|
||||
"^test_resize_downsample_scales_linear_align_corners", // results mismatch with onnx tests
|
||||
"^test_adam", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
|
||||
"^test_adam_multiple", // NOT_IMPLEMENTED : Could not find an implementation for the node Adam(1)
|
||||
"^test_dropout.*", // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
|
||||
"^test_training_dropout.*" // NOT_IMPLEMENTED : Could not find an implementation for the node Dropout(12) (Temporary, subsequent PR will add this -- we need training_mode change in the kernel)
|
||||
],
|
||||
"current_failing_tests_x86": [
|
||||
"^test_vgg19",
|
||||
"^test_zfnet512",
|
||||
"^test_bvlc_alexnet"
|
||||
],
|
||||
"current_failing_tests_NGRAPH": [
|
||||
"^test_clip.*",
|
||||
"^test_qlinearconv",
|
||||
"^test_depthtospace_crd.*",
|
||||
"^test_argmax_negative_axis.*",
|
||||
"^test_argmin_negative_axis.*",
|
||||
"^test_hardmax_negative_axis.*",
|
||||
"^test_gemm_default_no_bias",
|
||||
"^test_flatten_negative_axis.*",
|
||||
"^test_reduce_[a-z1-9_]*_negative_axes_.*",
|
||||
"test_squeeze_negative_axes",
|
||||
"test_unsqueeze_negative_axes",
|
||||
"test_constant_pad",
|
||||
"test_edge_pad",
|
||||
"test_reflect_pad",
|
||||
"^test_split_zero_size_splits_.*",
|
||||
"^test_argmax_keepdims_example_select_last_index",
|
||||
"^test_argmax_no_keepdims_example_select_last_index",
|
||||
"^test_argmin_no_keepdims_example_select_last_index",
|
||||
"^test_argmin_keepdims_example_select_last_index",
|
||||
"^test_pow_types_float32_int32_*",
|
||||
"^test_pow_types_float32_int64_*",
|
||||
"^test_pow_types_float_*",
|
||||
"^test_pow_types_int32_float32_*",
|
||||
"^test_pow_types_int_*",
|
||||
"^test_pow_types_int64_float32_*"
|
||||
],
|
||||
"current_failing_tests_DNNL": [
|
||||
"^test_range_float_type_positive_delta_expanded",
|
||||
"^test_range_int32_type_negative_delta_expanded",
|
||||
"^test_averagepool_2d_ceil",
|
||||
"^test_maxpool_2d_ceil",
|
||||
"^test_maxpool_2d_dilations",
|
||||
"^test_maxpool_2d_uint8",
|
||||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op.
|
||||
],
|
||||
"current_failing_tests_NNAPI": [
|
||||
"^test_maxpool_2d_uint8",
|
||||
"^test_negative_log_likelihood.*",
|
||||
"^test_softmax_cross_entropy.*"
|
||||
],
|
||||
"current_failing_tests_OPENVINO_GPU": [
|
||||
"^test_div",
|
||||
// temporarily exclude vgg19 test which comsumes too much memory, run out of memory on Upsquared device.
|
||||
// single test pass for vgg19, need furture investigation
|
||||
"^test_vgg19",
|
||||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op.
|
||||
],
|
||||
"current_failing_tests_OPENVINO_GPU_FP32": [
|
||||
"^test_operator_permute2",
|
||||
"^test_operator_repeat",
|
||||
"^test_operator_repeat_dim_overflow",
|
||||
"^test_add_bcast",
|
||||
"^test_batchnorm_epsilon",
|
||||
"^test_div_bcast",
|
||||
"^test_mul_bcast",
|
||||
"^test_pow_bcast_array",
|
||||
"^test_sub_bcast",
|
||||
"^test_batchnorm_example",
|
||||
"^test_clip_default_inbounds",
|
||||
"^test_resize_upsample_sizes_nearest_ceil_half_pixel",
|
||||
"^test_resize_upsample_sizes_nearest_floor_align_corners",
|
||||
"^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric",
|
||||
"^test_unique_not_sorted_without_axis",
|
||||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op.
|
||||
],
|
||||
"current_failing_tests_OPENVINO_CPU_FP32": [
|
||||
"^test_operator_permute2",
|
||||
"^test_operator_repeat",
|
||||
"^test_operator_repeat_dim_overflow",
|
||||
"^test_negative_log_likelihood.*", // Does not support 5-D or above tensors for SUB op.
|
||||
"^test_softmax_cross_entropy.*" // Does not support 5-D or above tensors for SUB op.
|
||||
],
|
||||
// ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported
|
||||
"tests_with_pre_opset7_dependencies": [
|
||||
"^test_AvgPool1d",
|
||||
"^test_AvgPool1d_stride",
|
||||
"^test_AvgPool2d",
|
||||
"^test_AvgPool2d_stride",
|
||||
"^test_AvgPool3d",
|
||||
"^test_AvgPool3d_stride1_pad0_gpu_input",
|
||||
"^test_AvgPool3d_stride",
|
||||
"^test_BatchNorm1d_3d_input_eval",
|
||||
"^test_BatchNorm2d_eval",
|
||||
"^test_BatchNorm2d_momentum_eval",
|
||||
"^test_BatchNorm3d_eval",
|
||||
"^test_BatchNorm3d_momentum_eval",
|
||||
"^test_GLU",
|
||||
"^test_GLU_dim",
|
||||
"^test_Linear",
|
||||
"^test_PReLU_1d",
|
||||
"^test_PReLU_1d_multiparam",
|
||||
"^test_PReLU_2d",
|
||||
"^test_PReLU_2d_multiparam",
|
||||
"^test_PReLU_3d",
|
||||
"^test_PReLU_3d_multiparam",
|
||||
"^test_PoissonNLLLLoss_no_reduce",
|
||||
"^test_Softsign",
|
||||
"^test_operator_add_broadcast",
|
||||
"^test_operator_add_size1_broadcast",
|
||||
"^test_operator_add_size1_right_broadcast",
|
||||
"^test_operator_add_size1_singleton_broadcast",
|
||||
"^test_operator_addconstant",
|
||||
"^test_operator_addmm",
|
||||
"^test_operator_basic",
|
||||
"^test_operator_mm",
|
||||
"^test_operator_non_float_params",
|
||||
"^test_operator_params",
|
||||
"^test_operator_pow"
|
||||
],
|
||||
"unsupported_usages": [
|
||||
"^test_convtranspose_1d", // ConvTransponse supports 4-D only
|
||||
"^test_convtranspose_3d"
|
||||
],
|
||||
"failing_permanently": [
|
||||
// Numpy float to string has unexpected rounding for some results given numpy default precision is meant to be 8.
|
||||
// e.g. 0.296140194 -> "0.2961402" not "0.29614019". ORT produces the latter with precision set to 8, which
|
||||
// doesn"t match the expected output that was generated with numpy.
|
||||
"^test_cast_FLOAT_to_STRING"
|
||||
],
|
||||
"test_with_types_disabled_due_to_binary_size_concerns": [
|
||||
"^test_bitshift_right_uint16",
|
||||
"^test_bitshift_left_uint16"
|
||||
]
|
||||
}
|
||||
Loading…
Reference in a new issue