From b9e8aac5f6760acf849dec4c0bd13554bff45ce8 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Sat, 25 May 2019 08:41:43 -0700 Subject: [PATCH] Enable some more tests (#1101) Restructure a little so permanently disabled tests are clearer. --- .../test/python/onnx_backend_test_series.py | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 60b5a1057b..b3dd22418a 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -14,6 +14,7 @@ import onnxruntime.backend as c2 pytest_plugins = 'onnx.backend.test.report', + class OrtBackendTest(onnx.backend.test.BackendTest): def __init__(self, backend, parent_module=None): @@ -33,8 +34,55 @@ class OrtBackendTest(onnx.backend.test.BackendTest): rtol=1e-3, atol=1e-5) -def CreateBackendTest(testname=None): +# ORT first supported opset 7, so models with nodes that require versions prior to opset 7 are not supported +def tests_with_pre_opset7_dependencies_filters(): + filters = ('^test_AvgPool1d_cpu.*', + '^test_AvgPool1d_stride_cpu.*', + '^test_AvgPool2d_cpu.*', + '^test_AvgPool2d_stride_cpu.*', + '^test_AvgPool3d_cpu.*', + '^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*', + '^test_AvgPool3d_stride_cpu.*', + '^test_BatchNorm1d_3d_input_eval_cpu.*', + '^test_BatchNorm2d_eval_cpu.*', + '^test_BatchNorm2d_momentum_eval_cpu.*', + '^test_BatchNorm3d_eval_cpu.*', + '^test_BatchNorm3d_momentum_eval_cpu.*', + '^test_GLU_cpu.*', + '^test_GLU_dim_cpu.*', + '^test_Linear_cpu.*', + '^test_PReLU_1d_cpu.*', + '^test_PReLU_1d_multiparam_cpu.*', + '^test_PReLU_2d_cpu.*', + '^test_PReLU_2d_multiparam_cpu.*', + '^test_PReLU_3d_cpu.*', + '^test_PReLU_3d_multiparam_cpu.*', + '^test_PoissonNLLLLoss_no_reduce_cpu.*', + '^test_Softsign_cpu.*', + '^test_operator_add_broadcast_cpu.*', + '^test_operator_add_size1_broadcast_cpu.*', + '^test_operator_add_size1_right_broadcast_cpu.*', + '^test_operator_add_size1_singleton_broadcast_cpu.*', + '^test_operator_addconstant_cpu.*', + '^test_operator_addmm_cpu.*', + '^test_operator_basic_cpu.*', + '^test_operator_mm_cpu.*', + '^test_operator_non_float_params_cpu.*', + '^test_operator_params_cpu.*', + '^test_operator_pow_cpu.*') + + return filters + + +def unsupported_usages_filters(): + filters = ('^test_convtranspose_1d_cpu.*', # ConvTransponse supports 4-D only + '^test_convtranspose_3d_cpu.*') + + return filters + + +def create_backend_test(testname=None): backend_test = OrtBackendTest(c2, __name__) # Type not supported @@ -43,58 +91,19 @@ def CreateBackendTest(testname=None): if testname: backend_test.include(testname + '.*') else: - backend_test.exclude(r'(' - '^test_cast_DOUBLE_to_FLOAT_cpu.*' - '|^test_cast_FLOAT_to_DOUBLE_cpu.*' - '|^test_cast_FLOAT_to_STRING_cpu.*' - '|^test_cast_STRING_to_FLOAT_cpu.*' - '|^test_convtranspose_1d_cpu.*' - '|^test_convtranspose_3d_cpu.*' - '|^test_constantofshape_*.*' - '|^test_dequantizelinear_cpu.*' + # Tests that are failing temporarily and should be fixed + current_failing_tests = ('^test_cast_FLOAT_to_STRING_cpu.*', + '^test_constantofshape_*.*', + '^test_dequantizelinear_cpu.*', + '^test_shrink_cpu.*', + '^test_qlinearconv_cpu.*', + '^test_quantizelinear_cpu.*') - '|^test_AvgPool1d_cpu.*' - '|^test_AvgPool1d_stride_cpu.*' - '|^test_AvgPool2d_cpu.*' - '|^test_AvgPool2d_stride_cpu.*' - '|^test_AvgPool3d_cpu.*' - '|^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*' - '|^test_AvgPool3d_stride_cpu.*' - '|^test_BatchNorm1d_3d_input_eval_cpu.*' - '|^test_BatchNorm2d_eval_cpu.*' - '|^test_BatchNorm2d_momentum_eval_cpu.*' - '|^test_BatchNorm3d_eval_cpu.*' - '|^test_BatchNorm3d_momentum_eval_cpu.*' - '|^test_GLU_cpu.*' - '|^test_GLU_dim_cpu.*' - '|^test_Linear_cpu.*' - '|^test_PReLU_1d_cpu.*' - '|^test_PReLU_1d_multiparam_cpu.*' - '|^test_PReLU_2d_cpu.*' - '|^test_PReLU_2d_multiparam_cpu.*' - '|^test_PReLU_3d_cpu.*' - '|^test_PReLU_3d_multiparam_cpu.*' - '|^test_PoissonNLLLLoss_no_reduce_cpu.*' - '|^test_Softsign_cpu.*' - '|^test_operator_add_broadcast_cpu.*' - '|^test_operator_add_size1_broadcast_cpu.*' - '|^test_operator_add_size1_right_broadcast_cpu.*' - '|^test_operator_add_size1_singleton_broadcast_cpu.*' - '|^test_operator_addconstant_cpu.*' - '|^test_operator_addmm_cpu.*' - '|^test_operator_basic_cpu.*' - '|^test_operator_mm_cpu.*' - '|^test_operator_non_float_params_cpu.*' - '|^test_operator_params_cpu.*' - '|^test_operator_pow_cpu.*' - '|^test_shrink_cpu.*' - '|^test_vgg19_cpu.*' - '|^test_zfnet512_cpu.*' - '|^test_qlinearconv_cpu.*' - '|^test_quantizelinear_cpu.*' - '|^test_roialign_cpu.*' - '|^test_operator_repeat_dim_overflow_cpu.*' - ')') + filters = current_failing_tests + \ + tests_with_pre_opset7_dependencies_filters() + \ + unsupported_usages_filters() + + backend_test.exclude('(' + '|'.join(filters) + ')') # import all test cases at global scope to make # them visible to python.unittest. @@ -102,6 +111,7 @@ def CreateBackendTest(testname=None): return backend_test + def parse_args(): parser = argparse.ArgumentParser(os.path.basename(__file__), description='Run the ONNX backend tests using ONNXRuntime.') @@ -122,5 +132,5 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - backend_test = CreateBackendTest(args.testname) + backend_test = create_backend_test(args.testname) unittest.main()