From 0157b6c209db1ae0e516e618a1ba89b78b1ce6e7 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 23 May 2019 16:45:27 -0700 Subject: [PATCH] Enable specifying tests for onnx_backend_test_series.py to run via the command line (#1099) * onnx_backend_test_series.py update to allow specifying a single test to run. The python unittest filtering is to a test method not test name so can't be used directly. * Clarify help message. --- .../test/python/onnx_backend_test_series.py | 146 +++++++++++------- 1 file changed, 87 insertions(+), 59 deletions(-) diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index e517bc334d..60b5a1057b 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -1,14 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import argparse +import sys import os - +import platform import unittest + import onnx.backend.test import numpy as np import onnxruntime.backend as c2 -import platform pytest_plugins = 'onnx.backend.test.report', @@ -31,68 +33,94 @@ class OrtBackendTest(onnx.backend.test.BackendTest): rtol=1e-3, atol=1e-5) -backend_test = OrtBackendTest(c2, __name__) +def CreateBackendTest(testname=None): -# Type not supported -backend_test.exclude(r'(FLOAT16)') + backend_test = OrtBackendTest(c2, __name__) -backend_test.exclude(r'(' -'^test_cast_DOUBLE_to_FLOAT_cpu.*' -'|^test_cast_FLOAT_to_DOUBLE_cpu.*' -'|^test_cast_FLOAT_to_STRING_cpu.*' -'|^test_cast_STRING_to_FLOAT_cpu.*' -'|^test_convtranspose_1d_cpu.*' -'|^test_convtranspose_3d_cpu.*' -'|^test_constantofshape_*.*' -'|^test_dequantizelinear_cpu.*' + # Type not supported + backend_test.exclude(r'(FLOAT16)') -'|^test_AvgPool1d_cpu.*' -'|^test_AvgPool1d_stride_cpu.*' -'|^test_AvgPool2d_cpu.*' -'|^test_AvgPool2d_stride_cpu.*' -'|^test_AvgPool3d_cpu.*' -'|^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*' -'|^test_AvgPool3d_stride_cpu.*' -'|^test_BatchNorm1d_3d_input_eval_cpu.*' -'|^test_BatchNorm2d_eval_cpu.*' -'|^test_BatchNorm2d_momentum_eval_cpu.*' -'|^test_BatchNorm3d_eval_cpu.*' -'|^test_BatchNorm3d_momentum_eval_cpu.*' -'|^test_GLU_cpu.*' -'|^test_GLU_dim_cpu.*' -'|^test_Linear_cpu.*' -'|^test_PReLU_1d_cpu.*' -'|^test_PReLU_1d_multiparam_cpu.*' -'|^test_PReLU_2d_cpu.*' -'|^test_PReLU_2d_multiparam_cpu.*' -'|^test_PReLU_3d_cpu.*' -'|^test_PReLU_3d_multiparam_cpu.*' -'|^test_PoissonNLLLLoss_no_reduce_cpu.*' -'|^test_Softsign_cpu.*' -'|^test_operator_add_broadcast_cpu.*' -'|^test_operator_add_size1_broadcast_cpu.*' -'|^test_operator_add_size1_right_broadcast_cpu.*' -'|^test_operator_add_size1_singleton_broadcast_cpu.*' -'|^test_operator_addconstant_cpu.*' -'|^test_operator_addmm_cpu.*' -'|^test_operator_basic_cpu.*' -'|^test_operator_mm_cpu.*' -'|^test_operator_non_float_params_cpu.*' -'|^test_operator_params_cpu.*' -'|^test_operator_pow_cpu.*' -'|^test_shrink_cpu.*' -'|^test_vgg19_cpu.*' -'|^test_zfnet512_cpu.*' -'|^test_qlinearconv_cpu.*' -'|^test_quantizelinear_cpu.*' -'|^test_roialign_cpu.*' -'|^test_operator_repeat_dim_overflow_cpu.*' -')') + if testname: + backend_test.include(testname + '.*') + else: + backend_test.exclude(r'(' + '^test_cast_DOUBLE_to_FLOAT_cpu.*' + '|^test_cast_FLOAT_to_DOUBLE_cpu.*' + '|^test_cast_FLOAT_to_STRING_cpu.*' + '|^test_cast_STRING_to_FLOAT_cpu.*' + '|^test_convtranspose_1d_cpu.*' + '|^test_convtranspose_3d_cpu.*' + '|^test_constantofshape_*.*' + '|^test_dequantizelinear_cpu.*' -# import all test cases at global scope to make -# them visible to python.unittest. -globals().update(backend_test.enable_report().test_cases) + '|^test_AvgPool1d_cpu.*' + '|^test_AvgPool1d_stride_cpu.*' + '|^test_AvgPool2d_cpu.*' + '|^test_AvgPool2d_stride_cpu.*' + '|^test_AvgPool3d_cpu.*' + '|^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*' + '|^test_AvgPool3d_stride_cpu.*' + '|^test_BatchNorm1d_3d_input_eval_cpu.*' + '|^test_BatchNorm2d_eval_cpu.*' + '|^test_BatchNorm2d_momentum_eval_cpu.*' + '|^test_BatchNorm3d_eval_cpu.*' + '|^test_BatchNorm3d_momentum_eval_cpu.*' + '|^test_GLU_cpu.*' + '|^test_GLU_dim_cpu.*' + '|^test_Linear_cpu.*' + '|^test_PReLU_1d_cpu.*' + '|^test_PReLU_1d_multiparam_cpu.*' + '|^test_PReLU_2d_cpu.*' + '|^test_PReLU_2d_multiparam_cpu.*' + '|^test_PReLU_3d_cpu.*' + '|^test_PReLU_3d_multiparam_cpu.*' + '|^test_PoissonNLLLLoss_no_reduce_cpu.*' + '|^test_Softsign_cpu.*' + '|^test_operator_add_broadcast_cpu.*' + '|^test_operator_add_size1_broadcast_cpu.*' + '|^test_operator_add_size1_right_broadcast_cpu.*' + '|^test_operator_add_size1_singleton_broadcast_cpu.*' + '|^test_operator_addconstant_cpu.*' + '|^test_operator_addmm_cpu.*' + '|^test_operator_basic_cpu.*' + '|^test_operator_mm_cpu.*' + '|^test_operator_non_float_params_cpu.*' + '|^test_operator_params_cpu.*' + '|^test_operator_pow_cpu.*' + '|^test_shrink_cpu.*' + '|^test_vgg19_cpu.*' + '|^test_zfnet512_cpu.*' + '|^test_qlinearconv_cpu.*' + '|^test_quantizelinear_cpu.*' + '|^test_roialign_cpu.*' + '|^test_operator_repeat_dim_overflow_cpu.*' + ')') + + # import all test cases at global scope to make + # them visible to python.unittest. + globals().update(backend_test.enable_report().test_cases) + + return backend_test + +def parse_args(): + parser = argparse.ArgumentParser(os.path.basename(__file__), + description='Run the ONNX backend tests using ONNXRuntime.') + + # Add an argument to match a single test name, by adding the name to the 'include' filter. + # Using -k with python unittest (https://docs.python.org/3/library/unittest.html#command-line-options) + # doesn't work as it filters on the test method name (Runner._add_model_test) rather than inidividual test case names. + parser.add_argument('-t', '--test-name', dest='testname', type=str, + help="Only run tests that match this value. Matching is regex based, and '.*' is automatically appended") + + # parse just our args. python unittest has its own args and arg parsing, and that runs inside unittest.main() + args, left = parser.parse_known_args() + sys.argv = sys.argv[:1] + left + + return args if __name__ == '__main__': + args = parse_args() + + backend_test = CreateBackendTest(args.testname) unittest.main()