mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Enable specifying tests for onnx_backend_test_series.py to run via the command line (#1099)
* onnx_backend_test_series.py update to allow specifying a single test to run. The python unittest filtering is to a test method not test name so can't be used directly. * Clarify help message.
This commit is contained in:
Scott McKay
GitHub
parent
7421755198
commit
0157b6c209
1 changed files with 87 additions and 59 deletions
|
|
@ -1,14 +1,16 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
|
||||
import platform
|
||||
import unittest
|
||||
|
||||
import onnx.backend.test
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime.backend as c2
|
||||
import platform
|
||||
|
||||
pytest_plugins = 'onnx.backend.test.report',
|
||||
|
||||
|
|
@ -31,68 +33,94 @@ class OrtBackendTest(onnx.backend.test.BackendTest):
|
|||
rtol=1e-3,
|
||||
atol=1e-5)
|
||||
|
||||
backend_test = OrtBackendTest(c2, __name__)
|
||||
def CreateBackendTest(testname=None):
|
||||
|
||||
# Type not supported
|
||||
backend_test.exclude(r'(FLOAT16)')
|
||||
backend_test = OrtBackendTest(c2, __name__)
|
||||
|
||||
backend_test.exclude(r'('
|
||||
'^test_cast_DOUBLE_to_FLOAT_cpu.*'
|
||||
'|^test_cast_FLOAT_to_DOUBLE_cpu.*'
|
||||
'|^test_cast_FLOAT_to_STRING_cpu.*'
|
||||
'|^test_cast_STRING_to_FLOAT_cpu.*'
|
||||
'|^test_convtranspose_1d_cpu.*'
|
||||
'|^test_convtranspose_3d_cpu.*'
|
||||
'|^test_constantofshape_*.*'
|
||||
'|^test_dequantizelinear_cpu.*'
|
||||
# Type not supported
|
||||
backend_test.exclude(r'(FLOAT16)')
|
||||
|
||||
'|^test_AvgPool1d_cpu.*'
|
||||
'|^test_AvgPool1d_stride_cpu.*'
|
||||
'|^test_AvgPool2d_cpu.*'
|
||||
'|^test_AvgPool2d_stride_cpu.*'
|
||||
'|^test_AvgPool3d_cpu.*'
|
||||
'|^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*'
|
||||
'|^test_AvgPool3d_stride_cpu.*'
|
||||
'|^test_BatchNorm1d_3d_input_eval_cpu.*'
|
||||
'|^test_BatchNorm2d_eval_cpu.*'
|
||||
'|^test_BatchNorm2d_momentum_eval_cpu.*'
|
||||
'|^test_BatchNorm3d_eval_cpu.*'
|
||||
'|^test_BatchNorm3d_momentum_eval_cpu.*'
|
||||
'|^test_GLU_cpu.*'
|
||||
'|^test_GLU_dim_cpu.*'
|
||||
'|^test_Linear_cpu.*'
|
||||
'|^test_PReLU_1d_cpu.*'
|
||||
'|^test_PReLU_1d_multiparam_cpu.*'
|
||||
'|^test_PReLU_2d_cpu.*'
|
||||
'|^test_PReLU_2d_multiparam_cpu.*'
|
||||
'|^test_PReLU_3d_cpu.*'
|
||||
'|^test_PReLU_3d_multiparam_cpu.*'
|
||||
'|^test_PoissonNLLLLoss_no_reduce_cpu.*'
|
||||
'|^test_Softsign_cpu.*'
|
||||
'|^test_operator_add_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_right_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_singleton_broadcast_cpu.*'
|
||||
'|^test_operator_addconstant_cpu.*'
|
||||
'|^test_operator_addmm_cpu.*'
|
||||
'|^test_operator_basic_cpu.*'
|
||||
'|^test_operator_mm_cpu.*'
|
||||
'|^test_operator_non_float_params_cpu.*'
|
||||
'|^test_operator_params_cpu.*'
|
||||
'|^test_operator_pow_cpu.*'
|
||||
'|^test_shrink_cpu.*'
|
||||
'|^test_vgg19_cpu.*'
|
||||
'|^test_zfnet512_cpu.*'
|
||||
'|^test_qlinearconv_cpu.*'
|
||||
'|^test_quantizelinear_cpu.*'
|
||||
'|^test_roialign_cpu.*'
|
||||
'|^test_operator_repeat_dim_overflow_cpu.*'
|
||||
')')
|
||||
if testname:
|
||||
backend_test.include(testname + '.*')
|
||||
else:
|
||||
backend_test.exclude(r'('
|
||||
'^test_cast_DOUBLE_to_FLOAT_cpu.*'
|
||||
'|^test_cast_FLOAT_to_DOUBLE_cpu.*'
|
||||
'|^test_cast_FLOAT_to_STRING_cpu.*'
|
||||
'|^test_cast_STRING_to_FLOAT_cpu.*'
|
||||
'|^test_convtranspose_1d_cpu.*'
|
||||
'|^test_convtranspose_3d_cpu.*'
|
||||
'|^test_constantofshape_*.*'
|
||||
'|^test_dequantizelinear_cpu.*'
|
||||
|
||||
# import all test cases at global scope to make
|
||||
# them visible to python.unittest.
|
||||
globals().update(backend_test.enable_report().test_cases)
|
||||
'|^test_AvgPool1d_cpu.*'
|
||||
'|^test_AvgPool1d_stride_cpu.*'
|
||||
'|^test_AvgPool2d_cpu.*'
|
||||
'|^test_AvgPool2d_stride_cpu.*'
|
||||
'|^test_AvgPool3d_cpu.*'
|
||||
'|^test_AvgPool3d_stride1_pad0_gpu_input_cpu.*'
|
||||
'|^test_AvgPool3d_stride_cpu.*'
|
||||
'|^test_BatchNorm1d_3d_input_eval_cpu.*'
|
||||
'|^test_BatchNorm2d_eval_cpu.*'
|
||||
'|^test_BatchNorm2d_momentum_eval_cpu.*'
|
||||
'|^test_BatchNorm3d_eval_cpu.*'
|
||||
'|^test_BatchNorm3d_momentum_eval_cpu.*'
|
||||
'|^test_GLU_cpu.*'
|
||||
'|^test_GLU_dim_cpu.*'
|
||||
'|^test_Linear_cpu.*'
|
||||
'|^test_PReLU_1d_cpu.*'
|
||||
'|^test_PReLU_1d_multiparam_cpu.*'
|
||||
'|^test_PReLU_2d_cpu.*'
|
||||
'|^test_PReLU_2d_multiparam_cpu.*'
|
||||
'|^test_PReLU_3d_cpu.*'
|
||||
'|^test_PReLU_3d_multiparam_cpu.*'
|
||||
'|^test_PoissonNLLLLoss_no_reduce_cpu.*'
|
||||
'|^test_Softsign_cpu.*'
|
||||
'|^test_operator_add_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_right_broadcast_cpu.*'
|
||||
'|^test_operator_add_size1_singleton_broadcast_cpu.*'
|
||||
'|^test_operator_addconstant_cpu.*'
|
||||
'|^test_operator_addmm_cpu.*'
|
||||
'|^test_operator_basic_cpu.*'
|
||||
'|^test_operator_mm_cpu.*'
|
||||
'|^test_operator_non_float_params_cpu.*'
|
||||
'|^test_operator_params_cpu.*'
|
||||
'|^test_operator_pow_cpu.*'
|
||||
'|^test_shrink_cpu.*'
|
||||
'|^test_vgg19_cpu.*'
|
||||
'|^test_zfnet512_cpu.*'
|
||||
'|^test_qlinearconv_cpu.*'
|
||||
'|^test_quantizelinear_cpu.*'
|
||||
'|^test_roialign_cpu.*'
|
||||
'|^test_operator_repeat_dim_overflow_cpu.*'
|
||||
')')
|
||||
|
||||
# import all test cases at global scope to make
|
||||
# them visible to python.unittest.
|
||||
globals().update(backend_test.enable_report().test_cases)
|
||||
|
||||
return backend_test
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(os.path.basename(__file__),
|
||||
description='Run the ONNX backend tests using ONNXRuntime.')
|
||||
|
||||
# Add an argument to match a single test name, by adding the name to the 'include' filter.
|
||||
# Using -k with python unittest (https://docs.python.org/3/library/unittest.html#command-line-options)
|
||||
# doesn't work as it filters on the test method name (Runner._add_model_test) rather than inidividual test case names.
|
||||
parser.add_argument('-t', '--test-name', dest='testname', type=str,
|
||||
help="Only run tests that match this value. Matching is regex based, and '.*' is automatically appended")
|
||||
|
||||
# parse just our args. python unittest has its own args and arg parsing, and that runs inside unittest.main()
|
||||
args, left = parser.parse_known_args()
|
||||
sys.argv = sys.argv[:1] + left
|
||||
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_args()
|
||||
|
||||
backend_test = CreateBackendTest(args.testname)
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in a new issue