bwruntests: Improve various aspects

* Allow dumping stdout of subprocess to terminal stdout. Synchronized
  with a lock.
* Add support for yaml file based test definitions (path + command per
  test)
* Make C-c terminate subprocesses properly
* Print table based summary at the end
This commit is contained in:
bluew 2020-01-17 13:37:52 +00:00
parent 2e5e3c80e7
commit 810d596ade

View file

@ -30,18 +30,21 @@
import argparse import argparse
from subprocess import (Popen, TimeoutExpired, from subprocess import (Popen, TimeoutExpired,
CalledProcessError, PIPE) CalledProcessError, PIPE)
from threading import Lock
import shlex
import sys import sys
import signal import signal
import os import os
import multiprocessing import multiprocessing
import errno import errno
import pprint import pprint
import time
runtest = argparse.ArgumentParser(prog='bwruntests', runtest = argparse.ArgumentParser(prog='bwruntests',
description="""Run PULP tests in parallel""") description="""Run PULP tests in parallel""")
runtest.version = '0.1' runtest.version = '0.2'
runtest.add_argument('test_file', type=str, runtest.add_argument('test_file', type=str,
help='file defining tests to be run') help='file defining tests to be run')
@ -55,42 +58,68 @@ runtest.add_argument('-t', '--timeout', type=float,
help="""Timeout for all processes in seconds""") help="""Timeout for all processes in seconds""")
runtest.add_argument('-v', '--verbose', action='store_true', runtest.add_argument('-v', '--verbose', action='store_true',
help="""Enable verbose output""") help="""Enable verbose output""")
runtest.add_argument('-s', '--proc-verbose', action='store_true',
help="""Write processes' stdout and stderr to shell stdout
after they terminate""")
runtest.add_argument('--report-junit', action='store_true', runtest.add_argument('--report-junit', action='store_true',
help="""Generate a junit report""") help="""Generate a junit report""")
runtest.add_argument('--disable-junit-pp', action='store_false', runtest.add_argument('--disable-junit-pp', action='store_true',
help="""Disable pretty print of junit report""") help="""Disable pretty print of junit report""")
runtest.add_argument('--disable-results-pp', action='store_true',
help="""Disable printing test results""")
runtest.add_argument('-y,', '--yaml', action='store_true',
help="""Read tests from yaml file instead of executing
from a list of commands""")
runtest.add_argument('-o,', '--output', type=str, runtest.add_argument('-o,', '--output', type=str,
help="""Write to file instead of stdout""") help="""Write junit.xml to file instead of stdout""")
stdout_lock = Lock()
class FinishedProcess(object): class FinishedProcess(object):
"""A process that has finished running. """A process that has finished running.
""" """
def __init__(self, args, returncode, stdout=None, stderr=None): def __init__(self, name, cwd, args, returncode,
stdout=None, stderr=None, time=None):
self.name = name
self.cwd = cwd
self.args = args self.args = args
self.returncode = returncode self.returncode = returncode
self.stdout = stdout self.stdout = stdout
self.stderr = stderr self.stderr = stderr
self.time = time
def __repr__(self): def __repr__(self):
args = ['args={!r}'.format(self.args), args = ['name={!r}'.format(self.name)]
'returncode={!r}'.format(self.returncode)] args += ['cwd={!r}'.format(self.cwd)]
args += ['args={!r}'.format(self.args),
'returncode={!r}'.format(self.returncode)]
if self.stdout is not None: if self.stdout is not None:
args.append('stdout={!r}'.format(self.stdout)) args.append('stdout={!r}'.format(self.stdout))
if self.stderr is not None: if self.stderr is not None:
args.append('stderr={!r}'.format(self.stderr)) args.append('stderr={!r}'.format(self.stderr))
if self.time is not None:
args.append('time={!r}'.format(self.time))
return "{}({})".format(type(self).__name__, ', '.join(args)) return "{}({})".format(type(self).__name__, ', '.join(args))
def fork(*popenargs, check=False, shell=True, def fork(name, cwd, *popenargs, check=False, shell=True,
**kwargs): **kwargs):
"""Run subprocess and return process args, error code, stdout and stderr """Run subprocess and return process args, error code, stdout and stderr
""" """
def proc_out(cwd, stdout, stderr):
print('cwd={}'.format(cwd))
print('stdout=')
print(stdout.decode('utf-8'))
print('stderr=')
print(stderr.decode('utf-8'))
kwargs['stdout'] = PIPE kwargs['stdout'] = PIPE
kwargs['stderr'] = PIPE kwargs['stderr'] = PIPE
with Popen(*popenargs, preexec_fn=os.setpgrp, **kwargs) as process: with Popen(*popenargs, preexec_fn=os.setpgrp, cwd=cwd,
**kwargs) as process:
try: try:
# Child and parent are racing for setting/using the pgid so we have # Child and parent are racing for setting/using the pgid so we have
# to set it in both processes. See glib manual. # to set it in both processes. See glib manual.
@ -99,7 +128,8 @@ def fork(*popenargs, check=False, shell=True,
except OSError as e: except OSError as e:
if e.errno != errno.EACCES: if e.errno != errno.EACCES:
raise raise
# measure runtime
start = time.time()
stdout, stderr = process.communicate(input, timeout=args.timeout) stdout, stderr = process.communicate(input, timeout=args.timeout)
except TimeoutExpired: except TimeoutExpired:
pgid = os.getpgid(process.pid) pgid = os.getpgid(process.pid)
@ -109,10 +139,20 @@ def fork(*popenargs, check=False, shell=True,
# (make -> vsim -> etc). We need to make a process group and kill # (make -> vsim -> etc). We need to make a process group and kill
# that # that
stdout, stderr = process.communicate() stdout, stderr = process.communicate()
return FinishedProcess(process.args, 1, timeoutmsg = 'TIMEOUT after {:f}s'.format(args.timeout)
if args.proc_verbose:
stdout_lock.acquire()
print(name)
print(timeoutmsg)
proc_out(cwd, stdout, stderr)
stdout_lock.release()
return FinishedProcess(name, cwd, process.args, 1,
stdout.decode('utf-8'), stdout.decode('utf-8'),
'TIMEOUT after {:f}s\n'.format(args.timeout) timeoutmsg + '\n'
+ stderr.decode('utf-8')) + stderr.decode('utf-8'),
time.time() - start)
# Including KeyboardInterrupt, communicate handled that. # Including KeyboardInterrupt, communicate handled that.
except: # noqa: E722 except: # noqa: E722
pgid = os.getpgid(process.pid) pgid = os.getpgid(process.pid)
@ -123,9 +163,16 @@ def fork(*popenargs, check=False, shell=True,
if check and retcode: if check and retcode:
raise CalledProcessError(retcode, process.args, raise CalledProcessError(retcode, process.args,
output=stdout, stderr=stderr) output=stdout, stderr=stderr)
return FinishedProcess(process.args, retcode, if args.proc_verbose:
stdout_lock.acquire()
print(name)
proc_out(cwd, stdout, stderr)
stdout_lock.release()
return FinishedProcess(name, cwd, process.args, retcode,
stdout.decode('utf-8'), stdout.decode('utf-8'),
stderr.decode('utf-8')) stderr.decode('utf-8'),
time.time() - start)
if __name__ == '__main__': if __name__ == '__main__':
@ -137,48 +184,112 @@ if __name__ == '__main__':
try: try:
from junit_xml import TestSuite, TestCase from junit_xml import TestSuite, TestCase
except ImportError: except ImportError:
print("""The --report-junit option requires print("""Error: The --report-junit option requires
the junit_xml module which is not installed.""", the junit_xml library which is not installed.""",
file=sys.stderr) file=sys.stderr)
exit(1) exit(1)
# load command list # lazy import PrettyTable for displaying results
if not(args.disable_results_pp):
try:
from prettytable import PrettyTable
except ImportError:
print("""Warning: Displaying results requires the PrettyTable
library which is not installed""")
testnames = []
shellcmds = []
cwds = []
tests = [] tests = []
with open(args.test_file) as f:
tests = list(map(str.rstrip, f))
if args.verbose: # load tests (yaml or command list)
print('Tests which we are running:') if args.yaml:
pp.pprint(tests) try:
import yaml
except ImportError:
print("""Error: The --yaml option requires
the pyyaml library which is not installed.""",
file=sys.stderr)
exit(1)
with open(args.test_file) as f:
testyaml = yaml.load(f)
for testsetname, testv in testyaml.items():
for testname, insn in testv.items():
cmd = shlex.split(insn['command'])
cwd = insn['path']
testnames.append(testsetname + ':' + testname)
shellcmds.append(cmd)
cwds.append(cwd)
tests.append((testsetname + ':' + testname, cwd, cmd))
if args.verbose:
pp.pprint(tests)
else: # (command list)
with open(args.test_file) as f:
testnames = list(map(str.rstrip, f))
shellcmds = [shlex.split(e) for e in testnames]
cwds = ['./' for e in testnames]
tests = list(zip(testnames, cwds, shellcmds))
if args.verbose:
print('Tests which we are running:')
pp.pprint(tests)
pp.pprint(shellcmds)
# list of commands to be run # Spawning process pool
shellcmds = [['make', '-C', e, 'clean', 'all', 'run'] for e in tests] # Disable signals to prevent race. Child processes inherit SIGINT handler
if args.verbose: original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
print('Generated shell commands:')
pp.pprint(shellcmds)
# by default we use the number of available cores to limit the number of
# concurrently spawned process
pool = multiprocessing.Pool(processes=args.max_procs) pool = multiprocessing.Pool(processes=args.max_procs)
procresults = pool.map(fork, shellcmds) # Restore SIGINT handler
pp.pprint(procresults) signal.signal(signal.SIGINT, original_sigint_handler)
try:
procresults = pool.starmap(fork, tests)
except KeyboardInterrupt:
print("\nTerminating bwruntest.py")
pool.terminate()
pool.join()
exit(1)
if args.verbose:
pp.pprint(procresults)
pool.close()
pool.join()
# Generate junit.xml file. Junit.xml differentiates between failure and # Generate junit.xml file. Junit.xml differentiates between failure and
# errors but we treat everything as errors. # errors but we treat everything as errors.
if args.report_junit: if args.report_junit:
testcases = [] testcases = []
for p in procresults: for p in procresults:
testcase = TestCase(' '.join(p.args), stdout=p.stdout, testcase = TestCase(p.name,
stderr=p.stderr) classname=p.name,
stdout=p.stdout,
stderr=p.stderr,
elapsed_sec=p.time)
if p.returncode != 0: if p.returncode != 0:
testcase.add_error_info(p.stderr) testcase.add_failure_info(p.stderr)
testcases.append(testcase) testcases.append(testcase)
testsuite = TestSuite("bwruntests", testcases) testsuite = TestSuite('bwruntests', testcases)
if args.output: if args.output:
with open(args.output, 'w') as f: with open(args.output, 'w') as f:
TestSuite.to_file(f, [testsuite], TestSuite.to_file(f, [testsuite],
prettyprint=args.disable_junit_pp) prettyprint=not(args.disable_junit_pp))
else: else:
print(TestSuite.to_xml_string([testsuite], print(TestSuite.to_xml_string([testsuite],
prettyprint=args.disable_junit_pp)) prettyprint=(args.disable_junit_pp)))
# print summary of test results
if not(args.disable_results_pp):
testcount = sum(1 for x in testnames)
testfailcount = sum(1 for p in procresults if p.returncode != 0)
testpassedcount = testcount - testfailcount
resulttable = PrettyTable(['test', 'config', 'time', 'passed/total'])
resulttable.align['test'] = "l"
resulttable.align['config'] = "l"
resulttable.add_row(['bwruntest', '', '', '{0:d}/{1:d}'.
format(testpassedcount, testcount)])
for p in procresults:
testpassed = 1 if p.returncode == 0 else 0
testname = p.name
resulttable.add_row([testname, '',
'{0:.2f}s'.format(p.time),
'{0:d}/{1:d}'.format(testpassed, 1)])
print(resulttable)