From 2e5e3c80e79055784b92fb3dc579cc5439a04a3f Mon Sep 17 00:00:00 2001 From: bluew Date: Wed, 15 Jan 2020 12:08:59 +0000 Subject: [PATCH] Add simple script to run multiple tests and gather results --- bwruntests.py | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 bwruntests.py diff --git a/bwruntests.py b/bwruntests.py new file mode 100755 index 0000000..f7450ae --- /dev/null +++ b/bwruntests.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +# Copyright 2020 ETH Zurich and University of Bologna +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Run shell commands listed in a file separated by newlines in a parallel +# fashion. If requested the results (tuples consisting of command, stdout, +# stderr and returncode) will be gathered in a junit.xml file. There a few +# knobs to tune the number of spawned processes and the junit.xml formatting. + +# Author: Robert Balas (balasr@iis.ee.ethz.ch) + +import argparse +from subprocess import (Popen, TimeoutExpired, + CalledProcessError, PIPE) +import sys +import signal +import os +import multiprocessing +import errno +import pprint + + +runtest = argparse.ArgumentParser(prog='bwruntests', + description="""Run PULP tests in parallel""") + +runtest.version = '0.1' + +runtest.add_argument('test_file', type=str, + help='file defining tests to be run') +runtest.add_argument('-p', '--max-procs', type=int, + default=multiprocessing.cpu_count(), + help="""Number of parallel + processes used to run test. + Default is number of cpu cores.""") +runtest.add_argument('-t', '--timeout', type=float, + default=None, + help="""Timeout for all processes in seconds""") +runtest.add_argument('-v', '--verbose', action='store_true', + help="""Enable verbose output""") +runtest.add_argument('--report-junit', action='store_true', + help="""Generate a junit report""") +runtest.add_argument('--disable-junit-pp', action='store_false', + help="""Disable pretty print of junit report""") +runtest.add_argument('-o,', '--output', type=str, + help="""Write to file instead of stdout""") + + +class FinishedProcess(object): + """A process that has finished running. + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + +def fork(*popenargs, check=False, shell=True, + **kwargs): + """Run subprocess and return process args, error code, stdout and stderr + """ + + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE + + with Popen(*popenargs, preexec_fn=os.setpgrp, **kwargs) as process: + try: + # Child and parent are racing for setting/using the pgid so we have + # to set it in both processes. See glib manual. + try: + os.setpgid(process.pid, process.pid) + except OSError as e: + if e.errno != errno.EACCES: + raise + + stdout, stderr = process.communicate(input, timeout=args.timeout) + except TimeoutExpired: + pgid = os.getpgid(process.pid) + os.killpg(pgid, signal.SIGKILL) + # process.kill() will only kill the immediate child but not its + # forks. This won't work since our commands will create a few forks + # (make -> vsim -> etc). We need to make a process group and kill + # that + stdout, stderr = process.communicate() + return FinishedProcess(process.args, 1, + stdout.decode('utf-8'), + 'TIMEOUT after {:f}s\n'.format(args.timeout) + + stderr.decode('utf-8')) + # Including KeyboardInterrupt, communicate handled that. + except: # noqa: E722 + pgid = os.getpgid(process.pid) + os.killpg(pgid, signal.SIGKILL) + # We don't call process.wait() as .__exit__ does that for us. + raise + retcode = process.poll() + if check and retcode: + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) + return FinishedProcess(process.args, retcode, + stdout.decode('utf-8'), + stderr.decode('utf-8')) + + +if __name__ == '__main__': + args = runtest.parse_args() + pp = pprint.PrettyPrinter(indent=4) + + # lazy importing so that we can work without junit_xml + if args.report_junit: + try: + from junit_xml import TestSuite, TestCase + except ImportError: + print("""The --report-junit option requires +the junit_xml module which is not installed.""", + file=sys.stderr) + exit(1) + + # load command list + tests = [] + with open(args.test_file) as f: + tests = list(map(str.rstrip, f)) + + if args.verbose: + print('Tests which we are running:') + pp.pprint(tests) + + # list of commands to be run + shellcmds = [['make', '-C', e, 'clean', 'all', 'run'] for e in tests] + if args.verbose: + print('Generated shell commands:') + pp.pprint(shellcmds) + + # by default we use the number of available cores to limit the number of + # concurrently spawned process + pool = multiprocessing.Pool(processes=args.max_procs) + procresults = pool.map(fork, shellcmds) + pp.pprint(procresults) + + # Generate junit.xml file. Junit.xml differentiates between failure and + # errors but we treat everything as errors. + if args.report_junit: + testcases = [] + for p in procresults: + testcase = TestCase(' '.join(p.args), stdout=p.stdout, + stderr=p.stderr) + if p.returncode != 0: + testcase.add_error_info(p.stderr) + testcases.append(testcase) + + testsuite = TestSuite("bwruntests", testcases) + if args.output: + with open(args.output, 'w') as f: + TestSuite.to_file(f, [testsuite], + prettyprint=args.disable_junit_pp) + else: + print(TestSuite.to_xml_string([testsuite], + prettyprint=args.disable_junit_pp))