From 2e5e3c80e79055784b92fb3dc579cc5439a04a3f Mon Sep 17 00:00:00 2001 From: bluew Date: Wed, 15 Jan 2020 12:08:59 +0000 Subject: [PATCH 1/9] Add simple script to run multiple tests and gather results --- bwruntests.py | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 bwruntests.py diff --git a/bwruntests.py b/bwruntests.py new file mode 100755 index 0000000..f7450ae --- /dev/null +++ b/bwruntests.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +# Copyright 2020 ETH Zurich and University of Bologna +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Run shell commands listed in a file separated by newlines in a parallel +# fashion. If requested the results (tuples consisting of command, stdout, +# stderr and returncode) will be gathered in a junit.xml file. There a few +# knobs to tune the number of spawned processes and the junit.xml formatting. + +# Author: Robert Balas (balasr@iis.ee.ethz.ch) + +import argparse +from subprocess import (Popen, TimeoutExpired, + CalledProcessError, PIPE) +import sys +import signal +import os +import multiprocessing +import errno +import pprint + + +runtest = argparse.ArgumentParser(prog='bwruntests', + description="""Run PULP tests in parallel""") + +runtest.version = '0.1' + +runtest.add_argument('test_file', type=str, + help='file defining tests to be run') +runtest.add_argument('-p', '--max-procs', type=int, + default=multiprocessing.cpu_count(), + help="""Number of parallel + processes used to run test. + Default is number of cpu cores.""") +runtest.add_argument('-t', '--timeout', type=float, + default=None, + help="""Timeout for all processes in seconds""") +runtest.add_argument('-v', '--verbose', action='store_true', + help="""Enable verbose output""") +runtest.add_argument('--report-junit', action='store_true', + help="""Generate a junit report""") +runtest.add_argument('--disable-junit-pp', action='store_false', + help="""Disable pretty print of junit report""") +runtest.add_argument('-o,', '--output', type=str, + help="""Write to file instead of stdout""") + + +class FinishedProcess(object): + """A process that has finished running. + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + +def fork(*popenargs, check=False, shell=True, + **kwargs): + """Run subprocess and return process args, error code, stdout and stderr + """ + + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE + + with Popen(*popenargs, preexec_fn=os.setpgrp, **kwargs) as process: + try: + # Child and parent are racing for setting/using the pgid so we have + # to set it in both processes. See glib manual. + try: + os.setpgid(process.pid, process.pid) + except OSError as e: + if e.errno != errno.EACCES: + raise + + stdout, stderr = process.communicate(input, timeout=args.timeout) + except TimeoutExpired: + pgid = os.getpgid(process.pid) + os.killpg(pgid, signal.SIGKILL) + # process.kill() will only kill the immediate child but not its + # forks. This won't work since our commands will create a few forks + # (make -> vsim -> etc). We need to make a process group and kill + # that + stdout, stderr = process.communicate() + return FinishedProcess(process.args, 1, + stdout.decode('utf-8'), + 'TIMEOUT after {:f}s\n'.format(args.timeout) + + stderr.decode('utf-8')) + # Including KeyboardInterrupt, communicate handled that. + except: # noqa: E722 + pgid = os.getpgid(process.pid) + os.killpg(pgid, signal.SIGKILL) + # We don't call process.wait() as .__exit__ does that for us. + raise + retcode = process.poll() + if check and retcode: + raise CalledProcessError(retcode, process.args, + output=stdout, stderr=stderr) + return FinishedProcess(process.args, retcode, + stdout.decode('utf-8'), + stderr.decode('utf-8')) + + +if __name__ == '__main__': + args = runtest.parse_args() + pp = pprint.PrettyPrinter(indent=4) + + # lazy importing so that we can work without junit_xml + if args.report_junit: + try: + from junit_xml import TestSuite, TestCase + except ImportError: + print("""The --report-junit option requires +the junit_xml module which is not installed.""", + file=sys.stderr) + exit(1) + + # load command list + tests = [] + with open(args.test_file) as f: + tests = list(map(str.rstrip, f)) + + if args.verbose: + print('Tests which we are running:') + pp.pprint(tests) + + # list of commands to be run + shellcmds = [['make', '-C', e, 'clean', 'all', 'run'] for e in tests] + if args.verbose: + print('Generated shell commands:') + pp.pprint(shellcmds) + + # by default we use the number of available cores to limit the number of + # concurrently spawned process + pool = multiprocessing.Pool(processes=args.max_procs) + procresults = pool.map(fork, shellcmds) + pp.pprint(procresults) + + # Generate junit.xml file. Junit.xml differentiates between failure and + # errors but we treat everything as errors. + if args.report_junit: + testcases = [] + for p in procresults: + testcase = TestCase(' '.join(p.args), stdout=p.stdout, + stderr=p.stderr) + if p.returncode != 0: + testcase.add_error_info(p.stderr) + testcases.append(testcase) + + testsuite = TestSuite("bwruntests", testcases) + if args.output: + with open(args.output, 'w') as f: + TestSuite.to_file(f, [testsuite], + prettyprint=args.disable_junit_pp) + else: + print(TestSuite.to_xml_string([testsuite], + prettyprint=args.disable_junit_pp)) From 810d596ade10f0fca21bbb12dd710d891ac279fa Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 13:37:52 +0000 Subject: [PATCH 2/9] bwruntests: Improve various aspects * Allow dumping stdout of subprocess to terminal stdout. Synchronized with a lock. * Add support for yaml file based test definitions (path + command per test) * Make C-c terminate subprocesses properly * Print table based summary at the end --- bwruntests.py | 187 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 149 insertions(+), 38 deletions(-) diff --git a/bwruntests.py b/bwruntests.py index f7450ae..1ca7673 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -30,18 +30,21 @@ import argparse from subprocess import (Popen, TimeoutExpired, CalledProcessError, PIPE) +from threading import Lock +import shlex import sys import signal import os import multiprocessing import errno import pprint +import time runtest = argparse.ArgumentParser(prog='bwruntests', description="""Run PULP tests in parallel""") -runtest.version = '0.1' +runtest.version = '0.2' runtest.add_argument('test_file', type=str, help='file defining tests to be run') @@ -55,42 +58,68 @@ runtest.add_argument('-t', '--timeout', type=float, help="""Timeout for all processes in seconds""") runtest.add_argument('-v', '--verbose', action='store_true', help="""Enable verbose output""") +runtest.add_argument('-s', '--proc-verbose', action='store_true', + help="""Write processes' stdout and stderr to shell stdout + after they terminate""") runtest.add_argument('--report-junit', action='store_true', help="""Generate a junit report""") -runtest.add_argument('--disable-junit-pp', action='store_false', +runtest.add_argument('--disable-junit-pp', action='store_true', help="""Disable pretty print of junit report""") +runtest.add_argument('--disable-results-pp', action='store_true', + help="""Disable printing test results""") +runtest.add_argument('-y,', '--yaml', action='store_true', + help="""Read tests from yaml file instead of executing + from a list of commands""") runtest.add_argument('-o,', '--output', type=str, - help="""Write to file instead of stdout""") + help="""Write junit.xml to file instead of stdout""") + +stdout_lock = Lock() class FinishedProcess(object): """A process that has finished running. """ - def __init__(self, args, returncode, stdout=None, stderr=None): + def __init__(self, name, cwd, args, returncode, + stdout=None, stderr=None, time=None): + self.name = name + self.cwd = cwd self.args = args self.returncode = returncode self.stdout = stdout self.stderr = stderr + self.time = time def __repr__(self): - args = ['args={!r}'.format(self.args), - 'returncode={!r}'.format(self.returncode)] + args = ['name={!r}'.format(self.name)] + args += ['cwd={!r}'.format(self.cwd)] + args += ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] if self.stdout is not None: args.append('stdout={!r}'.format(self.stdout)) if self.stderr is not None: args.append('stderr={!r}'.format(self.stderr)) + if self.time is not None: + args.append('time={!r}'.format(self.time)) return "{}({})".format(type(self).__name__, ', '.join(args)) -def fork(*popenargs, check=False, shell=True, +def fork(name, cwd, *popenargs, check=False, shell=True, **kwargs): """Run subprocess and return process args, error code, stdout and stderr """ + def proc_out(cwd, stdout, stderr): + print('cwd={}'.format(cwd)) + print('stdout=') + print(stdout.decode('utf-8')) + print('stderr=') + print(stderr.decode('utf-8')) + kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE - with Popen(*popenargs, preexec_fn=os.setpgrp, **kwargs) as process: + with Popen(*popenargs, preexec_fn=os.setpgrp, cwd=cwd, + **kwargs) as process: try: # Child and parent are racing for setting/using the pgid so we have # to set it in both processes. See glib manual. @@ -99,7 +128,8 @@ def fork(*popenargs, check=False, shell=True, except OSError as e: if e.errno != errno.EACCES: raise - + # measure runtime + start = time.time() stdout, stderr = process.communicate(input, timeout=args.timeout) except TimeoutExpired: pgid = os.getpgid(process.pid) @@ -109,10 +139,20 @@ def fork(*popenargs, check=False, shell=True, # (make -> vsim -> etc). We need to make a process group and kill # that stdout, stderr = process.communicate() - return FinishedProcess(process.args, 1, + timeoutmsg = 'TIMEOUT after {:f}s'.format(args.timeout) + + if args.proc_verbose: + stdout_lock.acquire() + print(name) + print(timeoutmsg) + proc_out(cwd, stdout, stderr) + stdout_lock.release() + + return FinishedProcess(name, cwd, process.args, 1, stdout.decode('utf-8'), - 'TIMEOUT after {:f}s\n'.format(args.timeout) - + stderr.decode('utf-8')) + timeoutmsg + '\n' + + stderr.decode('utf-8'), + time.time() - start) # Including KeyboardInterrupt, communicate handled that. except: # noqa: E722 pgid = os.getpgid(process.pid) @@ -123,9 +163,16 @@ def fork(*popenargs, check=False, shell=True, if check and retcode: raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) - return FinishedProcess(process.args, retcode, + if args.proc_verbose: + stdout_lock.acquire() + print(name) + proc_out(cwd, stdout, stderr) + stdout_lock.release() + + return FinishedProcess(name, cwd, process.args, retcode, stdout.decode('utf-8'), - stderr.decode('utf-8')) + stderr.decode('utf-8'), + time.time() - start) if __name__ == '__main__': @@ -137,48 +184,112 @@ if __name__ == '__main__': try: from junit_xml import TestSuite, TestCase except ImportError: - print("""The --report-junit option requires -the junit_xml module which is not installed.""", + print("""Error: The --report-junit option requires +the junit_xml library which is not installed.""", file=sys.stderr) exit(1) - # load command list + # lazy import PrettyTable for displaying results + if not(args.disable_results_pp): + try: + from prettytable import PrettyTable + except ImportError: + print("""Warning: Displaying results requires the PrettyTable +library which is not installed""") + + testnames = [] + shellcmds = [] + cwds = [] tests = [] - with open(args.test_file) as f: - tests = list(map(str.rstrip, f)) - if args.verbose: - print('Tests which we are running:') - pp.pprint(tests) + # load tests (yaml or command list) + if args.yaml: + try: + import yaml + except ImportError: + print("""Error: The --yaml option requires +the pyyaml library which is not installed.""", + file=sys.stderr) + exit(1) + with open(args.test_file) as f: + testyaml = yaml.load(f) + for testsetname, testv in testyaml.items(): + for testname, insn in testv.items(): + cmd = shlex.split(insn['command']) + cwd = insn['path'] + testnames.append(testsetname + ':' + testname) + shellcmds.append(cmd) + cwds.append(cwd) + tests.append((testsetname + ':' + testname, cwd, cmd)) + if args.verbose: + pp.pprint(tests) + else: # (command list) + with open(args.test_file) as f: + testnames = list(map(str.rstrip, f)) + shellcmds = [shlex.split(e) for e in testnames] + cwds = ['./' for e in testnames] + tests = list(zip(testnames, cwds, shellcmds)) + if args.verbose: + print('Tests which we are running:') + pp.pprint(tests) + pp.pprint(shellcmds) - # list of commands to be run - shellcmds = [['make', '-C', e, 'clean', 'all', 'run'] for e in tests] - if args.verbose: - print('Generated shell commands:') - pp.pprint(shellcmds) - - # by default we use the number of available cores to limit the number of - # concurrently spawned process + # Spawning process pool + # Disable signals to prevent race. Child processes inherit SIGINT handler + original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) pool = multiprocessing.Pool(processes=args.max_procs) - procresults = pool.map(fork, shellcmds) - pp.pprint(procresults) + # Restore SIGINT handler + signal.signal(signal.SIGINT, original_sigint_handler) + try: + procresults = pool.starmap(fork, tests) + except KeyboardInterrupt: + print("\nTerminating bwruntest.py") + pool.terminate() + pool.join() + exit(1) + + if args.verbose: + pp.pprint(procresults) + pool.close() + pool.join() # Generate junit.xml file. Junit.xml differentiates between failure and # errors but we treat everything as errors. if args.report_junit: testcases = [] for p in procresults: - testcase = TestCase(' '.join(p.args), stdout=p.stdout, - stderr=p.stderr) + testcase = TestCase(p.name, + classname=p.name, + stdout=p.stdout, + stderr=p.stderr, + elapsed_sec=p.time) if p.returncode != 0: - testcase.add_error_info(p.stderr) + testcase.add_failure_info(p.stderr) testcases.append(testcase) - testsuite = TestSuite("bwruntests", testcases) + testsuite = TestSuite('bwruntests', testcases) if args.output: with open(args.output, 'w') as f: TestSuite.to_file(f, [testsuite], - prettyprint=args.disable_junit_pp) + prettyprint=not(args.disable_junit_pp)) else: print(TestSuite.to_xml_string([testsuite], - prettyprint=args.disable_junit_pp)) + prettyprint=(args.disable_junit_pp))) + + # print summary of test results + if not(args.disable_results_pp): + testcount = sum(1 for x in testnames) + testfailcount = sum(1 for p in procresults if p.returncode != 0) + testpassedcount = testcount - testfailcount + resulttable = PrettyTable(['test', 'config', 'time', 'passed/total']) + resulttable.align['test'] = "l" + resulttable.align['config'] = "l" + resulttable.add_row(['bwruntest', '', '', '{0:d}/{1:d}'. + format(testpassedcount, testcount)]) + for p in procresults: + testpassed = 1 if p.returncode == 0 else 0 + testname = p.name + resulttable.add_row([testname, '', + '{0:.2f}s'.format(p.time), + '{0:d}/{1:d}'.format(testpassed, 1)]) + print(resulttable) From cea366dc66a9431b4138633e59798b21641e8c4d Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 13:57:21 +0000 Subject: [PATCH 3/9] bwruntests: Remove dead variables --- bwruntests.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bwruntests.py b/bwruntests.py index 1ca7673..9d17113 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -197,10 +197,7 @@ the junit_xml library which is not installed.""", print("""Warning: Displaying results requires the PrettyTable library which is not installed""") - testnames = [] - shellcmds = [] - cwds = [] - tests = [] + tests = [] # list of tuple (testname, working dir, command) # load tests (yaml or command list) if args.yaml: @@ -217,9 +214,6 @@ the pyyaml library which is not installed.""", for testname, insn in testv.items(): cmd = shlex.split(insn['command']) cwd = insn['path'] - testnames.append(testsetname + ':' + testname) - shellcmds.append(cmd) - cwds.append(cwd) tests.append((testsetname + ':' + testname, cwd, cmd)) if args.verbose: pp.pprint(tests) @@ -278,7 +272,7 @@ the pyyaml library which is not installed.""", # print summary of test results if not(args.disable_results_pp): - testcount = sum(1 for x in testnames) + testcount = sum(1 for x in tests) testfailcount = sum(1 for p in procresults if p.returncode != 0) testpassedcount = testcount - testfailcount resulttable = PrettyTable(['test', 'config', 'time', 'passed/total']) From 454075ed38df4cb652aa07f69aae5a7730a73a75 Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 14:16:49 +0000 Subject: [PATCH 4/9] bwruntests: Remove config column --- bwruntests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bwruntests.py b/bwruntests.py index 9d17113..699a6a9 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -275,15 +275,14 @@ the pyyaml library which is not installed.""", testcount = sum(1 for x in tests) testfailcount = sum(1 for p in procresults if p.returncode != 0) testpassedcount = testcount - testfailcount - resulttable = PrettyTable(['test', 'config', 'time', 'passed/total']) + resulttable = PrettyTable(['test', 'time', 'passed/total']) resulttable.align['test'] = "l" - resulttable.align['config'] = "l" - resulttable.add_row(['bwruntest', '', '', '{0:d}/{1:d}'. + resulttable.add_row(['bwruntest', '', '{0:d}/{1:d}'. format(testpassedcount, testcount)]) for p in procresults: testpassed = 1 if p.returncode == 0 else 0 testname = p.name - resulttable.add_row([testname, '', + resulttable.add_row([testname, '{0:.2f}s'.format(p.time), '{0:d}/{1:d}'.format(testpassed, 1)]) print(resulttable) From 504517a5f26e88ebb5ebee26cf8e9430a4683cbb Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 14:17:11 +0000 Subject: [PATCH 5/9] bwruntests: Improve classname in some cases --- bwruntests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bwruntests.py b/bwruntests.py index 699a6a9..b65ec84 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -252,8 +252,10 @@ the pyyaml library which is not installed.""", if args.report_junit: testcases = [] for p in procresults: + # we can either expect p.name = testsetname:testname + # or p.name = testname testcase = TestCase(p.name, - classname=p.name, + classname=((p.name).split(':'))[0], stdout=p.stdout, stderr=p.stderr, elapsed_sec=p.time) From c572069dd8b07735341a9a8816a88ad4c5ea6ba6 Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 15:12:15 +0000 Subject: [PATCH 6/9] bwruntests: Add version switch --- bwruntests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bwruntests.py b/bwruntests.py index b65ec84..b458c08 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -48,6 +48,8 @@ runtest.version = '0.2' runtest.add_argument('test_file', type=str, help='file defining tests to be run') +runtest.add_argument('--version', action='version', + version='%(prog)s ' + runtest.version) runtest.add_argument('-p', '--max-procs', type=int, default=multiprocessing.cpu_count(), help="""Number of parallel From 2d4dee1da104d5cd542b9822fc3cb8411e76927a Mon Sep 17 00:00:00 2001 From: bluew Date: Fri, 17 Jan 2020 15:12:26 +0000 Subject: [PATCH 7/9] bwruntests: Add usage explanation --- bwruntests.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/bwruntests.py b/bwruntests.py index b458c08..2da5f1c 100755 --- a/bwruntests.py +++ b/bwruntests.py @@ -41,8 +41,33 @@ import pprint import time -runtest = argparse.ArgumentParser(prog='bwruntests', - description="""Run PULP tests in parallel""") +runtest = argparse.ArgumentParser( + prog='bwruntests', + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""Run PULP tests in parallel""", + epilog=""" +Test_file needs to be either a .yaml file (set the --yaml switch) +which looks like this: + +mytests.yml +[...] +parallel_bare_tests: # name of the test set + parMatrixMul8: # name of the test + path: ./parallel_bare_tests/parMatrixMul8 # path to the test's folder + command: make clean all run # command to run in the test's folder +[...] + +or + +Test_file needs to be a list of commands to be executed. Each line corresponds +to a single command and a test + +commands.f +[...] +make -C ./ml_tests/mlGrad clean all run +make -C ./ml_tests/mlDct clean all run +[...] +""") runtest.version = '0.2' @@ -74,7 +99,6 @@ runtest.add_argument('-y,', '--yaml', action='store_true', from a list of commands""") runtest.add_argument('-o,', '--output', type=str, help="""Write junit.xml to file instead of stdout""") - stdout_lock = Lock() From 0b43d8256a825a346529fa2d0285f5b3699f0a80 Mon Sep 17 00:00:00 2001 From: bluew Date: Mon, 20 Jan 2020 13:48:48 +0000 Subject: [PATCH 8/9] bwruntest: Move to scripts folder --- bwruntests.py => scripts/bwruntests.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bwruntests.py => scripts/bwruntests.py (100%) diff --git a/bwruntests.py b/scripts/bwruntests.py similarity index 100% rename from bwruntests.py rename to scripts/bwruntests.py From af8918500cc3149f3a3e20eb5eb4c535ebe4f438 Mon Sep 17 00:00:00 2001 From: bluew Date: Mon, 20 Jan 2020 13:55:04 +0000 Subject: [PATCH 9/9] bwruntest: Add example command --- scripts/bwruntests.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/bwruntests.py b/scripts/bwruntests.py index 2da5f1c..2c8908b 100755 --- a/scripts/bwruntests.py +++ b/scripts/bwruntests.py @@ -67,7 +67,18 @@ commands.f make -C ./ml_tests/mlGrad clean all run make -C ./ml_tests/mlDct clean all run [...] -""") + +Example: +bwruntests.py --proc-verbose -v \\ + --report-junit -t 3600 --yaml \\ + -o simplified-runtime.xml runtime-tests.yaml + +This Runs a set of tests defined in runtime-tests.yaml and dumps the +resulting junit.xml into simplified-runtime.xml. The --proc-verbose +scripts makes sure to print the stdout of each process to the shell. To +prevent a broken process from running forever, a maximum timeout of 3600 +seconds was set. For debugging purposes we enabled -v (--verbose) which +shows the full set of commands being run.""") runtest.version = '0.2'