mirror of
https://github.com/saymrwulf/pulp-runtime.git
synced 2026-05-14 20:48:09 +00:00
327 lines
13 KiB
Python
Executable file
327 lines
13 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
# Copyright 2020 ETH Zurich and University of Bologna
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
# Run shell commands listed in a file separated by newlines in a parallel
|
|
# fashion. If requested the results (tuples consisting of command, stdout,
|
|
# stderr and returncode) will be gathered in a junit.xml file. There a few
|
|
# knobs to tune the number of spawned processes and the junit.xml formatting.
|
|
|
|
# Author: Robert Balas (balasr@iis.ee.ethz.ch)
|
|
|
|
import argparse
|
|
from subprocess import (Popen, TimeoutExpired,
|
|
CalledProcessError, PIPE)
|
|
from threading import Lock
|
|
import shlex
|
|
import sys
|
|
import signal
|
|
import os
|
|
import multiprocessing
|
|
import errno
|
|
import pprint
|
|
import time
|
|
|
|
|
|
runtest = argparse.ArgumentParser(
|
|
prog='bwruntests',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
description="""Run PULP tests in parallel""",
|
|
epilog="""
|
|
Test_file needs to be either a .yaml file (set the --yaml switch)
|
|
which looks like this:
|
|
|
|
mytests.yml
|
|
[...]
|
|
parallel_bare_tests: # name of the test set
|
|
parMatrixMul8: # name of the test
|
|
path: ./parallel_bare_tests/parMatrixMul8 # path to the test's folder
|
|
command: make clean all run # command to run in the test's folder
|
|
[...]
|
|
|
|
or
|
|
|
|
Test_file needs to be a list of commands to be executed. Each line corresponds
|
|
to a single command and a test
|
|
|
|
commands.f
|
|
[...]
|
|
make -C ./ml_tests/mlGrad clean all run
|
|
make -C ./ml_tests/mlDct clean all run
|
|
[...]
|
|
|
|
Example:
|
|
bwruntests.py --proc-verbose -v \\
|
|
--report-junit -t 3600 --yaml \\
|
|
-o simplified-runtime.xml runtime-tests.yaml
|
|
|
|
This Runs a set of tests defined in runtime-tests.yaml and dumps the
|
|
resulting junit.xml into simplified-runtime.xml. The --proc-verbose
|
|
scripts makes sure to print the stdout of each process to the shell. To
|
|
prevent a broken process from running forever, a maximum timeout of 3600
|
|
seconds was set. For debugging purposes we enabled -v (--verbose) which
|
|
shows the full set of commands being run.""")
|
|
|
|
runtest.version = '0.2'
|
|
|
|
runtest.add_argument('test_file', type=str,
|
|
help='file defining tests to be run')
|
|
runtest.add_argument('--version', action='version',
|
|
version='%(prog)s ' + runtest.version)
|
|
runtest.add_argument('-p', '--max-procs', type=int,
|
|
default=multiprocessing.cpu_count(),
|
|
help="""Number of parallel
|
|
processes used to run test.
|
|
Default is number of cpu cores.""")
|
|
runtest.add_argument('-t', '--timeout', type=float,
|
|
default=None,
|
|
help="""Timeout for all processes in seconds""")
|
|
runtest.add_argument('-v', '--verbose', action='store_true',
|
|
help="""Enable verbose output""")
|
|
runtest.add_argument('-s', '--proc-verbose', action='store_true',
|
|
help="""Write processes' stdout and stderr to shell stdout
|
|
after they terminate""")
|
|
runtest.add_argument('--report-junit', action='store_true',
|
|
help="""Generate a junit report""")
|
|
runtest.add_argument('--disable-junit-pp', action='store_true',
|
|
help="""Disable pretty print of junit report""")
|
|
runtest.add_argument('--disable-results-pp', action='store_true',
|
|
help="""Disable printing test results""")
|
|
runtest.add_argument('-y,', '--yaml', action='store_true',
|
|
help="""Read tests from yaml file instead of executing
|
|
from a list of commands""")
|
|
runtest.add_argument('-o,', '--output', type=str,
|
|
help="""Write junit.xml to file instead of stdout""")
|
|
stdout_lock = Lock()
|
|
|
|
|
|
class FinishedProcess(object):
|
|
"""A process that has finished running.
|
|
"""
|
|
def __init__(self, name, cwd, args, returncode,
|
|
stdout=None, stderr=None, time=None):
|
|
self.name = name
|
|
self.cwd = cwd
|
|
self.args = args
|
|
self.returncode = returncode
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
self.time = time
|
|
|
|
def __repr__(self):
|
|
args = ['name={!r}'.format(self.name)]
|
|
args += ['cwd={!r}'.format(self.cwd)]
|
|
args += ['args={!r}'.format(self.args),
|
|
'returncode={!r}'.format(self.returncode)]
|
|
if self.stdout is not None:
|
|
args.append('stdout={!r}'.format(self.stdout))
|
|
if self.stderr is not None:
|
|
args.append('stderr={!r}'.format(self.stderr))
|
|
if self.time is not None:
|
|
args.append('time={!r}'.format(self.time))
|
|
return "{}({})".format(type(self).__name__, ', '.join(args))
|
|
|
|
|
|
def fork(name, cwd, *popenargs, check=False, shell=True,
|
|
**kwargs):
|
|
"""Run subprocess and return process args, error code, stdout and stderr
|
|
"""
|
|
|
|
def proc_out(cwd, stdout, stderr):
|
|
print('cwd={}'.format(cwd))
|
|
print('stdout=')
|
|
print(stdout.decode('utf-8'))
|
|
print('stderr=')
|
|
print(stderr.decode('utf-8'))
|
|
|
|
kwargs['stdout'] = PIPE
|
|
kwargs['stderr'] = PIPE
|
|
|
|
with Popen(*popenargs, preexec_fn=os.setpgrp, cwd=cwd,
|
|
**kwargs) as process:
|
|
try:
|
|
# Child and parent are racing for setting/using the pgid so we have
|
|
# to set it in both processes. See glib manual.
|
|
try:
|
|
os.setpgid(process.pid, process.pid)
|
|
except OSError as e:
|
|
if e.errno != errno.EACCES:
|
|
raise
|
|
# measure runtime
|
|
start = time.time()
|
|
stdout, stderr = process.communicate(input, timeout=args.timeout)
|
|
except TimeoutExpired:
|
|
pgid = os.getpgid(process.pid)
|
|
os.killpg(pgid, signal.SIGKILL)
|
|
# process.kill() will only kill the immediate child but not its
|
|
# forks. This won't work since our commands will create a few forks
|
|
# (make -> vsim -> etc). We need to make a process group and kill
|
|
# that
|
|
stdout, stderr = process.communicate()
|
|
timeoutmsg = 'TIMEOUT after {:f}s'.format(args.timeout)
|
|
|
|
if args.proc_verbose:
|
|
stdout_lock.acquire()
|
|
print(name)
|
|
print(timeoutmsg)
|
|
proc_out(cwd, stdout, stderr)
|
|
stdout_lock.release()
|
|
|
|
return FinishedProcess(name, cwd, process.args, 1,
|
|
stdout.decode('utf-8'),
|
|
timeoutmsg + '\n'
|
|
+ stderr.decode('utf-8'),
|
|
time.time() - start)
|
|
# Including KeyboardInterrupt, communicate handled that.
|
|
except: # noqa: E722
|
|
pgid = os.getpgid(process.pid)
|
|
os.killpg(pgid, signal.SIGKILL)
|
|
# We don't call process.wait() as .__exit__ does that for us.
|
|
raise
|
|
retcode = process.poll()
|
|
if check and retcode:
|
|
raise CalledProcessError(retcode, process.args,
|
|
output=stdout, stderr=stderr)
|
|
if args.proc_verbose:
|
|
stdout_lock.acquire()
|
|
print(name)
|
|
proc_out(cwd, stdout, stderr)
|
|
stdout_lock.release()
|
|
|
|
return FinishedProcess(name, cwd, process.args, retcode,
|
|
stdout.decode('utf-8'),
|
|
stderr.decode('utf-8'),
|
|
time.time() - start)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
args = runtest.parse_args()
|
|
pp = pprint.PrettyPrinter(indent=4)
|
|
|
|
# lazy importing so that we can work without junit_xml
|
|
if args.report_junit:
|
|
try:
|
|
from junit_xml import TestSuite, TestCase
|
|
except ImportError:
|
|
print("""Error: The --report-junit option requires
|
|
the junit_xml library which is not installed.""",
|
|
file=sys.stderr)
|
|
exit(1)
|
|
|
|
# lazy import PrettyTable for displaying results
|
|
if not(args.disable_results_pp):
|
|
try:
|
|
from prettytable import PrettyTable
|
|
except ImportError:
|
|
print("""Warning: Displaying results requires the PrettyTable
|
|
library which is not installed""")
|
|
|
|
tests = [] # list of tuple (testname, working dir, command)
|
|
|
|
# load tests (yaml or command list)
|
|
if args.yaml:
|
|
try:
|
|
import yaml
|
|
except ImportError:
|
|
print("""Error: The --yaml option requires
|
|
the pyyaml library which is not installed.""",
|
|
file=sys.stderr)
|
|
exit(1)
|
|
with open(args.test_file) as f:
|
|
testyaml = yaml.load(f, Loader=yaml.Loader)
|
|
for testsetname, testv in testyaml.items():
|
|
for testname, insn in testv.items():
|
|
cmd = shlex.split(insn['command'])
|
|
cwd = insn['path']
|
|
tests.append((testsetname + ':' + testname, cwd, cmd))
|
|
if args.verbose:
|
|
pp.pprint(tests)
|
|
else: # (command list)
|
|
with open(args.test_file) as f:
|
|
testnames = list(map(str.rstrip, f))
|
|
shellcmds = [shlex.split(e) for e in testnames]
|
|
cwds = ['./' for e in testnames]
|
|
tests = list(zip(testnames, cwds, shellcmds))
|
|
if args.verbose:
|
|
print('Tests which we are running:')
|
|
pp.pprint(tests)
|
|
pp.pprint(shellcmds)
|
|
|
|
# Spawning process pool
|
|
# Disable signals to prevent race. Child processes inherit SIGINT handler
|
|
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
pool = multiprocessing.Pool(processes=args.max_procs)
|
|
# Restore SIGINT handler
|
|
signal.signal(signal.SIGINT, original_sigint_handler)
|
|
try:
|
|
procresults = pool.starmap(fork, tests)
|
|
except KeyboardInterrupt:
|
|
print("\nTerminating bwruntest.py")
|
|
pool.terminate()
|
|
pool.join()
|
|
exit(1)
|
|
|
|
if args.verbose:
|
|
pp.pprint(procresults)
|
|
pool.close()
|
|
pool.join()
|
|
|
|
# Generate junit.xml file. Junit.xml differentiates between failure and
|
|
# errors but we treat everything as errors.
|
|
if args.report_junit:
|
|
testcases = []
|
|
for p in procresults:
|
|
# we can either expect p.name = testsetname:testname
|
|
# or p.name = testname
|
|
testcase = TestCase(p.name,
|
|
classname=((p.name).split(':'))[0],
|
|
stdout=p.stdout,
|
|
stderr=p.stderr,
|
|
elapsed_sec=p.time)
|
|
if p.returncode != 0:
|
|
testcase.add_failure_info(p.stderr)
|
|
testcases.append(testcase)
|
|
|
|
testsuite = TestSuite('bwruntests', testcases)
|
|
if args.output:
|
|
with open(args.output, 'w') as f:
|
|
TestSuite.to_file(f, [testsuite],
|
|
prettyprint=not(args.disable_junit_pp))
|
|
else:
|
|
print(TestSuite.to_xml_string([testsuite],
|
|
prettyprint=(args.disable_junit_pp)))
|
|
|
|
# print summary of test results
|
|
if not(args.disable_results_pp):
|
|
testcount = sum(1 for x in tests)
|
|
testfailcount = sum(1 for p in procresults if p.returncode != 0)
|
|
testpassedcount = testcount - testfailcount
|
|
resulttable = PrettyTable(['test', 'time', 'passed/total'])
|
|
resulttable.align['test'] = "l"
|
|
resulttable.add_row(['bwruntest', '', '{0:d}/{1:d}'.
|
|
format(testpassedcount, testcount)])
|
|
for p in procresults:
|
|
testpassed = 1 if p.returncode == 0 else 0
|
|
testname = p.name
|
|
resulttable.add_row([testname,
|
|
'{0:.2f}s'.format(p.time),
|
|
'{0:d}/{1:d}'.format(testpassed, 1)])
|
|
print(resulttable)
|