diff --git a/tools/pddbci-btest.py b/tools/pddbci-btest.py new file mode 100644 index 000000000..e3d33d755 --- /dev/null +++ b/tools/pddbci-btest.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python3 +import argparse +import os +import sys +import logging +import subprocess +import time +import signal + +def main(): + parser = argparse.ArgumentParser(description="Regression tester for PDDB Basis/FSCB") + parser.add_argument( + "--loglevel", required=False, help="set logging level (INFO/DEBUG/WARNING/ERROR)", type=str, default="INFO", + ) + parser.add_argument( + "--runs", required=False, help="sets the number of runs", default='501' + ) + args = parser.parse_args() + + numeric_level = getattr(logging, args.loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % args.loglevel) + logging.basicConfig(level=numeric_level) + + pass_log = {} + err_log = [] + timeout = 240 # a bit longer to allow for a compilation to happen + for seed in range(0, int(args.runs)): + # remove the previous runs analysis file + try: + os.remove('./tools/pddb-images/hosted.bin') + except OSError as e: + print('Error removing previous run output: {}'.format(e.strerror)) + + err_log.append("Starting seed {}".format(seed)) + seed_env = os.environ + seed_env["XOUS_SEED"] = str(seed) + # result = subprocess.run(['tools/pddbdbg.py', '--name', 'patterne'], env=seed_env, capture_output=True, text=True, universal_newlines=True) + #result = subprocess.run(['cargo', 'xtask', 'run'], env=seed_env, capture_output=True, text=True, universal_newlines=True) + #for line in result.stdout.split('\n'): + # if 'Seed' in line: + # logging.info(line) + + passing = 'FAIL' + proc = subprocess.Popen( + ['cargo', 'xtask', 'pddb-btest'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + #shell=True, + encoding='utf-8', + errors='replace' + ) + start_time = time.time() + while True: + realtime_output = proc.stdout.readline() + if (realtime_output == '' and proc.poll() is not None) or (time.time() - start_time > 60): + proc.kill() + if time.time() - start_time > timeout: + logging.debug("timeout on generation") + passing = 'FAIL TIMEOUT' + break + if realtime_output: + if 'Seed' in realtime_output: + logging.info(realtime_output.strip()) # flush=True for print version + if 'basis stress test passed' in realtime_output: + passing = 'PASS' + logging.info(realtime_output.strip()) + if 'total_free' in realtime_output: + logging.info(realtime_output.strip()) + if 'CI done' in realtime_output: + logging.info(realtime_output.strip()) + proc.kill() + if 'Ran out of memory' in realtime_output: + err_log.append(realtime_output) + logging.debug("ran out of space") + # passing = False # not a fail, because it's the test condition that's wrong, not the code + passing = 'OOM' + proc.kill() + if "couldn't allocate memory" in realtime_output: + err_log.append(realtime_output) + logging.debug("ran out of space") + # passing = False # not a fail, because it's the test condition that's wrong, not the code + passing = 'OOM' + proc.kill() + if "No free space" in realtime_output: + err_log.append(realtime_output) + logging.debug("ran out of space") + # passing = False # not a fail, because it's the test condition that's wrong, not the code + passing = 'OOM' + proc.kill() + if 'Decryption auth error' in realtime_output: + err_log.append(realtime_output) + logging.debug("decryption auth error") + passing = 'FAIL AUTH' + proc.kill() + + logging.info("Seed {} {}".format(seed, passing)) + pass_log[seed] = passing + + for line in err_log: + logging.info(line) + err_log = [] + + # summary report + passing = True + for items in pass_log.items(): + logging.info(items) + if items[1] != 'PASS': + passing = False + if passing: + logging.info("Overall pass, exiting with 0") + exit(0) + else: + logging.info("A failure was detected, exiting with 1") + exit(1) + +if __name__ == "__main__": + main()