add a fuzzer for basis lock/unlocking

this commits a script that will repeatedly rerun the `btest`
which creates several Bases, fills them with pseudorandom datat,
and goes through a lock/unlock cycle.

should be run with the disk size set to 4MiB for maximum stress
testing (the larger disk won't trigger an FSCB refill because its
hard to run out free space on the larger disk size)
This commit is contained in:
bunnie 2022-07-04 17:53:29 +08:00
parent 97ae56915e
commit ae8fe799bc

118
tools/pddbci-btest.py Normal file
View file

@ -0,0 +1,118 @@
#! /usr/bin/env python3
import argparse
import os
import sys
import logging
import subprocess
import time
import signal
def main():
parser = argparse.ArgumentParser(description="Regression tester for PDDB Basis/FSCB")
parser.add_argument(
"--loglevel", required=False, help="set logging level (INFO/DEBUG/WARNING/ERROR)", type=str, default="INFO",
)
parser.add_argument(
"--runs", required=False, help="sets the number of runs", default='501'
)
args = parser.parse_args()
numeric_level = getattr(logging, args.loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % args.loglevel)
logging.basicConfig(level=numeric_level)
pass_log = {}
err_log = []
timeout = 240 # a bit longer to allow for a compilation to happen
for seed in range(0, int(args.runs)):
# remove the previous runs analysis file
try:
os.remove('./tools/pddb-images/hosted.bin')
except OSError as e:
print('Error removing previous run output: {}'.format(e.strerror))
err_log.append("Starting seed {}".format(seed))
seed_env = os.environ
seed_env["XOUS_SEED"] = str(seed)
# result = subprocess.run(['tools/pddbdbg.py', '--name', 'patterne'], env=seed_env, capture_output=True, text=True, universal_newlines=True)
#result = subprocess.run(['cargo', 'xtask', 'run'], env=seed_env, capture_output=True, text=True, universal_newlines=True)
#for line in result.stdout.split('\n'):
# if 'Seed' in line:
# logging.info(line)
passing = 'FAIL'
proc = subprocess.Popen(
['cargo', 'xtask', 'pddb-btest'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
#shell=True,
encoding='utf-8',
errors='replace'
)
start_time = time.time()
while True:
realtime_output = proc.stdout.readline()
if (realtime_output == '' and proc.poll() is not None) or (time.time() - start_time > 60):
proc.kill()
if time.time() - start_time > timeout:
logging.debug("timeout on generation")
passing = 'FAIL TIMEOUT'
break
if realtime_output:
if 'Seed' in realtime_output:
logging.info(realtime_output.strip()) # flush=True for print version
if 'basis stress test passed' in realtime_output:
passing = 'PASS'
logging.info(realtime_output.strip())
if 'total_free' in realtime_output:
logging.info(realtime_output.strip())
if 'CI done' in realtime_output:
logging.info(realtime_output.strip())
proc.kill()
if 'Ran out of memory' in realtime_output:
err_log.append(realtime_output)
logging.debug("ran out of space")
# passing = False # not a fail, because it's the test condition that's wrong, not the code
passing = 'OOM'
proc.kill()
if "couldn't allocate memory" in realtime_output:
err_log.append(realtime_output)
logging.debug("ran out of space")
# passing = False # not a fail, because it's the test condition that's wrong, not the code
passing = 'OOM'
proc.kill()
if "No free space" in realtime_output:
err_log.append(realtime_output)
logging.debug("ran out of space")
# passing = False # not a fail, because it's the test condition that's wrong, not the code
passing = 'OOM'
proc.kill()
if 'Decryption auth error' in realtime_output:
err_log.append(realtime_output)
logging.debug("decryption auth error")
passing = 'FAIL AUTH'
proc.kill()
logging.info("Seed {} {}".format(seed, passing))
pass_log[seed] = passing
for line in err_log:
logging.info(line)
err_log = []
# summary report
passing = True
for items in pass_log.items():
logging.info(items)
if items[1] != 'PASS':
passing = False
if passing:
logging.info("Overall pass, exiting with 0")
exit(0)
else:
logging.info("A failure was detected, exiting with 1")
exit(1)
if __name__ == "__main__":
main()