From 6230c922ac46bf80d5f93493aab13a308cae1ff8 Mon Sep 17 00:00:00 2001 From: bunnie Date: Sun, 9 Oct 2022 02:36:48 +0800 Subject: [PATCH] fix backup/restore scripts --- tools/backalyzer.py | 49 ++++++++++++++++++++++++++++++--- tools/backup.py | 45 +++++++++++++++++++++---------- tools/restore.py | 66 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 139 insertions(+), 21 deletions(-) diff --git a/tools/backalyzer.py b/tools/backalyzer.py index 5d0d10716..17c6169b5 100755 --- a/tools/backalyzer.py +++ b/tools/backalyzer.py @@ -70,9 +70,12 @@ def main(): parser.add_argument( "-f", "--file", help="Input file (defaults to backup.pddb; note that hosted is in pddb-images/hosted.bin).", type=str, default="backup.pddb" ) + parser.add_argument( + "-c", "--checksum-only", help="Only perform checks for media errors. Does not authenticate the backup; does not require passwords.", action="store_true" + ) args = parser.parse_args() - if args.hosted == False and args.pin == None: + if args.hosted == False and args.pin == None and args.checksum_only == False: print("Unlock PIN argument is required, please specified with `-p [pin]`") exit(0) @@ -193,6 +196,7 @@ def main(): i += 8 # padding because align=8 if backup_version == 0x10001: + checksum_errors = False logging.info("Doing hash verification of pt+ct metadata") hasher = SHA512.new(truncate="256") # this is where the backup upcode is located. It should be 1 @@ -201,9 +205,48 @@ def main(): hasher.update(check_region) computed_checksum = hasher.digest() if computed_checksum != checksum: - print("Header failed hash integrity check!") + logging.error("Header failed hash integrity check!") + logging.error("Calculated: {}".format(computed_checksum.hex())) + logging.error("Expected: {}".format(checksum.hex())) + exit(1) + else: + logging.info("Header passed integrity check.") - # TODO: sector checks + if total_checksums != 0: + raw_checksums = backup[header_total_size-(total_checksums * 16):header_total_size] + checksums = [raw_checksums[i:i+16] for i in range(0, len(raw_checksums), 16)] + check_block_num = 0 + while check_block_num < total_checksums: + hasher = SHA512.new(truncate="256") + hasher.update( + backup[ + header_total_size + check_block_num * checksum_region_len: + header_total_size + (check_block_num + 1) * checksum_region_len + ]) + sum = hasher.digest() + if sum[:16] != checksums[check_block_num]: + logging.error("Bad checksum on block {} at offset 0x{:x}".format(check_block_num, check_block_num * checksum_region_len)) + logging.error(" Calculated: {}".format(sum[:16].hex())) + logging.error(" Expected: {}".format(checksums[check_block_num].hex())) + checksum_errors = True + check_block_num += 1 + + if checksum_errors: + logging.error("Media errors were detected! Backup may be unusable.") + exit(1) + else: + logging.info("No media errors detected, {} blocks passed checksum tests".format(total_checksums)) + if args.checksum_only: + exit(0) + else: + if args.checksum_only: + logging.error("Can't perform checksum verification on backups that do not include checksums") + exit(1) + else: + logging.info("Backup has no checksum block, skipping media integrity checks") + elif args.checksum_only: + logging.error("Can't perform checksum verification on backups with a version older than 1.1") + exit(1) keyrom = pt_data[i:i+1024] user_key_enc = get_key(40, keyrom, 32) diff --git a/tools/backup.py b/tools/backup.py index 62159b618..a8e2b77ce 100755 --- a/tools/backup.py +++ b/tools/backup.py @@ -531,7 +531,7 @@ def check_header(backup): if header_total_size != len(backup): print("Header length does not match expected length") return (False, checksums) - raw_checksums = backup[:-(total_checksums * 16)] + raw_checksums = backup[-(total_checksums * 16):] checksums = [raw_checksums[i:i+16] for i in range(0, len(raw_checksums), 16)] op = int.from_bytes(backup[i:i+4], 'little') @@ -622,6 +622,7 @@ def main(): pc_usb.poke(vexdbg_addr, 0x00020000) header_checked = False + checksum_error = False flash_region = int(pc_usb.regions['spiflash'][0], 0) if args.peek: pc_usb.peek(args.peek + flash_region, display=True) @@ -640,6 +641,31 @@ def main(): if amount_read % (block_size * 16) == 0: pc_usb.ping_wdt() backup = pc_usb.burst_read(start_addr + amount_read + flash_region, block_size) + + # block check logic + if len(checksum) > 0: # this check inherently skips the header block, while also suppressing the check if no checksums are provided + check_block[check_block_offset:check_block_offset + block_size] = backup + check_block_offset += block_size + if check_block_offset >= len(check_block): + hasher = SHA512.new(truncate="256") + #print("hashing block of {} len".format(len(check_block))) + #print("start: {}".format(check_block[:64].hex())) + #print("end: {}".format(check_block[-64:].hex())) + hasher.update(check_block) + sum = hasher.digest() + if sum[:16] != checksum[check_block_num]: + print("Calculated: {}\nExpected: {}".format(sum[:16].hex(), bytes(checksum[check_block_num]).hex())) + print("Bad checksum on block {} at offset 0x{:x}. Re-run backup!".format(check_block_num, amount_read - block_size)) + # TODO: automatic retry of the download. However, for now, this condition *should* be very rare + # (not even sure how to trigger it for testing) that we just print the error message for now. + checksum_error = True + else: + # print("Block {}/{} downloaded OK".format(check_block_num + 1, len(checksum))) + pass + check_block_offset = 0 + check_block_num += 1 + + # header check - putting it after the above skips incorporating the header data into check_block if header_checked is False: (header_ok, checksum) = check_header(backup) if header_ok: @@ -650,19 +676,6 @@ def main(): if amount_read < total_length: progress.update(amount_read) - # block check logic - if len(checksum) > 0: # this check inherently skips the header block, while also suppressing the check if no checksums are provided - check_block[check_block_offset:check_block_offset + block_size] = backup - check_block_offset += block_size - if check_block_offset >= len(check_block): - hasher = SHA512.new(truncate="256") - hasher.update(check_block) - sum = hasher.digest() - if sum != checksum[check_block_num]: - print("Bad checksum on block {} at offset 0x{:x}. Re-run backup!".format(check_block_num, amount_read - block_size)) - check_block_offset = 0 - check_block_num += 1 - file.write(backup) progress.finish() file.close() @@ -670,6 +683,10 @@ def main(): print("Resuming CPU.") pc_usb.poke(vexdbg_addr, 0x02000000) + if checksum_error: + print("Link errors detected while downloading the backup. Data is likely corrupted, please try again!") + # Either that, or possibly the PDDB got into an inconsistent state after preparing the backup! + print("Resetting SOC...") try: pc_usb.poke(pc_usb.register('reboot_soc_reset'), 0xac, display=False) diff --git a/tools/restore.py b/tools/restore.py index 91f2b59a0..97ef86e1b 100644 --- a/tools/restore.py +++ b/tools/restore.py @@ -11,6 +11,7 @@ import csv import urllib.request from datetime import datetime import requests +from Crypto.Hash import SHA512 from progressbar.bar import ProgressBar @@ -545,7 +546,8 @@ def main(): backup = bytearray(backup_file.read()) i = 0 - print("Backup protocol version: 0x{:08x}".format(int.from_bytes(backup[i:i+4], 'little'))) + backup_version = int.from_bytes(backup[i:i+4], 'little') + print("Backup protocol version: 0x{:08x}".format(backup_version)) i += 4 print("Xous version: {}".format(bytes_to_semverstr(backup[i:i+16]))) backup_xous_ver = SemVer(backup[i:i+16]) @@ -571,9 +573,63 @@ def main(): i += 4 print("DNA: 0x{:x}".format(int.from_bytes(backup[i:i+8], 'little'))) i += 8 - i += 48 # reserved + checksum_region_len = int.from_bytes(backup[i:i+4], 'little') * 4096 + print("Checksum region length: 0x{:x}".format(checksum_region_len)) + i += 4 + total_checksums = int.from_bytes(backup[i:i+4], 'little') + print("Number of checksum blocks: {}".format(total_checksums)) # if this is 0, checksumming was skipped + i += 4 + header_total_size = int.from_bytes(backup[i:i+4], 'little') + print("Header total length in bytes: {}".format(header_total_size)) + i += 4 + i += 36 # reserved - assert(i == 0x90, "Plaintext operand offset calculated incorrectly! Check data structure sizes.") + if backup_version == 0x10001: + checksum_errors = False + check_region = backup[:0x5d0] + checksum = backup[0x5d0:0x5f0] + print("Doing hash verification of pt+ct metadata...") + hasher = SHA512.new(truncate="256") + hasher.update(check_region) + computed_checksum = hasher.digest() + if computed_checksum != checksum: + print("Header failed hash integrity check!") + print("Calculated: {}".format(computed_checksum.hex())) + print("Expected: {}".format(checksum.hex())) + exit(1) + else: + print("Header passed integrity check.") + + if total_checksums != 0: + raw_checksums = backup[header_total_size-(total_checksums * 16):header_total_size] + checksums = [raw_checksums[i:i+16] for i in range(0, len(raw_checksums), 16)] + check_block_num = 0 + while check_block_num < total_checksums: + hasher = SHA512.new(truncate="256") + hasher.update( + backup[ + header_total_size + check_block_num * checksum_region_len: + header_total_size + (check_block_num + 1) * checksum_region_len + ]) + sum = hasher.digest() + if sum[:16] != checksums[check_block_num]: + print("Bad checksum on block {} at offset 0x{:x}".format(check_block_num, check_block_num * checksum_region_len)) + print(" Calculated: {}".format(sum[:16].hex())) + print(" Expected: {}".format(checksums[check_block_num].hex())) + checksum_errors = True + check_block_num += 1 + + if checksum_errors: + print("Media errors were detected! Backup may be unusable, aborting restore.") + exit(1) + else: + print("No media errors detected, {} blocks passed checksum tests".format(total_checksums)) + + if i != 0x90: + # this is a sanity check for myself, and python doesn't like it when i make this an assert + # because it does the math and realizes it's always true but the point is I don't want to do the math. + print("Plaintext operand offset calculated incorrectly! Check data structure sizes.") + exit(1) backup[i:i+4] = (2).to_bytes(4, 'little') op = int.from_bytes(backup[i:i+4], 'little') print("Opcode (should be 2, to trigger the next phase of restore): {}".format(op)) @@ -582,8 +638,8 @@ def main(): # now try to download all the artifacts and check their versions # this list should visit kernels in order from newest to oldest. URL_LIST = [ - 'https://ci.betrusted.io/releases/v0.9.9/', 'https://ci.betrusted.io/latest-ci/', # prefer stable, but fallback to latest-ci if the kernels are too old + 'https://ci.betrusted.io/releases/v0.9.9/', 'https://ci.betrusted.io/releases/v0.9.8/', 'https://ci.betrusted.io/releases/v0.9.7/' ] @@ -612,6 +668,8 @@ def main(): # alright, we should now be at a base URL where everything is a match. We skip checking # the rest of the versions because it "should" be an ensemble. curver = SemVer(candidate_kernel[curver_loc:curver_loc + 16]) + # print("Min ver: {}".format(min_compat_kernel.as_str())) + # print("Cur ver: {}".format(curver.as_str())) print("Found viable kernel!") kernel = candidate_kernel print("Downloading loader...")