add an entrails parser for the PDDB

This commit is contained in:
bunnie 2021-11-14 19:02:34 +08:00
parent 93fd991dff
commit 94a86c542d
2 changed files with 579 additions and 0 deletions

318
tools/pddbdbg.py Executable file
View file

@ -0,0 +1,318 @@
#! /usr/bin/env python3
import argparse
import sys
from Crypto.Cipher import AES
from rfc8452 import AES_GCM_SIV
import binascii
import hashlib
SYSTEM_BASIS = '.System'
# from https://github.com/wc-duck/pymmh3/blob/master/pymmh3.py
def xrange( a, b, c ):
return range( a, b, c )
def xencode(x):
if isinstance(x, bytes) or isinstance(x, bytearray):
return x
else:
return x.encode()
def mm3_hash( key, seed = 0x0 ):
''' Implements 32bit murmur3 hash. '''
key = bytearray( xencode(key) )
def fmix( h ):
h ^= h >> 16
h = ( h * 0x85ebca6b ) & 0xFFFFFFFF
h ^= h >> 13
h = ( h * 0xc2b2ae35 ) & 0xFFFFFFFF
h ^= h >> 16
return h
length = len( key )
nblocks = int( length / 4 )
h1 = seed
c1 = 0xcc9e2d51
c2 = 0x1b873593
# body
for block_start in xrange( 0, nblocks * 4, 4 ):
# ??? big endian?
k1 = key[ block_start + 3 ] << 24 | \
key[ block_start + 2 ] << 16 | \
key[ block_start + 1 ] << 8 | \
key[ block_start + 0 ]
k1 = ( c1 * k1 ) & 0xFFFFFFFF
k1 = ( k1 << 15 | k1 >> 17 ) & 0xFFFFFFFF # inlined ROTL32
k1 = ( c2 * k1 ) & 0xFFFFFFFF
h1 ^= k1
h1 = ( h1 << 13 | h1 >> 19 ) & 0xFFFFFFFF # inlined ROTL32
h1 = ( h1 * 5 + 0xe6546b64 ) & 0xFFFFFFFF
# tail
tail_index = nblocks * 4
k1 = 0
tail_size = length & 3
if tail_size >= 3:
k1 ^= key[ tail_index + 2 ] << 16
if tail_size >= 2:
k1 ^= key[ tail_index + 1 ] << 8
if tail_size >= 1:
k1 ^= key[ tail_index + 0 ]
if tail_size > 0:
k1 = ( k1 * c1 ) & 0xFFFFFFFF
k1 = ( k1 << 15 | k1 >> 17 ) & 0xFFFFFFFF # inlined ROTL32
k1 = ( k1 * c2 ) & 0xFFFFFFFF
h1 ^= k1
#finalization
return fmix( h1 ^ length )
# weird, this code breaks things compared to the reference Rust implementation
unsigned_val = fmix( h1 ^ length )
if unsigned_val & 0x80000000 == 0:
return unsigned_val
else:
return -( (unsigned_val ^ 0xFFFFFFFF) + 1 )
def main():
parser = argparse.ArgumentParser(description="Debug PDDB Images")
parser.add_argument(
"--pddb-image", required=False, help="pddb disk image", type=str, nargs='?', metavar=('pddb image'), const='./pddb.bin'
)
parser.add_argument(
"--pddb-keys", required=False, help="known pddb keys", type=str, nargs='?', metavar=('pddb keys'), const='./pddb.key'
)
args = parser.parse_args()
if args.pddb_keys == None:
keyfile = './tools/pddb.key'
else:
keyfile = args.pddb_keys
if args.pddb_image == None:
imagefile = './tools/pddb.bin'
else:
imagefile = args.pddb_image
keys = {}
with open(keyfile, 'rb') as key_f:
raw_key = key_f.read()
num_keys = int.from_bytes(raw_key[:4], 'little')
for i in range(num_keys):
name_all = raw_key[4 + i*96 : 4 + i*96 + 64]
name_bytes = bytearray(b'')
for b in name_all:
if b != 0:
name_bytes.append(b)
else:
break
name = name_bytes.decode('utf8', errors='ignore')
key = raw_key[4 + i*96 + 64 : 4 + i*96 + 96]
keys[name] = key
print("Found basis keys:")
print(keys)
MBBB_PAGES = 10
FSCB_PAGES = 16
KEY_PAGES = 1
PAGE_SIZE = 4096
with open(imagefile, 'rb') as img_f:
raw_img = img_f.read()
pddb_len = len(raw_img)
pddb_size_pages = pddb_len // PAGE_SIZE
img_index = 0
tables = decode_pagetable(raw_img, pddb_size_pages, keys)
img_index += pddb_size_pages * Pte.PTE_LEN
rawkeys = raw_img[img_index : img_index + PAGE_SIZE * KEY_PAGES]
img_index += PAGE_SIZE * KEY_PAGES
mbbb = raw_img[img_index : img_index + PAGE_SIZE * MBBB_PAGES]
img_index += PAGE_SIZE * MBBB_PAGES
fscb = decode_fscb(raw_img[img_index: img_index + PAGE_SIZE * FSCB_PAGES], keys)
class Pte:
PTE_LEN = 16
def __init__(self, bytes):
assert(len(bytes) == self.PTE_LEN)
self.bytes = bytes
def len_bytes(self):
return(self.bytes.len())
def as_bytes(self):
return self.bytes
def addr(self):
return int.from_bytes(self.bytes[:6], 'little')
def flags(self):
if self.bytes[6] == 1:
return 'CLN'
else:
return 'INV'
def nonce(self):
return int.from_bytes(self.bytes[8:12], 'little')
def checksum(self):
return int.from_bytes(self.bytes[12:], 'little')
# checks the checksum
def is_valid(self):
return (self.checksum() == mm3_hash(self.bytes[:12], self.nonce()))
def print(self, ppage=None):
#print('checksum: {:08x}, hash: {:08x}'.format(self.checksum(), mm3_hash(self.bytes[:12], self.nonce())))
if ppage != None:
print('{:08x}: a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(ppage, self.addr(), self.flags(), self.nonce(), self.checksum()))
else:
print('a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(self.addr(), self.flags(), self.nonce(), self.checksum()))
def decode_pagetable(img, entries, keys):
key_table = {}
for name, key in keys.items():
print("Pages for basis {}".format(name))
print("key: {}".format(key.hex()))
cipher = AES.new(key, AES.MODE_ECB)
encrypted_ptes = [img[i:i+Pte.PTE_LEN] for i in range(0, entries * Pte.PTE_LEN, Pte.PTE_LEN)]
page = 0
v2p_table = {}
p2v_table = {}
for pte in encrypted_ptes:
maybe_pte = Pte(cipher.decrypt(pte))
if maybe_pte.is_valid():
maybe_pte.print(page)
p_addr = page * (4096 / Pte.PTE_LEN)
v2p_table[maybe_pte.addr()] = p_addr
p2v_table[p_addr] = maybe_pte.addr()
page += Pte.PTE_LEN
key_table[name] = [v2p_table, p2v_table]
return key_table
class PhysPage:
def __init__(self, bytes):
self.pp = int.from_bytes(bytes[:4], 'little')
def page_number(self):
return self.pp & 0xF_FFFF
def clean(self):
if (self.pp & 0x10_0000) != 0:
return True
else:
return False
def valid(self):
if (self.pp & 0x20_0000) != 0:
return True
else:
return False
def space_state(self):
code = (self.pp >> 22) & 3
if code == 0:
return 'FREE'
elif code == 1:
return 'MUSE'
elif code == 2:
return 'USED'
else:
return 'DIRT'
def journal(self):
return (self.pp >> 24) & 0xF
def print(self):
print('a_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal()))
class Fscb:
def __init__(self, bytes):
FASTSPACE_PAGES = 2
NONCE_LEN = 12
TAG_LEN = 16
PP_LEN = 4
FREE_POOL_LEN = ((4096 * FASTSPACE_PAGES) - (NONCE_LEN + TAG_LEN)) // PP_LEN
assert(len(bytes) == FREE_POOL_LEN * PP_LEN)
raw_pps = [bytes[i:i+PP_LEN] for i in range(0, FREE_POOL_LEN * PP_LEN, PP_LEN)]
self.free_space = {}
for raw_pp in raw_pps:
pp = PhysPage(raw_pp)
if pp.valid():
self.free_space[pp.page_number() * 4096] = pp
def at_phys_addr(self, addr):
try:
return self.free_pool[addr]
except:
None
def print(self):
for pagenum, pp in self.free_space.items():
pp.print()
# this is the "AAD" used to encrypt the FastSpace
def aad(version=0, dna=0):
return bytearray([46, 70, 97, 115, 116, 83, 112, 97, 99, 101]) + version.to_bytes(2, 'little') + dna.to_bytes(8, 'little')
# img is clipped to just the fscb region of the overall disk image
def decode_fscb(img, keys, FSCB_LEN_PAGES=2):
global SYSTEM_BASIS
if SYSTEM_BASIS in keys:
key = keys[SYSTEM_BASIS]
pages = [img[i:i+4096] for i in range(0, len(img), 4096)]
space_update = []
fscb_start = None
pg = 0
for page in pages:
if bytearray(page[:32]) == bytearray([0xff] * 32):
# page is blank
pass
elif bytearray(page[:16] == bytearray([0xff] * 16)):
space_update.append(page)
else:
if fscb_start == None:
fscb_start = pg
pg += 1
if fscb_start:
print("found FSCB at {:x}".format(fscb_start))
fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096]
try:
nonce = fscb_enc[:12]
print("key: {}, nonce: {}".format(key.hex(), nonce.hex()))
cipher = AES_GCM_SIV(key, nonce)
print("aad: {}".format(Fscb.aad().hex()))
print("mac: {}".format(fscb_enc[-16:].hex()))
#print("data: {}".format(fscb_enc[12:-16].hex()))
fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad())
print("decrypted")
fscb = Fscb(fscb_dec)
except KeyError:
print("couldn't decrypt")
fscb = None
if fscb:
fscb.print()
for update in space_update:
pass
if __name__ == "__main__":
main()
exit(0)
# 00 00 00 00 00 00 00 00 00 00 00
# 00 00 00 00 00 00 00 00 00 00
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0

261
tools/rfc8452.py Normal file
View file

@ -0,0 +1,261 @@
# -*- coding: utf-8 -*-
# Copyright (c) Bjorn Edstrom <be@bjrn.se> 2019
# Reference implementation of AES-GCM-SIV based on the IRTF draft.
# Do not use.
# Vendored from https://github.com/bjornedstrom/aes-gcm-siv-py on Nov 14 2021.
# Commit bc9c8f484405437720f7372850e8e8cadad08190
# And slightly modified to be compatible with the Pycryptodome crate as used elsewhere by
# the project.
#
# As of Nov 2021 Pycryptodome doesn't have an RFC8452 (aes-gcm-siv) implementation (the
# thing called Galois mode (GCM) is a different RFC). This code is labelled "do not use"
# by the author, but we use it only to diagnose/test a production
# Rust crate, so, in the worst case, we just get some invalid diagnostic results.
import Crypto.Cipher.AES as AES
import binascii
import six
import struct
class Field(object):
# The field is defined by the irreducible polynomial
# x^128 + x^127 + x^126 + x^121 + 1
_MOD = sum((1 << a) for a in [0, 121, 126, 127, 128])
# x^-128 is equal to x^127 + x^124 + x^121 + x^114 + 1
_INV = sum((1 << a) for a in [0, 114, 121, 124, 127])
@staticmethod
def add(x, y):
assert x < (1 << 128)
assert y < (1 << 128)
return x ^ y
@staticmethod
def mul(x, y):
assert x < (1 << 128), x
assert y < (1 << 128), y
res = 0
for bit in range(128):
if (y >> bit) & 1:
res ^= (2 ** bit) * x
return Field.mod(res, Field._MOD)
@staticmethod
def dot(a, b):
return Field.mul(Field.mul(a, b), Field._INV)
@staticmethod
def mod(a, m):
m2 = m
i = 0
while m2 < a:
m2 <<= 1
i += 1
while i >= 0:
a2 = a ^ m2
if a2 < a:
a = a2
m2 >>= 1
i -= 1
return a
def polyval(h, xs):
"""POLYVAL takes a field element, H, and a series of field elements X_1,
..., X_s. Its result is S_s, where S is defined by the iteration S_0
= 0; S_j = dot(S_{j-1} + X_j, H), for j = 1..s"""
s = 0
for x in xs:
s = Field.dot(Field.add(s, x), h)
return s
class PolyvalIUF(object):
"""Polyval implemented as an IUF construction, specifically in the context
of AES-GCM_SIV."""
def __init__(self, h, nonce):
self._s = 0
self._h = b2i(h)
self._nonce = bytearray(nonce)
# TODO: update() is a bit sensitive w.r.t zero-padding, make sure
# it's called so there is no superfluous zero-padding added in the middle
# of the input due to splitting etc.
def update(self, inp):
def update16(inp):
assert len(inp) == 16
self._s = Field.dot(Field.add(self._s, b2i(inp)), self._h)
def split16(s):
return [s[i:i+16] for i in range(0, len(s), 16)]
def _right_pad_to_16(b):
while len(b) % 16 != 0:
b += b'\x00'
return b
for block in split16(inp):
update16(_right_pad_to_16(block))
def digest(self):
S_s = bytearray(i2b(self._s))
for i in range(12):
S_s[i] ^= self._nonce[i]
S_s[15] &= 0x7f
return S_s
def b2i(s):
res = 0
for c in reversed(s):
res <<= 8
res |= (ord(c) if six.PY2 else c)
return res
def i2b(i):
if i == 0:
return b'\x00'*16
s = b''
while i:
s += chr(i & 0xff) if six.PY2 else bytes([i & 0xff])
i >>= 8
return bytes(s)
def s2i(s):
return b2i(binascii.unhexlify(s))
def i2s(i):
return binascii.hexlify(i2b(i))
def le_uint32(i):
return struct.pack(b'<L', i)
def read_le_uint32(b):
return struct.unpack(b'<L', b[0:4])[0]
def le_uint64(i):
return struct.pack(b'<Q', i)
class AES_GCM_SIV(object):
def __init__(self, key_gen_key, nonce):
aes_obj = AES.new(key_gen_key, AES.MODE_ECB)
msg_auth_key = aes_obj.encrypt(le_uint32(0) + nonce)[0:8] + \
aes_obj.encrypt(le_uint32(1) + nonce)[0:8]
msg_enc_key = aes_obj.encrypt(le_uint32(2) + nonce)[0:8] + \
aes_obj.encrypt(le_uint32(3) + nonce)[0:8]
if len(key_gen_key) == 32:
msg_enc_key += aes_obj.encrypt(le_uint32(4) + nonce)[0:8] + \
aes_obj.encrypt(le_uint32(5) + nonce)[0:8]
self.msg_auth_key = msg_auth_key
self.msg_enc_key = msg_enc_key
self.nonce = nonce
def _right_pad_to_16(self, inp):
while len(inp) % 16 != 0:
inp += b'\x00'
return inp
def _aes_ctr(self, key, initial_block, inp):
block = initial_block
output = b''
while len(inp) > 0:
keystream_block = AES.new(key, AES.MODE_ECB).encrypt(block)
block = le_uint32((read_le_uint32(block[0:4]) + 1) & 0xffffffff) + block[4:]
todo = min(len(inp), len(keystream_block))
for j in range(todo):
if six.PY2:
output += chr(ord(keystream_block[j]) ^ ord(inp[j]))
else:
output += bytes([keystream_block[j] ^ inp[j]])
inp = inp[todo:]
return output
def _polyval_calc(self, plaintext, additional_data):
# Instead of calculating S_s inline using the RFC polyval() function,
# we redesign polyval as an IUF "hash". The old/RFC way would be as below:
#
# """
# padded_plaintext = self._right_pad_to_16(plaintext)
# padded_ad = self._right_pad_to_16(additional_data)
# S_s = bytearray(
# i2b(polyval(b2i(self.msg_auth_key),
# map(b2i, split16(padded_ad) + split16(padded_plaintext) + [length_block]))))
# nonce = bytearray(self.nonce)
# for i in range(12):
# S_s[i] ^= nonce[i]
# S_s[15] &= 0x7f
# assert S_s == S_s_new
# """
pvh = PolyvalIUF(self.msg_auth_key, self.nonce)
pvh.update(additional_data)
pvh.update(plaintext)
length_block = le_uint64(len(additional_data) * 8) + \
le_uint64(len(plaintext) * 8)
pvh.update(length_block)
return pvh.digest()
def encrypt(self, plaintext, additional_data):
"""Encrypt"""
if len(plaintext) > 2**36:
raise ValueError('plaintext too large')
if len(additional_data) > 2**36:
raise ValueError('additional_data too large')
# Polyval/tag calculation
S_s = self._polyval_calc(plaintext, additional_data)
tag = AES.new(self.msg_enc_key).encrypt(bytes(S_s))
# Encrypt
counter_block = bytearray(tag)
counter_block[15] |= 0x80
return self._aes_ctr(self.msg_enc_key, bytes(counter_block), plaintext) + bytes(tag)
def decrypt(self, ciphertext, additional_data):
"""Decrypt"""
if len(ciphertext) < 16 or len(ciphertext) > 2**36 + 16:
raise ValueError('ciphertext too small or too large')
if len(additional_data) > 2**36:
raise ValueError('additional_data too large')
ciphertext, tag = ciphertext[0:-16], ciphertext[-16:]
counter_block = bytearray(tag)
counter_block[15] |= 0x80
plaintext = self._aes_ctr(self.msg_enc_key, bytes(counter_block), ciphertext)
# Polyval/tag calculation
S_s = self._polyval_calc(plaintext, additional_data)
expected_tag = bytearray(AES.new(self.msg_enc_key, AES.MODE_ECB).encrypt(bytes(S_s)))
# Check tag
actual_tag = bytearray(tag)
xor_sum = 0
for i in range(len(expected_tag)):
xor_sum |= expected_tag[i] ^ actual_tag[i]
if xor_sum != 0:
raise ValueError('auth fail')
return plaintext