From a68eb7cc734b696cb67ece4dbcd580d8e600becc Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 28 Jul 2022 04:47:02 +0800 Subject: [PATCH] add missing files for backalyzer; add PTE conflict resolution --- tools/backalyzer.py | 3 +- tools/bcrypt.py | 666 +++++++++++++++++++++++++++++++++++++++ tools/pddbcommon.py | 746 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1414 insertions(+), 1 deletion(-) create mode 100644 tools/bcrypt.py create mode 100644 tools/pddbcommon.py diff --git a/tools/backalyzer.py b/tools/backalyzer.py index 3629954d0..922b4d451 100755 --- a/tools/backalyzer.py +++ b/tools/backalyzer.py @@ -243,6 +243,7 @@ def main(): keys = {} keys[SYSTEM_BASIS] = [key_pt, key_data] + # extract the secret basis keys for name, pw in basis_credentials.items(): bname_copy = [0]*64 plaintext_pw = [0]*73 @@ -281,7 +282,6 @@ def main(): logging.info("MBBB: 0x{:x}".format(mbbb_offset)) img_index = 0 - tables = decode_pagetable(pddb, pddb_size_pages, keys, pddb[mbbb_offset:mbbb_offset + MBBB_PAGES * PAGE_SIZE]) img_index += pddb_size_pages * Pte.PTE_LEN if img_index & (PAGE_SIZE - 1) != 0: img_index = (img_index + PAGE_SIZE) & 0xFFFF_F000 @@ -300,6 +300,7 @@ def main(): logging.debug("Data: 0x{:x}".format(img_index)) data = pddb[img_index:] + tables = decode_pagetable(pddb, pddb_size_pages, keys, pddb[mbbb_offset:mbbb_offset + MBBB_PAGES * PAGE_SIZE], dna=dna_int, data=data) # iterate through found Bases and print their contents for name, key in keys.items(): diff --git a/tools/bcrypt.py b/tools/bcrypt.py new file mode 100644 index 000000000..be0b62065 --- /dev/null +++ b/tools/bcrypt.py @@ -0,0 +1,666 @@ +# Vendored in from https://github.com/mwielgoszewski/python-bcrypt +# Original license is OpenBSD. It says "do not use", but it is perfect for a CI situation. + +''' +*************************** DO NOT USE THIS MODULE *************************** + +This is a pure Python implementation of BCrypt, based on +Damien Miller's jBCrypt library . + +Please use if you wish to use +BCrypt in Python. + +WARNING: This code is not suitable for cryptographic use due to serious +performance issues when using a log_rounds value that is necessary for +today's systems. Utilizing a lower log_rounds value in this module will not +equate to the same computation time in C and Java implementations, thus +considerably negating the benefits of using an adaptive crytographic hash +function. + +******************************************************************************** +''' +#!/usr/bin/env python +import copy +import os + +# it's fine, we are using this in a CI/diagnostics/recovery tool, not to secure any data. +# import warnings +#warnings.warn('This module is not suitable for cryptographic use! ' +# 'It has been implemented in pure Python as an academic ' +# 'exercise only. DO NOT USE IN PRODUCTION SYSTEMS.', +# RuntimeWarning, +# stacklevel=2) + +__usage__ = '%s ' + +GENSALT_DEFAULT_LOG2_ROUNDS = 7 +BCRYPT_SALT_LEN = 16 +BLOWFISH_NUM_ROUNDS = 16 + +P_orig = ( + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, + 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, + 0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, + 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, + 0x9216d5d9, 0x8979fb1b + ) + + +S_orig = ( + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, + 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, + 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, + 0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, + 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, + 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, + 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, + 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, + 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, + 0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677, + 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, + 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, + 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, + 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, + 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, + 0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, + 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, + 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, + 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, + 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, + 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, + 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09, + 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, + 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, + 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, + 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, + 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, + 0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, + 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, + 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, + 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, + 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, + 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, + 0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1, + 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, + 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, + 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, + 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, + 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, + 0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, + 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, + 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, + 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, + 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266, + 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, + 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, + 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, + 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1, + 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, + 0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, + 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, + 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, + 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, + 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf, + 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, + 0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87, + 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, + 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, + 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, + 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509, + 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, + 0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, + 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, + 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, + 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, + 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, + 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, + 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf, + 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, + 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, + 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, + 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281, + 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, + 0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128, + 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, + 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, + 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, + 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, + 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, + 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061, + 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, + 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, + 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, + 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340, + 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, + 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, + 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, + 0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, + 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, + 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, + 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, + 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, + 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, + 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb, + 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, + 0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, + 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, + 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc, + 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, + 0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, + 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, + 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, + 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, + 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, + 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, + 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b, + 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, + 0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, + 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, + 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, + 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, + 0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, + 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, + 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, + 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, + 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, + 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, + 0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633, + 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, + 0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, + 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, + 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, + 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, + 0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, + 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, + 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, + 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0, + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, + 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe, + 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, + 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, + 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, + 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, + 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, + 0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, + 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, + 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51, + 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, + 0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, + 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, + 0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd, + 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, + 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, + 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, + 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, + 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, + 0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, + 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, + 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, + 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, + 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, + 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, + 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd, + 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, + 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, + 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, + 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, + 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, + 0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964, + 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, + 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d, + 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, + 0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, + 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, + 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a, + 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, + 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, + 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, + 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, + 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 + ) + +bf_crypt_ciphertext = ( + 0x4f727068, 0x65616e42, 0x65686f6c, + 0x64657253, 0x63727944, 0x6f756274 + ) + +base64_code = ( + '.', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', + 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9' + ) + +index_64 = ( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0, 1, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, -1, -1, + -1, -1, -1, -1, -1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + -1, -1, -1, -1, -1, -1, 28, 29, 30, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, + 51, 52, 53, -1, -1, -1, -1, -1 + ) + + +def char64(x): + ''' + Look up the 3 bits base64-encoded by the specified character, + range-checking againt conversion table. + + :param x: the base64-encoded value + :returns: the decoded value of x + ''' + if ord(x) < 0 or ord(x) > len(index_64): + return -1 + return index_64[ord(x)] + + +def encode_base64(d, length): + ''' + Encode a byte array using bcrypt's slightly-modified base64 + encoding scheme. Note that this is *not* compatible with + the standard MIME-base64 encoding. + + :param d: the byte array to encode + :param length: the number of bytes to encode + :returns: base64-encoded string + :raises: ValueError if the length is invalid + ''' + d = bytearray(d) + #if isinstance(d, basestring): + # d = map(ord, d) + off = 0 + rs = '' + if length <= 0 or length > len(d): + raise ValueError('Invalid length') + while off < length: + c1 = d[off] & 0xff; off += 1 + rs += base64_code[(c1 >> 2) & 0x3f] + c1 = (c1 & 0x03) << 4 + if off >= length: + rs += base64_code[c1 & 0x3f] + break + c2 = d[off] & 0xff; off += 1 + c1 |= (c2 >> 4) & 0x0f + rs += base64_code[c1 & 0x3f] + c1 = (c2 & 0x0f) << 2 + if off >= length: + rs += base64_code[c1 & 0x3f] + break + c2 = d[off] & 0xff; off += 1 + c1 |= (c2 >> 6) & 0x03 + rs += base64_code[c1 & 0x3f] + rs += base64_code[c2 & 0x3f] + return rs + + +def decode_base64(s, maxlen): + ''' + Decode a string encoded using bcrypt's base64 scheme to a + byte array. Note that this is *not* compatible with + the standard MIME-base64 encoding. + + :param s: the string to decode + :param maxlen: the maximum number of bytes to decode + :returns: an array containing the decoded bytes + :raises: ValueError if maxlen is invalid + ''' + off = 0 + slen = len(s) + olen = 0 + ret = [] + rs = '' + + if maxlen <= 0: + raise ValueError('Invalid maxlen') + + while off < (slen-1) and olen < maxlen: + c1 = char64(s[off]); off += 1 + c2 = char64(s[off]); off += 1 + + if c1 == -1 or c2 == -1: + break + + o = c1 << 2 + o |= (c2 & 0x30) >> 4 + rs += chr(o) + olen += 1 + + if olen >= maxlen or off >= slen: + break + + c3 = char64(s[off]); off += 1 + + if c3 == -1: + break + + o = ((c2 & 0x0f) << 4) + o |= (c3 & 0x3c) >> 2 + rs += chr(o) + olen += 1 + + if olen >= maxlen or off >= slen: + break + + c4 = char64(s[off]); off += 1 + o = ((c3 & 0x03) << 6) + o |= c4 + rs += chr(o) + olen += 1 + + return bytearray(rs) + + +def streamtoword(data, offp): + word = 0 + off = offp[0] + + for i in range(4): + word = (word << 8) | (data[off] & 0xff) + off = (off + 1) % len(data) + + offp[0] = off + return word + + +class BCrypt: + def __init__(self): + pass + + def __init_key(self): + ''' + Initialize the Blowfish key schedule. + ''' + self.P = list(copy.copy(P_orig)) + self.S = list(copy.copy(S_orig)) + self.slen = len(P_orig) + self.plen = len(S_orig) + + def __key(self, key): + ''' + Key the Blowfish cipher. + + :param key: An array containing the key + ''' + koffp = [0] + lr = [0, 0] + plen = len(self.P) + slen = len(self.S) + p = self.P + s = self.S + + for i in range(plen): + p[i] ^= streamtoword(key, koffp) + + for i in range(0, plen, 2): + self.__encipher(lr, 0) + p[i], p[i+1] = lr + + for i in range(0, slen, 2): + self.__encipher(lr, 0) + s[i], s[i+1] = lr + + def __ekskey(self, data, key): + ''' + Perform the "enhanced key schedule" step described by + Provos and Mazieres in "A Future-Adaptable Password Scheme" + http://www.openbsd.org/papers/bcrypt-paper.ps + + :param data: salt information + :param key: password information + ''' + koffp = [0] + doffp = [0] + lr = [0, 0] + plen = len(self.P) + slen = len(self.S) + + for i in range(plen): + self.P[i] ^= streamtoword(key, koffp) + + for i in range(0, plen, 2): + lr[0] ^= streamtoword(data, doffp) + lr[1] ^= streamtoword(data, doffp) + self.__encipher(lr, 0) + self.P[i], self.P[i+1] = lr + + for i in range(0, slen, 2): + lr[0] ^= streamtoword(data, doffp) + lr[1] ^= streamtoword(data, doffp) + self.__encipher(lr, 0) + self.S[i], self.S[i+1] = lr + + def __encipher(self, lr, off): + ''' + This function is so slow, it's pathetic. + Don't do bcrypt in pure python. + ''' + l = lr[off] + r = lr[off + 1] + + s = self.S + p = self.P + l ^= p[0] + + for i in range(0, BLOWFISH_NUM_ROUNDS - 1, 2): + # These two lines need serious help. + r ^= (((s[(l >> 24) & 0xff] + s[0x100 | ((l >> 16) & 0xff)]) ^ s[0x200 | ((l >> 8) & 0xff)]) + s[0x300 | (l & 0xff)]) ^ p[i+1] + l ^= (((s[(r >> 24) & 0xff] + s[0x100 | ((r >> 16) & 0xff)]) ^ s[0x200 | ((r >> 8) & 0xff)]) + s[0x300 | (r & 0xff)]) ^ p[i+2] + + lr[off] = r ^ p[BLOWFISH_NUM_ROUNDS + 1] + lr[off + 1] = l + + def crypt_raw(self, password, salt, log_rounds): + ''' + Perform the central password hashing step in the + bcrypt scheme. + + :param password: the password to hash + :param salt: the binary salt to hash with the password + :param log_rounds: the binary logarithm of the number + of rounds of hashing to apply + :returns: an array containing the binary hashed password + ''' + cdata = list(copy.copy(bf_crypt_ciphertext)) + clen = len(cdata) + + if log_rounds < 4 or log_rounds > 31: + raise ValueError('Bad number of rounds') + + rounds = 1 << log_rounds + + if len(salt) != BCRYPT_SALT_LEN: + raise ValueError('Bad salt length') + + self.__init_key() + self.__ekskey(salt, password) + + for i in range(rounds): + self.__key(password) + self.__key(salt) + + for i in range(64): + for j in range(clen >> 1): + self.__encipher(cdata, j << 1) + + ret = bytearray(clen*4) + + j = 0 + for i in range(clen): + ret[j] = (cdata[i] >> 24) & 0xff; j += 1 + ret[j] = (cdata[i] >> 16) & 0xff; j += 1 + ret[j] = (cdata[i] >> 8) & 0xff; j += 1 + ret[j] = cdata[i] & 0xff; j += 1 + + return ret + + +def hashpw(password, salt): + ''' + Hash the specified password and the salt using the OpenBSD + Blowfish password hashing algorithm. Returns the hashed password. + + :param password: The password to hash. + :param salt: The salt, returned by :func:`gensalt`. + :returns: The hashed password. + :raises: ValueError if salt version or number of salt rounds is + invalid. + ''' + if salt[0] != '$' or salt[1] != '2': + raise ValueError('Invalid salt version %r', (salt, )) + + if salt[2] == '$': + off = 3 + else: + minor = salt[2] + if minor != 'a' or salt[3] != '$': + raise ValueError('Invalid salt version') + off = 4 + + if salt[off + 2] > '$': + raise ValueError('Missing salt rounds') + + rounds = int(salt[off:off+2]) + + real_salt = salt[off+3:off+25] + + passwordb = (password + (minor >= 'a' and '\000' or '')).decode('utf-8') + passwordb = map(ord, passwordb) + saltb = decode_base64(real_salt, BCRYPT_SALT_LEN) + + bcrypter = BCrypt() + hashed = bcrypter.crypt_raw(passwordb, saltb, rounds) + + rs = "$2%s$%02d$%s%s" % ( + minor >= 'a' and minor or '', + rounds, + encode_base64(saltb, len(saltb)), + encode_base64(hashed, (len(bf_crypt_ciphertext) * 4) - 1)) + + return rs + + +def gensalt(log_rounds=GENSALT_DEFAULT_LOG2_ROUNDS): + ''' + Generate a random text salt for use with hashpw(). *log_rounds* + defines the complexity of the hashing, increasing the cost as + 2**log_rounds. + ''' + rnd = os.urandom(BCRYPT_SALT_LEN) + rs = '$2a$%02d$%s' % (log_rounds, encode_base64(rnd, len(rnd))) + return rs + + +def checkpw(plaintext, hashed): + return hashpw(plaintext, hashed) == hashed + + +if __name__ == '__main__': + import sys + + if len(sys.argv[1:]) != 2: + print(__usage__ % (sys.argv[0], )) + sys.exit(1) + + password = sys.argv[1] + rounds = int(sys.argv[2]) + + hashed = hashpw(password, gensalt(log_rounds=rounds)) + + print('Hashed: %s' % (hashed, )) + assert checkpw(password, hashed) == True, 'Password hash incorrect' + diff --git a/tools/pddbcommon.py b/tools/pddbcommon.py new file mode 100644 index 000000000..ce36facad --- /dev/null +++ b/tools/pddbcommon.py @@ -0,0 +1,746 @@ +import sys +from Crypto.Cipher import AES +from rfc8452 import AES_GCM_SIV +from Crypto.Hash import SHA512 +import binascii + +import logging + +SYSTEM_BASIS = '.System' +PAGE_SIZE = 4096 +VPAGE_SIZE = 4064 +MBBB_PAGES = 10 +DO_CI_TESTS = True +VERSION = 0x02_01 +DNA=0 +#DNA=0x1_0000_0000 + +# build a table mapping all non-printable characters to None +NOPRINT_TRANS_TABLE = { + i: None for i in range(0, sys.maxunicode + 1) if not chr(i).isprintable() +} + +def set_ci_tests_flag(setting): + global DO_CI_TESTS + DO_CI_TESTS = setting + +def make_printable(s): + global NOPRINT_TRANS_TABLE + """Replace non-printable characters in a string.""" + + # the translate method on str removes characters + # that map to None from the string + return s.translate(NOPRINT_TRANS_TABLE) + +# from https://github.com/wc-duck/pymmh3/blob/master/pymmh3.py +def xrange( a, b, c ): + return range( a, b, c ) +def xencode(x): + if isinstance(x, bytes) or isinstance(x, bytearray): + return x + else: + return x.encode() + +def mm3_hash( key, seed = 0x0 ): + ''' Implements 32bit murmur3 hash. ''' + + key = bytearray( xencode(key) ) + + def fmix( h ): + h ^= h >> 16 + h = ( h * 0x85ebca6b ) & 0xFFFFFFFF + h ^= h >> 13 + h = ( h * 0xc2b2ae35 ) & 0xFFFFFFFF + h ^= h >> 16 + return h + + length = len( key ) + nblocks = int( length / 4 ) + + h1 = seed + + c1 = 0xcc9e2d51 + c2 = 0x1b873593 + + # body + for block_start in xrange( 0, nblocks * 4, 4 ): + # ??? big endian? + k1 = key[ block_start + 3 ] << 24 | \ + key[ block_start + 2 ] << 16 | \ + key[ block_start + 1 ] << 8 | \ + key[ block_start + 0 ] + + k1 = ( c1 * k1 ) & 0xFFFFFFFF + k1 = ( k1 << 15 | k1 >> 17 ) & 0xFFFFFFFF # inlined ROTL32 + k1 = ( c2 * k1 ) & 0xFFFFFFFF + + h1 ^= k1 + h1 = ( h1 << 13 | h1 >> 19 ) & 0xFFFFFFFF # inlined ROTL32 + h1 = ( h1 * 5 + 0xe6546b64 ) & 0xFFFFFFFF + + # tail + tail_index = nblocks * 4 + k1 = 0 + tail_size = length & 3 + + if tail_size >= 3: + k1 ^= key[ tail_index + 2 ] << 16 + if tail_size >= 2: + k1 ^= key[ tail_index + 1 ] << 8 + if tail_size >= 1: + k1 ^= key[ tail_index + 0 ] + + if tail_size > 0: + k1 = ( k1 * c1 ) & 0xFFFFFFFF + k1 = ( k1 << 15 | k1 >> 17 ) & 0xFFFFFFFF # inlined ROTL32 + k1 = ( k1 * c2 ) & 0xFFFFFFFF + h1 ^= k1 + + #finalization + return fmix( h1 ^ length ) + + # weird, this code breaks things compared to the reference Rust implementation + unsigned_val = fmix( h1 ^ length ) + if unsigned_val & 0x80000000 == 0: + return unsigned_val + else: + return -( (unsigned_val ^ 0xFFFFFFFF) + 1 ) + +def keycommit_decrypt(key, aad, pp_data): + # print([hex(d) for d in pp_data[:32]]) + nonce = pp_data[:12] + ct = pp_data[12:12+4004] + kcom_nonce = pp_data[12+4004:12+4004+32] + kcom = pp_data[12+4004+32:12+4004+32+32] + mac = pp_data[-16:] + + #print([hex(d) for d in kcom_nonce]) + # print([hex(d) for d in kcom]) + #h_test = SHA512.new(truncate="256") # just something to make sure our hash functions are sane + #h_test.update(bytes([0x43, 0x6f, 0x6, 0xd6, 0xd, 0x69, 0x74, 0x01, 0x01])) + #print(h_test.hexdigest()) + + h_enc = SHA512.new(truncate="256") + h_enc.update(key) + h_enc.update(bytes([0x43, 0x6f, 0x6, 0xd6, 0xd, 0x69, 0x74, 0x01, 0x01])) + h_enc.update(kcom_nonce) + k_enc = h_enc.digest() + + h_com = SHA512.new(truncate="256") + h_com.update(key) + h_com.update(bytes([0x43, 0x6f, 0x6, 0xd6, 0xd, 0x69, 0x74, 0x01, 0x02])) + h_com.update(kcom_nonce) + k_com_derived = h_com.digest() + logging.debug('kcom_stored: ' + binascii.hexlify(kcom).decode('utf-8')) + logging.debug('kcom_derived: ' + binascii.hexlify(k_com_derived).decode('utf-8')) + + cipher = AES_GCM_SIV(k_enc, nonce) + pt_data = cipher.decrypt(ct + mac, aad) + if k_com_derived != kcom: + logging.error("basis failed key commit test") + raise Exception(ValueError) + else: + logging.debug("basis passed commitment test") + return pt_data + +def basis_aad(name, version=VERSION, dna=DNA): + name_bytes = bytearray(name, 'utf-8') + # name_bytes += bytearray(([0] * (Basis.MAX_NAME_LEN - len(name)))) + name_bytes += version.to_bytes(4, 'little') + name_bytes += dna.to_bytes(8, 'little') + + return name_bytes + +PRINTED_FULL = False +class KeyDescriptor: + MAX_NAME_LEN = 95 + def __init__(self, record, v2p, disk, key, name, dna=DNA): + # print(record.hex()) + i = 0 + self.start = int.from_bytes(record[i:i+8], 'little') + i += 8 + self.len = int.from_bytes(record[i:i+8], 'little') + i += 8 + self.reserved = int.from_bytes(record[i:i+8], 'little') + i += 8 + self.flags_code = int.from_bytes(record[i:i+4], 'little') + i += 4 + self.age = int.from_bytes(record[i:i+4], 'little') + i += 4 + self.name = record[i+1:i+1+record[i]].decode('utf8', errors='ignore') + if self.flags_code & 1 != 0: + self.valid = True + else: + self.valid = False + if self.flags_code & 2 != 0: + self.unresolved = True + else: + self.unresolved = False + if self.valid: + # print('valid key') + page_addr = self.start + self.data = bytearray() + remaining = self.len + read_start = self.start % VPAGE_SIZE # reads can start not page-aligned + while page_addr < self.start + self.len: + data_base = (page_addr // VPAGE_SIZE) * VPAGE_SIZE + try: + pp_start = v2p[data_base] + except KeyError: + logging.error("key {} is missing data allocation at va {:x}".format(self.name, data_base)) + self.ci_ok = False + raise KeyError + pp_data = disk[pp_start:pp_start + PAGE_SIZE] + try: + cipher = AES_GCM_SIV(key, pp_data[:12]) + pt_data = cipher.decrypt(pp_data[12:], basis_aad(name, dna=dna))[4:] # skip the journal, first 4 bytes + if read_start + remaining > VPAGE_SIZE: + read_end = VPAGE_SIZE + else: + read_end = read_start + remaining + self.data += pt_data[read_start:read_end] + bytes_read = read_end - read_start + remaining -= bytes_read + read_start = (read_start + bytes_read) % VPAGE_SIZE + if remaining > 0: + assert(read_start == 0) + except ValueError: + logging.error("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(page_addr), pp_start) + page_addr += VPAGE_SIZE + global DO_CI_TESTS + if DO_CI_TESTS: + # CI check -- if it doesn't pass, it doesn't mean we've failed -- could also just be a "normal" record that doesn't have the checksum appended + check_data = self.data[:-4] + while len(check_data) % 4 != 0: + check_data += bytes([0]) + checksum = mm3_hash(check_data) + refcheck = int.from_bytes(self.data[len(self.data)-4:], 'little') + if checksum == refcheck: + self.ci_ok = True + else: + self.ci_ok = False + logging.error('checksum: {:x}, refchecksum: {:x}\n'.format(checksum, refcheck)) + else: + # print('invalid key') + pass + + + def as_str(self, indent=''): + PRINT_LEN = 64 + global PRINTED_FULL + global DO_CI_TESTS + desc = '' + if self.start > 0x7e_fff02_0000: + desc += indent + 'Start: 0x{:x} (lg)\n'.format(self.start) + else: + dict = (self.start - 0x3f_8000_0000) // 0xFE_0000 + pool = ((self.start - 0x3f_8000_0000) - dict * 0xFE_0000) // 0xFE0 + desc += indent + 'Start: 0x{:x} | dict_index {} | pool {}\n'.format(self.start, dict, pool) + desc += indent + 'Len: {}/0x{:x}\n'.format(self.len, self.reserved) + desc += indent + 'Flags: ' + if self.valid: + desc += 'VALID' + if self.unresolved: + desc += indent + 'UNRESOLVED' + + desc += ' | Age: {}'.format(self.age) + if DO_CI_TESTS: + desc += ' | CI OK: {}'.format(self.ci_ok) + desc += '\n' + if len(self.data) < PRINT_LEN: + print_len = len(self.data) + extra = '' + else: + if DO_CI_TESTS and (PRINTED_FULL == False) and (self.ci_ok == False): + print_len = len(self.data) + extra = '' + PRINTED_FULL = True + else: + print_len = PRINT_LEN + extra = '...' + desc += indent + 'Data (hex): {}{}\n'.format(self.data[:print_len].hex(), extra) + desc += indent + 'Data (txt): {}{}'.format(make_printable(self.data[:print_len].decode('utf-8', errors='ignore')), extra) + '\n' + return (desc) + + +class BasisDicts: + DICT_VSTRIDE = 0xFE_0000 + KV_STRIDE = 127 + MAX_NAME_LEN = 111 + def __init__(self, index, v2p, disk, key, name, dna=DNA): + global PAGE_SIZE + global VPAGE_SIZE + dict_header_vaddr = self.DICT_VSTRIDE * (index + 1) + self.index = index + self.vaddr = dict_header_vaddr + if dict_header_vaddr in v2p: + self.valid = True + pp = v2p[dict_header_vaddr] + self.keys = {} + try: + logging.debug('dict decrypt @ pp {:x}, nonce: {}'.format(pp, disk[pp:pp+12].hex())) + cipher = AES_GCM_SIV(key, disk[pp:pp+12]) + pt_data = cipher.decrypt(disk[pp+12:pp+PAGE_SIZE], basis_aad(name, dna=dna)) + # print('raw pt_data: {}'.format(pt_data[:127].hex())) + i = 0 + self.journal = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.flags = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.age = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.num_keys = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.free_key_index = int.from_bytes(pt_data[i:i+4], 'little') + i += 4 + self.name = pt_data[i+1:i+1+pt_data[i]].decode('utf8', errors='ignore') + i += BasisDicts.MAX_NAME_LEN + logging.info("decrypt dict '{}' with {} keys and {} free_key_index".format(self.name, self.num_keys, self.free_key_index)) + logging.debug("dict header len: {}".format(i-4)) # subtract 4 because of the journal + except ValueError: + logging.error("basisdicts: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(dict_header_vaddr, v2p[dict_header_vaddr])) + + if self.num_keys > 0: + keys_found = 0 + keys_tried = 1 + prev_index = 128 # hack to optimize effort by only decrypting a new page when the key index resets + while (keys_found < self.num_keys) and keys_tried < 131071: + keyindex_start_vaddr = self.DICT_VSTRIDE * (index + 1) + (keys_tried * self.KV_STRIDE) + #if self.name == 'dict2' and keyindex_start_vaddr < (0x1fc2fa0 + 0xfe0): + # try: + # print('keys_tried {}'.format(keys_tried)) + # print('ki_vaddr {:x}'.format((keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE)) + # pp = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE] + # print('ki_paddr {:x}'.format(pp)) + # except KeyError: + # pass + try: + pp_start = v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE] + pp_data = disk[pp_start:pp_start + PAGE_SIZE] + except KeyError: + # the page wasn't allocated, so let's just assume all the key entries are invalid, and were "tried" + keys_tried += VPAGE_SIZE // self.KV_STRIDE + continue + try: + key_index = 4 + keys_tried % (VPAGE_SIZE // self.KV_STRIDE) * self.KV_STRIDE + if key_index < prev_index: + cipher = AES_GCM_SIV(key, pp_data[:12]) + pt_data = cipher.decrypt(pp_data[12:], basis_aad(name, dna=dna)) + prev_index = key_index + #if self.name == 'dict2': + # print('key_index {:x}/{}/{:x}/{:x}'.format(key_index, keys_tried, pp, (keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE)) + # print('{:x}'.format(v2p[(keyindex_start_vaddr // VPAGE_SIZE) * VPAGE_SIZE])) + try: + maybe_key = KeyDescriptor(pt_data[key_index:key_index + self.KV_STRIDE], v2p, disk, key, name, dna=dna) + if maybe_key.valid: + self.keys[maybe_key.name] = maybe_key + keys_found += 1 + #if self.name == 'dict2': + # print('found {}: {}'.format(keys_found, maybe_key.name)) + except KeyError: + logging.error("Key @ offset {} is missing data allocation".format(key_index)) + + except ValueError: + logging.error("key: couldn't decrypt vpage @ {:x} ppage @ {:x}".format(keyindex_start_vaddr, pp_start)) + keys_tried += 1 + if keys_found < self.num_keys: + logging.error("Expected {} keys but only found {}".format(self.num_keys, keys_found)) + self.found_all_keys = False + else: + self.found_all_keys = True + #if self.name == 'dict2': + # for v, p in v2p.items(): + # print('map: v{:16x} | p{:8x}'.format(v, p)) + + else: + self.valid = False + + def as_str(self): + desc = '' + desc += ' Name: {} / index {} / vaddr: {:x}\n'.format(self.name, self.index, self.vaddr) + desc += ' Age: {}\n'.format(self.age) + desc += ' Flags: {}\n'.format(self.flags) + desc += ' Key count: {}\n'.format(self.num_keys) + desc += ' Free key index: {}\n'.format(self.free_key_index) + desc += ' Keys:\n' + for (name, key) in self.keys.items(): + desc += ' {}:\n'.format(name) + desc += key.as_str(' ') + + return desc + + def ci_check(self): + check_ok = True + for (name, key) in self.keys.items(): + namecheck = name.split('|') + if namecheck[1] != self.name: + logging.warning(" Key/dict mismatch: {}/{}".format(key.name, self.name)) + keylen = int(namecheck[3][3:]) + if keylen > 12: + if keylen != key.len and keylen + 4 != key.len: # the second case is what happens with the length extension test, it's actually not an error + logging.warning(" Key named len vs disk len mismatch: {}/{}".format(keylen, key.len)) + else: + if key.len != 17 and key.len != 12: # 12 means we weren't extended. But, the test patch data extends a 12-length key to 13, so with the checksum its final length is 17. + logging.warning(" Key named len vs disk len mismatch: {}/{} (extension did not work correctly)".format(keylen, key.len)) + if key.ci_ok == False: + logging.warning(" CI failed on key {}:".format(name)) + logging.warning(key.as_str(' ')) + check_ok = False + if self.found_all_keys == False: + logging.warning(' Dictionary was missing keys') + check_ok = False + if check_ok: + logging.info(' Dictionary {} CI check: OK'.format(self.name)) + else: + logging.error(' Dictionary {} CI check: FAIL'.format(self.name)) + + +class Basis: + MAX_NAME_LEN = 64 + def __init__(self, i_bytes): + self.bytes = i_bytes + i = 0 + self.journal = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.magic = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.version = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.age = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.name = i_bytes[i+1:i+1+i_bytes[i]].rstrip(b'\x00').decode('utf8', errors='ignore') + i += Basis.MAX_NAME_LEN + #self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little') + #i += 8 + #self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little') + #i += 8 + """ rkyv unpacker + i = 0 + self.journal = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.rkyvpos = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + print("alloc on disk: {}".format(i_bytes[i:i+8].hex())) + self.prealloc_open_end = int.from_bytes(i_bytes[i:i+8], 'little') + i += 8 + self.age = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.num_dicts = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.version = int.from_bytes(i_bytes[i:i+2], 'little') + i += 2 + self.magic = int.from_bytes(i_bytes[i:i+4], 'little') + i += 4 + self.name = i_bytes[i:i+Basis.MAX_NAME_LEN].rstrip(b'\x00').decode('utf8', errors='ignore') + i += Basis.MAX_NAME_LEN + i += 10 # mysterious padding in the rkyv format!! + self.dict_ptr = int.from_bytes(i_bytes[i:i+8], 'little') + i += 8 + """ + + def as_str(self): + desc = '' + desc += ' Journal rev: {}\n'.format(self.journal) + #desc += ' Rkyv pos: {}\n'.format(self.rkyvpos) + desc += ' Magic: {:x}\n'.format(self.magic) + desc += ' Version: {:x}\n'.format(self.version) + desc += ' Age: {:x}\n'.format(self.age) + desc += ' Name: {}\n'.format(self.name) + #desc += ' Alloc: {:x}\n'.format(self.prealloc_open_end) + desc += ' NumDicts: {:x}\n'.format(self.num_dicts) + #desc += ' DictPtr: {:x}\n'.format(self.dict_ptr) + return desc + +class Pte: + PTE_LEN = 16 + def __init__(self, i_bytes): + assert(len(i_bytes) == self.PTE_LEN) + self.bytes = i_bytes + + def len_bytes(self): + return(self.bytes.len()) + def as_bytes(self): + return self.bytes + + def addr(self): + return int.from_bytes(self.bytes[:7], 'little') * VPAGE_SIZE + def flags(self): + if self.bytes[6] == 1: + return 'CLN' + elif self.bytes[6] == 2: + return 'CHK' + elif self.bytes[6] == 3: + return 'COK' + else: + return 'INV' + def nonce(self): + return int.from_bytes(self.bytes[8:12], 'little') + def checksum(self): + return int.from_bytes(self.bytes[12:], 'little') + + # checks the checksum + def is_valid(self): + return (self.checksum() == mm3_hash(self.bytes[:12], self.nonce())) + + def as_str(self, ppage=None): + #print('checksum: {:08x}, hash: {:08x}'.format(self.checksum(), mm3_hash(self.bytes[:12], self.nonce()))) + if ppage != None: + return '{:08x}: a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(ppage, self.addr(), self.flags(), self.nonce(), self.checksum()) + else: + return 'a_{:012x} | f_{} | n_{:08x} | c_{:08x}'.format(self.addr(), self.flags(), self.nonce(), self.checksum()) + +def find_mbbb(mbbb): + global MBBB_PAGES + pages = [mbbb[i:i+PAGE_SIZE] for i in range(0, MBBB_PAGES * PAGE_SIZE, PAGE_SIZE)] + candidates = [] + for page in pages: + if bytearray(page[:16]) != bytearray([0xff] * 16): + candidates.append(page) + + if len(candidates) > 1: + logging.error("More than one MBBB found, this is an error!") + elif len(candidates) == 1: + return candidates[0] + else: + return None + +def decode_pagetable(img, entries, keys, mbbb, dna=DNA, data=None): + global PAGE_SIZE + key_table = {} + for name, key in keys.items(): + logging.debug("Pages for basis '{}'".format(name)) + logging.debug("key: {}".format(key[0].hex())) + cipher = AES.new(key[0], AES.MODE_ECB) + pages = [img[i:i+PAGE_SIZE] for i in range(0, entries * Pte.PTE_LEN, PAGE_SIZE)] + page_num = 0 + v2p_table = {} + p2v_table = {} + for page in pages: + if bytearray(page[:16]) == bytearray([0xff] * 16): + # decrypt from mbbb + logging.debug("Found a blank PTE page, falling back to MBBB!") + maybe_page = find_mbbb(mbbb) + if maybe_page != None: + page = maybe_page + else: + logging.warning("Blank entry in PTE found, but no corresponding MBBB entry found. Filesystem likely corrupted.") + encrypted_ptes = [page[i:i+Pte.PTE_LEN] for i in range(0, PAGE_SIZE, Pte.PTE_LEN)] + for pte in encrypted_ptes: + maybe_pte = Pte(cipher.decrypt(pte)) + if maybe_pte.is_valid(): + logging.debug(maybe_pte.as_str(page_num // Pte.PTE_LEN)) + p_addr = page_num * (4096 // Pte.PTE_LEN) + if maybe_pte.addr() in v2p_table: + logging.info("duplicate V2P PTE entry {:x}:{:x}<->{:x}".format(maybe_pte.addr(), v2p_table[maybe_pte.addr()], p_addr)) + if data == None: + logging.warn("No data section passed, ignoring") + else: + prev_data = data[v2p_table[maybe_pte.addr()]:v2p_table[maybe_pte.addr()] + PAGE_SIZE] + new_data = data[p_addr:p_addr + PAGE_SIZE] + if maybe_pte.addr() == 0xFE0: + # it's a root page table + logging.debug("Try to resolve PTE conflict") + try: + prev = keycommit_decrypt(key[1], basis_aad(name, dna=dna), prev_data) + except ValueError: + logging.error("Previously indexed PTE is bogus, using new entry without checking") + p2v_table[p_addr] = maybe_pte.addr() + continue + try: + new = keycommit_decrypt(key[1], basis_aad(name, dna=dna), new_data) + except ValueError: + logging.error("Suggested new PTE is invalid. Retaining existing mapping") + continue + prev_journal = int.from_bytes(prev[:4], 'little') + new_journal = int.from_bytes(new[:4], 'little') + else: + # it's data + logging.debug("Try to resolve data conflict") + prev_cipher = AES_GCM_SIV(key[1], prev_data[:12]) + try: + prev_data = prev_cipher.decrypt(prev_data[12:], basis_aad(name, dna=dna)) + except ValueError: + logging.error("Previously indexed data is bogus, using new entry without checking") + p2v_table[p_addr] = maybe_pte.addr() + continue + new_cipher = AES_GCM_SIV(key[1], new_data[:12]) + try: + new_data = new_cipher.decrypt(new_data[12:], basis_aad(name, dna=dna)) + except ValueError: + logging.error("Suggested new data is invalid. Retaining existing mapping") + continue + prev_journal = int.from_bytes(prev_data[:4], 'little') + new_journal = int.from_bytes(new_data[:4], 'little') + if new_journal > prev_journal: + logging.info("New entry has journal {}, prev is {}; evicting prev".format(new_journal, prev_journal)) + p2v_table[p_addr] = maybe_pte.addr() + continue + elif new_journal == prev_journal: + logging.warn("Journal numbers conflict; arbitrarily sticking with the first entry found") + else: + logging.info("New entry has journal {}, prev is {}; keeping prev".format(new_journal, prev_journal)) + continue + else: + v2p_table[maybe_pte.addr()] = p_addr + if p_addr in p2v_table: + logging.warning("duplicate P2V PTE entry, evicting {:x}:{:x}".format(p_addr, p2v_table[p_addr])) + p2v_table[p_addr] = maybe_pte.addr() + + page_num += Pte.PTE_LEN + key_table[name] = [v2p_table, p2v_table] + return key_table + +class PhysPage: + PP_LEN = 4 + def __init__(self, i_bytes): + self.pp = int.from_bytes(i_bytes[:4], 'little') + + def page_number(self): + return self.pp & 0xF_FFFF + + def clean(self): + if (self.pp & 0x10_0000) != 0: + return True + else: + return False + + def valid(self): + if (self.pp & 0x20_0000) != 0: + return True + else: + return False + + def space_state(self): + code = (self.pp >> 22) & 3 + if code == 0: + return 'FREE' + elif code == 1: + return 'MUSE' + elif code == 2: + return 'USED' + else: + return 'DIRT' + + def journal(self): + return (self.pp >> 24) & 0xF + + def as_str(self): + return 'pp_{:05x} | c_{} | v_{} | ss_{} | j_{:02}'.format(self.page_number(), self.clean(), self.valid(), self.space_state(), self.journal()) + +class Fscb: + def __init__(self, i_bytes, FASTSPACE_PAGES=2): + NONCE_LEN = 12 + TAG_LEN = 16 + PP_LEN = 4 + FREE_POOL_LEN = ((4096 * FASTSPACE_PAGES) - (NONCE_LEN + TAG_LEN)) // PP_LEN + assert(len(i_bytes) == FREE_POOL_LEN * PP_LEN) + raw_pps = [i_bytes[i:i+PP_LEN] for i in range(0, FREE_POOL_LEN * PP_LEN, PP_LEN)] + self.free_space = {} + for raw_pp in raw_pps: + pp = PhysPage(raw_pp) + if pp.valid(): + self.free_space[pp.page_number() * 4096] = pp + + def at_phys_addr(self, addr): + try: + return self.free_pool[addr] + except: + None + + def len(self): + return len(self.free_space) + + def print(self): + for pagenum, pp in self.free_space.items(): + logging.debug(pp.as_str()) + + def try_replace(self, candidate): + if candidate.is_valid(): + pp = candidate.get_pp() + try: + cur_pp = self.free_space[pp.page_number() * 4096] + except KeyError: + logging.info("SpaceUpdate with no FSCB entry; this is normal after the FSCB is regenerated: {}".format(pp.as_str())) + return + if pp.valid() and (cur_pp.journal() < pp.journal()): + logging.debug("FSCB replace {} /with/ {}".format(cur_pp.as_str(), pp.as_str())) + self.free_space[pp.page_number() * 4096] = pp + elif cur_pp.journal() == pp.journal(): + logging.warning("Duplicate journal number found:\n {} (in table)\n {} (incoming)".format(cur_pp.as_str(), pp.as_str())) + + # this is the "AAD" used to encrypt the FastSpace + def aad(version=VERSION, dna=DNA): + return bytearray([46, 70, 97, 115, 116, 83, 112, 97, 99, 101]) + version.to_bytes(4, 'little') + dna.to_bytes(8, 'little') + +class SpaceUpdate: + SU_LEN = 16 + def __init__(self, i_bytes): + self.nonce = i_bytes[:8] + self.page_number = PhysPage(i_bytes[8:12]) + self.checksum = int.from_bytes(i_bytes[12:], 'little') + hash = mm3_hash(i_bytes[:12], int.from_bytes(i_bytes[4:8], 'big')) + if self.checksum == hash: + self.valid = True + else: + self.valid = False + + def is_valid(self): + return self.valid + + def get_pp(self): + return self.page_number + +# img is clipped to just the fscb region of the overall disk image +def decode_fscb(img, keys, FSCB_LEN_PAGES=2, dna=DNA): + global SYSTEM_BASIS + fscb = None + spaceupdate_count = 0 + if SYSTEM_BASIS in keys: + key = keys[SYSTEM_BASIS] + pages = [img[i:i+4096] for i in range(0, len(img), 4096)] + space_update = [] + fscb_start = None + pg = 0 + for page in pages: + if bytearray(page[:32]) == bytearray([0xff] * 32): + # page is blank + pass + elif bytearray(page[:16]) == bytearray([0xff] * 16): + space_update.append(page[16:]) + else: + if fscb_start == None: + fscb_start = pg + pg += 1 + + if fscb_start: + logging.debug("Found FSCB at {:x}".format(fscb_start)) + fscb_enc = img[fscb_start * 4096 : (fscb_start + FSCB_LEN_PAGES) * 4096] + # print("data: {}".format(fscb_enc.hex())) + try: + nonce = fscb_enc[:12] + logging.info("key: {}, nonce: {}".format(key[1].hex(), nonce.hex())) + cipher = AES_GCM_SIV(key[1], nonce) + logging.info("aad: {}".format(Fscb.aad(dna=dna).hex())) + logging.info("mac: {}".format(fscb_enc[-16:].hex())) + fscb_dec = cipher.decrypt(fscb_enc[12:], Fscb.aad(dna=dna)) + fscb = Fscb(fscb_dec, FASTSPACE_PAGES=FSCB_LEN_PAGES) + except KeyError: + logging.error("couldn't decrypt") + fscb = None + + if fscb != None: + logging.info("Total FreeSpace entries found: {}".format(fscb.len())) + logging.debug("key: {}".format(key[1].hex())) + cipher = AES.new(key[0], AES.MODE_ECB) + for update in space_update: + entries = [update[i:i+SpaceUpdate.SU_LEN] for i in range(0, len(update), SpaceUpdate.SU_LEN)] + for entry in entries: + if entry == bytearray([0xff] * 16): + break + else: + fscb.try_replace(SpaceUpdate(cipher.decrypt(entry))) + spaceupdate_count += 1 + logging.info("Total SpaceUpdate entries found: {}".format(spaceupdate_count)) + return fscb