initial attempt to make hosted mode more deterministic

replace all threadRng with a seeded Rng
This commit is contained in:
bunnie 2021-12-25 15:58:27 +08:00
parent c1c57edfd6
commit 66606ed3e0
13 changed files with 123 additions and 34 deletions

9
Cargo.lock generated
View file

@ -1460,6 +1460,7 @@ dependencies = [
"num-derive",
"num-traits",
"rand 0.8.4",
"rand_chacha 0.3.1",
"rand_core 0.5.1",
"rkyv",
"root-keys",
@ -1593,7 +1594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [
"libc",
"rand_chacha 0.3.0",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
"rand_hc 0.3.0",
]
@ -1610,9 +1611,9 @@ dependencies = [
[[package]]
name = "rand_chacha"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.3",
@ -2385,6 +2386,8 @@ dependencies = [
"log-server",
"num-derive",
"num-traits",
"rand 0.8.4",
"rand_chacha 0.3.1",
"rand_core 0.5.1",
"rkyv",
"susres",

64
kernel/Cargo.lock generated
View file

@ -239,9 +239,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.1.16"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [
"cfg-if 1.0.0",
"libc",
@ -286,7 +286,8 @@ dependencies = [
"gdbstub_arch",
"hex",
"lazy_static",
"rand",
"rand 0.8.4",
"rand_chacha 0.3.1",
"riscv",
"sha3",
"stats_alloc",
@ -406,14 +407,24 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"libc",
"rand_chacha",
"rand_core",
"rand_hc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc 0.2.0",
"rand_pcg",
]
[[package]]
name = "rand"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.3",
"rand_hc 0.3.1",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
@ -421,7 +432,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core",
"rand_core 0.5.1",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.3",
]
[[package]]
@ -429,6 +450,12 @@ name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
[[package]]
name = "rand_core"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
"getrandom",
]
@ -439,7 +466,16 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core",
"rand_core 0.5.1",
]
[[package]]
name = "rand_hc"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
dependencies = [
"rand_core 0.6.3",
]
[[package]]
@ -448,7 +484,7 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
dependencies = [
"rand_core",
"rand_core 0.5.1",
]
[[package]]
@ -577,9 +613,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "which"
@ -630,6 +666,6 @@ version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"rand",
"rand 0.7.3",
"syn",
]

View file

@ -43,7 +43,8 @@ crossbeam-channel = "0.5"
debug-here = "0.2.2"
hex = "0.4"
lazy_static = "1.4"
rand = "0.7"
rand = "0.8.4"
rand_chacha = "0.3.1"
[profile.release]
codegen-units = 1 # 1 better optimizations

View file

@ -62,14 +62,22 @@ pub fn set_send_addr(send_addr: Sender<SocketAddr>) {
});
}
#[cfg(not(test))]
use core::sync::atomic::{AtomicU64, Ordering};
#[cfg(not(test))]
static LOCAL_RNG_STATE: AtomicU64 = AtomicU64::new(xous_kernel::TESTING_RNG_SEED);
#[cfg(not(test))]
fn generate_pid_key() -> [u8; 16] {
use ::rand::{thread_rng, Rng};
use ::rand::prelude::*;
use rand_chacha::ChaCha8Rng;
let mut process_key = [0u8; 16];
let mut rng = thread_rng();
let mut rng = ChaCha8Rng::seed_from_u64(LOCAL_RNG_STATE.load(Ordering::SeqCst));
for b in process_key.iter_mut() {
*b = rng.gen();
}
LOCAL_RNG_STATE.store(rng.next_u64(), Ordering::SeqCst);
process_key
}

View file

@ -1,6 +1,15 @@
// SPDX-FileCopyrightText: 2020 Sean Cross <sean@xobs.io>
// SPDX-License-Identifier: Apache-2.0
use core::sync::atomic::Ordering;
use crate::arch::hosted::LOCAL_RNG_STATE;
pub fn get_u32() -> u32 {
::rand::random()
use ::rand::prelude::*;
use rand_chacha::ChaCha8Rng;
let mut rng = ChaCha8Rng::seed_from_u64(LOCAL_RNG_STATE.load(Ordering::SeqCst));
let r = rng.next_u32();
LOCAL_RNG_STATE.store(rng.next_u64(), Ordering::SeqCst);
r
}

View file

@ -16,6 +16,9 @@ static GLOBAL: &StatsAlloc<std::alloc::System> = &INSTRUMENTED_SYSTEM;
const SERVER_SPEC: &str = "127.0.0.1:0";
use core::sync::atomic::{AtomicU64, Ordering};
static RNG_LOCAL_STATE: AtomicU64 = AtomicU64::new(xous::TESTING_RNG_SEED);
fn start_kernel(server_spec: &str) -> JoinHandle<()> {
assert!(
std::env::var("XOUS_LISTEN_ADDR").is_err(),
@ -26,12 +29,14 @@ fn start_kernel(server_spec: &str) -> JoinHandle<()> {
"XOUS_SERVER environment variable must be unset to run tests"
);
use rand::{thread_rng, Rng};
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
let mut pid1_key = [0u8; 16];
let mut rng = thread_rng();
let mut rng = ChaCha8Rng::seed_from_u64(RNG_LOCAL_STATE.load(Ordering::SeqCst));
for b in pid1_key.iter_mut() {
*b = rng.gen();
}
RNG_LOCAL_STATE.store(rng.next_u64(), Ordering::SeqCst);
xous_kernel::arch::set_process_key(&pid1_key);
let server_addr = server_spec

View file

@ -44,6 +44,7 @@ utralib = { path = "../../utralib"}
[target.'cfg(any(windows,unix))'.dependencies]
rand = "0.8.4"
rand_chacha = "0.3.1"
[features]
# when selected, physical disk addresses are set to 64 bits, otherwise, they are 32 bits.

View file

@ -1,11 +1,15 @@
use rand::Rng;
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
use crate::*;
use core::sync::atomic::{AtomicU64, Ordering};
const UPPER_BOUND: usize = 9000;
const LOWER_BOUND: usize = 12; // needs to be big enough to compute murmur3 hash + hold checksum
static RNG_LOCAL_STATE: AtomicU64 = AtomicU64::new(xous::TESTING_RNG_SEED);
fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bound: usize) -> (String, Vec::<u8>) {
let mut rng = rand::thread_rng();
let mut rng = ChaCha8Rng::seed_from_u64(RNG_LOCAL_STATE.load(Ordering::SeqCst));
// we want roughly half our keys to be in the small bin, and half in the large bin
let keylen = if rng.gen_bool(0.5) {
rng.gen_range(lower_size_bound..VPAGE_SIZE)
@ -31,6 +35,7 @@ fn gen_key(dictname: &str, keynum: usize, lower_size_bound: usize, upper_size_bo
}
let checksum = murmur3_32(&checkdata, 0);
keydata.append(&mut checksum.to_le_bytes().to_vec());
RNG_LOCAL_STATE.store(rng.next_u64(), Ordering::SeqCst);
(keyname, keydata)
}

View file

@ -324,6 +324,12 @@ impl Drop for Spinor {
}
}
#[cfg(test)]
use core::sync::atomic::AtomicU64;
#[cfg(test)]
static TEST_RNG_STATE: AtomicU64 = AtomicU64::new(xous::TESTING_RNG_SEED);
// run with `cargo test -- --nocapture --test-threads=1`:
// --nocapture to see the print output (while debugging)
// --test-threads=1 because the FLASH ROM is a shared state object
@ -411,11 +417,13 @@ mod tests {
}
}
fn flash_fill_rand() {
use rand::Rng;
let mut rng = rand::thread_rng();
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
let mut rng = ChaCha8Rng::seed_from_u64(TEST_RNG_STATE.load(Ordering::SeqCst));;
for byte in EMU_FLASH.lock().unwrap().iter_mut() {
*byte = rng.gen::<u8>();
}
TEST_RNG_STATE.store(rng.next_u64(), Ordering::SeqCst);
}
/*

View file

@ -21,6 +21,10 @@ rand_core = "0.5"
[target.'cfg(not(any(windows,unix)))'.dependencies]
utralib = { path = "../../utralib"}
[target.'cfg(any(windows,unix))'.dependencies]
rand = "0.8.4"
rand_chacha = "0.3.1"
[features]
debugprint = []
avalanchetest = []

View file

@ -37,6 +37,11 @@ pub struct TrngErrors {
pub pending_mask: u32,
}
/// Performance issue just noticed: the data field is exactly 4096 bytes long, which means
/// the "len" field overflows the structure to be 2 pages. This will cause a lot of extra
/// zero-ing of pages, thrashing the cache and also pegging the CPU for useless work.
/// Consider revising the data field down to 1023 words in length, but need to revisit the
/// library implemnetations to make sure this doesn't break any existing code.
#[derive(Debug, Copy, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)]
pub struct TrngBuf {
pub data: [u32; 1024],

View file

@ -467,9 +467,12 @@ mod implementation {
// a stub to try to avoid breaking hosted mode for as long as possible.
#[cfg(not(any(target_os = "none", target_os = "xous")))]
mod implementation {
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
use crate::api::{HealthTests, TrngBuf, TrngErrors};
pub struct Trng {
rng: ChaCha8Rng,
seed: u32,
msgcount: u16, // re-print the message every time we rollover
}
@ -477,6 +480,7 @@ mod implementation {
impl Trng {
pub fn new(_xns: &xous_names::XousNames) -> Trng {
Trng {
rng: ChaCha8Rng::seed_from_u64(xous::TESTING_RNG_SEED),
seed: 0x1afe_cafe,
msgcount: 0,
}
@ -497,15 +501,12 @@ mod implementation {
log::info!("hosted mode TRNG is *not* random, it is a deterministic LFSR");
}
self.msgcount += 1;
let mut tb = TrngBuf {
data: [0; 1024],
let mut data = [0; 1024];
self.rng.fill(&mut data);
TrngBuf {
data,
len,
};
for i in 0..len as usize {
self.seed = self.move_lfsr(self.seed);
tb.data[i] = self.seed;
}
tb
}
pub fn get_trng(&mut self, _count: usize) -> [u32; 2] {

View file

@ -39,6 +39,9 @@ pub const EC_REGION_LEN: u32 = 0x0008_0000;
pub const PDDB_LOC: u32 = 0x0100_0000; // PDDB start
pub const PDDB_LEN: u32 = EC_REGION_LOC - PDDB_LOC;
#[cfg(all(any(windows, unix), not(feature = "processes-as-threads")))]
pub const TESTING_RNG_SEED: u64 = 0;
pub mod exceptions;
pub use exceptions::*;