diff --git a/xtask/src/builder.rs b/xtask/src/builder.rs index 77ddf8e64..8fbb9ddc8 100644 --- a/xtask/src/builder.rs +++ b/xtask/src/builder.rs @@ -29,6 +29,8 @@ pub(crate) enum CrateSpec { CratesIo(String, String), /// a prebuilt package: (name of executable, URL for download) Prebuilt(String, String), + /// a prebuilt binary, done using command line tools + BinaryFile(String), /// an empty entry None, } @@ -38,6 +40,7 @@ impl Clone for CrateSpec { CrateSpec::Local(s) => CrateSpec::Local(s.to_string()), CrateSpec::CratesIo(n, v) => CrateSpec::CratesIo(n.to_string(), v.to_string()), CrateSpec::Prebuilt(n, u) => CrateSpec::Prebuilt(n.to_string(), u.to_string()), + CrateSpec::BinaryFile(path) => CrateSpec::BinaryFile(path.to_string()), CrateSpec::None => CrateSpec::None, } } @@ -59,6 +62,12 @@ impl CrateSpec { name.to_string(), url.to_string() ) + // local files are specified as paths, which, at a minimum include one directory separator "/" or "\" + // i.e. "./local_file" + // Note that this is after a test for the '#' character, so that disambiguates URL slashes + // It does mean that files with a '#' character in them are mistaken for URL coded paths, and '@' as remote crates. + } else if spec.contains('/') || spec.contains('\\') { + CrateSpec::BinaryFile(spec.to_string()) } else { CrateSpec::Local(spec.to_string()) } @@ -214,9 +223,8 @@ impl Builder { self } /// Add a list of services - #[allow(dead_code)] - pub fn add_services<'a>(&'a mut self, service_list: &Vec::<&str>) -> &'a mut Builder { - for &service in service_list { + pub fn add_services<'a>(&'a mut self, service_list: &Vec::) -> &'a mut Builder { + for service in service_list { self.services.push(CrateSpec::from_str(service)); } self @@ -229,7 +237,6 @@ impl Builder { self } /// Add a list of apps - #[allow(dead_code)] pub fn add_apps<'a>(&'a mut self, app_list: &Vec::) -> &'a mut Builder { for app in app_list { self.apps.push(CrateSpec::from_str(app)); @@ -242,6 +249,17 @@ impl Builder { self.features.push(feature.into()); self } + /// add a feature to be passed on to just the loader + pub fn add_loader_feature<'a>(&'a mut self, feature: &str) -> &'a mut Builder { + self.loader_features.push(feature.into()); + self + } + /// add a feature to be passed on to just the loader + #[allow(dead_code)] + pub fn add_kernel_feature<'a>(&'a mut self, feature: &str) -> &'a mut Builder { + self.kernel_features.push(feature.into()); + self + } /// The builder sets up all the cargo arguments to build a set of packages with features for a respective /// target and stream. It also runs the build as well. It's meant to be called only by the `build()` @@ -475,6 +493,11 @@ impl Builder { for service in services_path.iter() { hosted_args.push(service); } + // jam in any pre-built local binary files that were specified + let binary_files_string = self.enumerate_binary_files()?; + let mut binary_files: Vec<&str> = binary_files_string.iter().map(|s| s.as_ref()).collect(); + hosted_args.append(&mut binary_files); + let mut dir = project_root(); dir.push("kernel"); println!("Starting hosted mode..."); @@ -560,6 +583,7 @@ impl Builder { // ---------- install any pre-built packages ---------- services_path.append(&mut self.fetch_prebuilds()?); + services_path.append(&mut self.enumerate_binary_files()?); // --------- package up and sign a binary image ---------- let output_bundle = self.create_image( @@ -735,6 +759,19 @@ impl Builder { Ok(paths) } + fn enumerate_binary_files(&self) -> Result, DynError> { + let mut paths = Vec::::new(); + for item in [&self.services[..], &self.apps[..]].concat() { + match item { + CrateSpec::BinaryFile(path) => { + paths.push(path); + } + _ => {} + } + } + Ok(paths) + } + fn locale_override(&mut self) { if let Some(locale) = &self.locale_override { { // stash the existing locale diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 370548bf5..c4833707b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,5 +1,4 @@ mod app_manifest; -use app_manifest::*; mod versioning; use versioning::*; mod utils; @@ -7,49 +6,344 @@ use utils::*; mod builder; use builder::*; -use std::fs::OpenOptions; use std::{ env, - io::{Read, Write}, - path::{Path, PathBuf, MAIN_SEPARATOR}, - process::Command, + path::{Path, PathBuf}, }; -// This is the default SVD file to be used to generate Xous. This must be manually -// updated every time the SoC version is bumped. -const SOC_SVD_VERSION: &str = "precursor-c809403"; -// gitrev of the current precursor SoC version targeted by this build +/// gitrev of the current precursor SoC version targeted by this build. This must +/// be manually updated every time the SoC version is bumped. const PRECURSOR_SOC_VERSION: &str = "c809403"; -// This is the minimum Xous version required to read a PDDB backup generated -// by the current kernel revision. +/// This is the minimum Xous version required to read a PDDB backup generated +/// by the current kernel revision. const MIN_XOUS_VERSION: &str = "v0.9.8-791"; -type DynError = Box; - -const PROGRAM_TARGET: &str = "riscv32imac-unknown-xous-elf"; -const KERNEL_TARGET: &str = "riscv32imac-unknown-xous-elf"; +/// target triple for precursor builds const TARGET_TRIPLE: &str = "riscv32imac-unknown-xous-elf"; +fn main() -> Result<(), Box> { + let mut builder = Builder::new(); + // encodes a timestamp into the build, unless '--no-timestamp' is passed + generate_version(env::args().filter(|x| x == "--no-timestamp").count() == 0); + + // A base set of packages. This is all you need for a normal + // operating system that can run libstd + let base_pkgs = [ + "ticktimer-server", // "well known" server: thread scheduling + "log-server", // "well known" server: debug logging + "xous-names", // "well known" server: manage inter-server connection lookup + "trng", // most process will need to generate secure ID numbers + "susres", // suspend/resume manager + ].to_vec(); + // minimal set of packages to do bare-iron graphical I/O + let gfx_dev_pkgs = [ + &base_pkgs[..], + &[ + "graphics-server", // raw (unprotected) frame buffer primitives + "keyboard", // required by graphics-server + "spinor", // required by keyboard - to save key mapping + "llio", // required by spinor + ] + ].concat(); + // packages in the user image + let user_pkgs = [ + &base_pkgs[..], + &gfx_dev_pkgs[..], + &[ + // net services + "com", + "net", + "dns", + // UX abstractions + "gam", + "ime-frontend", + "ime-plugin-shell", + "codec", + "modals", + // security + "root-keys", + "sha2", + "engine-25519", + "jtag", + // GUI front end + "status", + "shellchat", + // filesystem + "pddb", + // usb services + "usb-device-xous", + ] + ].concat(); + // for fast testing of compilation targets of the PDDB to real hardware + let pddb_dev_pkgs = [ + &base_pkgs[..], + &[ + "pddb", + "sha2", + ], + ].concat(); + // for fast checking of AES hardware accelerator + let aestest_pkgs = ["ticktimer-server", "log-server", "aes-test"].to_vec(); + + // ---- extract position independent args ---- + let lkey = get_flag("--lkey")?; + if lkey.len() != 0 { + builder.loader_key_file(lkey[0].to_string()); + } + let kkey = get_flag("--kkey")?; + if kkey.len() != 0 { + builder.kernel_key_file(kkey[0].to_string()); + } + + let extra_apps = get_flag("--app")?; + builder.add_apps(&extra_apps); + let extra_services = get_flag("--service")?; + builder.add_services(&extra_services); + + // ---- now process the verb plus position dependent arguments ---- + let mut args = env::args(); + let task = args.nth(1); + match task.as_deref() { + Some("install-toolkit") | Some("install-toolchain") => { + let arg = env::args().nth(2); + ensure_compiler( + &Some(TARGET_TRIPLE), + true, + arg.map(|x| x == "--force").unwrap_or(false), + )? + } + // ----- renode configs -------- + Some("renode-image") => { + builder.target_renode() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_apps(&get_cratespecs()); + } + Some("renode-image-debug") => { + builder.target_renode() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .stream(BuildStream::Debug) + .add_apps(&get_cratespecs()); + } + Some("renode-test") => { + builder.target_renode() + .add_services(&base_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + } + Some("libstd-test") => { + builder.target_renode() + .add_services(&base_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + builder.add_loader_feature("renode-bypass"); + } + Some("libstd-net") => { + builder.target_renode() + .add_services(&base_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + builder.add_loader_feature("renode-bypass") + .add_loader_feature("renode-minimal"); + builder.add_service("net") + .add_service("com") + .add_service("llio") + .add_service("dns"); + } + Some("renode-aes-test") => { + builder.target_renode() + .add_services(&aestest_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + } + + // ------- hosted mode configs ------- + Some("run") => { + builder.target_hosted() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("pddbtest") + .add_feature("ditherpunk") + .add_feature("tracking-alloc") + .add_feature("tls") + // .add_feature("test-rekey") + .add_apps(&get_cratespecs()); + } + Some("pddb-ci") => { + builder.target_hosted() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("pddb/ci") + .add_feature("pddb/deterministic"); + } + Some("pddb-btest") => { + builder.target_hosted() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("pddbtest") + .add_feature("autobasis") // this will make secret basis tracking synthetic and automated for stress testing + .add_feature("autobasis-ci") + .add_feature("pddb/deterministic"); + } + Some("hosted-debug") => { + builder.target_hosted() + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("pddbtest") + .add_feature("ditherpunk") + .add_feature("tracking-alloc") + .add_feature("tls") + .stream(BuildStream::Debug) + .add_apps(&get_cratespecs()); + } + Some("gfx-dev") => { + builder.target_hosted() + .add_services(&gfx_dev_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()) + .add_feature("graphics-server/testing"); + }, + + // ------ Precursor hardware image configs ------ + Some("app-image") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_apps(&get_cratespecs()); + } + Some("perf-image") => { + // note: to use this image, you need to load a version of the SOC that has the performance counters built in. + // this can be generated using the command `python3 .\betrusted_soc.py -e .\dummy.nky --perfcounter` in the betrusted-soc repo. + // + // to read out performance monitoring data, use the `usb_update.py` script as follows: + // ` python3 .\..\usb_update.py --dump v2p.txt --dump-file .\ring_aes_8.bin` + // where the `v2p.txt` file contains a virtual to physical mapping that is generated by the `perflib` framework and + // formatted in a fashion that can be automatically extracted by the usb_update script. + builder.target_precursor("c809403-perflib") + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_apps(&get_cratespecs()) + .add_feature("perfcounter") + .add_kernel_feature("v2p"); + } + Some("tts") => { + builder.target_precursor(PRECURSOR_SOC_VERSION); + + let mut pkgs = user_pkgs.to_vec(); + pkgs.push("tts-frontend"); + pkgs.push("ime-plugin-tts"); + pkgs.retain(|&pkg| pkg != "ime-plugin-shell"); + + builder.add_services(&pkgs.into_iter().map(String::from).collect()) + .add_apps(&get_cratespecs()) + .add_service("espeak-embedded#https://ci.betrusted.io/job/espeak-embedded/lastSuccessfulBuild/artifact/target/riscv32imac-unknown-xous-elf/release/espeak-embedded") + .override_locale("en-tts") + .add_feature("tts") + .add_feature("braille"); + } + Some("usbdev") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&gfx_dev_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + builder.add_service("usb-test"); + } + Some("ffi-test") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&gfx_dev_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + builder.add_service("ffi-test"); + builder.add_loader_feature("renode-bypass"); + } + Some("pddb-dev") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&pddb_dev_pkgs.into_iter().map(String::from).collect()) + .add_services(&get_cratespecs()); + }, + Some("trng-test") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("urandomtest"); + }, + Some("ro-test") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("ringosctest"); + } + Some("av-test") => { + builder.target_precursor(PRECURSOR_SOC_VERSION) + .add_services(&user_pkgs.into_iter().map(String::from).collect()) + .add_feature("avalanchetest"); + } + + // ---- other single-purpose commands ---- + Some("generate-locales") => generate_locales()?, + Some("wycheproof-import") => whycheproof_import()?, + _ => print_help(), + } + builder.build() +} + +fn print_help() { + eprintln!( +"cargo xtask [verb] [cratespecs ..] + [--feature [feature name]] + [--lkey [loader key]] [--kkey [kernel key]] + [--app [cratespec]] + [--service [cratespec]] + [--no-timestamp] + +[cratespecs] is a list of 0 or more items of the following syntax: + [name] crate 'name' to be built from local source + [name@version] crate 'name' to be fetched from crates.io at the specified version + [name#URL] pre-built binary crate of 'name' downloaded from a server at 'URL' + [path-to-binary] file path to a prebuilt binary image on local machine. + Files in '.' must be specified as './file' to avoid confusion with local source + +The [cratespecs] list is treated as apps or services based on the context of [verb]. Additional crates can +be merged in with explicit app/service treatment with the following flags: + [--app] [cratespec] [cratespec] is treated as an additional app + [--service] [cratespec] [cratespec] is treated as an additional service + +[--lkey] and [--kkey] Paths to alternate private key files for loader and kernel key signing (defaults to developer key) +[--no-timestamp] Do not include a timestamp in the build. By default, `ticktimer` is rebuilt on every run to encode a timestamp. + +- An 'app' must be enumerated in apps/manifest.json. + A pre-processor configures the launch menu based on the list of specified apps. +- A 'service' is merged into the device image without any pre-processing. + +[verb] options: +Hardware images: + app-image Precursor user image. [cratespecs] are apps + perf-image Precursor user image, with performance profiling. [cratespecs] are apps + tts builds an image with text to speech support via externally linked C executable. [cratespecs] are apps + ffi-test builds an image for testing C-FFI bindings and integration. [cratespecs] are services + usbdev minimal, insecure build for new USB core bringup. [cratespecs] are services + trng-test automation framework for TRNG testing (CPRNG seeded by RO^AV). [cratespecs] ignored. + ro-test automation framework for TRNG testing (RO directly, no CPRNG). [cratespecs] ignored. + av-test automation framework for TRNG testing (AV dircetly, no CPRNG). [cratespecs] ignored. + +Hosted emulation: + run Run user image in hosted mode with release flags. [cratespecs] are apps + pddb-ci PDDB config for CI testing (eg: TRNG->deterministic for reproducible errors). [cratespecs] ignored. + pddb-btest PDDB stress tester for secret basis creation/deletion [cratespecs] ignored. + hosted-debug Run user image in hosted mode with debug flags. [cratespecs] are apps + gfx-dev Testing mode for graphics primitves. [cratespecs] are services + pddb-dev Testing for compilation errors on hardware targets on the PDDB. + +Renode emulation: + renode-image Renode user image. Unspecified [cratespecs] are apps + renode-test Renode test image. Unspecified [cratespecs] are services + renode-image-debug Renode user image with --debug flag set + libstd-test Renode test image that includes the minimum packages. [cratespecs] are services + Bypasses sig checks, keys locked out. + libstd-net Renode test image for testing network functions. Bypasses sig checks, keys locked out. + renode-aes-test Renode image for AES emulation development. Extremely minimal. + +Other commands: + generate-locales (re)generate the locales include for the language selected in xous-rs/src/locale.rs + wycheproof-import generate binary test vectors for engine-25519 from whycheproof-import/x25519.json + install-toolkit installs Xous toolkit with no prompt, useful in CI. Specify `--force` to remove existing toolchains + +Note: By default, the `ticktimer` will get rebuilt every time. You can skip this by appending `--no-timestamp` to the command. +" + ) +} + +type DynError = Box; + enum MemorySpec { SvdFile(String), } -#[derive(Debug)] -enum BuildError { - PathConversionError, -} - -impl std::fmt::Display for BuildError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match *self { - BuildError::PathConversionError => write!(f, "could not convert path to UTF-8"), - } - } -} - -impl std::error::Error for BuildError {} - fn cargo() -> String { env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()) } @@ -62,977 +356,40 @@ fn project_root() -> PathBuf { .to_path_buf() } -// Filter out all arguments, which start with "-" -fn get_packages() -> Vec { +/// [cratespecs] are positional arguments, and is a list of 0 to N tokens that immediately +/// follow [verb] +fn get_cratespecs() -> Vec { + let mut cratespecs = Vec::::new(); let mut args = env::args(); - args.nth(1); - // skip everything past -- - let mut pkgs = Vec::::new(); + args.nth(1); // skip the verb for arg in args { - if arg == "--" { + if arg.starts_with('-') { + // stop processing the list as soon as first named argument is found break; } - pkgs.push(arg); + cratespecs.push(arg) } - pkgs.into_iter().filter(|x| !x.starts_with("-")).collect() + cratespecs } -fn main() -> Result<(), Box> { - let mut builder = Builder::new(); - generate_version(env::args().filter(|x| x == "--no-timestamp").count() == 0); // really brute force way to try and get a version into the build system. - - let hw_pkgs = [ - // core OS services - "gam", - "status", - "shellchat", - "ime-frontend", - "ime-plugin-shell", - "graphics-server", - "ticktimer-server", - "log-server", - "com", - "xous-names", - "keyboard", - "trng", - "llio", - "susres", - "codec", - "sha2", - "engine-25519", - "spinor", - "root-keys", - "jtag", - "net", - "dns", - "pddb", - "modals", - "usb-device-xous", - ]; - let minimal_pkgs = [ - "ticktimer-server", - "log-server", - "xous-names", - "trng", - "llio", - "susres", - "com", - ]; - // A base set of packages. This is all you need for a normal - // operating system that can run libstd - let base_pkgs = [ - "ticktimer-server", - "log-server", - "susres", - "xous-names", - "trng", - ]; - let pddb_dev_pkgs = [ - // just for checking compilation - "ticktimer-server", - "log-server", - "susres", - "xous-names", - "trng", - "pddb", - "sha2", - /* - "llio", - "root-keys", - "jtag", - "rtc", - "com", - "gam", - "graphics-server", - "keyboard", - "ime-frontend", - "ime-plugin-shell", - "status", - */ - ]; - let gfx_dev_pkgs = [ - "ticktimer-server", - "log-server", - "xous-names", - "trng", - "llio", - "susres", - "com", - "graphics-server", - "keyboard", - "spinor", - ]; - let aestest_pkgs = ["ticktimer-server", "log-server", "aes-test"]; - - let mut args = env::args(); - let task = args.nth(1); - // extract lkey/kkey args only after a "--" separator - let mut next_is_lkey = false; - let mut next_is_kkey = false; - let mut lkey: Option = None; - let mut kkey: Option = None; +fn get_flag(flag: &str) -> Result, DynError> { + let mut list = Vec::::new(); + let args = env::args(); + let mut flag_found = false; for arg in args { - if next_is_kkey { - kkey = Some(arg); - next_is_kkey = false; - continue; + if arg == flag { + flag_found = true; + continue } - if next_is_lkey { - lkey = Some(arg); - next_is_lkey = false; - next_is_kkey = true; - continue; - } - if arg == "--" { - next_is_lkey = true; - continue; + if flag_found { + if arg.starts_with('-') { + eprintln!("Malformed arguments. Expected argument after flag {}, but found {}", flag, arg); + return Err("Bad arguments".into()); + } + list.push(arg); + flag_found = false; + continue } } - match task.as_deref() { - Some("install-toolkit") | Some("install-toolchain") => { - let arg = env::args().nth(2); - ensure_compiler( - &Some(PROGRAM_TARGET), - true, - arg.map(|x| x == "--force").unwrap_or(false), - )? - } - Some("renode-image") => { - builder.target_renode() - .add_services(&hw_pkgs.to_vec()) - .add_apps(&get_packages()); - } - Some("renode-test") => { - builder.target_renode() - .add_services(&minimal_pkgs.to_vec()) - .add_apps(&get_packages()); - } - Some("tts") => { - builder.target_precursor(PRECURSOR_SOC_VERSION); - - let mut pkgs = hw_pkgs.to_vec(); - pkgs.push("tts-frontend"); - pkgs.push("ime-plugin-tts"); - pkgs.retain(|&pkg| pkg != "ime-plugin-shell"); - - builder.add_services(&pkgs) - .add_apps(&get_packages()) - .add_service("espeak-embedded#https://ci.betrusted.io/job/espeak-embedded/lastSuccessfulBuild/artifact/target/riscv32imac-unknown-xous-elf/release/espeak-embedded") - .override_locale("en-tts") - .add_feature("tts") - .add_feature("braille"); - } - Some("usbdev") => { - let mut args = env::args(); - args.nth(1); - let mut pkgs = base_pkgs.to_vec(); - pkgs.push("usb-test"); - let args: Vec = args.collect(); - let mut extra_packages = vec![]; - for program in &args { - extra_packages.push(program.as_str()); - } - if false { - renode_image( - false, - &pkgs, - extra_packages.as_slice(), - None, - Some(&["--no-default-features", "--features", "renode-bypass", "--features", "debug-print"]), - )?; - } else { - build_hw_image(false, None, &pkgs, - None, - None, - None, - extra_packages.as_slice(), None, None)?; - } - } - Some("libstd-test") => { - let mut args = env::args(); - args.nth(1); - let pkgs = base_pkgs.to_vec(); - let args: Vec = get_packages(); - let mut extra_packages = vec![]; - for program in &args { - extra_packages.push(program.as_str()); - } - renode_image( - false, - &pkgs, - extra_packages.as_slice(), - None, - Some(&["--features", "renode-bypass"]), - )?; - } - Some("ffi-test") => { - let mut args = env::args(); - args.nth(1); - //let mut pkgs = hw_pkgs.to_vec(); - let mut pkgs = gfx_dev_pkgs.to_vec(); - pkgs.push("ffi-test"); - let args: Vec = get_packages(); - let mut extra_packages = vec![]; - for program in &args { - extra_packages.push(program.as_str()); - } - build_hw_image( - false, - None, - &pkgs, - None, - None, - None, - &[], - Some(&["--features", "renode-bypass"]), - None, - )? - //renode_image(false, &pkgs, extra_packages.as_slice(), - //None, Some(&["--features", "renode-bypass"]))?; - } - Some("libstd-net") => { - let mut args = env::args(); - args.nth(1); - let mut pkgs = base_pkgs.to_vec(); - pkgs.push("net"); - pkgs.push("com"); - pkgs.push("llio"); - pkgs.push("dns"); - let args: Vec = get_packages(); - let mut extra_packages = vec![]; - for program in &args { - extra_packages.push(program.as_str()); - } - renode_image( - false, - &pkgs, - extra_packages.as_slice(), - Some(&["--features", "renode-minimal"]), - Some(&["--features", "renode-bypass"]), - )?; - } - Some("renode-aes-test") => { - generate_app_menus(&Vec::::new()); - renode_image(false, &aestest_pkgs, &[], None, None)? - } - Some("renode-image-debug") => { - generate_app_menus(&vec!["ball".to_string()]); - renode_image(true, &hw_pkgs, &[], None, None)? - } - Some("pddb-ci") => { - generate_app_menus(&Vec::::new()); - run( - false, - &hw_pkgs, - Some(&["--features", "pddb/ci", "--features", "pddb/deterministic"]), - false, - )? - } - Some("pddb-btest") => { - generate_app_menus(&Vec::::new()); - // for hosted runs, compile in the pddb test routines by default...for now. - run(false, &hw_pkgs, - Some(&[ - "--features", "pddbtest", - "--features", "autobasis", // this will make secret basis tracking synthetic and automated for stress testing - "--features", "pddb/deterministic", - "--features", "autobasis-ci", - ]), false)? - } - Some("run") => { - builder.target_hosted() - .add_services(&hw_pkgs.to_vec()) - .add_feature("pddbtest") - .add_feature("ditherpunk") - .add_feature("tracking-alloc") - .add_feature("tls") - // .add_feature("test-rekey") - .add_apps(&get_packages()); - } - Some("hosted-ci") => { - let mut pkgs = hw_pkgs.to_vec(); - let mut apps: Vec = get_packages(); - apps.push("ball".to_string()); - apps.push("repl".to_string()); - for app in &apps { - pkgs.push(app); - } - generate_app_menus(&apps); - run(false, &pkgs, None, true)? - } - Some("debug") => { - let mut args = env::args(); - args.nth(1); - let mut pkgs = hw_pkgs.to_vec(); - let mut apps: Vec = get_packages(); - if apps.len() == 0 { - // add the standard demo apps if none are specified - println!("No apps specified, adding default apps..."); - apps.push("ball".to_string()); - apps.push("repl".to_string()); - } - for app in &apps { - pkgs.push(app); - } - generate_app_menus(&apps); - run(true, &pkgs, None, false)? - } - Some("app-image") => { - builder.target_precursor(PRECURSOR_SOC_VERSION) - .add_services(&hw_pkgs.to_vec()) - .add_apps(&get_packages()); - } - Some("perf-image") => { - // note: to use this image, you need to load a version of the SOC that has the performance counters built in. - // this can be generated using the command `python3 .\betrusted_soc.py -e .\dummy.nky --perfcounter` in the betrusted-soc repo. - // - // to read out performance monitoring data, use the `usb_update.py` script as follows: - // ` python3 .\..\usb_update.py --dump v2p.txt --dump-file .\ring_aes_8.bin` - // where the `v2p.txt` file contains a virtual to physical mapping that is generated by the `perflib` framework and - // formatted in a fashion that can be automatically extracted by the usb_update script. - let mut args = env::args(); - args.nth(1); - let mut pkgs = hw_pkgs.to_vec(); - let apps = get_packages(); - for app in &apps { - pkgs.push(app); - } - generate_app_menus(&apps); - build_hw_image( - false, - Some("precursor-c809403-perflib".to_string()), // this is the name of the *feature* flag to utralib - &pkgs, - lkey, - kkey, - Some(&[ - "--features", "perfcounter", - ]), - &[], - None, - Some(&[ - "--features", "v2p", - ]), - )? - } - Some("gfx-dev") => run( - true, - &gfx_dev_pkgs, - Some(&["--features", "graphics-server/testing"]), - false, - )?, - Some("pddb-dev") => build_hw_image( - false, - env::args().nth(2), - &pddb_dev_pkgs, - lkey, - kkey, - None, - &[], - None, None, - )?, - Some("pddb-hosted") => run(false, &pddb_dev_pkgs, None, false)?, - Some("minimal") => build_hw_image( - false, - env::args().nth(2), - &minimal_pkgs, - lkey, - kkey, - None, - &[], - None, None, - )?, - Some("trng-test") => { - generate_app_menus(&Vec::::new()); - build_hw_image( - false, - env::args().nth(2), - &hw_pkgs, - lkey, - kkey, - Some(&["--features", "urandomtest"]), - &[], - None, None, - )? - } - Some("ro-test") => { - generate_app_menus(&Vec::::new()); - build_hw_image( - false, - env::args().nth(2), - &hw_pkgs, - lkey, - kkey, - Some(&["--features", "ringosctest"]), - &[], - None, None, - )? - } - Some("av-test") => { - generate_app_menus(&Vec::::new()); - build_hw_image( - false, - env::args().nth(2), - &hw_pkgs, - lkey, - kkey, - Some(&["--features", "avalanchetest"]), - &[], - None, None, - )? - } - Some("generate-locales") => generate_locales()?, - Some("wycheproof-import") => whycheproof_import()?, - _ => print_help(), - } - builder.build() -} - -fn print_help() { - eprintln!( - "cargo xtask [base config] [[pkg or app 1] [pkg or app 2] ...] - [--feature [feature name]] [--lkey [loader key]] [--kkey [kernel key]] [--no-timestamp] [--force] -Hardware images: - app-image [app1] [..] builds an image for real hardware of baseline kernel + specified apps - perf-image [app1] [..] builds an image for real hardware assuming a performance counter variant of the SOC. - -Hosted emulation: - run [app1] [..] runs a release build using a hosted environment plus specified apps - -Renode emulation: - renode-image builds a functional image for renode - renode-test builds a test image for renode - renode-image-debug builds a test image for renode in debug mode - libstd-test [pkg1] [..] builds a test image that includes the minimum packages, plus those - specified on the command line (e.g. built externally). Bypasses sig checks, keys locked out. - libstd-net [pkg1] [..] builds a test image for testing network functions. Bypasses sig checks, keys locked out. - -Locale (re-)generation: - generate-locales (re)generate the locales include for the language selected in xous-rs/src/locale.rs - -Various debug configurations (high chance of bitrot): - tts builds an image with text to speech support via externally linked C executable - wycheproof-import generate binary test vectors for engine-25519 from whycheproof-import/x25519.json - debug runs a debug build using a hosted environment - minimal builds a minimal image for API testing - ffi-test builds an image for testing C-FFI bindings and integration - gfx-dev minimal configuration for graphics primitive testing - pddb-dev PDDB testing only for live hardware - pddb-hosted PDDB testing in a hosted environment - pddb-ci PDDB config for CI testing (eg: TRNG->deterministic for reproducible errors) - usbdev minimal, insecure build for new USB core bringup - trng-test builds an image for TRNG testing - urandom source seeded by TRNG+AV - ro-test builds an image for ring oscillator only TRNG testing - av-test builds an image for avalanche generater only TRNG testing - install-toolkit installs Xous toolkit with no prompt, useful in CI. Specify `--force` to remove existing toolchains - -Note: By default, the `ticktimer` will get rebuilt every time. You can skip this by appending `--no-timestamp` to the command. -" - ) -} - -fn build_hw_image( - debug: bool, - svd: Option, - packages: &[&str], - lkey: Option, - kkey: Option, - extra_args: Option<&[&str]>, - extra_packages: &[&str], - loader_features: Option<&[&str]>, - kernel_features: Option<&[&str]>, -) -> Result<(), DynError> { - // ------ configure UTRA generation feature flags ------ - // note: once we switch over to hosted/precursor as first-class flags, the ["--features", "precursor"] should be added here - // for now that throws an error because we don't use that flag anywhere. - let mut svd_feat = if let Some(spec) = &svd { - if spec.contains("renode") { - vec!["--features"] - } else { - vec!["--features", "utralib/precursor", "--features"] - } - } else { - vec!["--features", "utralib/precursor", "--features"] - }; - let mut svd_path = String::from("utralib/"); - let svd_filename: String; - match svd { - Some(s) => { - svd_path.push_str(&s); - svd_filename = s.to_string(); - svd_feat.push(&svd_path); - }, - None => { - svd_path.push_str(SOC_SVD_VERSION); - svd_filename = SOC_SVD_VERSION.to_string(); - svd_feat.push(&svd_path); - }, - }; - - // LAST_CONFIG tracks the last SVD configuration. It's used by utralib to track if it - // should rebuild itself based on a change in SVD configs. Note that for some reason - // it takes two consecutive builds with the same SVD config before the build system - // figures out that it doesn't need to rebuild everything. After then, it behaves as expected. - let stream = if debug { "debug" } else { "release" }; - let last_config = format!("target/{}/{}/build/LAST_CONFIG", PROGRAM_TARGET, stream); - std::fs::create_dir_all(format!("target/{}/{}/build/", PROGRAM_TARGET, stream)).unwrap(); - let changed = match OpenOptions::new() - .read(true) - .open(&last_config) { - Ok(mut file) => { - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); - if contents != svd_filename { - true - } else { - false - } - } - _ => true - }; - if changed { - let mut file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&last_config).unwrap(); - write!(file, "{}", svd_filename).unwrap(); - } - - // concatenate the passed in feature flags with the computed utra feature flags - let mut kernel_features_finalized = Vec::<&str>::new(); - let mut loader_features_finalized = Vec::<&str>::new(); - for &s in svd_feat.iter() { - kernel_features_finalized.push(s); - loader_features_finalized.push(s); - } - if let Some(lf) = loader_features { - for &s in lf { - loader_features_finalized.push(s); - } - } - if let Some(kf) = kernel_features { - for &s in kf { - kernel_features_finalized.push(s); - } - } - - // std::env::set_var("RUST_LOG", "debug"); // set this to debug the image creation process - - // extract key file names; replace with defaults if not specified - let loaderkey_file = lkey.unwrap_or_else(|| "devkey/dev.key".into()); - let kernelkey_file = kkey.unwrap_or_else(|| "devkey/dev.key".into()); - - // ------ build the kernel ------ - let kernel = build_kernel(debug, Some(&kernel_features_finalized))?; - - // ------ build the services ------ - svd_feat.push("--features"); - svd_feat.push("utralib/std"); - let mut init = vec![]; - let base_path = build( - packages, - debug, - Some(PROGRAM_TARGET), - None, - extra_args, - Some(&svd_feat), - )?; - for pkg in packages { - let mut pkg_path = base_path.clone(); - // some packages may have a colon-delimited version after it to clarify crates.io patches. - // Strip off the version number before passing to the image builder. - let pkg_maybe_version: Vec<&str> = pkg.split(':').collect(); - let pkg_root = if pkg_maybe_version.len() > 1 { - pkg_maybe_version[pkg_maybe_version.len() - 2] - } else { - pkg - }; - pkg_path.push(pkg_root); - init.push(pkg_path); - } - for pkg in extra_packages { - let mut pkg_path = project_root(); - pkg_path.push(pkg); - init.push(pkg_path); - } - - // ------ build the loader ------ - // stash any LTO settings applied to the kernel; proper layout of the loader - // block depends on the loader being compact and highly optimized. - let existing_lto = std::env::var("CARGO_PROFILE_RELEASE_LTO") - .map(|v| Some(v)) - .unwrap_or(None); - let existing_codegen_units = std::env::var("CARGO_PROFILE_RELEASE_CODEGEN_UNITS") - .map(|v| Some(v)) - .unwrap_or(None); - // these settings will generate the most compact code (but also the hardest to debug) - std::env::set_var("CARGO_PROFILE_RELEASE_LTO", "true"); - std::env::set_var("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"); - let mut loader = build( - &["loader"], - debug, - Some(KERNEL_TARGET), - None, - None, - Some(&loader_features_finalized), - )?; - // restore the LTO settings - if let Some(existing) = existing_lto { - std::env::set_var("CARGO_PROFILE_RELEASE_LTO", existing); - } - if let Some(existing) = existing_codegen_units { - std::env::set_var("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", existing); - } - loader.push(PathBuf::from("loader")); - - // ---------- extract SVD file path, as computed by utralib ---------- - let svd_spec_path = format!("target/{}/{}/build/SVD_PATH", PROGRAM_TARGET, stream); - let mut svd_spec_file = OpenOptions::new() - .read(true) - .open(&svd_spec_path)?; - let mut svd_path = String::new(); - svd_spec_file.read_to_string(&mut svd_path)?; - - // --------- package up and sign a binary image ---------- - let output_bundle = create_image(&kernel, &init, debug, - MemorySpec::SvdFile(svd_path) - )?; - println!(); - println!( - "Kernel+Init bundle is available at {}", - output_bundle.display() - ); - - let mut loader_bin = output_bundle.parent().unwrap().to_owned(); - loader_bin.push("loader.bin"); - let mut loader_presign = output_bundle.parent().unwrap().to_owned(); - loader_presign.push("loader_presign.bin"); - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&[ - "run", - "--package", - "tools", - "--bin", - "copy-object", - "--", - loader.as_os_str().to_str().unwrap(), - loader_presign.as_os_str().to_str().unwrap(), - ]) - .status()?; - if !status.success() { - return Err("cargo build failed".into()); - } - - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&[ - "run", - "--package", - "tools", - "--bin", - "sign-image", - "--", - "--loader-image", - loader_presign.to_str().unwrap(), - "--loader-key", - loaderkey_file.as_str(), - "--loader-output", - loader_bin.to_str().unwrap(), - "--min-xous-ver", - MIN_XOUS_VERSION, - ]) - .status()?; - if !status.success() { - return Err("loader image sign failed".into()); - } - - let mut xous_img_path = output_bundle.parent().unwrap().to_owned(); - let mut xous_img_presign_path = xous_img_path.clone(); - xous_img_path.push("xous.img"); - xous_img_presign_path.push("xous_presign.img"); - let mut xous_img = - std::fs::File::create(&xous_img_presign_path).expect("couldn't create xous.img"); - let mut bundle_file = std::fs::File::open(output_bundle).expect("couldn't open output bundle"); - let mut buf = vec![]; - bundle_file - .read_to_end(&mut buf) - .expect("couldn't read output bundle file"); - xous_img - .write_all(&buf) - .expect("couldn't write bundle file to xous.img"); - println!("Bundled image file created at {}", xous_img_path.display()); - - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&[ - "run", - "--package", - "tools", - "--bin", - "sign-image", - "--", - "--kernel-image", - xous_img_presign_path.to_str().unwrap(), - "--kernel-key", - kernelkey_file.as_str(), - "--kernel-output", - xous_img_path.to_str().unwrap(), - "--min-xous-ver", - MIN_XOUS_VERSION, - // "--defile", - ]) - .status()?; - if !status.success() { - return Err("kernel image sign failed".into()); - } - - println!(); - println!("Signed loader at {}", loader_bin.display()); - println!("Signed kernel at {}", xous_img_path.display()); - - Ok(()) -} - -fn renode_image( - debug: bool, - packages: &[&str], - extra_packages: &[&str], - xous_features: Option<&[&str]>, - loader_features: Option<&[&str]>, -) -> Result<(), DynError> { - // Regenerate the Platform file - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&[ - "run", - "-p", - "svd2repl", - "--", - "utralib/renode/renode.svd", - "emulation/soc/betrusted-soc.repl", - ]) - .status()?; - if !status.success() { - return Err("Unable to regenerate Renode platform file".into()); - } - build_hw_image( - debug, - Some("renode".to_string()), - packages, - None, - None, - xous_features, - extra_packages, - loader_features, - None, - ) -} - -fn run( - debug: bool, - init: &[&str], - features: Option<&[&str]>, - dry_run: bool, -) -> Result<(), DynError> { - let stream = if debug { "debug" } else { "release" }; - - build(init, debug, None, None, None, features)?; - - // Build and run the kernel - let mut args = vec!["run"]; - if !debug { - args.push("--release"); - } - - args.push("--"); - - let mut paths = vec![]; - for i in init { - let tmp: PathBuf = if cfg!(windows) { - Path::new(&format!( - "..{}target{}{}{}{}.exe", - MAIN_SEPARATOR, MAIN_SEPARATOR, stream, MAIN_SEPARATOR, i - )) - .to_owned() - } else { - Path::new(&format!( - "..{}target{}{}{}{}", - MAIN_SEPARATOR, MAIN_SEPARATOR, stream, MAIN_SEPARATOR, i - )) - .to_owned() - }; - paths.push(tmp); - } - for t in &paths { - args.push(t.to_str().ok_or(BuildError::PathConversionError)?); - } - - let mut dir = project_root(); - dir.push("kernel"); - - if !dry_run { - println!("Building and running kernel..."); - print!(" Command: cargo"); - for arg in &args { - print!(" {}", arg); - } - println!(); - let status = Command::new(cargo()) - .current_dir(dir) - .args(&args) - .status()?; - if !status.success() { - return Err("cargo build failed".into()); - } - } - - Ok(()) -} - -fn build_kernel(debug: bool, features: Option<&[&str]>) -> Result { - let mut path = build(&["xous-kernel"], debug, Some(KERNEL_TARGET), None, None, features)?; - path.push("xous-kernel"); - Ok(path) - - /* - // cargo install --target riscv32imac-unknown-xous-elf --target-dir target xous-kernel --version 0.9.0 --features utralib/precursor-c809403 --root .\target\riscv32imac-unknown-xous-elf\release\ - // cargo install --list --root .\target\riscv32imac-unknown-xous-elf\release\ - // drop the --target spec and you'll get hosted mode binaries. - let stream = if debug { "debug" } else { "release" }; - let mut dir = project_root(); - let mut args = vec!["install"]; - let target_path = format!("target/{}/{}/", KERNEL_TARGET, stream); - args.push("--root"); - args.push(&target_path); - args.push("--target"); - args.push(KERNEL_TARGET); - args.push("xous-kernel"); - args.push("--version"); - args.push("0.9.0"); - args.push("--features"); - args.push("utralib/precursor-c809403"); - - let status = Command::new(cargo()) - .current_dir(dir) - .args(&args) - .status()?; - - if !status.success() { - return Err("cargo build failed".into()); - } - Ok(project_root().join(&target_path).join("bin/xous-kernel")) - //Ok(PathBuf::from_str("C:\\Users\\bunnie\\.cargo\\bin\\xous-kernel").unwrap()) - */ -} - - -fn build( - packages: &[&str], - debug: bool, - target: Option<&str>, - directory: Option, - extra_args: Option<&[&str]>, - features: Option<&[&str]>, -) -> Result { - ensure_compiler(&target, false, false)?; - let stream = if debug { "debug" } else { "release" }; - let mut args = vec!["build"]; - print!("Building"); - - if let Some(feature) = features { - for &a in feature { - args.push(a); - } - } - for package in packages { - print!(" {}", package); - args.push("--package"); - args.push(package); - } - println!(); - let mut target_path = "".to_owned(); - if let Some(t) = target { - args.push("--target"); - args.push(t); - target_path = format!("{}/", t); - } - - if !debug { - args.push("--release"); - } - - if let Some(extra) = extra_args { - for &a in extra { - args.push(a); - } - } - - let mut dir = project_root(); - if let Some(subdir) = &directory { - dir.push(subdir); - } - - print!(" Command: cargo"); - for arg in &args { - print!(" {}", arg); - } - println!(); - let status = Command::new(cargo()) - .current_dir(dir) - .args(&args) - .status()?; - - if !status.success() { - return Err("cargo build failed".into()); - } - - if let Some(base_dir) = &directory { - Ok(project_root().join(&format!( - "{}/target/{}{}/", - base_dir.to_str().ok_or(BuildError::PathConversionError)?, - target_path, - stream, - ))) - } else { - Ok(project_root().join(&format!("target/{}{}/", target_path, stream))) - } -} - -fn create_image( - kernel: &Path, - init: &[PathBuf], - debug: bool, - memory_spec: MemorySpec, -) -> Result { - let stream = if debug { "debug" } else { "release" }; - let mut args = vec!["run", "--package", "tools", "--bin", "create-image", "--"]; - - let output_file = format!("target/{}/{}/args.bin", PROGRAM_TARGET, stream); - args.push(&output_file); - - args.push("--kernel"); - args.push(kernel.to_str().ok_or(BuildError::PathConversionError)?); - - for i in init { - args.push("--init"); - args.push(i.to_str().ok_or(BuildError::PathConversionError)?); - } - - match memory_spec { - MemorySpec::SvdFile(ref s) => { - args.push("--svd"); - args.push(s); - } - } - - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&args) - .status()?; - - if !status.success() { - return Err("cargo build failed".into()); - } - Ok(project_root().join(&format!("target/{}/{}/args.bin", PROGRAM_TARGET, stream))) + Ok(list) } diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs index 69ff09348..90e4b317a 100644 --- a/xtask/src/utils.rs +++ b/xtask/src/utils.rs @@ -5,7 +5,7 @@ use std::{ process::Command, }; -use crate::{cargo, project_root, PROGRAM_TARGET}; +use crate::{cargo, project_root, TARGET_TRIPLE}; const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases"; @@ -85,12 +85,12 @@ pub(crate) fn ensure_compiler( } // If the sysroot exists, then we're good. - let target = target.unwrap_or(PROGRAM_TARGET); + let target = target.unwrap_or(TARGET_TRIPLE); if let Some(path) = get_sysroot(Some(target))? { let mut version_path = PathBuf::from(&path); version_path.push("lib"); version_path.push("rustlib"); - version_path.push(PROGRAM_TARGET); + version_path.push(TARGET_TRIPLE); if remove_existing { println!("Target path exists, removing it"); std::fs::remove_dir_all(version_path) @@ -98,13 +98,13 @@ pub(crate) fn ensure_compiler( println!("Also removing target directories for existing toolchain"); let mut target_main = project_root(); target_main.push("target"); - target_main.push(PROGRAM_TARGET); + target_main.push(TARGET_TRIPLE); std::fs::remove_dir_all(target_main).ok(); let mut target_loader = project_root(); target_loader.push("loader"); target_loader.push("target"); - target_loader.push(PROGRAM_TARGET); + target_loader.push(TARGET_TRIPLE); std::fs::remove_dir_all(target_loader).ok(); } else { DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst);