From 8af0ef796143440c6b61cae8a0d36dd9574b823a Mon Sep 17 00:00:00 2001 From: bunnie Date: Thu, 29 Sep 2022 19:05:24 +0800 Subject: [PATCH] split xtask into file modules --- xtask/src/app_manifest.rs | 245 +++++++++++++ xtask/src/main.rs | 755 +++----------------------------------- xtask/src/utils.rs | 376 +++++++++++++++++++ xtask/src/versioning.rs | 94 +++++ 4 files changed, 764 insertions(+), 706 deletions(-) create mode 100644 xtask/src/app_manifest.rs create mode 100644 xtask/src/utils.rs create mode 100644 xtask/src/versioning.rs diff --git a/xtask/src/app_manifest.rs b/xtask/src/app_manifest.rs new file mode 100644 index 000000000..2c29c8b05 --- /dev/null +++ b/xtask/src/app_manifest.rs @@ -0,0 +1,245 @@ +// This module supports generating the app menus from the JSON manifest in the apps/ directory. + +use std::{ + fs::{OpenOptions, File}, + io::{Read, Write}, + string::String, + fmt::Write as StdWrite, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; + +#[derive(Deserialize, Serialize, Debug)] +struct AppManifest { + context_name: String, + menu_name: HashMap>, + submenu: Option::, +} +#[derive(Deserialize, Serialize, Debug)] +struct Locales { + locales: HashMap>, +} + +pub(crate) fn generate_app_menus(apps: &Vec) { + let file = File::open("apps/manifest.json").expect("Failed to open the manifest file"); + let mut reader = std::io::BufReader::new(file); + let mut content = String::new(); + reader + .read_to_string(&mut content) + .expect("Failed to read the file"); + let manifest: HashMap = + serde_json::from_str(&content).expect("Cannot parse manifest file"); + + // localization file + // inject all the localization strings into the i18n file, which in theory reduces the churn on other crates that depend + // on the global i18n file between build variants + let mut l = BTreeMap::>::new(); + for (_app, manifest) in manifest.iter() { + for (name, translations) in &manifest.menu_name { + let mut map = BTreeMap::::new(); + for (language, phrase) in translations { + map.insert(language.to_string(), phrase.to_string()); + } + l.insert(name.to_string(), map); + } + } + // output a JSON localizations file, if things have changed + let new_i18n = serde_json::to_string(&l).unwrap(); + overwrite_if_changed(&new_i18n, "apps/i18n.json"); + + // output the Rust manifests - tailored just for the apps requested + let mut working_set = BTreeMap::::new(); + // derive a working_set that is just the apps we requested + for app in apps { + if let Some(manifest) = manifest.get(app) { + working_set.insert(app.to_string(), &manifest); + } + } + + // construct the gam_tokens + let mut gam_tokens = String::new(); + writeln!( + gam_tokens, + "// This file is auto-generated by xtask/main.rs generate_app_menus()" + ) + .unwrap(); + for (app_name, manifest) in working_set.iter() { + writeln!( + gam_tokens, + "pub const APP_NAME_{}: &'static str = \"{}\";", + app_name.to_uppercase(), + manifest.context_name, + ) + .unwrap(); + if let Some(menu_count) = manifest.submenu { + for i in 0..menu_count { + writeln!( + gam_tokens, + "pub const APP_MENU_{}_{}: &'static str = \"{} Submenu {}\";", + i, + app_name.to_uppercase(), + manifest.context_name, + i, + ) + .unwrap(); + } + } + } + writeln!( + gam_tokens, + "\npub const EXPECTED_APP_CONTEXTS: &[&'static str] = &[" + ) + .unwrap(); + for (app_name, manifest) in working_set.iter() { + writeln!(gam_tokens, " APP_NAME_{},", app_name.to_uppercase(),).unwrap(); + if let Some(menu_count) = manifest.submenu { + for i in 0..menu_count { + writeln!( + gam_tokens, + " APP_MENU_{}_{},", + i, + app_name.to_uppercase(), + ) + .unwrap(); + } + } + } + writeln!(gam_tokens, "];").unwrap(); + overwrite_if_changed(&gam_tokens, "services/gam/src/apps.rs"); + + // construct the app menu + let mut menu = String::new(); + writeln!( + menu, + "// This file is auto-generated by xtask/main.rs generate_app_menus()" + ) + .unwrap(); + if apps.len() == 0 { + writeln!(menu, "// NO APPS SELECTED: suppressing warning messages!").unwrap(); + writeln!(menu, "#![allow(dead_code)]").unwrap(); + writeln!(menu, "#![allow(unused_imports)]").unwrap(); + writeln!(menu, "#![allow(unused_variables)]").unwrap(); + } + writeln!(menu, r####"use crate::StatusOpcode; +use gam::{{MenuItem, MenuPayload}}; +use locales::t; +use num_traits::*; +use std::{{error::Error, fmt}}; + +#[derive(Debug)] +pub enum AppDispatchError {{ + IndexNotFound(usize), +}} + +impl Error for AppDispatchError {{}} + +impl fmt::Display for AppDispatchError {{ + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{ + match self {{ + AppDispatchError::IndexNotFound(app_index) => write!(f, "Index {{}} not found", app_index), + }} + }} +}} + +pub(crate) fn app_dispatch(gam: &gam::Gam, token: [u32; 4], index: usize) -> Result<(), AppDispatchError> {{ + match index {{"####).unwrap(); + for (index, (app_name, _manifest)) in working_set.iter().enumerate() { + writeln!( + menu, + " {} => {{ + gam.switch_to_app(gam::APP_NAME_{}, token).expect(\"couldn't raise app\"); + Ok(()) + }},", + index, + app_name.to_uppercase() + ) + .unwrap(); + } + writeln!( + menu, + r####" _ => Err(AppDispatchError::IndexNotFound(index)), + }} +}} + +pub(crate) fn app_index_to_name(index: usize) -> Result<&'static str, AppDispatchError> {{ + match index {{"#### + ) + .unwrap(); + for (index, (_, _manifest)) in working_set.iter().enumerate() { + for name in _manifest.menu_name.keys() { + writeln!( + menu, + " {} => Ok(t!(\"{}\", xous::LANG)),", + index, name, + ) + .unwrap(); + } + } + writeln!( + menu, + r####" _ => Err(AppDispatchError::IndexNotFound(index)), + }} +}} + +pub(crate) fn app_menu_items(menu_items: &mut Vec::, status_conn: u32) {{ +"#### + ) + .unwrap(); + for (index, (_app_name, manifest)) in working_set.iter().enumerate() { + writeln!(menu, " menu_items.push(MenuItem {{",).unwrap(); + assert!( + manifest.menu_name.len() == 1, + "Improper menu name record entry" + ); + for name in manifest.menu_name.keys() { + writeln!( + menu, + " name: xous_ipc::String::from_str(t!(\"{}\", xous::LANG)),", + name + ) + .unwrap(); + } + writeln!(menu, " action_conn: Some(status_conn),",).unwrap(); + writeln!( + menu, + " action_opcode: StatusOpcode::SwitchToApp.to_u32().unwrap(),", + ) + .unwrap(); + writeln!( + menu, + " action_payload: MenuPayload::Scalar([{}, 0, 0, 0]),", + index + ) + .unwrap(); + writeln!(menu, " close_on_select: true,",).unwrap(); + writeln!(menu, " }});\n",).unwrap(); + } + writeln!(menu, "}}").unwrap(); + overwrite_if_changed(&menu, "services/status/src/app_autogen.rs"); +} + +fn overwrite_if_changed(new_string: &String, old_file: &str) { + let original = match OpenOptions::new().read(true).open(old_file) { + Ok(mut ref_file) => { + let mut buf = String::new(); + ref_file + .read_to_string(&mut buf) + .expect("UTF-8 error in previous localization file"); + buf + } + _ => String::new(), + }; + if &original != new_string { + // println!("file change in i18n.json detected:"); + // println!("Old: {}", original); + // println!("New: {}", new_string); + let mut new_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(old_file) + .expect("Can't open our gam manifest for writing"); + write!(new_file, "{}", new_string).unwrap() + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 836bdb036..fd9567a7a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,10 +1,13 @@ -use chrono::Local; -use std::collections::{BTreeMap, HashMap}; -use std::fmt::Write as StdWrite; +mod app_manifest; +use app_manifest::*; +mod versioning; +use versioning::*; +mod utils; +use utils::*; + use std::fs::OpenOptions; use std::{ env, - fs::File, io::{Read, Write}, path::{Path, PathBuf, MAIN_SEPARATOR}, process::Command, @@ -22,7 +25,6 @@ type DynError = Box; const PROGRAM_TARGET: &str = "riscv32imac-unknown-xous-elf"; const KERNEL_TARGET: &str = "riscv32imac-unknown-xous-elf"; -const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases"; enum MemorySpec { SvdFile(String), @@ -43,6 +45,18 @@ impl std::fmt::Display for BuildError { impl std::error::Error for BuildError {} +fn cargo() -> String { + env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()) +} + +fn project_root() -> PathBuf { + Path::new(&env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(1) + .unwrap() + .to_path_buf() +} + // Filter out all arguments, which start with "-" fn get_packages() -> Vec { let mut args = env::args(); @@ -992,329 +1006,38 @@ fn build_kernel(debug: bool, features: Option<&[&str]>) -> Result, - force_install: bool, - remove_existing: bool, -) -> Result<(), String> { - use std::process::Stdio; - if DONE_COMPILER_CHECK.load(std::sync::atomic::Ordering::SeqCst) { - return Ok(()); - } - - /// Return the sysroot for the given target. If the target does not exist, - /// return None. - fn get_sysroot(target: Option<&str>) -> Result, String> { - let mut args = vec!["--print", "sysroot"]; - if let Some(target) = target { - args.push("--target"); - args.push(target); - } - - let sysroot_cmd = Command::new("rustc") - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .args(&args) - .spawn() - .expect("could not run rustc"); - let sysroot_output = sysroot_cmd.wait_with_output().unwrap(); - let have_toolchain = sysroot_output.status.success(); - - let toolchain_path = String::from_utf8(sysroot_output.stdout) - .map_err(|_| "Unable to find Rust sysroot".to_owned())? - .trim() - .to_owned(); - - // Look for the "RUST_VERSION" file to ensure it's compatible with this version. - if let Some(target) = target { - let mut version_path = PathBuf::from(&toolchain_path); - version_path.push("lib"); - version_path.push("rustlib"); - version_path.push(target); - version_path.push("RUST_VERSION"); - if let Ok(mut vp) = File::open(&version_path) { - let mut version_str = String::new(); - if let Err(_) = vp.read_to_string(&mut version_str) { - return Err("Unable to get version string".to_owned()); - } - - let rustc_version_str = format!("{}", rustc_version::version().unwrap()); - if version_str.trim() != rustc_version_str.trim() { - println!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str.trim(), rustc_version_str.trim()); - // return Err(format!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str, rustc_version_str)); - return Ok(None); - } - } else { - println!("Outdated toolchain installed."); - // return Err("Outdated toolchain installed".to_owned()); - return Ok(None); - } - } - - if have_toolchain { - Ok(Some(toolchain_path)) - } else { - Ok(None) - } - } - - // If the sysroot exists, then we're good. - let target = target.unwrap_or(PROGRAM_TARGET); - if let Some(path) = get_sysroot(Some(target))? { - let mut version_path = PathBuf::from(&path); - version_path.push("lib"); - version_path.push("rustlib"); - version_path.push(PROGRAM_TARGET); - if remove_existing { - println!("Target path exists, removing it"); - std::fs::remove_dir_all(version_path) - .or_else(|e| Err(format!("unable to remove existing toolchain: {}", e)))?; - println!("Also removing target directories for existing toolchain"); - let mut target_main = project_root(); - target_main.push("target"); - target_main.push(PROGRAM_TARGET); - std::fs::remove_dir_all(target_main).ok(); - - let mut target_loader = project_root(); - target_loader.push("loader"); - target_loader.push("target"); - target_loader.push(PROGRAM_TARGET); - std::fs::remove_dir_all(target_loader).ok(); - } else { - DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst); - return Ok(()); - } - } - - // Since no sysroot exists, we must download a new one. - let toolchain_path = - get_sysroot(None)?.ok_or_else(|| "default toolchain not installed".to_owned())?; - // If the terminal is a tty, or if toolchain installation is forced, - // download the latest toolchain. - if !atty::is(atty::Stream::Stdin) && !force_install { - return Err(format!("Toolchain for {} not found", target)); - } - - // Version 1.54 was the last major version that was released. - let ver = rustc_version::version().unwrap(); - if ver.major == 1 && ver.minor < 54 { - return Err("Rust 1.54 or higher is required".into()); - } - - // Ask the user if they want to install the toolchain. - let mut buffer = String::new(); - let stdin = std::io::stdin(); - let mut stdout = std::io::stdout(); - if force_install { - println!("Downloading toolchain"); - } else { - println!( - "Error: Toolchain for {} was not found on this system!", - target - ); - } - loop { - if force_install { - break; - } - - print!("Would you like this program to attempt to download and install it? [Y/n] "); - stdout.flush().unwrap(); - buffer.clear(); - stdin.read_line(&mut buffer).unwrap(); - - let trimmed = buffer.trim(); - - if trimmed == "n" || trimmed == "N" { - return Err(format!("Please install the {} toolchain", target)); - } - - if trimmed == "y" || trimmed == "Y" || trimmed.is_empty() { - break; - } - println!(); - } - - fn get_toolchain_url(major: u64, minor: u64, patch: u64) -> Result { - let j: serde_json::Value = ureq::get(TOOLCHAIN_RELEASE_URL) - .set("Accept", "application/vnd.github.v3+json") - .call() - .map_err(|e| format!("{}", e))? - .into_json() - .map_err(|e| format!("{}", e))?; - // let j: serde_json::Value = serde_json::from_str(CONTENT).expect("Cannot parse manifest file"); - - let releases = j.as_array().unwrap(); - let mut tag_urls = std::collections::BTreeMap::new(); - - let target_prefix = format!("{}.{}.{}", major, minor, patch); - for r in releases { - // println!(">>> Value: {}", r); - - let keys = match r.as_object() { - None => continue, - Some(r) => r, - }; - let release = match keys.get("tag_name") { - None => continue, - Some(s) => match s.as_str() { - None => continue, - Some(s) => s, - }, - }; - if !release.starts_with(&target_prefix) { - continue; - } - - let assets = match keys.get("assets") { - None => continue, - Some(s) => match s.as_array() { - None => continue, - Some(s) => s, - }, - }; - - let first_asset = match assets.get(0) { - None => continue, - Some(s) => s, - }; - - let download_url = match first_asset.get("browser_download_url") { - None => continue, - Some(s) => match s.as_str() { - None => continue, - Some(s) => s, - }, - }; - // println!("Candidate Release: {}", download_url); - tag_urls.insert(release.to_owned(), download_url.to_owned()); - } - - if let Some((_k, v)) = tag_urls.into_iter().last() { - // println!("Found candidate entry: v{} url {}", k, v); - return Ok(v); - } - Err(format!("No toolchains found for Rust {}", target_prefix)) - } - let toolchain_url = get_toolchain_url(ver.major, ver.minor, ver.patch)?; - - println!( - "Attempting to install toolchain for {} into {}", - target, toolchain_path - ); - println!("Downloading toolchain from {}...", toolchain_url); - - print!("Download in progress..."); - stdout.flush().unwrap(); - let mut zip_data = vec![]; - { - let agent = ureq::builder() - // .middleware(CounterMiddleware(shared_state.clone())) - .build(); - - let mut freader = agent - .get(&toolchain_url) - .call() - .map_err(|e| format!("{}", e))? - .into_reader(); - freader - .read_to_end(&mut zip_data) - .map_err(|e| format!("{}", e))?; - // |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| { - // // If either number is infinite, don't print anything and just continue. - // if total_bytes.is_infinite() || bytes_so_far.is_infinite() { - // return true; - // } - - // // Display progress. - // print!( - // "\rDownload progress: {:3.02}% ", - // bytes_so_far / total_bytes * 100.0 - // ); - // stdout.flush().unwrap(); - - // // Return `true` to continue the transfer. - // true - // }, - println!(); - } - println!( - "Download successful. Total data size is {} bytes", - zip_data.len() - ); - - /// Extract the zipfile to the target directory, ensuring that all files - /// contained within are created. - fn extract_zip>( - archive_data: P, - extract_to: P2, - ) -> Result<(), String> { - let mut archive = zip::ZipArchive::new(archive_data) - .map_err(|e| format!("unable to extract zip: {}", e))?; - for i in 0..archive.len() { - let mut entry_in_archive = archive - .by_index(i) - .map_err(|e| format!("unable to locate file index {}: {}", i, e))?; - // println!( - // "Trying to extract file {}", - // entry_in_archive.mangled_name().display() - // ); - - let output_path = extract_to.as_ref().join(entry_in_archive.mangled_name()); - if entry_in_archive.is_dir() { - std::fs::create_dir_all(&output_path).map_err(|e| { - format!( - "unable to create directory {}: {}", - output_path.display(), - e - ) - })?; - } else { - // Create the parent directory if necessary - if let Some(parent) = output_path.parent() { - if !parent.exists() { - std::fs::create_dir_all(&parent).map_err(|e| { - format!( - "unable to create directory {}: {}", - output_path.display(), - e - ) - })?; - } - } - let mut outfile = std::fs::File::create(&output_path).map_err(|e| { - format!("unable to create file {}: {}", output_path.display(), e) - })?; - std::io::copy(&mut entry_in_archive, &mut outfile).map_err(|e| { - format!( - "unable to write extracted file {}: {}", - output_path.display(), - e - ) - })?; - } - } - Ok(()) - } - println!("Extracting toolchain to {}...", toolchain_path); - extract_zip(std::io::Cursor::new(zip_data), &toolchain_path)?; - - println!("Toolchain successfully installed"); - - DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(()) -} fn build( packages: &[&str], @@ -1425,383 +1148,3 @@ fn create_image( } Ok(project_root().join(&format!("target/{}/{}/args.bin", PROGRAM_TARGET, stream))) } - -fn cargo() -> String { - env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()) -} - -fn project_root() -> PathBuf { - Path::new(&env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(1) - .unwrap() - .to_path_buf() -} - -/// Regenerate the locales files. This is only done when the command is explicitly run. -fn generate_locales() -> Result<(), std::io::Error> { - let ts = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - filetime::set_file_mtime("locales/src/lib.rs", ts)?; - let mut path = project_root(); - path.push("locales"); - let status = Command::new(cargo()) - .current_dir(path) - .args(&["build", "--package", "locales"]) - .status()?; - if !status.success() { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "Couldn't generate the locales", - )); - } - return Ok(()); -} - -fn whycheproof_import() -> Result<(), DynError> { - let input_file = "tools/wycheproof-import/x25519_test.json"; - let output_file = "services/shellchat/src/cmds/x25519_test.bin"; - let status = Command::new(cargo()) - .current_dir(project_root()) - .args(&[ - "run", - "--package", - "wycheproof-import", - "--", - input_file, - output_file, - ]) - .status()?; - if !status.success() { - return Err("wycheproof-import failed. If any, the output will not be usable.".into()); - } - - println!(); - println!("Wrote wycheproof x25519 testvectors to '{}'.", output_file); - - return Ok(()); -} - -////////////////////////// Versioning infrastructure -fn generate_version(add_timestamp: bool) { - let output = if cfg!(target_os = "windows") { - Command::new("cmd") - .args(["/C", "git describe --tags"]) - .output() - .expect("failed to execute process") - } else { - Command::new("sh") - .arg("-c") - .arg("git describe --tags") - .output() - .expect("failed to execute process") - }; - let gitver = output.stdout; - let semver = String::from_utf8_lossy(&gitver); - - let version_file = "services/ticktimer-server/src/version.rs"; - - // Read the existing file to see if it needs to be updated. - let mut existing_data = Vec::new(); - if let Ok(mut f) = std::fs::File::open(version_file) { - f.read_to_end(&mut existing_data).ok(); - } - - let mut new_data = Vec::new(); - print_header(&mut new_data); - if add_timestamp { - let now = Local::now(); - write!( - new_data, - "#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"{}\";\n", - now.to_rfc2822() - ) - .expect("couldn't add our timestamp"); - } else { - write!( - new_data, - "#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"unavailable\";\n" - ) - .expect("couldn't add our timestamp"); - } - write!( - new_data, - "pub const SEMVER: &'static str = \"{}\";\n", - semver - .strip_suffix("\r\n") - .or(semver.strip_suffix("\n")) - .unwrap_or(&semver) - ) - .expect("couldn't add our semver"); - - if existing_data != new_data { - let mut vfile = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(version_file) - .expect("Can't open our version file for writing"); - vfile - .write_all(&mut new_data) - .expect("couldn't write new timestamp to version.rs"); - } -} - -fn print_header(out: &mut U) { - let s = r####"// Versioning information is kept in a separate file, attached to a small, well-known server in the Xous System -// This is a trade-off between rebuild times and flexibility. -// This was autogenerated by xtask/src/main.rs:print_header(). Do not edit manually. - -pub(crate) fn get_version() -> crate::api::VersionString { - let mut v = crate::api::VersionString { - version: xous_ipc::String::new() - }; - v.version.append(SEMVER).ok(); - #[cfg(not(feature="no-timestamp"))] - v.version.append("\n").ok(); - #[cfg(not(feature="no-timestamp"))] - v.version.append(TIMESTAMP).ok(); - v -} -"####; - out.write_all(s.as_bytes()) - .expect("couldn't write our version template header"); -} - -////////////////////////// App manifest infrastructure -use serde::{Deserialize, Serialize}; -use std::string::String; -#[derive(Deserialize, Serialize, Debug)] -struct AppManifest { - context_name: String, - menu_name: HashMap>, - submenu: Option::, -} -#[derive(Deserialize, Serialize, Debug)] -struct Locales { - locales: HashMap>, -} - -fn generate_app_menus(apps: &Vec) { - let file = File::open("apps/manifest.json").expect("Failed to open the manifest file"); - let mut reader = std::io::BufReader::new(file); - let mut content = String::new(); - reader - .read_to_string(&mut content) - .expect("Failed to read the file"); - let manifest: HashMap = - serde_json::from_str(&content).expect("Cannot parse manifest file"); - - // localization file - // inject all the localization strings into the i18n file, which in theory reduces the churn on other crates that depend - // on the global i18n file between build variants - let mut l = BTreeMap::>::new(); - for (_app, manifest) in manifest.iter() { - for (name, translations) in &manifest.menu_name { - let mut map = BTreeMap::::new(); - for (language, phrase) in translations { - map.insert(language.to_string(), phrase.to_string()); - } - l.insert(name.to_string(), map); - } - } - // output a JSON localizations file, if things have changed - let new_i18n = serde_json::to_string(&l).unwrap(); - overwrite_if_changed(&new_i18n, "apps/i18n.json"); - - // output the Rust manifests - tailored just for the apps requested - let mut working_set = BTreeMap::::new(); - // derive a working_set that is just the apps we requested - for app in apps { - if let Some(manifest) = manifest.get(app) { - working_set.insert(app.to_string(), &manifest); - } - } - - // construct the gam_tokens - let mut gam_tokens = String::new(); - writeln!( - gam_tokens, - "// This file is auto-generated by xtask/main.rs generate_app_menus()" - ) - .unwrap(); - for (app_name, manifest) in working_set.iter() { - writeln!( - gam_tokens, - "pub const APP_NAME_{}: &'static str = \"{}\";", - app_name.to_uppercase(), - manifest.context_name, - ) - .unwrap(); - if let Some(menu_count) = manifest.submenu { - for i in 0..menu_count { - writeln!( - gam_tokens, - "pub const APP_MENU_{}_{}: &'static str = \"{} Submenu {}\";", - i, - app_name.to_uppercase(), - manifest.context_name, - i, - ) - .unwrap(); - } - } - } - writeln!( - gam_tokens, - "\npub const EXPECTED_APP_CONTEXTS: &[&'static str] = &[" - ) - .unwrap(); - for (app_name, manifest) in working_set.iter() { - writeln!(gam_tokens, " APP_NAME_{},", app_name.to_uppercase(),).unwrap(); - if let Some(menu_count) = manifest.submenu { - for i in 0..menu_count { - writeln!( - gam_tokens, - " APP_MENU_{}_{},", - i, - app_name.to_uppercase(), - ) - .unwrap(); - } - } - } - writeln!(gam_tokens, "];").unwrap(); - overwrite_if_changed(&gam_tokens, "services/gam/src/apps.rs"); - - // construct the app menu - let mut menu = String::new(); - writeln!( - menu, - "// This file is auto-generated by xtask/main.rs generate_app_menus()" - ) - .unwrap(); - if apps.len() == 0 { - writeln!(menu, "// NO APPS SELECTED: suppressing warning messages!").unwrap(); - writeln!(menu, "#![allow(dead_code)]").unwrap(); - writeln!(menu, "#![allow(unused_imports)]").unwrap(); - writeln!(menu, "#![allow(unused_variables)]").unwrap(); - } - writeln!(menu, r####"use crate::StatusOpcode; -use gam::{{MenuItem, MenuPayload}}; -use locales::t; -use num_traits::*; -use std::{{error::Error, fmt}}; - -#[derive(Debug)] -pub enum AppDispatchError {{ - IndexNotFound(usize), -}} - -impl Error for AppDispatchError {{}} - -impl fmt::Display for AppDispatchError {{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{ - match self {{ - AppDispatchError::IndexNotFound(app_index) => write!(f, "Index {{}} not found", app_index), - }} - }} -}} - -pub(crate) fn app_dispatch(gam: &gam::Gam, token: [u32; 4], index: usize) -> Result<(), AppDispatchError> {{ - match index {{"####).unwrap(); - for (index, (app_name, _manifest)) in working_set.iter().enumerate() { - writeln!( - menu, - " {} => {{ - gam.switch_to_app(gam::APP_NAME_{}, token).expect(\"couldn't raise app\"); - Ok(()) - }},", - index, - app_name.to_uppercase() - ) - .unwrap(); - } - writeln!( - menu, - r####" _ => Err(AppDispatchError::IndexNotFound(index)), - }} -}} - -pub(crate) fn app_index_to_name(index: usize) -> Result<&'static str, AppDispatchError> {{ - match index {{"#### - ) - .unwrap(); - for (index, (_, _manifest)) in working_set.iter().enumerate() { - for name in _manifest.menu_name.keys() { - writeln!( - menu, - " {} => Ok(t!(\"{}\", xous::LANG)),", - index, name, - ) - .unwrap(); - } - } - writeln!( - menu, - r####" _ => Err(AppDispatchError::IndexNotFound(index)), - }} -}} - -pub(crate) fn app_menu_items(menu_items: &mut Vec::, status_conn: u32) {{ -"#### - ) - .unwrap(); - for (index, (_app_name, manifest)) in working_set.iter().enumerate() { - writeln!(menu, " menu_items.push(MenuItem {{",).unwrap(); - assert!( - manifest.menu_name.len() == 1, - "Improper menu name record entry" - ); - for name in manifest.menu_name.keys() { - writeln!( - menu, - " name: xous_ipc::String::from_str(t!(\"{}\", xous::LANG)),", - name - ) - .unwrap(); - } - writeln!(menu, " action_conn: Some(status_conn),",).unwrap(); - writeln!( - menu, - " action_opcode: StatusOpcode::SwitchToApp.to_u32().unwrap(),", - ) - .unwrap(); - writeln!( - menu, - " action_payload: MenuPayload::Scalar([{}, 0, 0, 0]),", - index - ) - .unwrap(); - writeln!(menu, " close_on_select: true,",).unwrap(); - writeln!(menu, " }});\n",).unwrap(); - } - writeln!(menu, "}}").unwrap(); - overwrite_if_changed(&menu, "services/status/src/app_autogen.rs"); -} - -fn overwrite_if_changed(new_string: &String, old_file: &str) { - let original = match OpenOptions::new().read(true).open(old_file) { - Ok(mut ref_file) => { - let mut buf = String::new(); - ref_file - .read_to_string(&mut buf) - .expect("UTF-8 error in previous localization file"); - buf - } - _ => String::new(), - }; - if &original != new_string { - // println!("file change in i18n.json detected:"); - // println!("Old: {}", original); - // println!("New: {}", new_string); - let mut new_file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(old_file) - .expect("Can't open our gam manifest for writing"); - write!(new_file, "{}", new_string).unwrap() - } -} diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs new file mode 100644 index 000000000..69ff09348 --- /dev/null +++ b/xtask/src/utils.rs @@ -0,0 +1,376 @@ +use std::{ + fs::File, + io::{Read, Write}, + path::{Path, PathBuf}, + process::Command, +}; + +use crate::{cargo, project_root, PROGRAM_TARGET}; + +const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases"; + +/// Since we use the same TARGET for all calls to `build()`, +/// cache it inside an atomic boolean. If this is `true` then +/// it means we can assume the check passed already. +static DONE_COMPILER_CHECK: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Ensure we have a compatible compiler toolchain. We use a new Target, +/// and we want to give the user a friendly way of installing the latest +/// Rust toolchain. +pub(crate) fn ensure_compiler( + target: &Option<&str>, + force_install: bool, + remove_existing: bool, +) -> Result<(), String> { + use std::process::Stdio; + if DONE_COMPILER_CHECK.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + + /// Return the sysroot for the given target. If the target does not exist, + /// return None. + fn get_sysroot(target: Option<&str>) -> Result, String> { + let mut args = vec!["--print", "sysroot"]; + if let Some(target) = target { + args.push("--target"); + args.push(target); + } + + let sysroot_cmd = Command::new("rustc") + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .args(&args) + .spawn() + .expect("could not run rustc"); + let sysroot_output = sysroot_cmd.wait_with_output().unwrap(); + let have_toolchain = sysroot_output.status.success(); + + let toolchain_path = String::from_utf8(sysroot_output.stdout) + .map_err(|_| "Unable to find Rust sysroot".to_owned())? + .trim() + .to_owned(); + + // Look for the "RUST_VERSION" file to ensure it's compatible with this version. + if let Some(target) = target { + let mut version_path = PathBuf::from(&toolchain_path); + version_path.push("lib"); + version_path.push("rustlib"); + version_path.push(target); + version_path.push("RUST_VERSION"); + if let Ok(mut vp) = File::open(&version_path) { + let mut version_str = String::new(); + if let Err(_) = vp.read_to_string(&mut version_str) { + return Err("Unable to get version string".to_owned()); + } + + let rustc_version_str = format!("{}", rustc_version::version().unwrap()); + if version_str.trim() != rustc_version_str.trim() { + println!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str.trim(), rustc_version_str.trim()); + // return Err(format!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str, rustc_version_str)); + return Ok(None); + } + } else { + println!("Outdated toolchain installed."); + // return Err("Outdated toolchain installed".to_owned()); + return Ok(None); + } + } + + if have_toolchain { + Ok(Some(toolchain_path)) + } else { + Ok(None) + } + } + + // If the sysroot exists, then we're good. + let target = target.unwrap_or(PROGRAM_TARGET); + if let Some(path) = get_sysroot(Some(target))? { + let mut version_path = PathBuf::from(&path); + version_path.push("lib"); + version_path.push("rustlib"); + version_path.push(PROGRAM_TARGET); + if remove_existing { + println!("Target path exists, removing it"); + std::fs::remove_dir_all(version_path) + .or_else(|e| Err(format!("unable to remove existing toolchain: {}", e)))?; + println!("Also removing target directories for existing toolchain"); + let mut target_main = project_root(); + target_main.push("target"); + target_main.push(PROGRAM_TARGET); + std::fs::remove_dir_all(target_main).ok(); + + let mut target_loader = project_root(); + target_loader.push("loader"); + target_loader.push("target"); + target_loader.push(PROGRAM_TARGET); + std::fs::remove_dir_all(target_loader).ok(); + } else { + DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst); + return Ok(()); + } + } + + // Since no sysroot exists, we must download a new one. + let toolchain_path = + get_sysroot(None)?.ok_or_else(|| "default toolchain not installed".to_owned())?; + // If the terminal is a tty, or if toolchain installation is forced, + // download the latest toolchain. + if !atty::is(atty::Stream::Stdin) && !force_install { + return Err(format!("Toolchain for {} not found", target)); + } + + // Version 1.54 was the last major version that was released. + let ver = rustc_version::version().unwrap(); + if ver.major == 1 && ver.minor < 54 { + return Err("Rust 1.54 or higher is required".into()); + } + + // Ask the user if they want to install the toolchain. + let mut buffer = String::new(); + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + if force_install { + println!("Downloading toolchain"); + } else { + println!( + "Error: Toolchain for {} was not found on this system!", + target + ); + } + loop { + if force_install { + break; + } + + print!("Would you like this program to attempt to download and install it? [Y/n] "); + stdout.flush().unwrap(); + buffer.clear(); + stdin.read_line(&mut buffer).unwrap(); + + let trimmed = buffer.trim(); + + if trimmed == "n" || trimmed == "N" { + return Err(format!("Please install the {} toolchain", target)); + } + + if trimmed == "y" || trimmed == "Y" || trimmed.is_empty() { + break; + } + println!(); + } + + fn get_toolchain_url(major: u64, minor: u64, patch: u64) -> Result { + let j: serde_json::Value = ureq::get(TOOLCHAIN_RELEASE_URL) + .set("Accept", "application/vnd.github.v3+json") + .call() + .map_err(|e| format!("{}", e))? + .into_json() + .map_err(|e| format!("{}", e))?; + // let j: serde_json::Value = serde_json::from_str(CONTENT).expect("Cannot parse manifest file"); + + let releases = j.as_array().unwrap(); + let mut tag_urls = std::collections::BTreeMap::new(); + + let target_prefix = format!("{}.{}.{}", major, minor, patch); + for r in releases { + // println!(">>> Value: {}", r); + + let keys = match r.as_object() { + None => continue, + Some(r) => r, + }; + let release = match keys.get("tag_name") { + None => continue, + Some(s) => match s.as_str() { + None => continue, + Some(s) => s, + }, + }; + if !release.starts_with(&target_prefix) { + continue; + } + + let assets = match keys.get("assets") { + None => continue, + Some(s) => match s.as_array() { + None => continue, + Some(s) => s, + }, + }; + + let first_asset = match assets.get(0) { + None => continue, + Some(s) => s, + }; + + let download_url = match first_asset.get("browser_download_url") { + None => continue, + Some(s) => match s.as_str() { + None => continue, + Some(s) => s, + }, + }; + // println!("Candidate Release: {}", download_url); + tag_urls.insert(release.to_owned(), download_url.to_owned()); + } + + if let Some((_k, v)) = tag_urls.into_iter().last() { + // println!("Found candidate entry: v{} url {}", k, v); + return Ok(v); + } + Err(format!("No toolchains found for Rust {}", target_prefix)) + } + let toolchain_url = get_toolchain_url(ver.major, ver.minor, ver.patch)?; + + println!( + "Attempting to install toolchain for {} into {}", + target, toolchain_path + ); + println!("Downloading toolchain from {}...", toolchain_url); + + print!("Download in progress..."); + stdout.flush().unwrap(); + let mut zip_data = vec![]; + { + let agent = ureq::builder() + // .middleware(CounterMiddleware(shared_state.clone())) + .build(); + + let mut freader = agent + .get(&toolchain_url) + .call() + .map_err(|e| format!("{}", e))? + .into_reader(); + freader + .read_to_end(&mut zip_data) + .map_err(|e| format!("{}", e))?; + // |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| { + // // If either number is infinite, don't print anything and just continue. + // if total_bytes.is_infinite() || bytes_so_far.is_infinite() { + // return true; + // } + + // // Display progress. + // print!( + // "\rDownload progress: {:3.02}% ", + // bytes_so_far / total_bytes * 100.0 + // ); + // stdout.flush().unwrap(); + + // // Return `true` to continue the transfer. + // true + // }, + println!(); + } + println!( + "Download successful. Total data size is {} bytes", + zip_data.len() + ); + + /// Extract the zipfile to the target directory, ensuring that all files + /// contained within are created. + fn extract_zip>( + archive_data: P, + extract_to: P2, + ) -> Result<(), String> { + let mut archive = zip::ZipArchive::new(archive_data) + .map_err(|e| format!("unable to extract zip: {}", e))?; + for i in 0..archive.len() { + let mut entry_in_archive = archive + .by_index(i) + .map_err(|e| format!("unable to locate file index {}: {}", i, e))?; + // println!( + // "Trying to extract file {}", + // entry_in_archive.mangled_name().display() + // ); + + let output_path = extract_to.as_ref().join(entry_in_archive.mangled_name()); + if entry_in_archive.is_dir() { + std::fs::create_dir_all(&output_path).map_err(|e| { + format!( + "unable to create directory {}: {}", + output_path.display(), + e + ) + })?; + } else { + // Create the parent directory if necessary + if let Some(parent) = output_path.parent() { + if !parent.exists() { + std::fs::create_dir_all(&parent).map_err(|e| { + format!( + "unable to create directory {}: {}", + output_path.display(), + e + ) + })?; + } + } + let mut outfile = std::fs::File::create(&output_path).map_err(|e| { + format!("unable to create file {}: {}", output_path.display(), e) + })?; + std::io::copy(&mut entry_in_archive, &mut outfile).map_err(|e| { + format!( + "unable to write extracted file {}: {}", + output_path.display(), + e + ) + })?; + } + } + Ok(()) + } + println!("Extracting toolchain to {}...", toolchain_path); + extract_zip(std::io::Cursor::new(zip_data), &toolchain_path)?; + + println!("Toolchain successfully installed"); + + DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) +} + +/// Regenerate the locales files. This is only done when the command is explicitly run. +pub(crate) fn generate_locales() -> Result<(), std::io::Error> { + let ts = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + filetime::set_file_mtime("locales/src/lib.rs", ts)?; + let mut path = project_root(); + path.push("locales"); + let status = Command::new(cargo()) + .current_dir(path) + .args(&["build", "--package", "locales"]) + .status()?; + if !status.success() { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Couldn't generate the locales", + )); + } + return Ok(()); +} + +/// Import the Wycheproof test vectors +pub(crate) fn whycheproof_import() -> Result<(), crate::DynError> { + let input_file = "tools/wycheproof-import/x25519_test.json"; + let output_file = "services/shellchat/src/cmds/x25519_test.bin"; + let status = Command::new(cargo()) + .current_dir(project_root()) + .args(&[ + "run", + "--package", + "wycheproof-import", + "--", + input_file, + output_file, + ]) + .status()?; + if !status.success() { + return Err("wycheproof-import failed. If any, the output will not be usable.".into()); + } + + println!(); + println!("Wrote wycheproof x25519 testvectors to '{}'.", output_file); + + return Ok(()); +} diff --git a/xtask/src/versioning.rs b/xtask/src/versioning.rs new file mode 100644 index 000000000..488dc5242 --- /dev/null +++ b/xtask/src/versioning.rs @@ -0,0 +1,94 @@ +// Versioning infrastructure +// This generates the SEMVER data that is displayed when quering `xous ver` +// It also generates timestamps, if demanded. + +use std::io::{Write, Read}; +use std::fs::OpenOptions; +use std::process::Command; +use chrono::Local; + +pub(crate) fn generate_version(add_timestamp: bool) { + let output = if cfg!(target_os = "windows") { + Command::new("cmd") + .args(["/C", "git describe --tags"]) + .output() + .expect("failed to execute process") + } else { + Command::new("sh") + .arg("-c") + .arg("git describe --tags") + .output() + .expect("failed to execute process") + }; + let gitver = output.stdout; + let semver = String::from_utf8_lossy(&gitver); + + let version_file = "services/ticktimer-server/src/version.rs"; + + // Read the existing file to see if it needs to be updated. + let mut existing_data = Vec::new(); + if let Ok(mut f) = std::fs::File::open(version_file) { + f.read_to_end(&mut existing_data).ok(); + } + + let mut new_data = Vec::new(); + print_header(&mut new_data); + if add_timestamp { + let now = Local::now(); + write!( + new_data, + "#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"{}\";\n", + now.to_rfc2822() + ) + .expect("couldn't add our timestamp"); + } else { + write!( + new_data, + "#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"unavailable\";\n" + ) + .expect("couldn't add our timestamp"); + } + write!( + new_data, + "pub const SEMVER: &'static str = \"{}\";\n", + semver + .strip_suffix("\r\n") + .or(semver.strip_suffix("\n")) + .unwrap_or(&semver) + ) + .expect("couldn't add our semver"); + + if existing_data != new_data { + let mut vfile = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(version_file) + .expect("Can't open our version file for writing"); + vfile + .write_all(&mut new_data) + .expect("couldn't write new timestamp to version.rs"); + } +} + +fn print_header(out: &mut U) { + let s = r####"// Versioning information is kept in a separate file, attached to a small, well-known server in the Xous System +// This is a trade-off between rebuild times and flexibility. +// This was autogenerated by xtask/src/main.rs:print_header(). Do not edit manually. + +pub(crate) fn get_version() -> crate::api::VersionString { + let mut v = crate::api::VersionString { + version: xous_ipc::String::new() + }; + v.version.append(SEMVER).ok(); + #[cfg(not(feature="no-timestamp"))] + v.version.append("\n").ok(); + #[cfg(not(feature="no-timestamp"))] + v.version.append(TIMESTAMP).ok(); + v +} +"####; + out.write_all(s.as_bytes()) + .expect("couldn't write our version template header"); +}