split xtask into file modules

This commit is contained in:
bunnie 2022-09-29 19:05:24 +08:00
parent 884e19b9de
commit 8af0ef7961
4 changed files with 764 additions and 706 deletions

245
xtask/src/app_manifest.rs Normal file
View file

@ -0,0 +1,245 @@
// This module supports generating the app menus from the JSON manifest in the apps/ directory.
use std::{
fs::{OpenOptions, File},
io::{Read, Write},
string::String,
fmt::Write as StdWrite,
};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
#[derive(Deserialize, Serialize, Debug)]
struct AppManifest {
context_name: String,
menu_name: HashMap<String, HashMap<String, String>>,
submenu: Option::<u8>,
}
#[derive(Deserialize, Serialize, Debug)]
struct Locales {
locales: HashMap<String, HashMap<String, String>>,
}
pub(crate) fn generate_app_menus(apps: &Vec<String>) {
let file = File::open("apps/manifest.json").expect("Failed to open the manifest file");
let mut reader = std::io::BufReader::new(file);
let mut content = String::new();
reader
.read_to_string(&mut content)
.expect("Failed to read the file");
let manifest: HashMap<String, AppManifest> =
serde_json::from_str(&content).expect("Cannot parse manifest file");
// localization file
// inject all the localization strings into the i18n file, which in theory reduces the churn on other crates that depend
// on the global i18n file between build variants
let mut l = BTreeMap::<String, BTreeMap<String, String>>::new();
for (_app, manifest) in manifest.iter() {
for (name, translations) in &manifest.menu_name {
let mut map = BTreeMap::<String, String>::new();
for (language, phrase) in translations {
map.insert(language.to_string(), phrase.to_string());
}
l.insert(name.to_string(), map);
}
}
// output a JSON localizations file, if things have changed
let new_i18n = serde_json::to_string(&l).unwrap();
overwrite_if_changed(&new_i18n, "apps/i18n.json");
// output the Rust manifests - tailored just for the apps requested
let mut working_set = BTreeMap::<String, &AppManifest>::new();
// derive a working_set that is just the apps we requested
for app in apps {
if let Some(manifest) = manifest.get(app) {
working_set.insert(app.to_string(), &manifest);
}
}
// construct the gam_tokens
let mut gam_tokens = String::new();
writeln!(
gam_tokens,
"// This file is auto-generated by xtask/main.rs generate_app_menus()"
)
.unwrap();
for (app_name, manifest) in working_set.iter() {
writeln!(
gam_tokens,
"pub const APP_NAME_{}: &'static str = \"{}\";",
app_name.to_uppercase(),
manifest.context_name,
)
.unwrap();
if let Some(menu_count) = manifest.submenu {
for i in 0..menu_count {
writeln!(
gam_tokens,
"pub const APP_MENU_{}_{}: &'static str = \"{} Submenu {}\";",
i,
app_name.to_uppercase(),
manifest.context_name,
i,
)
.unwrap();
}
}
}
writeln!(
gam_tokens,
"\npub const EXPECTED_APP_CONTEXTS: &[&'static str] = &["
)
.unwrap();
for (app_name, manifest) in working_set.iter() {
writeln!(gam_tokens, " APP_NAME_{},", app_name.to_uppercase(),).unwrap();
if let Some(menu_count) = manifest.submenu {
for i in 0..menu_count {
writeln!(
gam_tokens,
" APP_MENU_{}_{},",
i,
app_name.to_uppercase(),
)
.unwrap();
}
}
}
writeln!(gam_tokens, "];").unwrap();
overwrite_if_changed(&gam_tokens, "services/gam/src/apps.rs");
// construct the app menu
let mut menu = String::new();
writeln!(
menu,
"// This file is auto-generated by xtask/main.rs generate_app_menus()"
)
.unwrap();
if apps.len() == 0 {
writeln!(menu, "// NO APPS SELECTED: suppressing warning messages!").unwrap();
writeln!(menu, "#![allow(dead_code)]").unwrap();
writeln!(menu, "#![allow(unused_imports)]").unwrap();
writeln!(menu, "#![allow(unused_variables)]").unwrap();
}
writeln!(menu, r####"use crate::StatusOpcode;
use gam::{{MenuItem, MenuPayload}};
use locales::t;
use num_traits::*;
use std::{{error::Error, fmt}};
#[derive(Debug)]
pub enum AppDispatchError {{
IndexNotFound(usize),
}}
impl Error for AppDispatchError {{}}
impl fmt::Display for AppDispatchError {{
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{
match self {{
AppDispatchError::IndexNotFound(app_index) => write!(f, "Index {{}} not found", app_index),
}}
}}
}}
pub(crate) fn app_dispatch(gam: &gam::Gam, token: [u32; 4], index: usize) -> Result<(), AppDispatchError> {{
match index {{"####).unwrap();
for (index, (app_name, _manifest)) in working_set.iter().enumerate() {
writeln!(
menu,
" {} => {{
gam.switch_to_app(gam::APP_NAME_{}, token).expect(\"couldn't raise app\");
Ok(())
}},",
index,
app_name.to_uppercase()
)
.unwrap();
}
writeln!(
menu,
r####" _ => Err(AppDispatchError::IndexNotFound(index)),
}}
}}
pub(crate) fn app_index_to_name(index: usize) -> Result<&'static str, AppDispatchError> {{
match index {{"####
)
.unwrap();
for (index, (_, _manifest)) in working_set.iter().enumerate() {
for name in _manifest.menu_name.keys() {
writeln!(
menu,
" {} => Ok(t!(\"{}\", xous::LANG)),",
index, name,
)
.unwrap();
}
}
writeln!(
menu,
r####" _ => Err(AppDispatchError::IndexNotFound(index)),
}}
}}
pub(crate) fn app_menu_items(menu_items: &mut Vec::<MenuItem>, status_conn: u32) {{
"####
)
.unwrap();
for (index, (_app_name, manifest)) in working_set.iter().enumerate() {
writeln!(menu, " menu_items.push(MenuItem {{",).unwrap();
assert!(
manifest.menu_name.len() == 1,
"Improper menu name record entry"
);
for name in manifest.menu_name.keys() {
writeln!(
menu,
" name: xous_ipc::String::from_str(t!(\"{}\", xous::LANG)),",
name
)
.unwrap();
}
writeln!(menu, " action_conn: Some(status_conn),",).unwrap();
writeln!(
menu,
" action_opcode: StatusOpcode::SwitchToApp.to_u32().unwrap(),",
)
.unwrap();
writeln!(
menu,
" action_payload: MenuPayload::Scalar([{}, 0, 0, 0]),",
index
)
.unwrap();
writeln!(menu, " close_on_select: true,",).unwrap();
writeln!(menu, " }});\n",).unwrap();
}
writeln!(menu, "}}").unwrap();
overwrite_if_changed(&menu, "services/status/src/app_autogen.rs");
}
fn overwrite_if_changed(new_string: &String, old_file: &str) {
let original = match OpenOptions::new().read(true).open(old_file) {
Ok(mut ref_file) => {
let mut buf = String::new();
ref_file
.read_to_string(&mut buf)
.expect("UTF-8 error in previous localization file");
buf
}
_ => String::new(),
};
if &original != new_string {
// println!("file change in i18n.json detected:");
// println!("Old: {}", original);
// println!("New: {}", new_string);
let mut new_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(old_file)
.expect("Can't open our gam manifest for writing");
write!(new_file, "{}", new_string).unwrap()
}
}

View file

@ -1,10 +1,13 @@
use chrono::Local;
use std::collections::{BTreeMap, HashMap};
use std::fmt::Write as StdWrite;
mod app_manifest;
use app_manifest::*;
mod versioning;
use versioning::*;
mod utils;
use utils::*;
use std::fs::OpenOptions;
use std::{
env,
fs::File,
io::{Read, Write},
path::{Path, PathBuf, MAIN_SEPARATOR},
process::Command,
@ -22,7 +25,6 @@ type DynError = Box<dyn std::error::Error>;
const PROGRAM_TARGET: &str = "riscv32imac-unknown-xous-elf";
const KERNEL_TARGET: &str = "riscv32imac-unknown-xous-elf";
const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases";
enum MemorySpec {
SvdFile(String),
@ -43,6 +45,18 @@ impl std::fmt::Display for BuildError {
impl std::error::Error for BuildError {}
fn cargo() -> String {
env::var("CARGO").unwrap_or_else(|_| "cargo".to_string())
}
fn project_root() -> PathBuf {
Path::new(&env!("CARGO_MANIFEST_DIR"))
.ancestors()
.nth(1)
.unwrap()
.to_path_buf()
}
// Filter out all arguments, which start with "-"
fn get_packages() -> Vec<String> {
let mut args = env::args();
@ -992,329 +1006,38 @@ fn build_kernel(debug: bool, features: Option<&[&str]>) -> Result<PathBuf, DynEr
let mut path = build(&["xous-kernel"], debug, Some(KERNEL_TARGET), None, None, features)?;
path.push("xous-kernel");
Ok(path)
/*
// cargo install --target riscv32imac-unknown-xous-elf --target-dir target xous-kernel --version 0.9.0 --features utralib/precursor-c809403 --root .\target\riscv32imac-unknown-xous-elf\release\
// cargo install --list --root .\target\riscv32imac-unknown-xous-elf\release\
// drop the --target spec and you'll get hosted mode binaries.
let stream = if debug { "debug" } else { "release" };
let mut dir = project_root();
let mut args = vec!["install"];
let target_path = format!("target/{}/{}/", KERNEL_TARGET, stream);
args.push("--root");
args.push(&target_path);
args.push("--target");
args.push(KERNEL_TARGET);
args.push("xous-kernel");
args.push("--version");
args.push("0.9.0");
args.push("--features");
args.push("utralib/precursor-c809403");
let status = Command::new(cargo())
.current_dir(dir)
.args(&args)
.status()?;
if !status.success() {
return Err("cargo build failed".into());
}
Ok(project_root().join(&target_path).join("bin/xous-kernel"))
//Ok(PathBuf::from_str("C:\\Users\\bunnie\\.cargo\\bin\\xous-kernel").unwrap())
*/
}
/// Since we use the same TARGET for all calls to `build()`,
/// cache it inside an atomic boolean. If this is `true` then
/// it means we can assume the check passed already.
static DONE_COMPILER_CHECK: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
/// Ensure we have a compatible compiler toolchain. We use a new Target,
/// and we want to give the user a friendly way of installing the latest
/// Rust toolchain.
fn ensure_compiler(
target: &Option<&str>,
force_install: bool,
remove_existing: bool,
) -> Result<(), String> {
use std::process::Stdio;
if DONE_COMPILER_CHECK.load(std::sync::atomic::Ordering::SeqCst) {
return Ok(());
}
/// Return the sysroot for the given target. If the target does not exist,
/// return None.
fn get_sysroot(target: Option<&str>) -> Result<Option<String>, String> {
let mut args = vec!["--print", "sysroot"];
if let Some(target) = target {
args.push("--target");
args.push(target);
}
let sysroot_cmd = Command::new("rustc")
.stderr(Stdio::null())
.stdout(Stdio::piped())
.args(&args)
.spawn()
.expect("could not run rustc");
let sysroot_output = sysroot_cmd.wait_with_output().unwrap();
let have_toolchain = sysroot_output.status.success();
let toolchain_path = String::from_utf8(sysroot_output.stdout)
.map_err(|_| "Unable to find Rust sysroot".to_owned())?
.trim()
.to_owned();
// Look for the "RUST_VERSION" file to ensure it's compatible with this version.
if let Some(target) = target {
let mut version_path = PathBuf::from(&toolchain_path);
version_path.push("lib");
version_path.push("rustlib");
version_path.push(target);
version_path.push("RUST_VERSION");
if let Ok(mut vp) = File::open(&version_path) {
let mut version_str = String::new();
if let Err(_) = vp.read_to_string(&mut version_str) {
return Err("Unable to get version string".to_owned());
}
let rustc_version_str = format!("{}", rustc_version::version().unwrap());
if version_str.trim() != rustc_version_str.trim() {
println!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str.trim(), rustc_version_str.trim());
// return Err(format!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str, rustc_version_str));
return Ok(None);
}
} else {
println!("Outdated toolchain installed.");
// return Err("Outdated toolchain installed".to_owned());
return Ok(None);
}
}
if have_toolchain {
Ok(Some(toolchain_path))
} else {
Ok(None)
}
}
// If the sysroot exists, then we're good.
let target = target.unwrap_or(PROGRAM_TARGET);
if let Some(path) = get_sysroot(Some(target))? {
let mut version_path = PathBuf::from(&path);
version_path.push("lib");
version_path.push("rustlib");
version_path.push(PROGRAM_TARGET);
if remove_existing {
println!("Target path exists, removing it");
std::fs::remove_dir_all(version_path)
.or_else(|e| Err(format!("unable to remove existing toolchain: {}", e)))?;
println!("Also removing target directories for existing toolchain");
let mut target_main = project_root();
target_main.push("target");
target_main.push(PROGRAM_TARGET);
std::fs::remove_dir_all(target_main).ok();
let mut target_loader = project_root();
target_loader.push("loader");
target_loader.push("target");
target_loader.push(PROGRAM_TARGET);
std::fs::remove_dir_all(target_loader).ok();
} else {
DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst);
return Ok(());
}
}
// Since no sysroot exists, we must download a new one.
let toolchain_path =
get_sysroot(None)?.ok_or_else(|| "default toolchain not installed".to_owned())?;
// If the terminal is a tty, or if toolchain installation is forced,
// download the latest toolchain.
if !atty::is(atty::Stream::Stdin) && !force_install {
return Err(format!("Toolchain for {} not found", target));
}
// Version 1.54 was the last major version that was released.
let ver = rustc_version::version().unwrap();
if ver.major == 1 && ver.minor < 54 {
return Err("Rust 1.54 or higher is required".into());
}
// Ask the user if they want to install the toolchain.
let mut buffer = String::new();
let stdin = std::io::stdin();
let mut stdout = std::io::stdout();
if force_install {
println!("Downloading toolchain");
} else {
println!(
"Error: Toolchain for {} was not found on this system!",
target
);
}
loop {
if force_install {
break;
}
print!("Would you like this program to attempt to download and install it? [Y/n] ");
stdout.flush().unwrap();
buffer.clear();
stdin.read_line(&mut buffer).unwrap();
let trimmed = buffer.trim();
if trimmed == "n" || trimmed == "N" {
return Err(format!("Please install the {} toolchain", target));
}
if trimmed == "y" || trimmed == "Y" || trimmed.is_empty() {
break;
}
println!();
}
fn get_toolchain_url(major: u64, minor: u64, patch: u64) -> Result<String, String> {
let j: serde_json::Value = ureq::get(TOOLCHAIN_RELEASE_URL)
.set("Accept", "application/vnd.github.v3+json")
.call()
.map_err(|e| format!("{}", e))?
.into_json()
.map_err(|e| format!("{}", e))?;
// let j: serde_json::Value = serde_json::from_str(CONTENT).expect("Cannot parse manifest file");
let releases = j.as_array().unwrap();
let mut tag_urls = std::collections::BTreeMap::new();
let target_prefix = format!("{}.{}.{}", major, minor, patch);
for r in releases {
// println!(">>> Value: {}", r);
let keys = match r.as_object() {
None => continue,
Some(r) => r,
};
let release = match keys.get("tag_name") {
None => continue,
Some(s) => match s.as_str() {
None => continue,
Some(s) => s,
},
};
if !release.starts_with(&target_prefix) {
continue;
}
let assets = match keys.get("assets") {
None => continue,
Some(s) => match s.as_array() {
None => continue,
Some(s) => s,
},
};
let first_asset = match assets.get(0) {
None => continue,
Some(s) => s,
};
let download_url = match first_asset.get("browser_download_url") {
None => continue,
Some(s) => match s.as_str() {
None => continue,
Some(s) => s,
},
};
// println!("Candidate Release: {}", download_url);
tag_urls.insert(release.to_owned(), download_url.to_owned());
}
if let Some((_k, v)) = tag_urls.into_iter().last() {
// println!("Found candidate entry: v{} url {}", k, v);
return Ok(v);
}
Err(format!("No toolchains found for Rust {}", target_prefix))
}
let toolchain_url = get_toolchain_url(ver.major, ver.minor, ver.patch)?;
println!(
"Attempting to install toolchain for {} into {}",
target, toolchain_path
);
println!("Downloading toolchain from {}...", toolchain_url);
print!("Download in progress...");
stdout.flush().unwrap();
let mut zip_data = vec![];
{
let agent = ureq::builder()
// .middleware(CounterMiddleware(shared_state.clone()))
.build();
let mut freader = agent
.get(&toolchain_url)
.call()
.map_err(|e| format!("{}", e))?
.into_reader();
freader
.read_to_end(&mut zip_data)
.map_err(|e| format!("{}", e))?;
// |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| {
// // If either number is infinite, don't print anything and just continue.
// if total_bytes.is_infinite() || bytes_so_far.is_infinite() {
// return true;
// }
// // Display progress.
// print!(
// "\rDownload progress: {:3.02}% ",
// bytes_so_far / total_bytes * 100.0
// );
// stdout.flush().unwrap();
// // Return `true` to continue the transfer.
// true
// },
println!();
}
println!(
"Download successful. Total data size is {} bytes",
zip_data.len()
);
/// Extract the zipfile to the target directory, ensuring that all files
/// contained within are created.
fn extract_zip<P: std::io::Read + std::io::Seek, P2: AsRef<Path>>(
archive_data: P,
extract_to: P2,
) -> Result<(), String> {
let mut archive = zip::ZipArchive::new(archive_data)
.map_err(|e| format!("unable to extract zip: {}", e))?;
for i in 0..archive.len() {
let mut entry_in_archive = archive
.by_index(i)
.map_err(|e| format!("unable to locate file index {}: {}", i, e))?;
// println!(
// "Trying to extract file {}",
// entry_in_archive.mangled_name().display()
// );
let output_path = extract_to.as_ref().join(entry_in_archive.mangled_name());
if entry_in_archive.is_dir() {
std::fs::create_dir_all(&output_path).map_err(|e| {
format!(
"unable to create directory {}: {}",
output_path.display(),
e
)
})?;
} else {
// Create the parent directory if necessary
if let Some(parent) = output_path.parent() {
if !parent.exists() {
std::fs::create_dir_all(&parent).map_err(|e| {
format!(
"unable to create directory {}: {}",
output_path.display(),
e
)
})?;
}
}
let mut outfile = std::fs::File::create(&output_path).map_err(|e| {
format!("unable to create file {}: {}", output_path.display(), e)
})?;
std::io::copy(&mut entry_in_archive, &mut outfile).map_err(|e| {
format!(
"unable to write extracted file {}: {}",
output_path.display(),
e
)
})?;
}
}
Ok(())
}
println!("Extracting toolchain to {}...", toolchain_path);
extract_zip(std::io::Cursor::new(zip_data), &toolchain_path)?;
println!("Toolchain successfully installed");
DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
fn build(
packages: &[&str],
@ -1425,383 +1148,3 @@ fn create_image(
}
Ok(project_root().join(&format!("target/{}/{}/args.bin", PROGRAM_TARGET, stream)))
}
fn cargo() -> String {
env::var("CARGO").unwrap_or_else(|_| "cargo".to_string())
}
fn project_root() -> PathBuf {
Path::new(&env!("CARGO_MANIFEST_DIR"))
.ancestors()
.nth(1)
.unwrap()
.to_path_buf()
}
/// Regenerate the locales files. This is only done when the command is explicitly run.
fn generate_locales() -> Result<(), std::io::Error> {
let ts = filetime::FileTime::from_system_time(std::time::SystemTime::now());
filetime::set_file_mtime("locales/src/lib.rs", ts)?;
let mut path = project_root();
path.push("locales");
let status = Command::new(cargo())
.current_dir(path)
.args(&["build", "--package", "locales"])
.status()?;
if !status.success() {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Couldn't generate the locales",
));
}
return Ok(());
}
fn whycheproof_import() -> Result<(), DynError> {
let input_file = "tools/wycheproof-import/x25519_test.json";
let output_file = "services/shellchat/src/cmds/x25519_test.bin";
let status = Command::new(cargo())
.current_dir(project_root())
.args(&[
"run",
"--package",
"wycheproof-import",
"--",
input_file,
output_file,
])
.status()?;
if !status.success() {
return Err("wycheproof-import failed. If any, the output will not be usable.".into());
}
println!();
println!("Wrote wycheproof x25519 testvectors to '{}'.", output_file);
return Ok(());
}
////////////////////////// Versioning infrastructure
fn generate_version(add_timestamp: bool) {
let output = if cfg!(target_os = "windows") {
Command::new("cmd")
.args(["/C", "git describe --tags"])
.output()
.expect("failed to execute process")
} else {
Command::new("sh")
.arg("-c")
.arg("git describe --tags")
.output()
.expect("failed to execute process")
};
let gitver = output.stdout;
let semver = String::from_utf8_lossy(&gitver);
let version_file = "services/ticktimer-server/src/version.rs";
// Read the existing file to see if it needs to be updated.
let mut existing_data = Vec::new();
if let Ok(mut f) = std::fs::File::open(version_file) {
f.read_to_end(&mut existing_data).ok();
}
let mut new_data = Vec::new();
print_header(&mut new_data);
if add_timestamp {
let now = Local::now();
write!(
new_data,
"#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"{}\";\n",
now.to_rfc2822()
)
.expect("couldn't add our timestamp");
} else {
write!(
new_data,
"#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"unavailable\";\n"
)
.expect("couldn't add our timestamp");
}
write!(
new_data,
"pub const SEMVER: &'static str = \"{}\";\n",
semver
.strip_suffix("\r\n")
.or(semver.strip_suffix("\n"))
.unwrap_or(&semver)
)
.expect("couldn't add our semver");
if existing_data != new_data {
let mut vfile = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(version_file)
.expect("Can't open our version file for writing");
vfile
.write_all(&mut new_data)
.expect("couldn't write new timestamp to version.rs");
}
}
fn print_header<U: Write>(out: &mut U) {
let s = r####"// Versioning information is kept in a separate file, attached to a small, well-known server in the Xous System
// This is a trade-off between rebuild times and flexibility.
// This was autogenerated by xtask/src/main.rs:print_header(). Do not edit manually.
pub(crate) fn get_version() -> crate::api::VersionString {
let mut v = crate::api::VersionString {
version: xous_ipc::String::new()
};
v.version.append(SEMVER).ok();
#[cfg(not(feature="no-timestamp"))]
v.version.append("\n").ok();
#[cfg(not(feature="no-timestamp"))]
v.version.append(TIMESTAMP).ok();
v
}
"####;
out.write_all(s.as_bytes())
.expect("couldn't write our version template header");
}
////////////////////////// App manifest infrastructure
use serde::{Deserialize, Serialize};
use std::string::String;
#[derive(Deserialize, Serialize, Debug)]
struct AppManifest {
context_name: String,
menu_name: HashMap<String, HashMap<String, String>>,
submenu: Option::<u8>,
}
#[derive(Deserialize, Serialize, Debug)]
struct Locales {
locales: HashMap<String, HashMap<String, String>>,
}
fn generate_app_menus(apps: &Vec<String>) {
let file = File::open("apps/manifest.json").expect("Failed to open the manifest file");
let mut reader = std::io::BufReader::new(file);
let mut content = String::new();
reader
.read_to_string(&mut content)
.expect("Failed to read the file");
let manifest: HashMap<String, AppManifest> =
serde_json::from_str(&content).expect("Cannot parse manifest file");
// localization file
// inject all the localization strings into the i18n file, which in theory reduces the churn on other crates that depend
// on the global i18n file between build variants
let mut l = BTreeMap::<String, BTreeMap<String, String>>::new();
for (_app, manifest) in manifest.iter() {
for (name, translations) in &manifest.menu_name {
let mut map = BTreeMap::<String, String>::new();
for (language, phrase) in translations {
map.insert(language.to_string(), phrase.to_string());
}
l.insert(name.to_string(), map);
}
}
// output a JSON localizations file, if things have changed
let new_i18n = serde_json::to_string(&l).unwrap();
overwrite_if_changed(&new_i18n, "apps/i18n.json");
// output the Rust manifests - tailored just for the apps requested
let mut working_set = BTreeMap::<String, &AppManifest>::new();
// derive a working_set that is just the apps we requested
for app in apps {
if let Some(manifest) = manifest.get(app) {
working_set.insert(app.to_string(), &manifest);
}
}
// construct the gam_tokens
let mut gam_tokens = String::new();
writeln!(
gam_tokens,
"// This file is auto-generated by xtask/main.rs generate_app_menus()"
)
.unwrap();
for (app_name, manifest) in working_set.iter() {
writeln!(
gam_tokens,
"pub const APP_NAME_{}: &'static str = \"{}\";",
app_name.to_uppercase(),
manifest.context_name,
)
.unwrap();
if let Some(menu_count) = manifest.submenu {
for i in 0..menu_count {
writeln!(
gam_tokens,
"pub const APP_MENU_{}_{}: &'static str = \"{} Submenu {}\";",
i,
app_name.to_uppercase(),
manifest.context_name,
i,
)
.unwrap();
}
}
}
writeln!(
gam_tokens,
"\npub const EXPECTED_APP_CONTEXTS: &[&'static str] = &["
)
.unwrap();
for (app_name, manifest) in working_set.iter() {
writeln!(gam_tokens, " APP_NAME_{},", app_name.to_uppercase(),).unwrap();
if let Some(menu_count) = manifest.submenu {
for i in 0..menu_count {
writeln!(
gam_tokens,
" APP_MENU_{}_{},",
i,
app_name.to_uppercase(),
)
.unwrap();
}
}
}
writeln!(gam_tokens, "];").unwrap();
overwrite_if_changed(&gam_tokens, "services/gam/src/apps.rs");
// construct the app menu
let mut menu = String::new();
writeln!(
menu,
"// This file is auto-generated by xtask/main.rs generate_app_menus()"
)
.unwrap();
if apps.len() == 0 {
writeln!(menu, "// NO APPS SELECTED: suppressing warning messages!").unwrap();
writeln!(menu, "#![allow(dead_code)]").unwrap();
writeln!(menu, "#![allow(unused_imports)]").unwrap();
writeln!(menu, "#![allow(unused_variables)]").unwrap();
}
writeln!(menu, r####"use crate::StatusOpcode;
use gam::{{MenuItem, MenuPayload}};
use locales::t;
use num_traits::*;
use std::{{error::Error, fmt}};
#[derive(Debug)]
pub enum AppDispatchError {{
IndexNotFound(usize),
}}
impl Error for AppDispatchError {{}}
impl fmt::Display for AppDispatchError {{
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{
match self {{
AppDispatchError::IndexNotFound(app_index) => write!(f, "Index {{}} not found", app_index),
}}
}}
}}
pub(crate) fn app_dispatch(gam: &gam::Gam, token: [u32; 4], index: usize) -> Result<(), AppDispatchError> {{
match index {{"####).unwrap();
for (index, (app_name, _manifest)) in working_set.iter().enumerate() {
writeln!(
menu,
" {} => {{
gam.switch_to_app(gam::APP_NAME_{}, token).expect(\"couldn't raise app\");
Ok(())
}},",
index,
app_name.to_uppercase()
)
.unwrap();
}
writeln!(
menu,
r####" _ => Err(AppDispatchError::IndexNotFound(index)),
}}
}}
pub(crate) fn app_index_to_name(index: usize) -> Result<&'static str, AppDispatchError> {{
match index {{"####
)
.unwrap();
for (index, (_, _manifest)) in working_set.iter().enumerate() {
for name in _manifest.menu_name.keys() {
writeln!(
menu,
" {} => Ok(t!(\"{}\", xous::LANG)),",
index, name,
)
.unwrap();
}
}
writeln!(
menu,
r####" _ => Err(AppDispatchError::IndexNotFound(index)),
}}
}}
pub(crate) fn app_menu_items(menu_items: &mut Vec::<MenuItem>, status_conn: u32) {{
"####
)
.unwrap();
for (index, (_app_name, manifest)) in working_set.iter().enumerate() {
writeln!(menu, " menu_items.push(MenuItem {{",).unwrap();
assert!(
manifest.menu_name.len() == 1,
"Improper menu name record entry"
);
for name in manifest.menu_name.keys() {
writeln!(
menu,
" name: xous_ipc::String::from_str(t!(\"{}\", xous::LANG)),",
name
)
.unwrap();
}
writeln!(menu, " action_conn: Some(status_conn),",).unwrap();
writeln!(
menu,
" action_opcode: StatusOpcode::SwitchToApp.to_u32().unwrap(),",
)
.unwrap();
writeln!(
menu,
" action_payload: MenuPayload::Scalar([{}, 0, 0, 0]),",
index
)
.unwrap();
writeln!(menu, " close_on_select: true,",).unwrap();
writeln!(menu, " }});\n",).unwrap();
}
writeln!(menu, "}}").unwrap();
overwrite_if_changed(&menu, "services/status/src/app_autogen.rs");
}
fn overwrite_if_changed(new_string: &String, old_file: &str) {
let original = match OpenOptions::new().read(true).open(old_file) {
Ok(mut ref_file) => {
let mut buf = String::new();
ref_file
.read_to_string(&mut buf)
.expect("UTF-8 error in previous localization file");
buf
}
_ => String::new(),
};
if &original != new_string {
// println!("file change in i18n.json detected:");
// println!("Old: {}", original);
// println!("New: {}", new_string);
let mut new_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(old_file)
.expect("Can't open our gam manifest for writing");
write!(new_file, "{}", new_string).unwrap()
}
}

376
xtask/src/utils.rs Normal file
View file

@ -0,0 +1,376 @@
use std::{
fs::File,
io::{Read, Write},
path::{Path, PathBuf},
process::Command,
};
use crate::{cargo, project_root, PROGRAM_TARGET};
const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases";
/// Since we use the same TARGET for all calls to `build()`,
/// cache it inside an atomic boolean. If this is `true` then
/// it means we can assume the check passed already.
static DONE_COMPILER_CHECK: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
/// Ensure we have a compatible compiler toolchain. We use a new Target,
/// and we want to give the user a friendly way of installing the latest
/// Rust toolchain.
pub(crate) fn ensure_compiler(
target: &Option<&str>,
force_install: bool,
remove_existing: bool,
) -> Result<(), String> {
use std::process::Stdio;
if DONE_COMPILER_CHECK.load(std::sync::atomic::Ordering::SeqCst) {
return Ok(());
}
/// Return the sysroot for the given target. If the target does not exist,
/// return None.
fn get_sysroot(target: Option<&str>) -> Result<Option<String>, String> {
let mut args = vec!["--print", "sysroot"];
if let Some(target) = target {
args.push("--target");
args.push(target);
}
let sysroot_cmd = Command::new("rustc")
.stderr(Stdio::null())
.stdout(Stdio::piped())
.args(&args)
.spawn()
.expect("could not run rustc");
let sysroot_output = sysroot_cmd.wait_with_output().unwrap();
let have_toolchain = sysroot_output.status.success();
let toolchain_path = String::from_utf8(sysroot_output.stdout)
.map_err(|_| "Unable to find Rust sysroot".to_owned())?
.trim()
.to_owned();
// Look for the "RUST_VERSION" file to ensure it's compatible with this version.
if let Some(target) = target {
let mut version_path = PathBuf::from(&toolchain_path);
version_path.push("lib");
version_path.push("rustlib");
version_path.push(target);
version_path.push("RUST_VERSION");
if let Ok(mut vp) = File::open(&version_path) {
let mut version_str = String::new();
if let Err(_) = vp.read_to_string(&mut version_str) {
return Err("Unable to get version string".to_owned());
}
let rustc_version_str = format!("{}", rustc_version::version().unwrap());
if version_str.trim() != rustc_version_str.trim() {
println!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str.trim(), rustc_version_str.trim());
// return Err(format!("Version upgrade. Compiler is version {}, the installed toolchain is for {}", version_str, rustc_version_str));
return Ok(None);
}
} else {
println!("Outdated toolchain installed.");
// return Err("Outdated toolchain installed".to_owned());
return Ok(None);
}
}
if have_toolchain {
Ok(Some(toolchain_path))
} else {
Ok(None)
}
}
// If the sysroot exists, then we're good.
let target = target.unwrap_or(PROGRAM_TARGET);
if let Some(path) = get_sysroot(Some(target))? {
let mut version_path = PathBuf::from(&path);
version_path.push("lib");
version_path.push("rustlib");
version_path.push(PROGRAM_TARGET);
if remove_existing {
println!("Target path exists, removing it");
std::fs::remove_dir_all(version_path)
.or_else(|e| Err(format!("unable to remove existing toolchain: {}", e)))?;
println!("Also removing target directories for existing toolchain");
let mut target_main = project_root();
target_main.push("target");
target_main.push(PROGRAM_TARGET);
std::fs::remove_dir_all(target_main).ok();
let mut target_loader = project_root();
target_loader.push("loader");
target_loader.push("target");
target_loader.push(PROGRAM_TARGET);
std::fs::remove_dir_all(target_loader).ok();
} else {
DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst);
return Ok(());
}
}
// Since no sysroot exists, we must download a new one.
let toolchain_path =
get_sysroot(None)?.ok_or_else(|| "default toolchain not installed".to_owned())?;
// If the terminal is a tty, or if toolchain installation is forced,
// download the latest toolchain.
if !atty::is(atty::Stream::Stdin) && !force_install {
return Err(format!("Toolchain for {} not found", target));
}
// Version 1.54 was the last major version that was released.
let ver = rustc_version::version().unwrap();
if ver.major == 1 && ver.minor < 54 {
return Err("Rust 1.54 or higher is required".into());
}
// Ask the user if they want to install the toolchain.
let mut buffer = String::new();
let stdin = std::io::stdin();
let mut stdout = std::io::stdout();
if force_install {
println!("Downloading toolchain");
} else {
println!(
"Error: Toolchain for {} was not found on this system!",
target
);
}
loop {
if force_install {
break;
}
print!("Would you like this program to attempt to download and install it? [Y/n] ");
stdout.flush().unwrap();
buffer.clear();
stdin.read_line(&mut buffer).unwrap();
let trimmed = buffer.trim();
if trimmed == "n" || trimmed == "N" {
return Err(format!("Please install the {} toolchain", target));
}
if trimmed == "y" || trimmed == "Y" || trimmed.is_empty() {
break;
}
println!();
}
fn get_toolchain_url(major: u64, minor: u64, patch: u64) -> Result<String, String> {
let j: serde_json::Value = ureq::get(TOOLCHAIN_RELEASE_URL)
.set("Accept", "application/vnd.github.v3+json")
.call()
.map_err(|e| format!("{}", e))?
.into_json()
.map_err(|e| format!("{}", e))?;
// let j: serde_json::Value = serde_json::from_str(CONTENT).expect("Cannot parse manifest file");
let releases = j.as_array().unwrap();
let mut tag_urls = std::collections::BTreeMap::new();
let target_prefix = format!("{}.{}.{}", major, minor, patch);
for r in releases {
// println!(">>> Value: {}", r);
let keys = match r.as_object() {
None => continue,
Some(r) => r,
};
let release = match keys.get("tag_name") {
None => continue,
Some(s) => match s.as_str() {
None => continue,
Some(s) => s,
},
};
if !release.starts_with(&target_prefix) {
continue;
}
let assets = match keys.get("assets") {
None => continue,
Some(s) => match s.as_array() {
None => continue,
Some(s) => s,
},
};
let first_asset = match assets.get(0) {
None => continue,
Some(s) => s,
};
let download_url = match first_asset.get("browser_download_url") {
None => continue,
Some(s) => match s.as_str() {
None => continue,
Some(s) => s,
},
};
// println!("Candidate Release: {}", download_url);
tag_urls.insert(release.to_owned(), download_url.to_owned());
}
if let Some((_k, v)) = tag_urls.into_iter().last() {
// println!("Found candidate entry: v{} url {}", k, v);
return Ok(v);
}
Err(format!("No toolchains found for Rust {}", target_prefix))
}
let toolchain_url = get_toolchain_url(ver.major, ver.minor, ver.patch)?;
println!(
"Attempting to install toolchain for {} into {}",
target, toolchain_path
);
println!("Downloading toolchain from {}...", toolchain_url);
print!("Download in progress...");
stdout.flush().unwrap();
let mut zip_data = vec![];
{
let agent = ureq::builder()
// .middleware(CounterMiddleware(shared_state.clone()))
.build();
let mut freader = agent
.get(&toolchain_url)
.call()
.map_err(|e| format!("{}", e))?
.into_reader();
freader
.read_to_end(&mut zip_data)
.map_err(|e| format!("{}", e))?;
// |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| {
// // If either number is infinite, don't print anything and just continue.
// if total_bytes.is_infinite() || bytes_so_far.is_infinite() {
// return true;
// }
// // Display progress.
// print!(
// "\rDownload progress: {:3.02}% ",
// bytes_so_far / total_bytes * 100.0
// );
// stdout.flush().unwrap();
// // Return `true` to continue the transfer.
// true
// },
println!();
}
println!(
"Download successful. Total data size is {} bytes",
zip_data.len()
);
/// Extract the zipfile to the target directory, ensuring that all files
/// contained within are created.
fn extract_zip<P: std::io::Read + std::io::Seek, P2: AsRef<Path>>(
archive_data: P,
extract_to: P2,
) -> Result<(), String> {
let mut archive = zip::ZipArchive::new(archive_data)
.map_err(|e| format!("unable to extract zip: {}", e))?;
for i in 0..archive.len() {
let mut entry_in_archive = archive
.by_index(i)
.map_err(|e| format!("unable to locate file index {}: {}", i, e))?;
// println!(
// "Trying to extract file {}",
// entry_in_archive.mangled_name().display()
// );
let output_path = extract_to.as_ref().join(entry_in_archive.mangled_name());
if entry_in_archive.is_dir() {
std::fs::create_dir_all(&output_path).map_err(|e| {
format!(
"unable to create directory {}: {}",
output_path.display(),
e
)
})?;
} else {
// Create the parent directory if necessary
if let Some(parent) = output_path.parent() {
if !parent.exists() {
std::fs::create_dir_all(&parent).map_err(|e| {
format!(
"unable to create directory {}: {}",
output_path.display(),
e
)
})?;
}
}
let mut outfile = std::fs::File::create(&output_path).map_err(|e| {
format!("unable to create file {}: {}", output_path.display(), e)
})?;
std::io::copy(&mut entry_in_archive, &mut outfile).map_err(|e| {
format!(
"unable to write extracted file {}: {}",
output_path.display(),
e
)
})?;
}
}
Ok(())
}
println!("Extracting toolchain to {}...", toolchain_path);
extract_zip(std::io::Cursor::new(zip_data), &toolchain_path)?;
println!("Toolchain successfully installed");
DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
/// Regenerate the locales files. This is only done when the command is explicitly run.
pub(crate) fn generate_locales() -> Result<(), std::io::Error> {
let ts = filetime::FileTime::from_system_time(std::time::SystemTime::now());
filetime::set_file_mtime("locales/src/lib.rs", ts)?;
let mut path = project_root();
path.push("locales");
let status = Command::new(cargo())
.current_dir(path)
.args(&["build", "--package", "locales"])
.status()?;
if !status.success() {
return Err(std::io::Error::new(
std::io::ErrorKind::Other,
"Couldn't generate the locales",
));
}
return Ok(());
}
/// Import the Wycheproof test vectors
pub(crate) fn whycheproof_import() -> Result<(), crate::DynError> {
let input_file = "tools/wycheproof-import/x25519_test.json";
let output_file = "services/shellchat/src/cmds/x25519_test.bin";
let status = Command::new(cargo())
.current_dir(project_root())
.args(&[
"run",
"--package",
"wycheproof-import",
"--",
input_file,
output_file,
])
.status()?;
if !status.success() {
return Err("wycheproof-import failed. If any, the output will not be usable.".into());
}
println!();
println!("Wrote wycheproof x25519 testvectors to '{}'.", output_file);
return Ok(());
}

94
xtask/src/versioning.rs Normal file
View file

@ -0,0 +1,94 @@
// Versioning infrastructure
// This generates the SEMVER data that is displayed when quering `xous ver`
// It also generates timestamps, if demanded.
use std::io::{Write, Read};
use std::fs::OpenOptions;
use std::process::Command;
use chrono::Local;
pub(crate) fn generate_version(add_timestamp: bool) {
let output = if cfg!(target_os = "windows") {
Command::new("cmd")
.args(["/C", "git describe --tags"])
.output()
.expect("failed to execute process")
} else {
Command::new("sh")
.arg("-c")
.arg("git describe --tags")
.output()
.expect("failed to execute process")
};
let gitver = output.stdout;
let semver = String::from_utf8_lossy(&gitver);
let version_file = "services/ticktimer-server/src/version.rs";
// Read the existing file to see if it needs to be updated.
let mut existing_data = Vec::new();
if let Ok(mut f) = std::fs::File::open(version_file) {
f.read_to_end(&mut existing_data).ok();
}
let mut new_data = Vec::new();
print_header(&mut new_data);
if add_timestamp {
let now = Local::now();
write!(
new_data,
"#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"{}\";\n",
now.to_rfc2822()
)
.expect("couldn't add our timestamp");
} else {
write!(
new_data,
"#[allow(dead_code)]\npub const TIMESTAMP: &'static str = \"unavailable\";\n"
)
.expect("couldn't add our timestamp");
}
write!(
new_data,
"pub const SEMVER: &'static str = \"{}\";\n",
semver
.strip_suffix("\r\n")
.or(semver.strip_suffix("\n"))
.unwrap_or(&semver)
)
.expect("couldn't add our semver");
if existing_data != new_data {
let mut vfile = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(version_file)
.expect("Can't open our version file for writing");
vfile
.write_all(&mut new_data)
.expect("couldn't write new timestamp to version.rs");
}
}
fn print_header<U: Write>(out: &mut U) {
let s = r####"// Versioning information is kept in a separate file, attached to a small, well-known server in the Xous System
// This is a trade-off between rebuild times and flexibility.
// This was autogenerated by xtask/src/main.rs:print_header(). Do not edit manually.
pub(crate) fn get_version() -> crate::api::VersionString {
let mut v = crate::api::VersionString {
version: xous_ipc::String::new()
};
v.version.append(SEMVER).ok();
#[cfg(not(feature="no-timestamp"))]
v.version.append("\n").ok();
#[cfg(not(feature="no-timestamp"))]
v.version.append(TIMESTAMP).ok();
v
}
"####;
out.write_all(s.as_bytes())
.expect("couldn't write our version template header");
}