From 9889856eded23194059c8e5da08c98e44a58ec5e Mon Sep 17 00:00:00 2001 From: bunnie Date: Tue, 7 Feb 2023 21:10:23 +0800 Subject: [PATCH] page-align processes within the kernel image --- tools/src/bin/create-image.rs | 29 ++++++- tools/src/tags/inie.rs | 3 + tools/src/tags/inif.rs | 85 ++++++++++++++++++++ tools/src/tags/mod.rs | 21 +++++ tools/src/tags/xkrn.rs | 4 + tools/src/xous_arguments.rs | 12 ++- xtask/src/builder.rs | 145 +++++++++++++++++++++++++++------- xtask/src/main.rs | 14 ++-- xtask/src/verifier.rs | 2 +- 9 files changed, 273 insertions(+), 42 deletions(-) create mode 100644 tools/src/tags/inif.rs diff --git a/tools/src/bin/create-image.rs b/tools/src/bin/create-image.rs index 0714d81d0..f1f4667fd 100644 --- a/tools/src/bin/create-image.rs +++ b/tools/src/bin/create-image.rs @@ -9,6 +9,7 @@ use std::fs::File; use tools::elf::{read_minielf, read_program}; use tools::tags::bflg::Bflg; use tools::tags::inie::IniE; +use tools::tags::inif::IniF; use tools::tags::memory::{MemoryRegion, MemoryRegions}; use tools::tags::pnam::ProcessNames; use tools::tags::xkrn::XousKernel; @@ -87,6 +88,15 @@ fn main() { .number_of_values(1) .help("Initial program to load"), ) + .arg( + Arg::with_name("inif") + .short("f") + .long("inif") + .takes_value(true) + .multiple(true) + .number_of_values(1) + .help("Initial program to load from FLASH"), + ) .arg( Arg::with_name("csv") .short("c") @@ -235,8 +245,8 @@ fn main() { .expect("unable to read kernel"); process_names.set(1, "kernel"); + let mut pid = 2; if let Some(init_paths) = matches.values_of("init") { - let mut pid = 2; for init_path in init_paths { let program_name = std::path::Path::new(init_path); process_names.set( @@ -253,6 +263,23 @@ fn main() { } } + if let Some(init_paths) = matches.values_of("inif") { + for init_path in init_paths { + let program_name = std::path::Path::new(init_path); + process_names.set( + pid, + program_name + .file_stem() + .expect("program had no name") + .to_str() + .expect("program name is not valid utf-8"), + ); + pid += 1; + let init = read_minielf(init_path).expect("couldn't parse init file"); + args.add(IniF::new(init.entry_point, init.sections, init.program)); + } + } + let xkrn = XousKernel::new( kernel.text_offset, kernel.text_size, diff --git a/tools/src/tags/inie.rs b/tools/src/tags/inie.rs index e99605ff9..521ce2218 100644 --- a/tools/src/tags/inie.rs +++ b/tools/src/tags/inie.rs @@ -60,6 +60,9 @@ impl XousArgument for IniE { fn finalize(&mut self, offset: usize) -> usize { self.load_offset = offset as u32; + + assert!(offset % crate::tags::PAGE_SIZE == 0, "IniE load offset is not aligned"); + self.data = crate::tags::align_data_up(&self.data); self.data.len() } diff --git a/tools/src/tags/inif.rs b/tools/src/tags/inif.rs new file mode 100644 index 000000000..d10c79ace --- /dev/null +++ b/tools/src/tags/inif.rs @@ -0,0 +1,85 @@ +use crate::elf::MiniElfSection; +use crate::xous_arguments::{XousArgument, XousArgumentCode, XousSize}; +use std::fmt; +use std::io; + +#[derive(Debug)] +pub struct IniF { + /// Address in flash + load_offset: u32, + + /// Virtual address entry point + entrypoint: u32, + + /// Array of minielf sections + sections: Vec, + + /// Actual program data + data: Vec, +} + +impl fmt::Display for IniF { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + " IniF: entrypoint @ {:08x}, loaded from {:08x}. Sections:", + self.entrypoint, self.load_offset + )?; + let mut load_offset = self.load_offset; + for section in &self.sections { + writeln!(f, " Loaded from {:08x} - {}", load_offset, section)?; + load_offset += section.size as u32; + } + Ok(()) + } +} + +impl IniF { + pub fn new(entrypoint: u32, sections: Vec, mut data: Vec) -> IniF { + // pad the data to 4 bytes + while data.len() & 3 != 0 { + data.push(0); + } + IniF { + load_offset: 0, + entrypoint, + sections, + data, + } + } +} + +impl XousArgument for IniF { + fn code(&self) -> XousArgumentCode { + u32::from_le_bytes(*b"IniF") + } + + fn length(&self) -> XousSize { + 4 + 4 + (self.sections.len() * 8) as XousSize + } + + fn finalize(&mut self, offset: usize) -> usize { + self.load_offset = offset as u32; + println!("{:x}", self.load_offset); + assert!(offset % crate::tags::PAGE_SIZE == 0, "IniF load offset is not aligned"); + self.data = crate::tags::align_data_up(&self.data); + crate::tags::align_size_up(self.data.len()) + } + + fn last_data(&self) -> &[u8] { + &self.data + } + + fn serialize(&self, output: &mut dyn io::Write) -> io::Result { + let mut written = 0; + written += output.write(&self.load_offset.to_le_bytes())?; + written += output.write(&self.entrypoint.to_le_bytes())?; + for section in &self.sections { + written += output.write(§ion.virt.to_le_bytes())?; + let mut word2 = section.size.to_le_bytes(); + word2[3] = section.flags.bits(); + written += output.write(&word2)?; + } + Ok(written) + } +} diff --git a/tools/src/tags/mod.rs b/tools/src/tags/mod.rs index 34b0fcb19..3028f8e3e 100644 --- a/tools/src/tags/mod.rs +++ b/tools/src/tags/mod.rs @@ -3,3 +3,24 @@ pub mod inie; pub mod memory; pub mod pnam; pub mod xkrn; +pub mod inif; + +pub (crate) const PAGE_SIZE: usize = 4096; + +pub fn align_size_up(offset: usize) -> usize { + if offset % PAGE_SIZE == 0 { + offset + } else { + (offset + PAGE_SIZE) & !(PAGE_SIZE - 1) + } +} + +pub fn align_data_up(data: &Vec) -> Vec:: { + if data.len() % PAGE_SIZE == 0 { + data.to_vec() + } else { + let padding = align_size_up(data.len()) - data.len(); + let pad = vec![0u8; padding]; + (&[&data[..], &pad[..]]).concat().to_vec() + } +} \ No newline at end of file diff --git a/tools/src/tags/xkrn.rs b/tools/src/tags/xkrn.rs index 6d5c8ffeb..1f4a550df 100644 --- a/tools/src/tags/xkrn.rs +++ b/tools/src/tags/xkrn.rs @@ -77,6 +77,10 @@ impl XousArgument for XousKernel { self.load_offset = offset as u32; assert!(self.text_offset > 0xff00_0000, "kernel text section is invalid: 0x{:08x} < 0xff000000 -- was it linked with a linker script?", self.text_offset); + + assert!(offset % crate::tags::PAGE_SIZE == 0, "XKrn load offset is not aligned"); + self.program = crate::tags::align_data_up(&self.program); + self.program.len() } diff --git a/tools/src/xous_arguments.rs b/tools/src/xous_arguments.rs index 807bd9941..72be3382d 100644 --- a/tools/src/xous_arguments.rs +++ b/tools/src/xous_arguments.rs @@ -1,5 +1,5 @@ use std::fmt; -use std::io::{Cursor, Result, Write}; +use std::io::{Cursor, Result, Write, Seek}; pub type XousArgumentCode = u32; pub type XousSize = u32; use crc::{crc16, Hasher16}; @@ -81,7 +81,7 @@ impl XousArguments { } pub fn finalize(&mut self) { - let mut running_offset = self.len() as usize; + let mut running_offset = crate::tags::align_size_up(self.len() as usize); for arg in &mut self.arguments { running_offset += arg.finalize(running_offset); } @@ -96,7 +96,7 @@ impl XousArguments { pub fn write(&mut self, mut w: T) -> Result<()> where - T: Write, + T: Write + Seek, { let total_length = self.len(); @@ -156,6 +156,12 @@ impl XousArguments { w.write_all(tag_data.get_ref())?; } + // pad to the nearest page + let pos = w.stream_position()?; + let pad_len = crate::tags::align_size_up(pos as usize) - pos as usize; + let pad = vec![0u8; pad_len]; + w.write_all(&pad)?; + // Write any pending data, such as payloads for arg in &self.arguments { w.write_all(arg.last_data()) diff --git a/xtask/src/builder.rs b/xtask/src/builder.rs index 7f29122d3..da9a87068 100644 --- a/xtask/src/builder.rs +++ b/xtask/src/builder.rs @@ -22,25 +22,55 @@ impl BuildStream { } } +#[derive(Debug)] pub enum CrateSpec { /// name of the crate - Local(String), + Local(String, bool), /// crates.io: (name of crate, version) - CratesIo(String, String), + CratesIo(String, String, bool), /// a prebuilt package: (name of executable, URL for download) - Prebuilt(String, String), + Prebuilt(String, String, bool), /// a prebuilt binary, done using command line tools - BinaryFile(String), + BinaryFile(String, bool), /// an empty entry None, } +impl CrateSpec { + pub fn is_xip(&self) -> bool { + match self { + CrateSpec::Local(_s, xip) => *xip, + CrateSpec::CratesIo(_n, _v, xip) => *xip, + CrateSpec::Prebuilt(_n, _u, xip) => *xip, + CrateSpec::BinaryFile(_path, xip) => *xip, + _ => false + } + } + pub fn set_xip(&mut self, xip: bool) { + *self = match self { + CrateSpec::Local(s, _xip) => CrateSpec::Local(s.to_string(), xip), + CrateSpec::CratesIo(n, v, _xip) => CrateSpec::CratesIo(n.to_string(), v.to_string(), xip), + CrateSpec::Prebuilt(n, u, _xip) => CrateSpec::Prebuilt(n.to_string(), u.to_string(), xip), + CrateSpec::BinaryFile(path, _xip) => CrateSpec::BinaryFile(path.to_string(), xip), + CrateSpec::None => CrateSpec::None, + } + } + pub fn name(&self) -> Option { + match self { + CrateSpec::Local(s, _xip) => Some(s.to_string()), + CrateSpec::CratesIo(n, _v, _xip) => Some(n.to_string()), + CrateSpec::Prebuilt(n, _u, _xip) => Some(n.to_string()), + CrateSpec::BinaryFile(path, _xip) => Some(path.to_string()), + _ => None, + } + } +} impl Clone for CrateSpec { fn clone(&self) -> CrateSpec { match self { - CrateSpec::Local(s) => CrateSpec::Local(s.to_string()), - CrateSpec::CratesIo(n, v) => CrateSpec::CratesIo(n.to_string(), v.to_string()), - CrateSpec::Prebuilt(n, u) => CrateSpec::Prebuilt(n.to_string(), u.to_string()), - CrateSpec::BinaryFile(path) => CrateSpec::BinaryFile(path.to_string()), + CrateSpec::Local(s, xip) => CrateSpec::Local(s.to_string(), *xip), + CrateSpec::CratesIo(n, v, xip) => CrateSpec::CratesIo(n.to_string(), v.to_string(), *xip), + CrateSpec::Prebuilt(n, u, xip) => CrateSpec::Prebuilt(n.to_string(), u.to_string(), *xip), + CrateSpec::BinaryFile(path, xip) => CrateSpec::BinaryFile(path.to_string(), *xip), CrateSpec::None => CrateSpec::None, } } @@ -52,7 +82,8 @@ impl From<&str> for CrateSpec { let (name, version) = spec.split_once('@').expect("couldn't parse crate specifier"); CrateSpec::CratesIo( name.to_string(), - version.to_string() + version.to_string(), + false ) // prebuilt crates are specified as "name#url" // i.e. "espeak-embedded#https://ci.betrusted.io/job/espeak-embedded/lastSuccessfulBuild/artifact/target/riscv32imac-unknown-xous-elf/release/" @@ -60,16 +91,17 @@ impl From<&str> for CrateSpec { let (name, url) = spec.split_once('#').expect("couldn't parse crate specifier"); CrateSpec::Prebuilt( name.to_string(), - url.to_string() + url.to_string(), + false ) // local files are specified as paths, which, at a minimum include one directory separator "/" or "\" // i.e. "./local_file" // Note that this is after a test for the '#' character, so that disambiguates URL slashes // It does mean that files with a '#' character in them are mistaken for URL coded paths, and '@' as remote crates. } else if spec.contains('/') || spec.contains('\\') { - CrateSpec::BinaryFile(spec.to_string()) + CrateSpec::BinaryFile(spec.to_string(), false) } else { - CrateSpec::Local(spec.to_string()) + CrateSpec::Local(spec.to_string(), false) } } } @@ -103,11 +135,11 @@ pub(crate) struct Builder { impl Builder { pub fn new() -> Builder { Builder { - loader: CrateSpec::Local("loader".to_string()), + loader: CrateSpec::Local("loader".to_string(), false), loader_features: Vec::new(), loader_key: "devkey/dev.key".into(), loader_disable_defaults: false, - kernel: CrateSpec::Local("xous-kernel".to_string()), + kernel: CrateSpec::Local("xous-kernel".to_string(), false), kernel_features: Vec::new(), kernel_key: "devkey/dev.key".into(), kernel_disable_defaults: false, @@ -189,8 +221,8 @@ impl Builder { self.stream = BuildStream::Release; self.run_svd2repl = true; self.utra_target = "renode".to_string(); - self.loader = CrateSpec::Local("loader".to_string()); - self.kernel = CrateSpec::Local("xous-kernel".to_string()); + self.loader = CrateSpec::Local("loader".to_string(), false); + self.kernel = CrateSpec::Local("xous-kernel".to_string(), false); self } /// Configure for precursor targets. This is the default, but it's good practice @@ -201,8 +233,8 @@ impl Builder { self.stream = BuildStream::Release; self.utra_target = format!("precursor-{}", soc_version).to_string(); self.run_svd2repl = false; - self.loader = CrateSpec::Local("loader".to_string()); - self.kernel = CrateSpec::Local("xous-kernel".to_string()); + self.loader = CrateSpec::Local("loader".to_string(), false); + self.kernel = CrateSpec::Local("xous-kernel".to_string(), false); self } @@ -220,9 +252,10 @@ impl Builder { } /// Add just one service - #[allow(dead_code)] - pub fn add_service<'a>(&'a mut self, service_spec: &str) -> &'a mut Builder { - self.services.push(service_spec.into()); + pub fn add_service<'a>(&'a mut self, service_spec: &str, xip: bool) -> &'a mut Builder { + let mut spec: CrateSpec = service_spec.into(); + spec.set_xip(xip); + self.services.push(spec); self } /// Add a list of services @@ -339,8 +372,8 @@ impl Builder { let mut remote_pkgs = Vec::<(&str, &str)>::new(); for pkg in packages.iter() { match pkg { - CrateSpec::Local(name) => local_pkgs.push(&name), - CrateSpec::CratesIo(name, version) => remote_pkgs.push((&name, &version)), + CrateSpec::Local(name, _xip) => local_pkgs.push(&name), + CrateSpec::CratesIo(name, version, _xip) => remote_pkgs.push((&name, &version)), _ => {}, } } @@ -404,6 +437,50 @@ impl Builder { Ok(artifacts) } + pub fn split_xip(&self, services: Vec) -> (Vec, Vec) { + let mut inie = Vec::::new(); + let mut inif = Vec::::new(); + for service in services.iter() { + let mut found = false; + for app in self.apps.iter() { + if let Some(n) = &app.name() { + if Path::new(service).file_name().unwrap_or_default().to_str().unwrap_or_default() == n { + if app.is_xip() { + inif.push(service.to_string()); + } else { + inie.push(service.to_string()); + } + found = true; + continue; + } + } + } + if found { + continue; + } + for serv in self.services.iter() { + if let Some(n) = &serv.name() { + if Path::new(service).file_name().unwrap_or_default().to_str().unwrap_or_default() == n { + if serv.is_xip() { + inif.push(service.to_string()); + } else { + inie.push(service.to_string()); + } + found = true; + continue; + } + } + } + if found { + continue; + } + // the service wasn't found in any of the other lists, mark it as non-xip + inie.push(service.to_string()); + } + assert!(inie.len() + inif.len() == services.len()); + (inie, inif) + } + /// Consume the builder and execute the configured build task. This handles dispatching all configurations, /// including renode, hosted, and hardware targets. pub fn build(mut self) -> Result<(), DynError> { @@ -441,8 +518,8 @@ impl Builder { let mut app_names = Vec::::new(); for app in self.apps.iter() { match app { - CrateSpec::Local(name) => app_names.push(name.into()), - CrateSpec::CratesIo(name, _version) => app_names.push(name.into()), + CrateSpec::Local(name, _xip) => app_names.push(name.into()), + CrateSpec::CratesIo(name, _version, _xip) => app_names.push(name.into()), _ => {} } } @@ -460,7 +537,7 @@ impl Builder { // throw a warning if prebuilts are specified for hosted mode for item in [&self.services[..], &self.apps[..]].concat() { match item { - CrateSpec::Prebuilt(name, _) => println!("Warning! Pre-built binaries not supported for hosted mode ({})", name), + CrateSpec::Prebuilt(name, _, _xip) => println!("Warning! Pre-built binaries not supported for hosted mode ({})", name), _ => {}, } } @@ -503,7 +580,7 @@ impl Builder { } else { // confirm the kernel can build before quitting let _ = self.builder( - &vec![CrateSpec::Local("xous-kernel".into())], + &vec![CrateSpec::Local("xous-kernel".into(), false)], &self.kernel_features, &target, self.stream, @@ -584,9 +661,11 @@ impl Builder { services_path.append(&mut self.enumerate_binary_files()?); // --------- package up and sign a binary image ---------- + let (inie, inif) = self.split_xip(services_path); let output_bundle = self.create_image( &kernel_path[0], - &services_path, + &inie, + &inif, MemorySpec::SvdFile(svd_path) )?; println!(); @@ -679,6 +758,7 @@ impl Builder { &self, kernel: &String, init: &[String], + inif: &[String], memory_spec: MemorySpec, ) -> Result { let stream = self.stream.to_str(); @@ -699,6 +779,11 @@ impl Builder { args.push(i); } + for i in inif { + args.push("--inif"); + args.push(i); + } + match memory_spec { MemorySpec::SvdFile(ref s) => { args.push("--svd"); @@ -721,7 +806,7 @@ impl Builder { let mut paths = Vec::::new(); for item in [&self.services[..], &self.apps[..]].concat() { match item { - CrateSpec::Prebuilt(name, url) => { + CrateSpec::Prebuilt(name, url, _xip) => { let exec_name = format!("target/{}/{}/{}", TARGET_TRIPLE, self.stream.to_str(), name); println!("Fetching {} executable from build server...", name); let mut exec_file = OpenOptions::new() @@ -752,7 +837,7 @@ impl Builder { let mut paths = Vec::::new(); for item in [&self.services[..], &self.apps[..]].concat() { match item { - CrateSpec::BinaryFile(path) => { + CrateSpec::BinaryFile(path, _xip) => { paths.push(path); } _ => {} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 85de6fa17..41ba60ca5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -193,10 +193,10 @@ fn main() -> Result<(), Box> { .add_services(&get_cratespecs()); builder.add_loader_feature("renode-bypass") .add_loader_feature("renode-minimal"); - builder.add_service("net") - .add_service("com") - .add_service("llio") - .add_service("dns"); + builder.add_service("net", false) + .add_service("com", false) + .add_service("llio", false) + .add_service("dns", false); } Some("renode-aes-test") => { builder.target_renode() @@ -207,7 +207,7 @@ fn main() -> Result<(), Box> { builder.target_renode() .add_services(&gfx_base_pkgs.into_iter().map(String::from).collect()) .add_services(&get_cratespecs()); - builder.add_service("ffi-test"); + builder.add_service("ffi-test", false); builder.add_loader_feature("renode-bypass"); } Some("renode-remote") => { @@ -316,7 +316,7 @@ fn main() -> Result<(), Box> { builder.add_services(&pkgs.into_iter().map(String::from).collect()) .add_apps(&get_cratespecs()) - .add_service("espeak-embedded#https://ci.betrusted.io/job/espeak-embedded/lastSuccessfulBuild/artifact/target/riscv32imac-unknown-xous-elf/release/espeak-embedded") + .add_service("espeak-embedded#https://ci.betrusted.io/job/espeak-embedded/lastSuccessfulBuild/artifact/target/riscv32imac-unknown-xous-elf/release/espeak-embedded", false) .override_locale("en-tts") .add_feature("tts") .add_feature("braille"); @@ -331,7 +331,7 @@ fn main() -> Result<(), Box> { .add_services(&base_pkgs.into_iter().map(String::from).collect()) .add_services(&get_cratespecs()); //builder.add_service("usb-test"); - builder.add_service("usb-device-xous"); + builder.add_service("usb-device-xous", false); } Some("pddb-dev") => { builder.target_precursor(PRECURSOR_SOC_VERSION) diff --git a/xtask/src/verifier.rs b/xtask/src/verifier.rs index 0b757a60a..6b917ee58 100644 --- a/xtask/src/verifier.rs +++ b/xtask/src/verifier.rs @@ -32,7 +32,7 @@ pub fn check_project_consistency() -> Result<(), DynError> { } pub fn verify(spec: CrateSpec) -> Result<(), DynError> { - if let CrateSpec::CratesIo(name, version) = spec { + if let CrateSpec::CratesIo(name, version, _xip) = spec { let mut cache_path = Path::new(&env::var("CARGO_HOME").unwrap()).to_path_buf(); cache_path.push("registry"); cache_path.push("src");