From 35857ab54357e06d4841c428999731d6a6f91bf3 Mon Sep 17 00:00:00 2001 From: Sean Cross Date: Tue, 29 Mar 2022 20:47:37 +0800 Subject: [PATCH 1/2] xtask: replace `curl` with `ureq` The `ureq` package is much smaller, and may be more reliable on Linux. Signed-off-by: Sean Cross --- Cargo.lock | 89 ----------------------------------------------- xtask/Cargo.toml | 5 ++- xtask/src/main.rs | 81 +++++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04c332d22..a9787967b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -695,36 +695,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" -[[package]] -name = "curl" -version = "0.4.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a872858e9cb9e3b96c80dd78774ad9e32e44d3b05dc31e142b858d14aebc82c" -dependencies = [ - "curl-sys", - "libc", - "openssl-probe", - "openssl-sys", - "schannel", - "socket2", - "winapi", -] - -[[package]] -name = "curl-sys" -version = "0.4.45+curl-7.78.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9e5a72b1c744eb5dd20b2be4d7eb84625070bb5c4ab9b347b70464ab1e62eb" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", - "winapi", -] - [[package]] name = "curve25519-dalek" version = "3.1.0" @@ -1502,18 +1472,6 @@ dependencies = [ "xous-names", ] -[[package]] -name = "libz-sys" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "llio" version = "0.1.0" @@ -1810,25 +1768,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" -[[package]] -name = "openssl-probe" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" - -[[package]] -name = "openssl-sys" -version = "0.9.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1996d2d305e561b70d1ee0c53f1542833f4e1ac6ce9a6708b6ff2738ca67dc82" -dependencies = [ - "autocfg", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "orbclient" version = "0.3.30" @@ -2399,16 +2338,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" -dependencies = [ - "lazy_static", - "winapi", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2660,17 +2589,6 @@ dependencies = [ "managed", ] -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi", -] - [[package]] name = "spectrum-analyzer" version = "0.5.2" @@ -3140,12 +3058,6 @@ dependencies = [ "svd2utra", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vec_map" version = "0.8.2" @@ -3556,7 +3468,6 @@ version = "0.1.0" dependencies = [ "atty", "chrono", - "curl", "filetime", "rustc_version 0.4.0", "serde", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 2e2333e97..767bd918b 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -7,13 +7,12 @@ edition = "2018" # Dependency policy: fully specify dependencies to the minor version number [dependencies] -curl = "0.4.35" atty = "0.2.14" filetime = "0.2.14" rustc_version = "0.4.0" zip = "0.5.13" chrono = "0.4" serde_json = "1.0.41" -serde = {version = "1.0.130", features = ["derive"]} +serde = { version = "1.0.130", features = ["derive"] } tempfile = "3.3.0" -ureq = "2.4.0" \ No newline at end of file +ureq = "2.4.0" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index cdd45f8f5..a87230477 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -188,13 +188,7 @@ fn main() -> Result<(), Box> { pkgs.push(app); } generate_app_menus(&apps); - renode_image( - false, - &hw_pkgs, - &[], - None, - None, - )? + renode_image(false, &hw_pkgs, &[], None, None)? } Some("renode-test") => { let mut args = env::args(); @@ -1113,7 +1107,6 @@ fn ensure_compiler( target_loader.push("target"); target_loader.push(PROGRAM_TARGET); std::fs::remove_dir_all(target_loader).ok(); - } else { DONE_COMPILER_CHECK.store(true, std::sync::atomic::Ordering::SeqCst); return Ok(()); @@ -1140,10 +1133,14 @@ fn ensure_compiler( let stdin = std::io::stdin(); let mut stdout = std::io::stdout(); println!(); - println!( - "Error: Toolchain for {} was not found on this system!", - target - ); + if force_install { + println!("Downloading toolchain"); + } else { + println!( + "Error: Toolchain for {} was not found on this system!", + target + ); + } loop { if force_install { break; @@ -1177,44 +1174,38 @@ fn ensure_compiler( ); println!("Downloading from {}...", toolchain_url); - print!("Download rogress: 0%"); + print!("Download in progress..."); stdout.flush().unwrap(); let mut zip_data = vec![]; { - let mut easy = curl::easy::Easy::new(); - easy.url(&toolchain_url).unwrap(); - easy.follow_location(true).unwrap(); - easy.progress(true).unwrap(); - let mut transfer = easy.transfer(); - transfer - .progress_function( - |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| { - // If either number is infinite, don't print anything and just continue. - if total_bytes.is_infinite() || bytes_so_far.is_infinite() { - return true; - } + let agent = ureq::builder() + // .middleware(CounterMiddleware(shared_state.clone())) + .build(); - // Display progress. - print!( - "\rDownload progress: {:3.02}% ", - bytes_so_far / total_bytes * 100.0 - ); - stdout.flush().unwrap(); + let mut freader = agent + .get(&toolchain_url) + .call() + .map_err(|e| format!("{}", e))? + .into_reader(); + freader + .read_to_end(&mut zip_data) + .map_err(|e| format!("{}", e))?; + // |total_bytes, bytes_so_far, _total_uploaded, _uploaded_so_far| { + // // If either number is infinite, don't print anything and just continue. + // if total_bytes.is_infinite() || bytes_so_far.is_infinite() { + // return true; + // } - // Return `true` to continue the transfer. - true - }, - ) - .unwrap(); - transfer - .write_function(|data| { - zip_data.extend_from_slice(data); - Ok(data.len()) - }) - .unwrap(); - transfer - .perform() - .map_err(|e| format!("Unable to download toolchain: {}", e))?; + // // Display progress. + // print!( + // "\rDownload progress: {:3.02}% ", + // bytes_so_far / total_bytes * 100.0 + // ); + // stdout.flush().unwrap(); + + // // Return `true` to continue the transfer. + // true + // }, println!(); } println!( From e6209302e7cc2cc4547f039b793709c1d29770e2 Mon Sep 17 00:00:00 2001 From: Sean Cross Date: Tue, 29 Mar 2022 21:46:18 +0800 Subject: [PATCH 2/2] xtask: use assets api for determining URL Now that we have full JSON parsing inside xtask, actually parse the assets API endpoint to determine the correct URL. Signed-off-by: Sean Cross --- Cargo.lock | 2 ++ xtask/Cargo.toml | 3 +- xtask/src/main.rs | 73 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9787967b..bed8bcbf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3034,6 +3034,8 @@ dependencies = [ "log", "once_cell", "rustls", + "serde", + "serde_json", "url", "webpki", "webpki-roots", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 767bd918b..06aa0299f 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -4,6 +4,7 @@ description = "Cargo-based build system for Xous" version = "0.1.0" authors = ["Sean Cross "] edition = "2018" +rust-version = "1.59" # Dependency policy: fully specify dependencies to the minor version number [dependencies] @@ -15,4 +16,4 @@ chrono = "0.4" serde_json = "1.0.41" serde = { version = "1.0.130", features = ["derive"] } tempfile = "3.3.0" -ureq = "2.4.0" +ureq = { version = "2.4.0", features = ["json"] } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index a87230477..78ab26b3b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -14,9 +14,7 @@ type DynError = Box; const PROGRAM_TARGET: &str = "riscv32imac-unknown-xous-elf"; const KERNEL_TARGET: &str = "riscv32imac-unknown-xous-elf"; -const TOOLCHAIN_URL_PREFIX: &str = - "https://github.com/betrusted-io/rust/releases/latest/download/riscv32imac-unknown-xous_"; -const TOOLCHAIN_URL_SUFFIX: &str = ".zip"; +const TOOLCHAIN_RELEASE_URL: &str = "https://api.github.com/repos/betrusted-io/rust/releases"; enum MemorySpec { SvdFile(String), @@ -1132,7 +1130,6 @@ fn ensure_compiler( let mut buffer = String::new(); let stdin = std::io::stdin(); let mut stdout = std::io::stdout(); - println!(); if force_install { println!("Downloading toolchain"); } else { @@ -1163,16 +1160,74 @@ fn ensure_compiler( println!(); } - let toolchain_url = format!( - "{}{}.{}.{}{}", - TOOLCHAIN_URL_PREFIX, ver.major, ver.minor, ver.patch, TOOLCHAIN_URL_SUFFIX - ); + fn get_toolchain_url(major: u64, minor: u64, patch: u64) -> Result { + let j: serde_json::Value = ureq::get(TOOLCHAIN_RELEASE_URL) + .set("Accept", "application/vnd.github.v3+json") + .call() + .map_err(|e| format!("{}", e))? + .into_json() + .map_err(|e| format!("{}", e))?; + // let j: serde_json::Value = serde_json::from_str(CONTENT).expect("Cannot parse manifest file"); + + let releases = j.as_array().unwrap(); + let mut tag_urls = std::collections::BTreeMap::new(); + + let target_prefix = format!("{}.{}.{}", major, minor, patch); + for r in releases { + // println!(">>> Value: {}", r); + + let keys = match r.as_object() { + None => continue, + Some(r) => r, + }; + let release = match keys.get("tag_name") { + None => continue, + Some(s) => match s.as_str() { + None => continue, + Some(s) => s, + }, + }; + if !release.starts_with(&target_prefix) { + continue; + } + + let assets = match keys.get("assets") { + None => continue, + Some(s) => match s.as_array() { + None => continue, + Some(s) => s, + }, + }; + + let first_asset = match assets.get(0) { + None => continue, + Some(s) => s, + }; + + let download_url = match first_asset.get("browser_download_url") { + None => continue, + Some(s) => match s.as_str() { + None => continue, + Some(s) => s, + }, + }; + // println!("Candidate Release: {}", download_url); + tag_urls.insert(release.to_owned(), download_url.to_owned()); + } + + if let Some((_k, v)) = tag_urls.into_iter().last() { + // println!("Found candidate entry: v{} url {}", k, v); + return Ok(v); + } + Err(format!("No toolchains found for Rust {}", target_prefix)) + } + let toolchain_url = get_toolchain_url(ver.major, ver.minor, ver.patch)?; println!( "Attempting to install toolchain for {} into {}", target, toolchain_path ); - println!("Downloading from {}...", toolchain_url); + println!("Downloading toolchain from {}...", toolchain_url); print!("Download in progress..."); stdout.flush().unwrap();