From e9415b43f4b7614beb5036a8204ef5e50228e3fd Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 23 Jun 2023 16:25:11 +1000 Subject: [PATCH] Migrate archive tool into runner Also fix minilints declaring a stamp it wasn't creating. The same approach is necessary with archives now too, as it no longer executes under a standard "runner run". For now, rustls is hard-coded - we could pass the desired TLS impl in from the ./ninja script, but the runner is not recompiled frequently anyway. --- Cargo.lock | 24 +++------ Cargo.toml | 1 - build/archives/Cargo.toml | 24 --------- build/configure/src/rust.rs | 4 +- build/ninja_gen/src/archives.rs | 34 +------------ build/ninja_gen/src/build.rs | 4 +- build/runner/Cargo.toml | 8 +++ .../src/main.rs => runner/src/archive.rs} | 51 +++++++++++-------- build/runner/src/main.rs | 6 +++ tools/minilints/src/main.rs | 6 ++- 10 files changed, 64 insertions(+), 98 deletions(-) delete mode 100644 build/archives/Cargo.toml rename build/{archives/src/main.rs => runner/src/archive.rs} (81%) diff --git a/Cargo.lock b/Cargo.lock index 0581147b7..d1bdd5144 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,22 +280,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "archives" -version = "0.0.0" -dependencies = [ - "anki_io", - "camino", - "flate2", - "reqwest", - "sha2", - "tar", - "tokio", - "xz2", - "zip", - "zstd 0.12.3+zstd.1.5.2", -] - [[package]] name = "arrayref" version = "0.3.7" @@ -3334,10 +3318,18 @@ dependencies = [ "anyhow", "camino", "clap 4.2.1", + "flate2", "itertools", "junction", + "reqwest", + "sha2", + "tar", "termcolor", + "tokio", "which", + "xz2", + "zip", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cbb2adf05..1cd463eac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ members = [ "pylib/rsbridge", "build/configure", "build/ninja_gen", - "build/archives", "build/runner", "ftl", "tools/minilints", diff --git a/build/archives/Cargo.toml b/build/archives/Cargo.toml deleted file mode 100644 index 36eb12a3b..000000000 --- a/build/archives/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "archives" -version.workspace = true -authors.workspace = true -edition.workspace = true -license.workspace = true -publish = false -rust-version.workspace = true - -[dependencies] -anki_io.workspace = true -camino.workspace = true -flate2.workspace = true -reqwest.workspace = true -sha2.workspace = true -tar.workspace = true -tokio.workspace = true -xz2.workspace = true -zip.workspace = true -zstd.workspace = true - -[features] -rustls = ["reqwest/rustls-tls", "reqwest/rustls-tls-native-roots"] -native-tls = ["reqwest/native-tls"] diff --git a/build/configure/src/rust.rs b/build/configure/src/rust.rs index 5c5bd6f11..fbf72527d 100644 --- a/build/configure/src/rust.rs +++ b/build/configure/src/rust.rs @@ -182,7 +182,7 @@ pub fn check_minilints(build: &mut Build) -> Result<()> { impl BuildAction for RunMinilints { fn command(&self) -> &str { - "$minilints_bin $fix" + "$minilints_bin $fix $stamp" } fn bypass_runner(&self) -> bool { @@ -192,7 +192,7 @@ pub fn check_minilints(build: &mut Build) -> Result<()> { fn files(&mut self, build: &mut impl FilesHandle) { build.add_inputs("minilints_bin", inputs![":build:minilints"]); build.add_inputs("", &self.deps); - build.add_variable("fix", if self.fix { "fix" } else { "" }); + build.add_variable("fix", if self.fix { "fix" } else { "check" }); build.add_output_stamp(format!("tests/minilints.{}", self.fix)); } diff --git a/build/ninja_gen/src/archives.rs b/build/ninja_gen/src/archives.rs index a6392e76a..6175ea820 100644 --- a/build/ninja_gen/src/archives.rs +++ b/build/ninja_gen/src/archives.rs @@ -8,9 +8,6 @@ use camino::Utf8Path; use camino::Utf8PathBuf; use crate::action::BuildAction; -use crate::cargo::CargoBuild; -use crate::cargo::RustOutput; -use crate::glob; use crate::input::BuildInput; use crate::inputs; use crate::Build; @@ -81,23 +78,18 @@ struct DownloadArchive { impl BuildAction for DownloadArchive { fn command(&self) -> &str { - "$archives_bin download $url $checksum $out" + "$runner archive download $url $checksum $out" } fn files(&mut self, build: &mut impl crate::build::FilesHandle) { let (_, filename) = self.archive.url.rsplit_once('/').unwrap(); let output_path = Utf8Path::new("download").join(filename); - build.add_order_only_inputs("archives_bin", inputs![":build:archives"]); build.add_variable("url", self.archive.url); build.add_variable("checksum", self.archive.sha256); build.add_outputs("out", &[output_path.into_string()]) } - fn on_first_instance(&self, build: &mut Build) -> Result<()> { - build_archive_tool(build) - } - fn check_output_timestamps(&self) -> bool { true } @@ -131,11 +123,10 @@ where I::Item: AsRef, { fn command(&self) -> &str { - "$archive_tool extract $in $extraction_folder" + "$runner archive extract $in $extraction_folder" } fn files(&mut self, build: &mut impl crate::build::FilesHandle) { - build.add_order_only_inputs("archive_tool", inputs![":build:archives"]); build.add_inputs("in", inputs![self.archive_path.clone()]); let folder = self.extraction_folder(); @@ -152,10 +143,6 @@ where build.add_output_stamp(folder.with_extension("marker")); } - fn on_first_instance(&self, build: &mut Build) -> Result<()> { - build_archive_tool(build) - } - fn name(&self) -> &'static str { "extract" } @@ -165,23 +152,6 @@ where } } -fn build_archive_tool(build: &mut Build) -> Result<()> { - build.once_only("build_archive_tool", |build| { - let features = Platform::tls_feature(); - build.add_action( - "build:archives", - CargoBuild { - inputs: inputs![glob!("build/archives/**/*")], - outputs: &[RustOutput::Binary("archives")], - target: None, - extra_args: &format!("-p archives --features {features}"), - release_override: Some(false), - }, - )?; - Ok(()) - }) -} - /// See [DownloadArchive] and [ExtractArchive]. pub fn download_and_extract( build: &mut Build, diff --git a/build/ninja_gen/src/build.rs b/build/ninja_gen/src/build.rs index 5ae5696b0..51e961875 100644 --- a/build/ninja_gen/src/build.rs +++ b/build/ninja_gen/src/build.rs @@ -371,7 +371,9 @@ pub trait FilesHandle { subgroup: bool, ); - /// Save an output stamp if the command completes successfully. + /// Save an output stamp if the command completes successfully. Note that + /// if you have bypassed the runner, you will need to create the file + /// yourself. fn add_output_stamp(&mut self, path: impl Into); /// Set an env var for the duration of the provided command(s). /// Note this is defined once for the rule, so if the value should change diff --git a/build/runner/Cargo.toml b/build/runner/Cargo.toml index 8d908170f..e8f35ab5b 100644 --- a/build/runner/Cargo.toml +++ b/build/runner/Cargo.toml @@ -12,7 +12,15 @@ anki_io.workspace = true anyhow.workspace = true camino.workspace = true clap.workspace = true +flate2.workspace = true itertools.workspace = true junction.workspace = true +reqwest = { workspace = true, features = ["rustls-tls", "rustls-tls-native-roots"] } +sha2.workspace = true +tar.workspace = true termcolor.workspace = true +tokio.workspace = true which.workspace = true +xz2.workspace = true +zip.workspace = true +zstd.workspace = true diff --git a/build/archives/src/main.rs b/build/runner/src/archive.rs similarity index 81% rename from build/archives/src/main.rs rename to build/runner/src/archive.rs index 3f6b4c724..8a78dd515 100644 --- a/build/archives/src/main.rs +++ b/build/runner/src/archive.rs @@ -1,38 +1,46 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use std::error::Error; use std::fs; use std::io::Read; use std::path::Path; +use std::path::PathBuf; use anki_io::read_file; +use anki_io::write_file; +use anyhow::Result; use camino::Utf8Path; +use clap::Args; +use clap::Subcommand; use sha2::Digest; -type Result = std::result::Result>; +#[derive(Subcommand)] +pub enum ArchiveArgs { + Download(DownloadArgs), + Extract(ExtractArgs), +} + +#[derive(Args)] +pub struct DownloadArgs { + archive_url: String, + checksum: String, + output_path: PathBuf, +} + +#[derive(Args)] +pub struct ExtractArgs { + archive_path: String, + output_folder: String, +} #[tokio::main] -async fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - - let action = args[1].as_str(); - match action { - "download" => { - let archive_url = &args[2]; - let checksum = &args[3]; - let output_path = Path::new(&args[4]); - download_and_check(archive_url, checksum, output_path).await?; +pub async fn archive_command(args: ArchiveArgs) -> Result<()> { + match args { + ArchiveArgs::Download(args) => { + download_and_check(&args.archive_url, &args.checksum, &args.output_path).await } - "extract" => { - let archive_path = &args[2]; - let output_folder = &args[3]; - extract(archive_path, output_folder)?; - } - _ => panic!("unexpected action"), + ArchiveArgs::Extract(args) => extract_archive(&args.archive_path, &args.output_folder), } - - Ok(()) } async fn download_and_check(archive_url: &str, checksum: &str, output_path: &Path) -> Result<()> { @@ -73,7 +81,7 @@ enum ArchiveKind { Zip, } -fn extract(archive_path: &str, output_folder: &str) -> Result<()> { +fn extract_archive(archive_path: &str, output_folder: &str) -> Result<()> { let archive_path = Utf8Path::new(archive_path); let archive_filename = archive_path.file_name().unwrap(); let mut components = archive_filename.rsplit('.'); @@ -140,5 +148,6 @@ fn extract(archive_path: &str, output_folder: &str) -> Result<()> { } else { fs::rename(output_tmp, output_folder)?; } + write_file(output_folder.with_extension("marker"), "")?; Ok(()) } diff --git a/build/runner/src/main.rs b/build/runner/src/main.rs index d044af95a..8fdf8f06f 100644 --- a/build/runner/src/main.rs +++ b/build/runner/src/main.rs @@ -5,6 +5,7 @@ //! silencing their output when they succeed. Most build actions implicitly use //! the 'run' command; we define separate commands for more complicated actions. +mod archive; mod build; mod bundle; mod paths; @@ -14,6 +15,8 @@ mod run; mod yarn; use anyhow::Result; +use archive::archive_command; +use archive::ArchiveArgs; use build::run_build; use build::BuildArgs; use bundle::artifacts::build_artifacts; @@ -48,6 +51,8 @@ enum Command { BuildArtifacts(BuildArtifactsArgs), BuildBundleBinary, BuildDistFolder(BuildDistFolderArgs), + #[clap(subcommand)] + Archive(ArchiveArgs), } fn main() -> Result<()> { @@ -60,6 +65,7 @@ fn main() -> Result<()> { Command::BuildArtifacts(args) => build_artifacts(args), Command::BuildBundleBinary => build_bundle_binary(), Command::BuildDistFolder(args) => build_dist_folder(args), + Command::Archive(args) => archive_command(args)?, }; Ok(()) } diff --git a/tools/minilints/src/main.rs b/tools/minilints/src/main.rs index 9c2315af8..1f8a206fc 100644 --- a/tools/minilints/src/main.rs +++ b/tools/minilints/src/main.rs @@ -43,7 +43,9 @@ const IGNORED_FOLDERS: &[&str] = &[ ]; fn main() -> Result<()> { - let want_fix = env::args().nth(1) == Some("fix".to_string()); + let mut args = env::args(); + let want_fix = args.nth(1) == Some("fix".to_string()); + let stamp = args.next().unwrap(); let mut ctx = LintContext::new(want_fix); ctx.check_contributors()?; ctx.check_rust_licenses()?; @@ -51,6 +53,8 @@ fn main() -> Result<()> { if ctx.found_problems { std::process::exit(1); } + write_file(stamp, "")?; + Ok(()) }