Migrate archive tool into runner

Also fix minilints declaring a stamp it wasn't creating. The same
approach is necessary with archives now too, as it no longer executes
under a standard "runner run".

For now, rustls is hard-coded - we could pass the desired TLS impl in
from the ./ninja script, but the runner is not recompiled frequently
anyway.

Cargo.lock

@@ -280,22 +280,6 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "archives"
-version = "0.0.0"
-dependencies = [
- "anki_io",
- "camino",
- "flate2",
- "reqwest",
- "sha2",
- "tar",
- "tokio",
- "xz2",
- "zip",
- "zstd 0.12.3+zstd.1.5.2",
-]
-
 [[package]]
 name = "arrayref"
 version = "0.3.7"
@@ -3334,10 +3318,18 @@ dependencies = [
  "anyhow",
  "camino",
  "clap 4.2.1",
+ "flate2",
  "itertools",
  "junction",
+ "reqwest",
+ "sha2",
+ "tar",
  "termcolor",
+ "tokio",
  "which",
+ "xz2",
+ "zip",
+ "zstd 0.12.3+zstd.1.5.2",
 ]
 
 [[package]]

Cargo.toml

@@ -17,7 +17,6 @@ members = [
   "pylib/rsbridge",
   "build/configure",
   "build/ninja_gen",
-  "build/archives",
   "build/runner",
   "ftl",
   "tools/minilints",

build/archives/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "archives"
-version.workspace = true
-authors.workspace = true
-edition.workspace = true
-license.workspace = true
-publish = false
-rust-version.workspace = true
-
-[dependencies]
-anki_io.workspace = true
-camino.workspace = true
-flate2.workspace = true
-reqwest.workspace = true
-sha2.workspace = true
-tar.workspace = true
-tokio.workspace = true
-xz2.workspace = true
-zip.workspace = true
-zstd.workspace = true
-
-[features]
-rustls = ["reqwest/rustls-tls", "reqwest/rustls-tls-native-roots"]
-native-tls = ["reqwest/native-tls"]

build/configure/src/rust.rs

@@ -182,7 +182,7 @@ pub fn check_minilints(build: &mut Build) -> Result<()> {
 
     impl BuildAction for RunMinilints {
         fn command(&self) -> &str {
-            "$minilints_bin $fix"
+            "$minilints_bin $fix $stamp"
         }
 
         fn bypass_runner(&self) -> bool {
@@ -192,7 +192,7 @@ pub fn check_minilints(build: &mut Build) -> Result<()> {
         fn files(&mut self, build: &mut impl FilesHandle) {
             build.add_inputs("minilints_bin", inputs![":build:minilints"]);
             build.add_inputs("", &self.deps);
-            build.add_variable("fix", if self.fix { "fix" } else { "" });
+            build.add_variable("fix", if self.fix { "fix" } else { "check" });
             build.add_output_stamp(format!("tests/minilints.{}", self.fix));
         }
 

build/ninja_gen/src/archives.rs

@@ -8,9 +8,6 @@ use camino::Utf8Path;
 use camino::Utf8PathBuf;
 
 use crate::action::BuildAction;
-use crate::cargo::CargoBuild;
-use crate::cargo::RustOutput;
-use crate::glob;
 use crate::input::BuildInput;
 use crate::inputs;
 use crate::Build;
@@ -81,23 +78,18 @@ struct DownloadArchive {
 
 impl BuildAction for DownloadArchive {
     fn command(&self) -> &str {
-        "$archives_bin download $url $checksum $out"
+        "$runner archive download $url $checksum $out"
     }
 
     fn files(&mut self, build: &mut impl crate::build::FilesHandle) {
         let (_, filename) = self.archive.url.rsplit_once('/').unwrap();
         let output_path = Utf8Path::new("download").join(filename);
 
-        build.add_order_only_inputs("archives_bin", inputs![":build:archives"]);
         build.add_variable("url", self.archive.url);
         build.add_variable("checksum", self.archive.sha256);
         build.add_outputs("out", &[output_path.into_string()])
     }
 
-    fn on_first_instance(&self, build: &mut Build) -> Result<()> {
-        build_archive_tool(build)
-    }
-
     fn check_output_timestamps(&self) -> bool {
         true
     }
@@ -131,11 +123,10 @@ where
     I::Item: AsRef<str>,
 {
     fn command(&self) -> &str {
-        "$archive_tool extract $in $extraction_folder"
+        "$runner archive extract $in $extraction_folder"
     }
 
     fn files(&mut self, build: &mut impl crate::build::FilesHandle) {
-        build.add_order_only_inputs("archive_tool", inputs![":build:archives"]);
         build.add_inputs("in", inputs![self.archive_path.clone()]);
 
         let folder = self.extraction_folder();
@@ -152,10 +143,6 @@ where
         build.add_output_stamp(folder.with_extension("marker"));
     }
 
-    fn on_first_instance(&self, build: &mut Build) -> Result<()> {
-        build_archive_tool(build)
-    }
-
     fn name(&self) -> &'static str {
         "extract"
     }
@@ -165,23 +152,6 @@ where
     }
 }
 
-fn build_archive_tool(build: &mut Build) -> Result<()> {
-    build.once_only("build_archive_tool", |build| {
-        let features = Platform::tls_feature();
-        build.add_action(
-            "build:archives",
-            CargoBuild {
-                inputs: inputs![glob!("build/archives/**/*")],
-                outputs: &[RustOutput::Binary("archives")],
-                target: None,
-                extra_args: &format!("-p archives --features {features}"),
-                release_override: Some(false),
-            },
-        )?;
-        Ok(())
-    })
-}
-
 /// See [DownloadArchive] and [ExtractArchive].
 pub fn download_and_extract<I>(
     build: &mut Build,

build/ninja_gen/src/build.rs

@@ -371,7 +371,9 @@ pub trait FilesHandle {
         subgroup: bool,
     );
 
-    /// Save an output stamp if the command completes successfully.
+    /// Save an output stamp if the command completes successfully. Note that
+    /// if you have bypassed the runner, you will need to create the file
+    /// yourself.
     fn add_output_stamp(&mut self, path: impl Into<String>);
     /// Set an env var for the duration of the provided command(s).
     /// Note this is defined once for the rule, so if the value should change

build/runner/Cargo.toml

@@ -12,7 +12,15 @@ anki_io.workspace = true
 anyhow.workspace = true
 camino.workspace = true
 clap.workspace = true
+flate2.workspace = true
 itertools.workspace = true
 junction.workspace = true
+reqwest = { workspace = true, features = ["rustls-tls", "rustls-tls-native-roots"] }
+sha2.workspace = true
+tar.workspace = true
 termcolor.workspace = true
+tokio.workspace = true
 which.workspace = true
+xz2.workspace = true
+zip.workspace = true
+zstd.workspace = true

build/runner/src/archive.rs

@@ -1,38 +1,46 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use std::error::Error;
 use std::fs;
 use std::io::Read;
 use std::path::Path;
+use std::path::PathBuf;
 
 use anki_io::read_file;
+use anki_io::write_file;
+use anyhow::Result;
 use camino::Utf8Path;
+use clap::Args;
+use clap::Subcommand;
 use sha2::Digest;
 
-type Result<T> = std::result::Result<T, Box<dyn Error>>;
+#[derive(Subcommand)]
+pub enum ArchiveArgs {
+    Download(DownloadArgs),
+    Extract(ExtractArgs),
+}
+
+#[derive(Args)]
+pub struct DownloadArgs {
+    archive_url: String,
+    checksum: String,
+    output_path: PathBuf,
+}
+
+#[derive(Args)]
+pub struct ExtractArgs {
+    archive_path: String,
+    output_folder: String,
+}
 
 #[tokio::main]
-async fn main() -> Result<()> {
-    let args: Vec<_> = std::env::args().collect();
-
-    let action = args[1].as_str();
-    match action {
-        "download" => {
-            let archive_url = &args[2];
-            let checksum = &args[3];
-            let output_path = Path::new(&args[4]);
-            download_and_check(archive_url, checksum, output_path).await?;
+pub async fn archive_command(args: ArchiveArgs) -> Result<()> {
+    match args {
+        ArchiveArgs::Download(args) => {
+            download_and_check(&args.archive_url, &args.checksum, &args.output_path).await
         }
-        "extract" => {
-            let archive_path = &args[2];
-            let output_folder = &args[3];
-            extract(archive_path, output_folder)?;
-        }
-        _ => panic!("unexpected action"),
+        ArchiveArgs::Extract(args) => extract_archive(&args.archive_path, &args.output_folder),
     }
-
-    Ok(())
 }
 
 async fn download_and_check(archive_url: &str, checksum: &str, output_path: &Path) -> Result<()> {
@@ -73,7 +81,7 @@ enum ArchiveKind {
     Zip,
 }
 
-fn extract(archive_path: &str, output_folder: &str) -> Result<()> {
+fn extract_archive(archive_path: &str, output_folder: &str) -> Result<()> {
     let archive_path = Utf8Path::new(archive_path);
     let archive_filename = archive_path.file_name().unwrap();
     let mut components = archive_filename.rsplit('.');
@@ -140,5 +148,6 @@ fn extract(archive_path: &str, output_folder: &str) -> Result<()> {
     } else {
         fs::rename(output_tmp, output_folder)?;
     }
+    write_file(output_folder.with_extension("marker"), "")?;
     Ok(())
 }

build/runner/src/main.rs

@@ -5,6 +5,7 @@
 //! silencing their output when they succeed. Most build actions implicitly use
 //! the 'run' command; we define separate commands for more complicated actions.
 
+mod archive;
 mod build;
 mod bundle;
 mod paths;
@@ -14,6 +15,8 @@ mod run;
 mod yarn;
 
 use anyhow::Result;
+use archive::archive_command;
+use archive::ArchiveArgs;
 use build::run_build;
 use build::BuildArgs;
 use bundle::artifacts::build_artifacts;
@@ -48,6 +51,8 @@ enum Command {
     BuildArtifacts(BuildArtifactsArgs),
     BuildBundleBinary,
     BuildDistFolder(BuildDistFolderArgs),
+    #[clap(subcommand)]
+    Archive(ArchiveArgs),
 }
 
 fn main() -> Result<()> {
@@ -60,6 +65,7 @@ fn main() -> Result<()> {
         Command::BuildArtifacts(args) => build_artifacts(args),
         Command::BuildBundleBinary => build_bundle_binary(),
         Command::BuildDistFolder(args) => build_dist_folder(args),
+        Command::Archive(args) => archive_command(args)?,
     };
     Ok(())
 }

tools/minilints/src/main.rs

@@ -43,7 +43,9 @@ const IGNORED_FOLDERS: &[&str] = &[
 ];
 
 fn main() -> Result<()> {
-    let want_fix = env::args().nth(1) == Some("fix".to_string());
+    let mut args = env::args();
+    let want_fix = args.nth(1) == Some("fix".to_string());
+    let stamp = args.next().unwrap();
     let mut ctx = LintContext::new(want_fix);
     ctx.check_contributors()?;
     ctx.check_rust_licenses()?;
@@ -51,6 +53,8 @@ fn main() -> Result<()> {
     if ctx.found_problems {
         std::process::exit(1);
     }
+    write_file(stamp, "")?;
+
     Ok(())
 }