From dd95f6f74906b998ade7f834124b751b6ec84fa4 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 17 Jun 2023 14:01:27 +1000 Subject: [PATCH] Check for stale licenses.json in minilints + Add an anki_process library with some helpers for command running. --- Cargo.lock | 11 +++ Cargo.toml | 1 + build/configure/src/rust.rs | 15 +++- cargo/README.md | 5 +- cargo/licenses.json | 9 --- cargo/update-licenses.sh | 8 -- rslib/io/src/lib.rs | 8 ++ rslib/process/Cargo.toml | 14 ++++ rslib/process/src/lib.rs | 126 ++++++++++++++++++++++++++++++++ tools/minilints/Cargo.toml | 2 + tools/minilints/src/main.rs | 49 ++++++++++++- tools/workspace-hack/Cargo.toml | 5 +- 12 files changed, 226 insertions(+), 27 deletions(-) delete mode 100755 cargo/update-licenses.sh create mode 100644 rslib/process/Cargo.toml create mode 100644 rslib/process/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index bee605ecf..e07630b40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,6 +200,14 @@ dependencies = [ "tempfile", ] +[[package]] +name = "anki_process" +version = "0.0.0" +dependencies = [ + "itertools", + "snafu", +] + [[package]] name = "anki_proto" version = "0.0.0" @@ -2331,6 +2339,8 @@ dependencies = [ name = "minilints" version = "0.0.0" dependencies = [ + "anki_io", + "anki_process", "anyhow", "camino", "once_cell", @@ -5059,6 +5069,7 @@ dependencies = [ "serde_json", "sha2", "snafu", + "snafu-derive", "syn 1.0.109", "syn 2.0.12", "time", diff --git a/Cargo.toml b/Cargo.toml index 365f85b87..19ed4b630 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "rslib/linkchecker", "rslib/proto", "rslib/io", + "rslib/process", "pylib/rsbridge", "build/configure", "build/ninja_gen", diff --git a/build/configure/src/rust.rs b/build/configure/src/rust.rs index a0ae0526d..5c5bd6f11 100644 --- a/build/configure/src/rust.rs +++ b/build/configure/src/rust.rs @@ -185,6 +185,10 @@ pub fn check_minilints(build: &mut Build) -> Result<()> { "$minilints_bin $fix" } + fn bypass_runner(&self) -> bool { + true + } + fn files(&mut self, build: &mut impl FilesHandle) { build.add_inputs("minilints_bin", inputs![":build:minilints"]); build.add_inputs("", &self.deps); @@ -206,10 +210,13 @@ pub fn check_minilints(build: &mut Build) -> Result<()> { } } - let files = inputs![glob![ - "**/*.{py,rs,ts,svelte,mjs}", - "{node_modules,qt/bundle/PyOxidizer}/**" - ]]; + let files = inputs![ + glob![ + "**/*.{py,rs,ts,svelte,mjs}", + "{node_modules,qt/bundle/PyOxidizer}/**" + ], + "Cargo.lock" + ]; build.add_action( "check:minilints", diff --git a/cargo/README.md b/cargo/README.md index 6c8d75bab..84ec7a0ea 100644 --- a/cargo/README.md +++ b/cargo/README.md @@ -1,5 +1,4 @@ -This folder used to contain Bazel-specific Rust integration, but now only -contains: +This folder contains: -- our license-checking script, which should be run when using cargo update. +- a list of Rust crate licenses, which is checked/updated with ./ninja [check|fix]:minilints - a nightly toolchain definition for formatting diff --git a/cargo/licenses.json b/cargo/licenses.json index 31b5628c4..c70571861 100644 --- a/cargo/licenses.json +++ b/cargo/licenses.json @@ -1637,15 +1637,6 @@ "license_file": null, "description": "A Rust crate for producing string-representations of numbers, formatted according to international standards" }, - { - "name": "num-integer", - "version": "0.1.45", - "authors": "The Rust Project Developers", - "repository": "https://github.com/rust-num/num-integer", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "Integer traits and functions" - }, { "name": "num-traits", "version": "0.2.15", diff --git a/cargo/update-licenses.sh b/cargo/update-licenses.sh deleted file mode 100755 index 230782746..000000000 --- a/cargo/update-licenses.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -cargo install cargo-license@0.5.1 -cargo-license --features rustls --features native-tls --json --manifest-path ../rslib/Cargo.toml >licenses.json - -cargo install cargo-deny@0.13.5 -(cd .. && cargo deny check -A duplicate && cargo hakari generate) - diff --git a/rslib/io/src/lib.rs b/rslib/io/src/lib.rs index 0c6738872..b28e19975 100644 --- a/rslib/io/src/lib.rs +++ b/rslib/io/src/lib.rs @@ -75,6 +75,14 @@ pub fn read_file(path: impl AsRef) -> Result> { }) } +/// See [std::fs::read_to_string]. +pub fn read_to_string(path: impl AsRef) -> Result { + std::fs::read_to_string(&path).context(FileIoSnafu { + path: path.as_ref(), + op: FileOp::Read, + }) +} + /// Like [read_file], but skips the section that is potentially locked by /// SQLite. pub fn read_locked_db_file(path: impl AsRef) -> Result> { diff --git a/rslib/process/Cargo.toml b/rslib/process/Cargo.toml new file mode 100644 index 000000000..eb0ee2420 --- /dev/null +++ b/rslib/process/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "anki_process" +publish = false +description = "Utils for better process error reporting" + +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +itertools = "0.10.5" +snafu = "0.7.4" diff --git a/rslib/process/src/lib.rs b/rslib/process/src/lib.rs new file mode 100644 index 000000000..66f4c224e --- /dev/null +++ b/rslib/process/src/lib.rs @@ -0,0 +1,126 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::ffi::OsStr; +use std::process::Command; +use std::string::FromUtf8Error; + +use itertools::Itertools; +use snafu::ensure; +use snafu::ResultExt; +use snafu::Snafu; + +#[derive(Debug, Snafu)] +pub enum Error { + DidNotExecute { + cmdline: String, + source: std::io::Error, + }, + ReturnedError { + cmdline: String, + code: Option, + }, + InvalidUtf8 { + cmdline: String, + source: FromUtf8Error, + }, +} + +pub type Result = std::result::Result; + +pub struct Utf8Output { + pub stdout: String, + pub stderr: String, +} + +pub trait CommandExt { + fn run(cmd_and_args: I) -> Result<()> + where + I: IntoIterator, + S: AsRef, + { + let mut all_args = cmd_and_args.into_iter(); + Command::new(all_args.next().unwrap()) + .args(all_args) + .ensure_success()?; + Ok(()) + } + fn run_with_output(cmd_and_args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut all_args = cmd_and_args.into_iter(); + Command::new(all_args.next().unwrap()) + .args(all_args) + .utf8_output() + } + + fn ensure_success(&mut self) -> Result<&mut Self>; + fn utf8_output(&mut self) -> Result; +} + +impl CommandExt for Command { + fn ensure_success(&mut self) -> Result<&mut Self> { + let status = self.status().with_context(|_| DidNotExecuteSnafu { + cmdline: get_cmdline(self), + })?; + ensure!( + status.success(), + ReturnedSnafu { + cmdline: get_cmdline(self), + code: status.code(), + } + ); + Ok(self) + } + + fn utf8_output(&mut self) -> Result { + let output = self.output().with_context(|_| DidNotExecuteSnafu { + cmdline: get_cmdline(self), + })?; + ensure!( + output.status.success(), + ReturnedSnafu { + cmdline: get_cmdline(self), + code: output.status.code(), + } + ); + Ok(Utf8Output { + stdout: String::from_utf8(output.stdout).with_context(|_| InvalidUtf8Snafu { + cmdline: get_cmdline(self), + })?, + stderr: String::from_utf8(output.stderr).with_context(|_| InvalidUtf8Snafu { + cmdline: get_cmdline(self), + })?, + }) + } +} + +fn get_cmdline(arg: &mut Command) -> String { + [arg.get_program().to_string_lossy()] + .into_iter() + .chain(arg.get_args().map(|arg| arg.to_string_lossy())) + .join(" ") +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_run() { + assert!(matches!( + Command::run(["fakefake", "1", "2"]), + Err(Error::DidNotExecute { + cmdline, + .. + }) if cmdline == "fakefake 1 2" + )); + #[cfg(not(windows))] + assert!(matches!( + Command::new("false").ensure_success(), + Err(Error::ReturnedError { code: Some(1), .. }) + )); + } +} diff --git a/tools/minilints/Cargo.toml b/tools/minilints/Cargo.toml index 40e80ac7f..5e329b385 100644 --- a/tools/minilints/Cargo.toml +++ b/tools/minilints/Cargo.toml @@ -9,6 +9,8 @@ license.workspace = true rust-version.workspace = true [dependencies] +anki_io = { version = "0.0.0", path = "../../rslib/io" } +anki_process = { version = "0.0.0", path = "../../rslib/process" } anyhow = "1.0.71" camino = "1.1.4" once_cell = "1.17.1" diff --git a/tools/minilints/src/main.rs b/tools/minilints/src/main.rs index bfd39b676..42d2a5684 100644 --- a/tools/minilints/src/main.rs +++ b/tools/minilints/src/main.rs @@ -10,6 +10,9 @@ use std::io::Write; use std::path::Path; use std::process::Command; +use anki_io::read_to_string; +use anki_io::write_file; +use anki_process::CommandExt; use anyhow::Context; use anyhow::Result; use camino::Utf8Path; @@ -43,6 +46,7 @@ fn main() -> Result<()> { let want_fix = env::args().nth(1) == Some("fix".to_string()); let mut ctx = LintContext::new(want_fix); ctx.check_contributors()?; + ctx.check_rust_licenses()?; ctx.walk_folders(Path::new("."))?; if ctx.found_problems { std::process::exit(1); @@ -147,7 +151,6 @@ impl LintContext { println!("Dependabot whitelisted."); return Ok(()); } else if all_contributors.contains(last_author.as_str()) { - println!("Author found in CONTRIBUTORS"); return Ok(()); } @@ -175,6 +178,35 @@ impl LintContext { std::process::exit(1); } + + fn check_rust_licenses(&mut self) -> Result<()> { + let license_path = Path::new("cargo/licenses.json"); + let licenses = generate_licences()?; + let existing_licenses = read_to_string(license_path)?; + if licenses != existing_licenses { + if self.want_fix { + check_cargo_deny()?; + update_hakari()?; + write_file(license_path, licenses)?; + } else { + println!("cargo/licenses.json is out of date; run ./ninja fix:minilints"); + self.found_problems = true; + } + } + Ok(()) + } +} + +fn check_cargo_deny() -> Result<()> { + Command::run(["cargo", "install", "-q", "cargo-deny@0.13.5"])?; + Command::run(["cargo", "deny", "check", "-A", "duplicate"])?; + Ok(()) +} + +fn update_hakari() -> Result<()> { + Command::run(["cargo", "install", "-q", "cargo-hakari@0.9.23"])?; + Command::run(["cargo", "hakari", "generate"])?; + Ok(()) } fn head_of_file(path: &Utf8Path) -> Result { @@ -223,3 +255,18 @@ fn check_for_unstaged_changes() { std::process::exit(1); } } + +fn generate_licences() -> Result { + Command::run(["cargo", "install", "-q", "cargo-license@0.5.1"])?; + let output = Command::run_with_output([ + "cargo-license", + "--features", + "rustls", + "--features", + "native-tls", + "--json", + "--manifest-path", + "rslib/Cargo.toml", + ])?; + Ok(output.stdout) +} diff --git a/tools/workspace-hack/Cargo.toml b/tools/workspace-hack/Cargo.toml index 9b4657225..3d9f7bb2d 100644 --- a/tools/workspace-hack/Cargo.toml +++ b/tools/workspace-hack/Cargo.toml @@ -39,7 +39,7 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_json = { version = "1", features = ["raw_value"] } sha2 = { version = "0.10" } -snafu = { version = "0.7", features = ["backtraces"] } +snafu = { version = "0.7", features = ["backtraces", "rust_1_61"] } time = { version = "0.3", features = ["formatting", "parsing"] } tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["codec", "io"] } @@ -80,7 +80,8 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive", "rc"] } serde_json = { version = "1", features = ["raw_value"] } sha2 = { version = "0.10" } -snafu = { version = "0.7", features = ["backtraces"] } +snafu = { version = "0.7", features = ["backtraces", "rust_1_61"] } +snafu-derive = { version = "0.7", default-features = false, features = ["rust_1_39", "rust_1_46", "rust_1_61"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit-mut"] } time = { version = "0.3", features = ["formatting", "parsing"] }