From f9126927b1c537f76b2e11f2f8a6003d2313245b Mon Sep 17 00:00:00 2001 From: RumovZ Date: Wed, 15 Mar 2023 06:46:03 +0100 Subject: [PATCH] Check URLs in TS code (#2436) * Move help page URLs in ts to new file * Unnest linkchecker test module * Check TS help pages * Add a comment (dae) --- Cargo.lock | 2 + rslib/linkchecker/Cargo.toml | 2 + rslib/linkchecker/tests/links.rs | 203 +++++++++++++++---------- ts/deck-options/AdvancedOptions.svelte | 15 +- ts/deck-options/AudioOptions.svelte | 3 +- ts/deck-options/BuryOptions.svelte | 3 +- ts/deck-options/DailyLimits.svelte | 9 +- ts/deck-options/DisplayOrder.svelte | 3 +- ts/deck-options/LapseOptions.svelte | 11 +- ts/deck-options/NewOptions.svelte | 11 +- ts/deck-options/TimerOptions.svelte | 3 +- ts/lib/help-page.ts | 36 +++++ 12 files changed, 196 insertions(+), 105 deletions(-) create mode 100644 ts/lib/help-page.ts diff --git a/Cargo.lock b/Cargo.lock index 8ea5b3c88..2f26a507f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2011,7 +2011,9 @@ dependencies = [ "anki", "futures", "itertools", + "lazy_static", "linkcheck", + "regex", "reqwest", "strum", "tokio", diff --git a/rslib/linkchecker/Cargo.toml b/rslib/linkchecker/Cargo.toml index 62321a087..599063ef2 100644 --- a/rslib/linkchecker/Cargo.toml +++ b/rslib/linkchecker/Cargo.toml @@ -19,6 +19,8 @@ linkcheck = { git = "https://github.com/ankitects/linkcheck.git", rev = "2f20798 futures = "0.3.25" itertools = "0.10.5" +lazy_static = "1.4.0" +regex = "1.7.1" strum = { version = "0.24.1", features = ["derive"] } tokio = { version = "1.24.2", features = ["full"] } workspace-hack = { version = "0.1", path = "../../tools/workspace-hack" } diff --git a/rslib/linkchecker/tests/links.rs b/rslib/linkchecker/tests/links.rs index 35c5ee23d..0b0d1001e 100644 --- a/rslib/linkchecker/tests/links.rs +++ b/rslib/linkchecker/tests/links.rs @@ -1,94 +1,137 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +#![cfg(test)] + +use std::borrow::Cow; +use std::env; +use std::iter; + use anki::links::HelpPage; +use futures::StreamExt; +use itertools::Itertools; +use lazy_static::lazy_static; +use linkcheck::validation::check_web; +use linkcheck::validation::Context; +use linkcheck::validation::Reason; +use linkcheck::BasicContext; +use regex::Regex; +use reqwest::Url; +use strum::IntoEnumIterator; -#[cfg(test)] -mod test { - use std::env; - use std::iter; +/// Aggregates [`Outcome`]s by collecting the error messages of the invalid +/// ones. +#[derive(Default)] +struct Outcomes(Vec); - use futures::StreamExt; - use itertools::Itertools; - use linkcheck::validation::check_web; - use linkcheck::validation::Context; - use linkcheck::validation::Reason; - use linkcheck::BasicContext; - use reqwest::Url; - use strum::IntoEnumIterator; +enum Outcome { + Valid, + Invalid(String), +} - use super::*; +#[derive(Clone)] +enum CheckableUrl { + HelpPage(HelpPage), + String(&'static str), +} - /// Aggregates [`Outcome`]s by collecting the error messages of the invalid - /// ones. - #[derive(Default)] - struct Outcomes(Vec); - - enum Outcome { - Valid, - Invalid(String), - } - - #[tokio::test] - async fn check_links() { - if env::var("ONLINE_TESTS").is_err() { - println!("test disabled; ONLINE_TESTS not set"); - return; - } - let ctx = BasicContext::default(); - let result = futures::stream::iter(HelpPage::iter()) - .map(|page| check_page(page, &ctx)) - .buffer_unordered(ctx.concurrency()) - .collect::() - .await; - if !result.0.is_empty() { - panic!("{}", result.message()); +impl CheckableUrl { + fn url(&self) -> Cow { + match *self { + Self::HelpPage(page) => page.to_link().into(), + Self::String(s) => s.into(), } } - async fn check_page(page: HelpPage, ctx: &BasicContext) -> Outcome { - let link = page.to_link(); - match Url::parse(&link) { - Ok(url) => { - if url.as_str() == link { - match check_web(&url, ctx).await { - Ok(()) => Outcome::Valid, - Err(Reason::Dom) => Outcome::Invalid(format!( - "'#{}' not found on '{}{}'", - url.fragment().unwrap(), - url.domain().unwrap(), - url.path(), - )), - Err(Reason::Web(err)) => Outcome::Invalid(err.to_string()), - _ => unreachable!(), - } - } else { - Outcome::Invalid(format!( - "'{}' is not a valid URL part", - page.to_link_suffix(), - )) - } - } - Err(err) => Outcome::Invalid(err.to_string()), - } - } - - impl Extend for Outcomes { - fn extend>(&mut self, items: T) { - for outcome in items { - match outcome { - Outcome::Valid => (), - Outcome::Invalid(err) => self.0.push(err), - } - } - } - } - - impl Outcomes { - fn message(&self) -> String { - iter::once("invalid links found:") - .chain(self.0.iter().map(String::as_str)) - .join("\n - ") + fn anchor(&self) -> Cow { + match *self { + Self::HelpPage(page) => page.to_link_suffix().into(), + Self::String(s) => s.split('#').last().unwrap_or_default().into(), } } } + +impl From for CheckableUrl { + fn from(value: HelpPage) -> Self { + Self::HelpPage(value) + } +} + +impl From<&'static str> for CheckableUrl { + fn from(value: &'static str) -> Self { + Self::String(value) + } +} + +fn ts_help_pages() -> impl Iterator { + lazy_static! { + static ref QUOTED_URL: Regex = Regex::new("\"(http.+)\"").unwrap(); + } + QUOTED_URL + .captures_iter(include_str!("../../../ts/lib/help-page.ts")) + .map(|caps| caps.get(1).unwrap().as_str()) +} + +#[tokio::test] +async fn check_links() { + if env::var("ONLINE_TESTS").is_err() { + println!("test disabled; ONLINE_TESTS not set"); + return; + } + let ctx = BasicContext::default(); + let result = futures::stream::iter( + HelpPage::iter() + .map(CheckableUrl::from) + .chain(ts_help_pages().map(CheckableUrl::from)), + ) + .map(|page| check_url(page, &ctx)) + .buffer_unordered(ctx.concurrency()) + .collect::() + .await; + if !result.0.is_empty() { + panic!("{}", result.message()); + } +} + +async fn check_url(page: CheckableUrl, ctx: &BasicContext) -> Outcome { + let link = page.url(); + match Url::parse(&link) { + Ok(url) => { + if url.as_str() == link { + match check_web(&url, ctx).await { + Ok(()) => Outcome::Valid, + Err(Reason::Dom) => Outcome::Invalid(format!( + "'#{}' not found on '{}{}'", + url.fragment().unwrap(), + url.domain().unwrap(), + url.path(), + )), + Err(Reason::Web(err)) => Outcome::Invalid(err.to_string()), + _ => unreachable!(), + } + } else { + Outcome::Invalid(format!("'{}' is not a valid URL part", page.anchor(),)) + } + } + Err(err) => Outcome::Invalid(err.to_string()), + } +} + +impl Extend for Outcomes { + fn extend>(&mut self, items: T) { + for outcome in items { + match outcome { + Outcome::Valid => (), + Outcome::Invalid(err) => self.0.push(err), + } + } + } +} + +impl Outcomes { + fn message(&self) -> String { + iter::once("invalid links found:") + .chain(self.0.iter().map(String::as_str)) + .join("\n - ") + } +} diff --git a/ts/deck-options/AdvancedOptions.svelte b/ts/deck-options/AdvancedOptions.svelte index f0e59c0f2..741fc8633 100644 --- a/ts/deck-options/AdvancedOptions.svelte +++ b/ts/deck-options/AdvancedOptions.svelte @@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -->