Check URLs in TS code (#2436)

* Move help page URLs in ts to new file

* Unnest linkchecker test module

* Check TS help pages

* Add a comment (dae)

Cargo.lock

@@ -2011,7 +2011,9 @@ dependencies = [
  "anki",
  "futures",
  "itertools",
+ "lazy_static",
  "linkcheck",
+ "regex",
  "reqwest",
  "strum",
  "tokio",

rslib/linkchecker/Cargo.toml

@@ -19,6 +19,8 @@ linkcheck = { git = "https://github.com/ankitects/linkcheck.git", rev = "2f20798
 
 futures = "0.3.25"
 itertools = "0.10.5"
+lazy_static = "1.4.0"
+regex = "1.7.1"
 strum = { version = "0.24.1", features = ["derive"] }
 tokio = { version = "1.24.2", features = ["full"] }
 workspace-hack = { version = "0.1", path = "../../tools/workspace-hack" }

rslib/linkchecker/tests/links.rs

@@ -1,94 +1,137 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+#![cfg(test)]
+
+use std::borrow::Cow;
+use std::env;
+use std::iter;
+
 use anki::links::HelpPage;
+use futures::StreamExt;
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use linkcheck::validation::check_web;
+use linkcheck::validation::Context;
+use linkcheck::validation::Reason;
+use linkcheck::BasicContext;
+use regex::Regex;
+use reqwest::Url;
+use strum::IntoEnumIterator;
 
-#[cfg(test)]
+/// Aggregates [`Outcome`]s by collecting the error messages of the invalid
-mod test {
+/// ones.
-    use std::env;
+#[derive(Default)]
-    use std::iter;
+struct Outcomes(Vec<String>);
 
-    use futures::StreamExt;
+enum Outcome {
-    use itertools::Itertools;
+    Valid,
-    use linkcheck::validation::check_web;
+    Invalid(String),
-    use linkcheck::validation::Context;
+}
-    use linkcheck::validation::Reason;
-    use linkcheck::BasicContext;
-    use reqwest::Url;
-    use strum::IntoEnumIterator;
 
-    use super::*;
+#[derive(Clone)]
+enum CheckableUrl {
+    HelpPage(HelpPage),
+    String(&'static str),
+}
 
-    /// Aggregates [`Outcome`]s by collecting the error messages of the invalid
+impl CheckableUrl {
-    /// ones.
+    fn url(&self) -> Cow<str> {
-    #[derive(Default)]
+        match *self {
-    struct Outcomes(Vec<String>);
+            Self::HelpPage(page) => page.to_link().into(),
-
+            Self::String(s) => s.into(),
-    enum Outcome {
-        Valid,
-        Invalid(String),
-    }
-
-    #[tokio::test]
-    async fn check_links() {
-        if env::var("ONLINE_TESTS").is_err() {
-            println!("test disabled; ONLINE_TESTS not set");
-            return;
-        }
-        let ctx = BasicContext::default();
-        let result = futures::stream::iter(HelpPage::iter())
-            .map(|page| check_page(page, &ctx))
-            .buffer_unordered(ctx.concurrency())
-            .collect::<Outcomes>()
-            .await;
-        if !result.0.is_empty() {
-            panic!("{}", result.message());
         }
     }
 
-    async fn check_page(page: HelpPage, ctx: &BasicContext) -> Outcome {
+    fn anchor(&self) -> Cow<str> {
-        let link = page.to_link();
+        match *self {
-        match Url::parse(&link) {
+            Self::HelpPage(page) => page.to_link_suffix().into(),
-            Ok(url) => {
+            Self::String(s) => s.split('#').last().unwrap_or_default().into(),
-                if url.as_str() == link {
-                    match check_web(&url, ctx).await {
-                        Ok(()) => Outcome::Valid,
-                        Err(Reason::Dom) => Outcome::Invalid(format!(
-                            "'#{}' not found on '{}{}'",
-                            url.fragment().unwrap(),
-                            url.domain().unwrap(),
-                            url.path(),
-                        )),
-                        Err(Reason::Web(err)) => Outcome::Invalid(err.to_string()),
-                        _ => unreachable!(),
-                    }
-                } else {
-                    Outcome::Invalid(format!(
-                        "'{}' is not a valid URL part",
-                        page.to_link_suffix(),
-                    ))
-                }
-            }
-            Err(err) => Outcome::Invalid(err.to_string()),
-        }
-    }
-
-    impl Extend<Outcome> for Outcomes {
-        fn extend<T: IntoIterator<Item = Outcome>>(&mut self, items: T) {
-            for outcome in items {
-                match outcome {
-                    Outcome::Valid => (),
-                    Outcome::Invalid(err) => self.0.push(err),
-                }
-            }
-        }
-    }
-
-    impl Outcomes {
-        fn message(&self) -> String {
-            iter::once("invalid links found:")
-                .chain(self.0.iter().map(String::as_str))
-                .join("\n  - ")
         }
     }
 }
+
+impl From<HelpPage> for CheckableUrl {
+    fn from(value: HelpPage) -> Self {
+        Self::HelpPage(value)
+    }
+}
+
+impl From<&'static str> for CheckableUrl {
+    fn from(value: &'static str) -> Self {
+        Self::String(value)
+    }
+}
+
+fn ts_help_pages() -> impl Iterator<Item = &'static str> {
+    lazy_static! {
+        static ref QUOTED_URL: Regex = Regex::new("\"(http.+)\"").unwrap();
+    }
+    QUOTED_URL
+        .captures_iter(include_str!("../../../ts/lib/help-page.ts"))
+        .map(|caps| caps.get(1).unwrap().as_str())
+}
+
+#[tokio::test]
+async fn check_links() {
+    if env::var("ONLINE_TESTS").is_err() {
+        println!("test disabled; ONLINE_TESTS not set");
+        return;
+    }
+    let ctx = BasicContext::default();
+    let result = futures::stream::iter(
+        HelpPage::iter()
+            .map(CheckableUrl::from)
+            .chain(ts_help_pages().map(CheckableUrl::from)),
+    )
+    .map(|page| check_url(page, &ctx))
+    .buffer_unordered(ctx.concurrency())
+    .collect::<Outcomes>()
+    .await;
+    if !result.0.is_empty() {
+        panic!("{}", result.message());
+    }
+}
+
+async fn check_url(page: CheckableUrl, ctx: &BasicContext) -> Outcome {
+    let link = page.url();
+    match Url::parse(&link) {
+        Ok(url) => {
+            if url.as_str() == link {
+                match check_web(&url, ctx).await {
+                    Ok(()) => Outcome::Valid,
+                    Err(Reason::Dom) => Outcome::Invalid(format!(
+                        "'#{}' not found on '{}{}'",
+                        url.fragment().unwrap(),
+                        url.domain().unwrap(),
+                        url.path(),
+                    )),
+                    Err(Reason::Web(err)) => Outcome::Invalid(err.to_string()),
+                    _ => unreachable!(),
+                }
+            } else {
+                Outcome::Invalid(format!("'{}' is not a valid URL part", page.anchor(),))
+            }
+        }
+        Err(err) => Outcome::Invalid(err.to_string()),
+    }
+}
+
+impl Extend<Outcome> for Outcomes {
+    fn extend<T: IntoIterator<Item = Outcome>>(&mut self, items: T) {
+        for outcome in items {
+            match outcome {
+                Outcome::Valid => (),
+                Outcome::Invalid(err) => self.0.push(err),
+            }
+        }
+    }
+}
+
+impl Outcomes {
+    fn message(&self) -> String {
+        iter::once("invalid links found:")
+            .chain(self.0.iter().map(String::as_str))
+            .join("\n  - ")
+    }
+}

ts/deck-options/AdvancedOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -29,32 +30,32 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         maximumInterval: {
             title: tr.schedulingMaximumInterval(),
             help: tr.deckConfigMaximumIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#maximum-interval",
+            url: HelpPage.DeckOptions.maximumInterval,
         },
         startingEase: {
             title: tr.schedulingStartingEase(),
             help: tr.deckConfigStartingEaseTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#starting-ease",
+            url: HelpPage.DeckOptions.startingEase,
         },
         easyBonus: {
             title: tr.schedulingEasyBonus(),
             help: tr.deckConfigEasyBonusTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#easy-bonus",
+            url: HelpPage.DeckOptions.easyBonus,
         },
         intervalModifier: {
             title: tr.schedulingIntervalModifier(),
             help: tr.deckConfigIntervalModifierTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#interval-modifier",
+            url: HelpPage.DeckOptions.intervalModifier,
         },
         hardInterval: {
             title: tr.schedulingHardInterval(),
             help: tr.deckConfigHardIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#hard-interval",
+            url: HelpPage.DeckOptions.hardInterval,
         },
         newInterval: {
             title: tr.schedulingNewInterval(),
             help: tr.deckConfigNewIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#new-interval",
+            url: HelpPage.DeckOptions.newInterval,
         },
         customScheduling: {
             title: tr.deckConfigCustomScheduling(),
@@ -76,7 +77,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.deckConfigAdvancedTitle()}>
     <HelpModal
         title={tr.deckConfigAdvancedTitle()}
-        url="https://docs.ankiweb.net/deck-options.html#advanced"
+        url={HelpPage.DeckOptions.advanced}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/AudioOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -46,7 +47,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.deckConfigAudioTitle()}>
     <HelpModal
         title={tr.deckConfigAudioTitle()}
-        url="https://docs.ankiweb.net/deck-options.html#audio"
+        url={HelpPage.DeckOptions.audio}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/BuryOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -54,7 +55,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.deckConfigBuryTitle()}>
     <HelpModal
         title={tr.deckConfigBuryTitle()}
-        url="https://docs.ankiweb.net/studying.html#siblings-and-burying"
+        url={HelpPage.Studying.siblingsAndBurying}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/DailyLimits.svelte

@@ -4,6 +4,7 @@
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -135,17 +136,17 @@
         newLimit: {
             title: tr.schedulingNewCardsday(),
             help: tr.deckConfigNewLimitTooltip() + v3Extra,
-            url: "https://docs.ankiweb.net/deck-options.html#new-cardsday",
+            url: HelpPage.DeckOptions.newCardsday,
         },
         reviewLimit: {
             title: tr.schedulingMaximumReviewsday(),
             help: tr.deckConfigReviewLimitTooltip() + reviewV3Extra,
-            url: "https://docs.ankiweb.net/deck-options.html#maximum-reviewsday",
+            url: HelpPage.DeckOptions.maximumReviewsday,
         },
         newCardsIgnoreReviewLimit: {
             title: tr.deckConfigNewCardsIgnoreReviewLimit(),
             help: newCardsIgnoreReviewLimitHelp,
-            url: "https://docs.ankiweb.net/deck-options.html#new-cardsday",
+            url: HelpPage.DeckOptions.newCardsday,
         },
     };
     const helpSections = Object.values(settings) as DeckOption[];
@@ -162,7 +163,7 @@
 <TitledContainer title={tr.deckConfigDailyLimits()}>
     <HelpModal
         title={tr.deckConfigDailyLimits()}
-        url="https://docs.ankiweb.net/deck-options.html#daily-limits"
+        url={HelpPage.DeckOptions.dailyLimits}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/DisplayOrder.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import { DeckConfig } from "@tslib/proto";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
@@ -127,7 +128,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.deckConfigOrderingTitle()}>
     <HelpModal
         title={tr.deckConfigOrderingTitle()}
-        url="https://docs.ankiweb.net/deck-options.html#display-order"
+        url={HelpPage.DeckOptions.displayOrder}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/LapseOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -42,22 +43,22 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         relearningSteps: {
             title: tr.deckConfigRelearningSteps(),
             help: tr.deckConfigRelearningStepsTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#relearning-steps",
+            url: HelpPage.DeckOptions.relearningSteps,
         },
         minimumInterval: {
             title: tr.schedulingMinimumInterval(),
             help: tr.deckConfigMinimumIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#minimum-interval",
+            url: HelpPage.DeckOptions.minimumInterval,
         },
         leechThreshold: {
             title: tr.schedulingLeechThreshold(),
             help: tr.deckConfigLeechThresholdTooltip(),
-            url: "https://docs.ankiweb.net/leeches.html#leeches",
+            url: HelpPage.Leeches.leeches,
         },
         leechAction: {
             title: tr.schedulingLeechAction(),
             help: tr.deckConfigLeechActionTooltip(),
-            url: "https://docs.ankiweb.net/leeches.html#waiting",
+            url: HelpPage.Leeches.waiting,
         },
     };
     const helpSections = Object.values(settings) as DeckOption[];
@@ -74,7 +75,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.schedulingLapses()}>
     <HelpModal
         title={tr.schedulingLapses()}
-        url="https://docs.ankiweb.net/deck-options.html#lapses"
+        url={HelpPage.DeckOptions.lapses}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/NewOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import { DeckConfig } from "@tslib/proto";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
@@ -58,22 +59,22 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         learningSteps: {
             title: tr.deckConfigLearningSteps(),
             help: tr.deckConfigLearningStepsTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#learning-steps",
+            url: HelpPage.DeckOptions.learningSteps,
         },
         graduatingInterval: {
             title: tr.schedulingGraduatingInterval(),
             help: tr.deckConfigGraduatingIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#graduating-interval",
+            url: HelpPage.DeckOptions.graduatingInterval,
         },
         easyInterval: {
             title: tr.schedulingEasyInterval(),
             help: tr.deckConfigEasyIntervalTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#easy-interval",
+            url: HelpPage.DeckOptions.easyInterval,
         },
         insertionOrder: {
             title: tr.deckConfigNewInsertionOrder(),
             help: tr.deckConfigNewInsertionOrderTooltip(),
-            url: "https://docs.ankiweb.net/deck-options.html#insertion-order",
+            url: HelpPage.DeckOptions.insertionOrder,
         },
     };
     const helpSections = Object.values(settings) as DeckOption[];
@@ -90,7 +91,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.schedulingNewCards()}>
     <HelpModal
         title={tr.schedulingNewCards()}
-        url="https://docs.ankiweb.net/deck-options.html#new-cards"
+        url={HelpPage.DeckOptions.newCards}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/deck-options/TimerOptions.svelte

@@ -4,6 +4,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
     import * as tr from "@tslib/ftl";
+    import { HelpPage } from "@tslib/help-page";
     import type Carousel from "bootstrap/js/dist/carousel";
     import type Modal from "bootstrap/js/dist/modal";
 
@@ -53,7 +54,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 <TitledContainer title={tr.deckConfigTimerTitle()}>
     <HelpModal
         title={tr.deckConfigTimerTitle()}
-        url="https://docs.ankiweb.net/deck-options.html#timer"
+        url={HelpPage.DeckOptions.timer}
         slot="tooltip"
         {helpSections}
         on:mount={(e) => {

ts/lib/help-page.ts

@@ -0,0 +1,36 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/// These links are checked in CI to ensure they are valid.
+export const HelpPage = {
+    DeckOptions: {
+        maximumInterval: "https://docs.ankiweb.net/deck-options.html#maximum-interval",
+        startingEase: "https://docs.ankiweb.net/deck-options.html#starting-ease",
+        easyBonus: "https://docs.ankiweb.net/deck-options.html#easy-bonus",
+        intervalModifier: "https://docs.ankiweb.net/deck-options.html#interval-modifier",
+        hardInterval: "https://docs.ankiweb.net/deck-options.html#hard-interval",
+        newInterval: "https://docs.ankiweb.net/deck-options.html#new-interval",
+        advanced: "https://docs.ankiweb.net/deck-options.html#advanced",
+        timer: "https://docs.ankiweb.net/deck-options.html#timer",
+        learningSteps: "https://docs.ankiweb.net/deck-options.html#learning-steps",
+        graduatingInterval: "https://docs.ankiweb.net/deck-options.html#graduating-interval",
+        easyInterval: "https://docs.ankiweb.net/deck-options.html#easy-interval",
+        insertionOrder: "https://docs.ankiweb.net/deck-options.html#insertion-order",
+        newCards: "https://docs.ankiweb.net/deck-options.html#new-cards",
+        relearningSteps: "https://docs.ankiweb.net/deck-options.html#relearning-steps",
+        minimumInterval: "https://docs.ankiweb.net/deck-options.html#minimum-interval",
+        lapses: "https://docs.ankiweb.net/deck-options.html#lapses",
+        displayOrder: "https://docs.ankiweb.net/deck-options.html#display-order",
+        maximumReviewsday: "https://docs.ankiweb.net/deck-options.html#maximum-reviewsday",
+        newCardsday: "https://docs.ankiweb.net/deck-options.html#new-cardsday",
+        dailyLimits: "https://docs.ankiweb.net/deck-options.html#daily-limits",
+        audio: "https://docs.ankiweb.net/deck-options.html#audio",
+    },
+    Leeches: {
+        leeches: "https://docs.ankiweb.net/leeches.html#leeches",
+        waiting: "https://docs.ankiweb.net/leeches.html#waiting",
+    },
+    Studying: {
+        siblingsAndBurying: "https://docs.ankiweb.net/studying.html#siblings-and-burying",
+    },
+};