Add support for offline builds (#2963)

* CONTRIBUTORS: Add myself to the contributors list

* Add support for offline builds

Downloading files during build time is a non-starter for FreeBSD ports
(and presumably for other *BSD ports and some Linux distros as well).

In order to still be able to build Anki successfully, two new
environment variables have been added that can be set accordingly:

* NO_VENV: If set, the Python system environment is used instead of
  a venv. This is necessary if there are no usable Python wheels for a
  platform, e.g. PyQt6.

* OFFLINE_BUILD: If set, the git repository synchronization (translation
  files, build hash, etc.) is skipped.

To successfully build Anki offline, following conditions must be met:

1. All required dependencies (node, Python, rust, yarn, etc.) must be
   present in the build environment.

2. The offline repositories for the translation files must be
   copied/linked to ftl/qt-repo and ftl/core-repo.

3. The Python pseudo venv needs to be setup:

   $ mkdir out/pyenv/bin
   $ ln -s /path/to/python out/pyenv/bin/python
   $ ln -s /path/to/protoc-gen-mypy out/pyenv/bin/protoc-gen-mypy

4. Create the offline cache for yarn and use its own environment
   variable YARN_CACHE_FOLDER to it:

   YARN_CACHE_FOLDER=/path/to/the/yarn/cache
   $ /path/to/yarn install --ignore-scripts

5. Build Anki:

   $ /path/to/cargo build --package runner --release --verbose --verbose
   $ OFFLINE_BUILD=1 \
     NO_VENV=1 \
     ${WRKSRC}/out/rust/release/runner build wheels

CONTRIBUTORS

@@ -157,6 +157,7 @@ Marko Sisovic <msisovic13@gmail.com>
 Viktor Ricci <ricci@primateer.de>
 Harvey Randall <harveyrandall2001@gmail.com>
 Pedro Lameiras <pedrolameiras@tecnico.ulisboa.pt>
+Kai Knoblich <kai@FreeBSD.org>
 
 ********************
 

build/configure/src/bundle.rs

@@ -1,6 +1,8 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+use std::env;
+
 use anyhow::Result;
 use ninja_gen::action::BuildAction;
 use ninja_gen::archives::download_and_extract;
@@ -250,10 +252,13 @@ fn install_anki_wheels(build: &mut Build) -> Result<()> {
 }
 
 fn build_pyoxidizer(build: &mut Build) -> Result<()> {
+    let offline_build = env::var("OFFLINE_BUILD").is_ok();
+
     build.add_action(
         "bundle:pyoxidizer:repo",
         SyncSubmodule {
             path: "qt/bundle/PyOxidizer",
+            offline_build,
         },
     )?;
     build.add_action(

build/configure/src/main.rs

@@ -9,6 +9,8 @@ mod python;
 mod rust;
 mod web;
 
+use std::env;
+
 use anyhow::Result;
 use aqt::build_and_check_aqt;
 use bundle::build_bundle;
@@ -22,6 +24,7 @@ use pylib::build_pylib;
 use pylib::check_pylib;
 use python::check_python;
 use python::setup_venv;
+use python::setup_venv_stub;
 use rust::build_rust;
 use rust::check_minilints;
 use rust::check_rust;
@@ -45,7 +48,13 @@ fn main() -> Result<()> {
     check_proto(build, inputs![glob!["proto/**/*.proto"]])?;
 
     setup_python(build)?;
-    setup_venv(build)?;
+
+    if env::var("NO_VENV").is_ok() {
+        println!("NO_VENV is set, using Python system environment.");
+        setup_venv_stub(build)?;
+    } else {
+        setup_venv(build)?;
+    }
 
     build_rust(build)?;
     build_pylib(build)?;
@@ -53,7 +62,10 @@ fn main() -> Result<()> {
     build_and_check_aqt(build)?;
     build_bundle(build)?;
 
-    setup_sphix(build)?;
+    if env::var("OFFLINE_BUILD").is_err() {
+        println!("OFFLINE_BUILD is set, skipping build of offline documentation.");
+        setup_sphix(build)?;
+    }
 
     check_rust(build)?;
     check_pylib(build)?;

build/configure/src/python.rs

@@ -13,6 +13,7 @@ use ninja_gen::input::BuildInput;
 use ninja_gen::inputs;
 use ninja_gen::python::python_format;
 use ninja_gen::python::PythonEnvironment;
+use ninja_gen::python::PythonEnvironmentStub;
 use ninja_gen::python::PythonLint;
 use ninja_gen::python::PythonTypecheck;
 use ninja_gen::rsync::RsyncFiles;
@@ -81,6 +82,25 @@ pub fn setup_venv(build: &mut Build) -> Result<()> {
     Ok(())
 }
 
+pub fn setup_venv_stub(build: &mut Build) -> Result<()> {
+    build.add_action(
+        "pyenv",
+        PythonEnvironmentStub {
+            folder: "pyenv",
+            extra_binary_exports: &[
+                "mypy",
+                "black",  // Required in some parts of the code, but not for build
+                "isort",  // dito
+                "pylint", // dito
+                "pytest", // dito
+                "protoc-gen-mypy",
+            ],
+        },
+    )?;
+
+    Ok(())
+}
+
 pub struct GenPythonProto {
     pub proto_files: BuildInput,
 }

build/configure/src/rust.rs

@@ -1,6 +1,8 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+use std::env;
+
 use anyhow::Result;
 use ninja_gen::action::BuildAction;
 use ninja_gen::build::BuildProfile;
@@ -26,17 +28,21 @@ pub fn build_rust(build: &mut Build) -> Result<()> {
 }
 
 fn prepare_translations(build: &mut Build) -> Result<()> {
+    let offline_build = env::var("OFFLINE_BUILD").is_ok();
+
     // ensure repos are checked out
     build.add_action(
         "ftl:repo:core",
         SyncSubmodule {
             path: "ftl/core-repo",
+            offline_build,
         },
     )?;
     build.add_action(
         "ftl:repo:qt",
         SyncSubmodule {
             path: "ftl/qt-repo",
+            offline_build,
         },
     )?;
     // build anki_i18n and spit out strings.json

build/ninja_gen/src/git.rs

@@ -10,19 +10,27 @@ use crate::input::BuildInput;
 
 pub struct SyncSubmodule {
     pub path: &'static str,
+    pub offline_build: bool,
 }
 
 impl BuildAction for SyncSubmodule {
     fn command(&self) -> &str {
-        "git -c protocol.file.allow=always submodule update --init $path"
+        if self.offline_build {
+            "echo OFFLINE_BUILD is set, skipping git repository update for $path"
+        } else {
+            "git -c protocol.file.allow=always submodule update --init $path"
+        }
     }
 
     fn files(&mut self, build: &mut impl build::FilesHandle) {
-        if let Some(head) = locate_git_head() {
+        if !self.offline_build {
-            build.add_inputs("", head);
+            if let Some(head) = locate_git_head() {
-        } else {
+                build.add_inputs("", head);
-            println!("Warning, .git/HEAD not found; submodules may be stale");
+            } else {
+                println!("Warning, .git/HEAD not found; submodules may be stale");
+            }
         }
+
         build.add_variable("path", self.path);
         build.add_output_stamp(format!("git/{}", self.path));
     }

build/ninja_gen/src/python.rs

@@ -88,6 +88,11 @@ pub struct PythonEnvironment {
     pub extra_binary_exports: &'static [&'static str],
 }
 
+pub struct PythonEnvironmentStub {
+    pub folder: &'static str,
+    pub extra_binary_exports: &'static [&'static str],
+}
+
 impl BuildAction for PythonEnvironment {
     fn command(&self) -> &str {
         "$runner pyenv $python_binary $builddir/$pyenv_folder $system_pkgs $base_requirements $requirements"
@@ -120,6 +125,35 @@ impl BuildAction for PythonEnvironment {
     }
 }
 
+impl BuildAction for PythonEnvironmentStub {
+    fn command(&self) -> &str {
+        "echo Running PythonEnvironmentStub..."
+    }
+
+    fn files(&mut self, build: &mut impl crate::build::FilesHandle) {
+        let bin_path = |binary: &str| -> Vec<String> {
+            let folder = self.folder;
+            let path = if cfg!(windows) {
+                format!("{folder}/scripts/{binary}.exe")
+            } else {
+                format!("{folder}/bin/{binary}")
+            };
+            vec![path]
+        };
+
+        build.add_inputs("python_binary", inputs![":python_binary"]);
+        build.add_variable("pyenv_folder", self.folder);
+        build.add_outputs_ext("bin", bin_path("python"), true);
+        for binary in self.extra_binary_exports {
+            build.add_outputs_ext(*binary, bin_path(binary), true);
+        }
+    }
+
+    fn check_output_timestamps(&self) -> bool {
+        true
+    }
+}
+
 pub struct PythonTypecheck {
     pub folders: &'static [&'static str],
     pub deps: BuildInput,

build/runner/src/build.rs

@@ -155,7 +155,7 @@ fn bootstrap_build() {
 fn maybe_update_buildhash(build_root: &Utf8Path) {
     // only updated on release builds
     let path = build_root.join("buildhash");
-    if env::var("RELEASE").is_ok() || !path.exists() {
+    if (env::var("RELEASE").is_ok() && env::var("OFFLINE_BUILD").is_err()) || !path.exists() {
         write_if_changed(&path, &get_buildhash())
     }
 }

build/runner/src/yarn.rs

@@ -1,6 +1,7 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+use std::env;
 use std::path::Path;
 use std::process::Command;
 
@@ -17,7 +18,18 @@ pub struct YarnArgs {
 pub fn setup_yarn(args: YarnArgs) {
     link_node_modules();
 
-    run_command(Command::new(&args.yarn_bin).arg("install"));
+    if env::var("OFFLINE_BUILD").is_ok() {
+        println!("OFFLINE_BUILD is set");
+        println!("Running yarn with '--offline' and '--ignore-scripts'.");
+        run_command(
+            Command::new(&args.yarn_bin)
+                .arg("install")
+                .arg("--offline")
+                .arg("--ignore-scripts"),
+        );
+    } else {
+        run_command(Command::new(&args.yarn_bin).arg("install"));
+    }
 
     std::fs::write(args.stamp, b"").unwrap();
 }