Refactor offline build process and add offline generation of Sphinx docs (#3082)

* Simplify the offline build

The two environment variables OFFLINE_BUILD and NO_VENV jointly provide
the ability to build Anki fully offline. This commit boils them down
into just one, namely OFFLINE_BUILD.

The rationale being that first, OFFLINE_BUILD implies the use of
a custom non-networked Python environment.
Second, building Anki with a custom Python environment in a networked
setting is a use case, that we currently do not support.
Developers in need of such a solution may want to give containerized
development environments a try. Users could also look into building
Anki fully offline instead.

* Add documentation for offline builds.

* Add support for offline generation of Sphinx documentation.

Control installation of Sphinx dependencies via the network through the
OFFLINE_BUILD environment variable.

* Add documentation for offline generation of Sphinx documentation.

build/configure/src/main.rs

@@ -24,14 +24,13 @@ use pylib::build_pylib;
 use pylib::check_pylib;
 use python::check_python;
 use python::setup_venv;
-use python::setup_venv_stub;
 use rust::build_rust;
 use rust::check_minilints;
 use rust::check_rust;
 use web::build_and_check_web;
 use web::check_sql;
 
-use crate::python::setup_sphix;
+use crate::python::setup_sphinx;
 
 fn anki_version() -> String {
     std::fs::read_to_string(".version")
@@ -47,25 +46,22 @@ fn main() -> Result<()> {
     setup_protoc(build)?;
     check_proto(build, inputs![glob!["proto/**/*.proto"]])?;
 
-    setup_python(build)?;
+    if env::var("OFFLINE_BUILD").is_err() {
-
+        setup_python(build)?;
-    if env::var("NO_VENV").is_ok() {
-        println!("NO_VENV is set, using Python system environment.");
-        setup_venv_stub(build)?;
-    } else {
-        setup_venv(build)?;
     }
+    setup_venv(build)?;
 
     build_rust(build)?;
     build_pylib(build)?;
     build_and_check_web(build)?;
     build_and_check_aqt(build)?;
-    build_bundle(build)?;
 
     if env::var("OFFLINE_BUILD").is_err() {
-        setup_sphix(build)?;
+        build_bundle(build)?;
     }
 
+    setup_sphinx(build)?;
+
     check_rust(build)?;
     check_pylib(build)?;
     check_python(build)?;

build/configure/src/python.rs

@@ -1,6 +1,8 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+use std::env;
+
 use anyhow::Result;
 use ninja_gen::action::BuildAction;
 use ninja_gen::archives::Platform;
@@ -13,7 +15,6 @@ use ninja_gen::input::BuildInput;
 use ninja_gen::inputs;
 use ninja_gen::python::python_format;
 use ninja_gen::python::PythonEnvironment;
-use ninja_gen::python::PythonEnvironmentStub;
 use ninja_gen::python::PythonLint;
 use ninja_gen::python::PythonTypecheck;
 use ninja_gen::rsync::RsyncFiles;
@@ -45,11 +46,11 @@ pub fn setup_venv(build: &mut Build) -> Result<()> {
                 "pip-compile",
                 "pip-sync",
                 "mypy",
-                "black",
+                "black", // Required for offline build
                 "isort",
                 "pylint",
                 "pytest",
-                "protoc-gen-mypy",
+                "protoc-gen-mypy", // ditto
             ],
         },
     )?;
@@ -82,25 +83,6 @@ pub fn setup_venv(build: &mut Build) -> Result<()> {
     Ok(())
 }
 
-pub fn setup_venv_stub(build: &mut Build) -> Result<()> {
-    build.add_action(
-        "pyenv",
-        PythonEnvironmentStub {
-            folder: "pyenv",
-            extra_binary_exports: &[
-                "mypy",
-                "black",  // Required in some parts of the code, but not for build
-                "isort",  // dito
-                "pylint", // dito
-                "pytest", // dito
-                "protoc-gen-mypy",
-            ],
-        },
-    )?;
-
-    Ok(())
-}
-
 pub struct GenPythonProto {
     pub proto_files: BuildInput,
 }
@@ -268,13 +250,19 @@ struct Sphinx {
 
 impl BuildAction for Sphinx {
     fn command(&self) -> &str {
-        "$pip install sphinx sphinx_rtd_theme sphinx-autoapi \
+        if env::var("OFFLINE_BUILD").is_err() {
-         && $python python/sphinx/build.py"
+            "$pip install sphinx sphinx_rtd_theme sphinx-autoapi \
+             && $python python/sphinx/build.py"
+        } else {
+            "$python python/sphinx/build.py"
+        }
     }
 
     fn files(&mut self, build: &mut impl FilesHandle) {
+        if env::var("OFFLINE_BUILD").is_err() {
+            build.add_inputs("pip", inputs![":pyenv:pip"]);
+        }
         build.add_inputs("python", inputs![":pyenv:bin"]);
-        build.add_inputs("pip", inputs![":pyenv:pip"]);
         build.add_inputs("", &self.deps);
         build.add_output_stamp("python/sphinx/stamp");
     }
@@ -284,7 +272,7 @@ impl BuildAction for Sphinx {
     }
 }
 
-pub(crate) fn setup_sphix(build: &mut Build) -> Result<()> {
+pub(crate) fn setup_sphinx(build: &mut Build) -> Result<()> {
     build.add_action(
         "python:sphinx:copy_conf",
         CopyFiles {

build/ninja_gen/src/python.rs

@@ -88,48 +88,15 @@ pub struct PythonEnvironment {
     pub extra_binary_exports: &'static [&'static str],
 }
 
-pub struct PythonEnvironmentStub {
-    pub folder: &'static str,
-    pub extra_binary_exports: &'static [&'static str],
-}
-
 impl BuildAction for PythonEnvironment {
     fn command(&self) -> &str {
-        "$runner pyenv $python_binary $builddir/$pyenv_folder $system_pkgs $base_requirements $requirements"
+        if env::var("OFFLINE_BUILD").is_err() {
-    }
+            "$runner pyenv $python_binary $builddir/$pyenv_folder $system_pkgs $base_requirements $requirements"
-
+        } else {
-    fn files(&mut self, build: &mut impl crate::build::FilesHandle) {
+            "echo 'OFFLINE_BUILD is set. Using the existing PythonEnvironment.'"
-        let bin_path = |binary: &str| -> Vec<String> {
-            let folder = self.folder;
-            let path = if cfg!(windows) {
-                format!("{folder}/scripts/{binary}.exe")
-            } else {
-                format!("{folder}/bin/{binary}")
-            };
-            vec![path]
-        };
-
-        build.add_inputs("python_binary", inputs![":python_binary"]);
-        build.add_inputs("base_requirements", &self.base_requirements_txt);
-        build.add_inputs("requirements", &self.requirements_txt);
-        build.add_variable("pyenv_folder", self.folder);
-        build.add_outputs_ext("bin", bin_path("python"), true);
-        build.add_outputs_ext("pip", bin_path("pip"), true);
-        for binary in self.extra_binary_exports {
-            build.add_outputs_ext(*binary, bin_path(binary), true);
         }
     }
 
-    fn check_output_timestamps(&self) -> bool {
-        true
-    }
-}
-
-impl BuildAction for PythonEnvironmentStub {
-    fn command(&self) -> &str {
-        "echo Running PythonEnvironmentStub..."
-    }
-
     fn files(&mut self, build: &mut impl crate::build::FilesHandle) {
         let bin_path = |binary: &str| -> Vec<String> {
             let folder = self.folder;
@@ -141,8 +108,13 @@ impl BuildAction for PythonEnvironmentStub {
             vec![path]
         };
 
-        build.add_inputs("python_binary", inputs![":python_binary"]);
+        if env::var("OFFLINE_BUILD").is_err() {
-        build.add_variable("pyenv_folder", self.folder);
+            build.add_inputs("python_binary", inputs![":python_binary"]);
+            build.add_variable("pyenv_folder", self.folder);
+            build.add_inputs("base_requirements", &self.base_requirements_txt);
+            build.add_inputs("requirements", &self.requirements_txt);
+            build.add_outputs_ext("pip", bin_path("pip"), true);
+        }
         build.add_outputs_ext("bin", bin_path("python"), true);
         for binary in self.extra_binary_exports {
             build.add_outputs_ext(*binary, bin_path(binary), true);

docs/linux.md

@@ -84,6 +84,52 @@ PYTHON_BINARY, NODE_BINARY, YARN_BINARY and/or PROTOC_BINARY to use locally-inst
 If rust-toolchain.toml is removed, newer Rust versions can be used. Older versions
 may or may not compile the code.
 
+To build Anki fully offline, set the following environment variables:
+
+- OFFLINE_BUILD: If set, the build does not run tools that may access
+  the network.
+
+- NODE_BINARY, YARN_BINARY and PROTOC_BINARY must also be set.
+
+With OFFLINE_BUILD defined, manual intervention is required for the
+offline build to succeed. The following conditions must be met:
+
+1. All required dependencies (node, Python, rust, yarn, etc.) must be
+   present in the build environment.
+
+2. The offline repositories for the translation files must be
+   copied/linked to ftl/qt-repo and ftl/core-repo.
+
+3. The Python pseudo venv must be set up:
+
+   ```
+   mkdir out/pyenv/bin
+   ln -s /path/to/python out/pyenv/bin/python
+   ln -s /path/to/protoc-gen-mypy out/pyenv/bin/protoc-gen-mypy
+   ```
+
+   Optionally, set up your environment to generate Sphinx documentation:
+
+   ```
+   ln -s /path/to/sphinx-apidoc out/pyenv/bin/sphinx-apidoc
+   ln -s /path/to/sphinx-build out/pyenv/bin/sphinx-build
+   ```
+
+   Note that the PYTHON_BINARY environment variable need not be set,
+   since it is only used when OFFLINE_BUILD is unset to automatically
+   create a network-dependent Python venv.
+
+4. Create the offline cache for yarn and use its own environment
+   variable YARN_CACHE_FOLDER to it:
+
+   ```
+   YARN_CACHE_FOLDER=/path/to/the/yarn/cache
+   /path/to/yarn install --ignore-scripts
+   ```
+
+You are now ready to build wheels and Sphinx documentation fully
+offline.
+
 ## More
 
 For info on running tests, building wheels and so on, please see [Development](./development.md).