From 42cc2c913ce53b5057ce890edb0b4b09d73c8dfe Mon Sep 17 00:00:00 2001 From: Kai Knoblich <43905002+knobix@users.noreply.github.com> Date: Wed, 31 Jan 2024 00:13:46 +0100 Subject: [PATCH] Add support for offline builds (#2963) * CONTRIBUTORS: Add myself to the contributors list * Add support for offline builds Downloading files during build time is a non-starter for FreeBSD ports (and presumably for other *BSD ports and some Linux distros as well). In order to still be able to build Anki successfully, two new environment variables have been added that can be set accordingly: * NO_VENV: If set, the Python system environment is used instead of a venv. This is necessary if there are no usable Python wheels for a platform, e.g. PyQt6. * OFFLINE_BUILD: If set, the git repository synchronization (translation files, build hash, etc.) is skipped. To successfully build Anki offline, following conditions must be met: 1. All required dependencies (node, Python, rust, yarn, etc.) must be present in the build environment. 2. The offline repositories for the translation files must be copied/linked to ftl/qt-repo and ftl/core-repo. 3. The Python pseudo venv needs to be setup: $ mkdir out/pyenv/bin $ ln -s /path/to/python out/pyenv/bin/python $ ln -s /path/to/protoc-gen-mypy out/pyenv/bin/protoc-gen-mypy 4. Create the offline cache for yarn and use its own environment variable YARN_CACHE_FOLDER to it: YARN_CACHE_FOLDER=/path/to/the/yarn/cache $ /path/to/yarn install --ignore-scripts 5. Build Anki: $ /path/to/cargo build --package runner --release --verbose --verbose $ OFFLINE_BUILD=1 \ NO_VENV=1 \ ${WRKSRC}/out/rust/release/runner build wheels --- CONTRIBUTORS | 1 + build/configure/src/bundle.rs | 5 +++++ build/configure/src/main.rs | 16 ++++++++++++++-- build/configure/src/python.rs | 20 ++++++++++++++++++++ build/configure/src/rust.rs | 6 ++++++ build/ninja_gen/src/git.rs | 18 +++++++++++++----- build/ninja_gen/src/python.rs | 34 ++++++++++++++++++++++++++++++++++ build/runner/src/build.rs | 2 +- build/runner/src/yarn.rs | 14 +++++++++++++- 9 files changed, 107 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 0c6e80436..ed44c40d6 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -157,6 +157,7 @@ Marko Sisovic Viktor Ricci Harvey Randall Pedro Lameiras +Kai Knoblich ******************** diff --git a/build/configure/src/bundle.rs b/build/configure/src/bundle.rs index 4b2552160..15369f389 100644 --- a/build/configure/src/bundle.rs +++ b/build/configure/src/bundle.rs @@ -1,6 +1,8 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use std::env; + use anyhow::Result; use ninja_gen::action::BuildAction; use ninja_gen::archives::download_and_extract; @@ -250,10 +252,13 @@ fn install_anki_wheels(build: &mut Build) -> Result<()> { } fn build_pyoxidizer(build: &mut Build) -> Result<()> { + let offline_build = env::var("OFFLINE_BUILD").is_ok(); + build.add_action( "bundle:pyoxidizer:repo", SyncSubmodule { path: "qt/bundle/PyOxidizer", + offline_build, }, )?; build.add_action( diff --git a/build/configure/src/main.rs b/build/configure/src/main.rs index 7c6f65133..b372fb77d 100644 --- a/build/configure/src/main.rs +++ b/build/configure/src/main.rs @@ -9,6 +9,8 @@ mod python; mod rust; mod web; +use std::env; + use anyhow::Result; use aqt::build_and_check_aqt; use bundle::build_bundle; @@ -22,6 +24,7 @@ use pylib::build_pylib; use pylib::check_pylib; use python::check_python; use python::setup_venv; +use python::setup_venv_stub; use rust::build_rust; use rust::check_minilints; use rust::check_rust; @@ -45,7 +48,13 @@ fn main() -> Result<()> { check_proto(build, inputs![glob!["proto/**/*.proto"]])?; setup_python(build)?; - setup_venv(build)?; + + if env::var("NO_VENV").is_ok() { + println!("NO_VENV is set, using Python system environment."); + setup_venv_stub(build)?; + } else { + setup_venv(build)?; + } build_rust(build)?; build_pylib(build)?; @@ -53,7 +62,10 @@ fn main() -> Result<()> { build_and_check_aqt(build)?; build_bundle(build)?; - setup_sphix(build)?; + if env::var("OFFLINE_BUILD").is_err() { + println!("OFFLINE_BUILD is set, skipping build of offline documentation."); + setup_sphix(build)?; + } check_rust(build)?; check_pylib(build)?; diff --git a/build/configure/src/python.rs b/build/configure/src/python.rs index c08c21c00..3ae66a416 100644 --- a/build/configure/src/python.rs +++ b/build/configure/src/python.rs @@ -13,6 +13,7 @@ use ninja_gen::input::BuildInput; use ninja_gen::inputs; use ninja_gen::python::python_format; use ninja_gen::python::PythonEnvironment; +use ninja_gen::python::PythonEnvironmentStub; use ninja_gen::python::PythonLint; use ninja_gen::python::PythonTypecheck; use ninja_gen::rsync::RsyncFiles; @@ -81,6 +82,25 @@ pub fn setup_venv(build: &mut Build) -> Result<()> { Ok(()) } +pub fn setup_venv_stub(build: &mut Build) -> Result<()> { + build.add_action( + "pyenv", + PythonEnvironmentStub { + folder: "pyenv", + extra_binary_exports: &[ + "mypy", + "black", // Required in some parts of the code, but not for build + "isort", // dito + "pylint", // dito + "pytest", // dito + "protoc-gen-mypy", + ], + }, + )?; + + Ok(()) +} + pub struct GenPythonProto { pub proto_files: BuildInput, } diff --git a/build/configure/src/rust.rs b/build/configure/src/rust.rs index 4fb85e551..880b16a95 100644 --- a/build/configure/src/rust.rs +++ b/build/configure/src/rust.rs @@ -1,6 +1,8 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use std::env; + use anyhow::Result; use ninja_gen::action::BuildAction; use ninja_gen::build::BuildProfile; @@ -26,17 +28,21 @@ pub fn build_rust(build: &mut Build) -> Result<()> { } fn prepare_translations(build: &mut Build) -> Result<()> { + let offline_build = env::var("OFFLINE_BUILD").is_ok(); + // ensure repos are checked out build.add_action( "ftl:repo:core", SyncSubmodule { path: "ftl/core-repo", + offline_build, }, )?; build.add_action( "ftl:repo:qt", SyncSubmodule { path: "ftl/qt-repo", + offline_build, }, )?; // build anki_i18n and spit out strings.json diff --git a/build/ninja_gen/src/git.rs b/build/ninja_gen/src/git.rs index 2b996a325..5499f8946 100644 --- a/build/ninja_gen/src/git.rs +++ b/build/ninja_gen/src/git.rs @@ -10,19 +10,27 @@ use crate::input::BuildInput; pub struct SyncSubmodule { pub path: &'static str, + pub offline_build: bool, } impl BuildAction for SyncSubmodule { fn command(&self) -> &str { - "git -c protocol.file.allow=always submodule update --init $path" + if self.offline_build { + "echo OFFLINE_BUILD is set, skipping git repository update for $path" + } else { + "git -c protocol.file.allow=always submodule update --init $path" + } } fn files(&mut self, build: &mut impl build::FilesHandle) { - if let Some(head) = locate_git_head() { - build.add_inputs("", head); - } else { - println!("Warning, .git/HEAD not found; submodules may be stale"); + if !self.offline_build { + if let Some(head) = locate_git_head() { + build.add_inputs("", head); + } else { + println!("Warning, .git/HEAD not found; submodules may be stale"); + } } + build.add_variable("path", self.path); build.add_output_stamp(format!("git/{}", self.path)); } diff --git a/build/ninja_gen/src/python.rs b/build/ninja_gen/src/python.rs index 2614b623d..a97050b17 100644 --- a/build/ninja_gen/src/python.rs +++ b/build/ninja_gen/src/python.rs @@ -88,6 +88,11 @@ pub struct PythonEnvironment { pub extra_binary_exports: &'static [&'static str], } +pub struct PythonEnvironmentStub { + pub folder: &'static str, + pub extra_binary_exports: &'static [&'static str], +} + impl BuildAction for PythonEnvironment { fn command(&self) -> &str { "$runner pyenv $python_binary $builddir/$pyenv_folder $system_pkgs $base_requirements $requirements" @@ -120,6 +125,35 @@ impl BuildAction for PythonEnvironment { } } +impl BuildAction for PythonEnvironmentStub { + fn command(&self) -> &str { + "echo Running PythonEnvironmentStub..." + } + + fn files(&mut self, build: &mut impl crate::build::FilesHandle) { + let bin_path = |binary: &str| -> Vec { + let folder = self.folder; + let path = if cfg!(windows) { + format!("{folder}/scripts/{binary}.exe") + } else { + format!("{folder}/bin/{binary}") + }; + vec![path] + }; + + build.add_inputs("python_binary", inputs![":python_binary"]); + build.add_variable("pyenv_folder", self.folder); + build.add_outputs_ext("bin", bin_path("python"), true); + for binary in self.extra_binary_exports { + build.add_outputs_ext(*binary, bin_path(binary), true); + } + } + + fn check_output_timestamps(&self) -> bool { + true + } +} + pub struct PythonTypecheck { pub folders: &'static [&'static str], pub deps: BuildInput, diff --git a/build/runner/src/build.rs b/build/runner/src/build.rs index e10c045fe..e1d4eefde 100644 --- a/build/runner/src/build.rs +++ b/build/runner/src/build.rs @@ -155,7 +155,7 @@ fn bootstrap_build() { fn maybe_update_buildhash(build_root: &Utf8Path) { // only updated on release builds let path = build_root.join("buildhash"); - if env::var("RELEASE").is_ok() || !path.exists() { + if (env::var("RELEASE").is_ok() && env::var("OFFLINE_BUILD").is_err()) || !path.exists() { write_if_changed(&path, &get_buildhash()) } } diff --git a/build/runner/src/yarn.rs b/build/runner/src/yarn.rs index 20fd144b6..9e1bd5b58 100644 --- a/build/runner/src/yarn.rs +++ b/build/runner/src/yarn.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use std::env; use std::path::Path; use std::process::Command; @@ -17,7 +18,18 @@ pub struct YarnArgs { pub fn setup_yarn(args: YarnArgs) { link_node_modules(); - run_command(Command::new(&args.yarn_bin).arg("install")); + if env::var("OFFLINE_BUILD").is_ok() { + println!("OFFLINE_BUILD is set"); + println!("Running yarn with '--offline' and '--ignore-scripts'."); + run_command( + Command::new(&args.yarn_bin) + .arg("install") + .arg("--offline") + .arg("--ignore-scripts"), + ); + } else { + run_command(Command::new(&args.yarn_bin).arg("install")); + } std::fs::write(args.stamp, b"").unwrap(); }