From bdbcb6d7aa96451015f8c94a0a63c98466a142da Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 15 Oct 2021 20:16:04 +1000 Subject: [PATCH] default to a vendored copy of Python Brings Python in line with our other dependencies, and means users no longer need to install it prior to building, or deal with issues caused by having the wrong version available. --- .bazelrc | 5 ++- docs/development.md | 32 +++++++++--------- docs/linux.md | 42 ++++++----------------- docs/mac.md | 33 +++++------------- docs/windows.md | 16 +-------- pylib/rsbridge/BUILD.bazel | 6 ++++ python.bzl | 68 +++++++++++++++++++++++++++++--------- 7 files changed, 97 insertions(+), 105 deletions(-) diff --git a/.bazelrc b/.bazelrc index 21547b9c6..c5803c7c5 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,9 +1,8 @@ common --enable_platform_specific_config common --experimental_repository_cache_hardlinks -# specify python path for pyo3 compile -build:windows --action_env="PYO3_PYTHON=c:/python/python.exe" -#build:linux --action_env="PYO3_PYTHON=/usr/local/bin/python3.9" +# specify custom python path +# build --action_env="PYO3_PYTHON=/usr/local/bin/python3.9" # only affects the ankihelper library #build:macos --macos_cpus=x86_64,arm64 diff --git a/docs/development.md b/docs/development.md index 23166bf06..95b0cb5fc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -11,13 +11,13 @@ https://betas.ankiweb.net/ Pre-built Python packages are available on PyPI. They are useful if you wish to: -- Run Anki from a local Python installation without building it yourself -- Get code completion when developing add-ons -- Make command line scripts that modify .anki2 files via Anki's Python libraries +- Run Anki from a local Python installation without building it yourself +- Get code completion when developing add-ons +- Make command line scripts that modify .anki2 files via Anki's Python libraries -You will need the 64 bit version of Python 3.9 or 3.10 installed. If you do not -have Python yet, please see the platform-specific instructions in the "Building -from source" section below for more info. +You will need the 64 bit version of Python 3.9 or 3.10 installed. 3.9 is recommended, +as Anki has only received minimal testing on 3.10 so far, and some dependencies have not +been fully updated yet. You can install Python from python.org or from your distro. **Mac/Linux**: @@ -51,13 +51,10 @@ c:\> \pyenv\scripts\anki Platform-specific instructions: -- [Windows](./windows.md) -- [Mac](./mac.md) -- [Linux](./linux.md) -- [Other Platforms](./new-platform.md) - -Don't name the Git checkout ~/Anki or ~/Documents/Anki, as those folders -were used on old Anki versions and will be automatically moved. +- [Windows](./windows.md) +- [Mac](./mac.md) +- [Linux](./linux.md) +- [Other Platforms](./new-platform.md) Before contributing code, please see [Contributing](./contributing.md). @@ -186,6 +183,7 @@ VS Code's "watcher exclude", and add `**/bazel-*`. After running 'code' from the project root, it may take a minute or two to be ready. + ## TypeScript editing Visual Studio Code seems to give the best experience. Use 'code ts' from the project @@ -235,7 +233,7 @@ Anki to load a specific profile. If you're using PyCharm: -- right click on the "run" file in the root of the PyCharm Anki folder -- click "Edit 'run'..." - in Script options and enter: - "-p [dev profile name]" without the quotes -- click "Ok" +- right click on the "run" file in the root of the PyCharm Anki folder +- click "Edit 'run'..." - in Script options and enter: + "-p [dev profile name]" without the quotes +- click "Ok" diff --git a/docs/linux.md b/docs/linux.md index cddbf59bc..a36f72efe 100644 --- a/docs/linux.md +++ b/docs/linux.md @@ -18,50 +18,30 @@ $ sudo apt install bash grep findutils curl gcc g++ git The 'find' utility is 'findutils' on Debian. -**Install Python 3.9**: +**Python**: -If you're on a modern distribution, you may be able to install Python from the repo: +For building and running from source, Python is required, but the version is +flexible - any version from 2.7 onwards should work. The build system expects to +find the command `python`, so if your system only has a `python3`, you'll need +to link it to `python`, or do something like `sudo apt install python-is-python3`. -``` -$ sudo apt install python3.9 -``` +The system Python is only used for running scripts, and the build system will +download a copy of Python 3.9 into a local folder as part of the build. -If you are using a packaged Python version that is installed in /usr/bin, you can jump -immediately to the next section after ensuring python3.9-distutils is installed. - -If Python 3.9 is not available in your distro, you can download it from -python.org, compile it, and install it in /usr/local. If you're on a basic -Debian install, make sure you have the following installed before building -Python: - -gcc g++ make libsqlite3-dev libreadline-dev libssl-dev zlib1g-dev libffi-dev - -Bazel does not look in /usr/local by default. If you've installed Python somewhere -other than /usr/bin, you'll need to put the following into a file called user.bazelrc -at the top of this repo before proceeding: +You can have it use a locally installed Python instead, by putting something +like the following into a file called user.bazelrc at the top of this repo +before proceeding: ``` build --action_env=PYO3_PYTHON=/usr/local/bin/python3.9 ``` -If you're building Anki from a docker container or distro that has no `python` command in -/usr/bin, you'll need to symlink `python` to `/usr/bin/python`. `/usr/bin/python` does not -need to be Python 3.9; any version will do. `apt install python-is-python3` can also be -used to link python3 to python. - -If your system only has Python 3.9, you should be able to build Anki with it, -but the pylint tests may fail. - -Anki's build system will not place packages in system locations, so you do not -need to build with an active Python virtual environment, and building outside -of one is recommended. - **Install Bazelisk**: Download it under the name 'bazel': ``` -$ curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.7.4/bazelisk-linux-amd64 -o ./bazel +$ curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-linux-amd64 -o ./bazel ``` And put it on your path: diff --git a/docs/mac.md b/docs/mac.md index 16febc02c..f28aa8b5a 100644 --- a/docs/mac.md +++ b/docs/mac.md @@ -17,33 +17,18 @@ Then install deps: $ brew install rsync bazelisk ``` -**Install Python 3.9**: +**Python**: -Install Python 3.9 from . We have heard reports -of issues with pyenv and homebrew, so the package from python.org is -the only recommended approach. +The build system will automatically download a copy of Python 3.9 as part +of the build. -Python 3.9 is not currently recommended, as pylint does not support it yet. +It is also possible to override the Python 3.9 that the build system uses. +We only recommend you do this if you have downloaded Python from python.org, +as we have heard reports of things failing when using a Python 3 from macOS +or Homebrew. -You do not need to set up a Python venv prior to building Anki. - -When you run "python" in a shell, if it shows Python 2.x, you may get a -bunch of hashlib warnings during build. You can work around this by -pointing python to python3.9: - -``` -$ ln -sf /usr/local/bin/{python3.9,python} -``` - -This linking will not work if you're using the system Python from Big Sur, -which is one of the reasons why we recommend using Python from python.org. - -But even so, if you still have a Python 2 binary on your system linked to -/usr/bin/python, it can happen that Bazel uses it and the build process -errors. In that case you might want to try forcing Bazel to use the -Python 3 binary by putting the following into a file called user.bazelrc at -the top of this repo (assuming /usr/local/bin/python links to your Python -3.9 binary). +To override Python, put the following into a file called user.bazelrc at the top +of this repo (assuming /usr/local/bin/python links to your Python 3.9 binary). ``` build --action_env=PYO3_PYTHON=/usr/local/bin/python diff --git a/docs/windows.md b/docs/windows.md index c855e2801..913c5d0e8 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -15,17 +15,6 @@ Install the [Visual Studio build tools](https://visualstudio.microsoft.com/downl Make sure the "C++ build tools" box is selected, and leave the default optional components enabled on the right. -**Python 3.9**: - -Download the 64 bit Python 3.9 from . Run the installer, -and customize the installation. Select "install for all users", and choose -the install path as c:\python. Currently the build scripts require -Python to be installed in that location. - -When the install is done, click on the "remove the path limit" button. - -Python 3.9 is not currently recommended, as pylint does not support it yet. - **MSYS**: Install [msys2](https://www.msys2.org/) into the default folder location. @@ -49,12 +38,9 @@ PS> cd \bazel Then grab Bazelisk: ``` -PS> \msys64\usr\bin\curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.7.4/bazelisk-windows-amd64.exe -o bazel.exe +PS> \msys64\usr\bin\curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe -o bazel.exe ``` -NOTE: At the time of writing, Windows Defender is claiming this file has a virus. If it disappears -the first time you run Bazel, restoring it from the Defender settings should allow you to proceed. - **Source folder**: Anki's source files do not need to be in a specific location other than on the diff --git a/pylib/rsbridge/BUILD.bazel b/pylib/rsbridge/BUILD.bazel index 66502a1c5..b3579abae 100644 --- a/pylib/rsbridge/BUILD.bazel +++ b/pylib/rsbridge/BUILD.bazel @@ -6,6 +6,12 @@ load("//rslib:rustfmt.bzl", "rustfmt_fix", "rustfmt_test") cargo_build_script( name = "build_script", srcs = ["build.rs"], + build_script_env = { + "PYO3_PYTHON": "$(location @python)", + }, + data = [ + "@python", + ], ) rust_library( diff --git a/python.bzl b/python.bzl index aa67d8aa4..6add734f8 100644 --- a/python.bzl +++ b/python.bzl @@ -1,23 +1,61 @@ +_python_distros = { + "macos_amd64": struct( + url = "https://github.com/indygreg/python-build-standalone/releases/download/20211012/cpython-3.9.7-x86_64-apple-darwin-install_only-20211011T1926.tar.gz", + sha256 = "43cb1a83919f49b1ce95e42f9c461d8a9fb00ff3957bebef9cffe61a5f362377", + ), + "macos_arm64": struct( + url = "https://github.com/indygreg/python-build-standalone/releases/download/20211012/cpython-3.9.7-aarch64-apple-darwin-install_only-20211011T1926.tar.gz", + sha256 = "213d64f08f82184c9ad398537becf9b341ee85b1c406cfb2b4cbcb78cdb27a2c", + ), + "linux_amd64": struct( + # pending https://github.com/indygreg/python-build-standalone/issues/95 + url = "https://github.com/ankitects/python-build-standalone/releases/download/anki-2021-10-15/cpython-3.9.7-x86_64-unknown-linux-gnu-install_only-20211013T1538.tar.gz", + sha256 = "6a42fe15950f4e42000f5c68ebefacf8fce024f4d80a446789c18b174efdec1b", + ), + "windows_amd64": struct( + url = "https://github.com/indygreg/python-build-standalone/releases/download/20211012/cpython-3.9.7-x86_64-pc-windows-msvc-shared-install_only-20211011T1926.tar.gz", + sha256 = "80370f232fd63d5cb3ff9418121acb87276228b0dafbeee3c57af143aca11f89", + ), +} + +def get_platform(rctx): + if rctx.os.name == "mac os x": + result = rctx.execute(["arch"]) + if result.return_code: + fail("invoking arch failed", result.stderr) + arch = result.stdout.strip() + if arch == "i386": + return "macos_amd64" + elif arch == "arm64": + return "macos_arm64" + else: + fail("unexpected arch:", arch) + elif rctx.os.name == "linux": + return "linux_amd64" + elif rctx.os.name.startswith("windows"): + return "windows_amd64" + else: + fail("unexpected platform", rctx.os.name) + def _impl(rctx): - # locate python on path, and export it - names = [ - "python3.9", - "python3.10", - "python.exe", - ] - path = None + # bundled python overriden? if rctx.os.environ.get("PYO3_PYTHON"): path = rctx.os.environ.get("PYO3_PYTHON") + rctx.symlink(path, "python") else: - for name in names: - path = rctx.which(name) - if path: - break + platform = get_platform(rctx) + distro = _python_distros.get(platform) + rctx.download_and_extract( + url = distro.url, + sha256 = distro.sha256, + stripPrefix = "python", + ) + if platform.startswith("windows"): + path = rctx.path("python.exe") + else: + path = rctx.path("bin/python3") + rctx.symlink(path, "python") - if not path: - fail("python3 or python.exe not found on path, and PYO3_PYTHON not set") - - rctx.symlink(path, "python") rctx.file("BUILD.bazel", """ load("@rules_python//python:defs.bzl", "py_runtime_pair")