diff --git a/docs/Dockerfile b/docs/Dockerfile deleted file mode 100644 index e5a69a1b2..000000000 --- a/docs/Dockerfile +++ /dev/null @@ -1,110 +0,0 @@ -# Warning: -# This file needs updating to work with Anki's updated build system. -# - -ARG PYTHON_VERSION="3.8" - -FROM python:$PYTHON_VERSION AS dependencies - -# Allow non-root users to install things and modify installations in /opt. -RUN chmod 777 /opt && chmod a+s /opt - -# Install rust. -ENV CARGO_HOME="/opt/cargo" \ - RUSTUP_HOME="/opt/rustup" -ENV PATH="$CARGO_HOME/bin:$PATH" -RUN mkdir $CARGO_HOME $RUSTUP_HOME \ - && chmod a+rws $CARGO_HOME $RUSTUP_HOME \ - && curl -fsSL --proto '=https' --tlsv1.2 https://sh.rustup.rs \ - | sh -s -- -y --quiet --no-modify-path \ - && rustup update \ - && cargo install ripgrep - -# Install system dependencies. -RUN apt-get update \ - && apt-get install --yes --no-install-recommends \ - gettext \ - lame \ - libnss3 \ - libxcb-icccm4 \ - libxcb-image0 \ - libxcb-keysyms1 \ - libxcb-randr0 \ - libxcb-render-util0 \ - libxcb-xinerama0 \ - libxcb-xkb1 \ - libxkbcommon-x11-0 \ - libxcomposite1 \ - mpv \ - portaudio19-dev \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -# Install node and npm. -WORKDIR /opt/node -RUN curl -fsSL --proto '=https' https://nodejs.org/dist/v12.18.3/node-v12.18.3-linux-x64.tar.xz \ - | tar xJ --strip-components 1 -ENV PATH="/opt/node/bin:$PATH" - -# Install protoc. -WORKDIR /opt/protoc -RUN curl -fsSL --proto '=https' -O https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip \ - && unzip protoc-3.11.4-linux-x86_64.zip -x readme.txt \ - && rm protoc-3.11.4-linux-x86_64.zip -ENV PATH="/opt/protoc/bin:$PATH" - -# Allow non-root users to install toolchains and update rust crates. -RUN chmod 777 $RUSTUP_HOME/toolchains $RUSTUP_HOME/update-hashes $CARGO_HOME/registry \ - && chmod -R a+rw $CARGO_HOME/registry \ - # Necessary for TypeScript. - && chmod a+w /home - -# Build anki. Use a separate image so users can build an image with build-time -# dependencies. -FROM dependencies AS builder -WORKDIR /opt/anki -COPY . . -RUN make develop - -FROM builder AS pythonbuilder -RUN make build - -# Build final image. -FROM python:${PYTHON_VERSION}-slim - -# Install system dependencies. -RUN apt-get update \ - && apt-get install --yes --no-install-recommends \ - gettext \ - lame \ - libnss3 \ - libxcb-icccm4 \ - libxcb-image0 \ - libxcb-keysyms1 \ - libxcb-randr0 \ - libxcb-render-util0 \ - libxcb-xinerama0 \ - libxcb-xkb1 \ - libxkbcommon-x11-0 \ - libxcomposite1 \ - mpv \ - portaudio19-dev \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -# Install pre-compiled Anki. -COPY --from=pythonbuilder /opt/anki/dist/ /opt/anki/ -RUN python -m pip install --no-cache-dir \ - PyQtWebEngine \ - /opt/anki/*.whl \ - # Create an anki executable. - && printf "#!/usr/bin/env python\nimport aqt\naqt.run()\n" > /usr/local/bin/anki \ - && chmod +x /usr/local/bin/anki \ - # Create non-root user. - && useradd --create-home anki - -USER anki - -ENTRYPOINT ["/usr/local/bin/anki"] - -LABEL maintainer="Jakub Kaczmarzyk " diff --git a/docs/contributing.md b/docs/contributing.md index 1a0a2a3a3..efbedc19c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,7 +1,3 @@ -# Needs updating - -Some of the below references the old build system, and needs updating. - # Contributing Code For info on contributing things other than code, such as translations, decks @@ -53,9 +49,9 @@ and lists, and they can be difficult to fully type. Don't worry too much about getting the types perfect - even a partial type like Dict[str, Any] or List[Tuple] is an improvement over no types at all. -Anki bundles Qt stubs, but they are not perfect, so you'll find when doing -things like connecting signals, you may have to add the following to the end -of a line to silence the spurious errors. +Qt's stubs are not perfect, so you'll find when doing things like connecting +signals, you may have to add the following to the end of a line to silence the +spurious errors. ``` # type: ignore @@ -108,8 +104,7 @@ In most cases, hooks are better added in the GUI code than in pylib. The hook code is automatically generated using the definitions in pylib/tools/genhooks.py and qt/tools/genhooks_gui.py. Adding a new definition -in one of those files and running 'make develop' will update pylib/anki/hooks -.py or qt/aqt/gui_hooks.py. +in one of those files will update the generated files. ## Translations @@ -122,16 +117,12 @@ https://ankitects.github.io/translating/#/anki/developers ## Tests Must Pass -Please make sure 'make check' completes successfully before submitting code. +Please make sure 'bazel test //...' completes successfully before submitting code. You can do this automatically by adding the following into .git/hooks/pre-commit or .git/hooks/pre-push and making it executable. #!/bin/bash -set -eu -o pipefail \${SHELLFLAGS} -make check - -You may need to adjust the PATH variable so that things like a local install -of cargo can be found. +bazel test //... If your change is to anki/ and not covered by the existing unit tests, please consider adding a unit test at the same time. @@ -144,8 +135,6 @@ variables that use camelCaps. Variables local to a function are safer to rename, but please do so only when a function needs to be changed for other reasons as well. -Code formatting is automatically done when you use "make fix". - ## Do One Thing A patch or pull request should be the minimum necessary to address one issue. diff --git a/docs/development.md b/docs/development.md index 181afc673..2ddebe4da 100644 --- a/docs/development.md +++ b/docs/development.md @@ -11,17 +11,27 @@ You are welcome to run Anki from source instead, but it is expected that you can sort out issues by yourself - we are not able to provide support for problems you encounter when running from source. -## Python wheels +## Pre-built Python wheels If you want to run Anki from a local Python installation but don't want to make changes to the source code, you can install pre-built packages from PyPI. +For older versions: + ``` $ python -m venv pyenv $ pyenv/bin/pip install aqt anki ankirspy pyqt5 pyqtwebengine $ pyenv/bin/python -c 'import aqt; aqt.run()' ``` +From Anki 2.1.36 onwards: + +``` +$ python -m venv pyenv +$ pyenv/bin/pip install aqt anki pyqtwebengine +$ pyenv/bin/python -c 'import aqt; aqt.run()' +``` + ## Building from source Platform-specific instructions: @@ -37,213 +47,51 @@ Before contributing code, please see [Contributing](./contributing.md). If you'd like to contribute translations, please see . +## Running tests + +From inside the source folder: + +``` +bazel test //... +``` + +## Fixing formatting + +If the format tests fail, most can be fixed by running format_fix +in the relevant folder: + +``` +bazel run //rslib:format_fix +bazel run //pylib:format_fix +bazel run //pylib/rsbridge:format_fix +bazel run //qt:format_fix +``` + +Currently the typescript code needs to be formatted differently: + +``` +cd ts +./node_modules/.bin/prettier --write . +``` + +## Building redistributable wheels + +``` +bazel build -c opt //pylib/anki:wheel +bazel build -c opt //qt/aqt:wheel +``` + +## Tracing build problems + +You can run bazel with '-s' to print the commands that are being executed. + ## Subcomponents -- pylib contains a Python module (anki) with the non-GUI Python code. +- pylib contains a Python module (anki) with the non-GUI Python code, + and a bridge to the Rust code. - qt contains the Qt GUI implementation (aqt). -- rspy contains a Python module (ankirspy) for accessing the Rust code. - rslib contains the parts of the code implemented in Rust. -- proto contains the interface used to communicate between different - languages. - -# Obsolete instructions that need updating - -The text below was written before the build system was changed, -and these instructions need updating or are obsolete. - -The pyenv folder is created when running make for the first time. -It is a Python virtual environment that contains Anki's libraries -and all the required dependencies. - -## Makefile - -Use 'make check' to run unit tests, type checking and linting on all of the -subcomponents. - -Use 'make fix' to fix any formatting issues that were found with 'make check'. - -Use 'make build' to output Python wheels of the subcomponents into the dist/ -folder. - -Use 'make clean' to remove some generated files. - -To see all commands run by make or any shell script, export the environment -variable SHELLFLAGS with '-x' to tell shell to print all commands run by it. -For example, 'export SHELLFLAGS=-x' on Linux or 'set SHELLFLAGS=-x' on Windows. - -## PyQt - -The build scripts will use PyQt/Qt from PyPI by default. If you wish to use a -system install, you will need to set up the pyenv folder yourself, making sure -you have PyQt5, the WebEngine module and development tools (eg pyqt5-dev-tools) -installed as well. You'll need to create the venv with --system-site-packages. - -## Studying Anki on your Dev machine? - -It's advisable to create a new Anki dev profile for debug and testing so that your study collection doesn't become -incompatible with your installed version: - run Anki - File | Switch profile > Add > provide [dev profile name] > Ok - make a note of that [dev profile name] - -Because you now have at least two profiles the Profiles dialog will open when Anki runs to allow you to choose the -profile to load. - -Alternatively you can set your Anki project to load your [dev profile name] profile automatically with the startup -options described here: https://docs.ankiweb.net/#/files?id=startup-options - -To do this in PyCharm: - right click on the "run" file in the root of the PyCharm Anki folder - click "Edit 'run'..." - in Script options enter: - "-p [dev profile name]" - without the quotes - click "Ok" - -## Debian/Ubuntu users - -Install Python 3.7+ if it's not installed. - -Install other dependencies: - -``` -sudo apt install portaudio19-dev mpv lame npm rsync gcc gettext git curl python3-dev python3-venv libxcb-xinerama0 -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source $HOME/.cargo/env -wget https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip -sudo unzip protoc-3.11.4-linux-x86_64.zip -d /usr/local/ -x readme.txt -rustup update -cargo install ripgrep -``` - -Another way to build Anki on Linux is to use Docker - please see README.docker -for more information. - -## Mac users - -You can use homebrew to install some dependencies: - -\$ brew install python mpv lame portaudio protobuf npm rustup-init gettext rename ripgrep - -\$ brew link gettext --force - -## Windows users (using Visual Studio) - -The build scripts assume a UNIX-like environment, so on Windows you will -need to use WSL or Cygwin to use them. - -User-contributed instructions for building using Cygwin: - -1. Download and install Cygwin and put its `/bin/` directory on your system path (This PC > Properties > Advancded system settings > Environment Variables > double-click Path > New). -1. Install the Cygwin Packages: `apt-cyg install rsync make` OR select rsync package during Cygwin installation - 1. Download `gettext` 0.20.1 or superior and put its `bin` directory on your system path. - 1. https://mlocati.github.io/articles/gettext-iconv-windows.html -1. Download and install Python for Windows (not from Cygwin) and put `python.exe` (not `python3.exe`) on your system path. -1. Download and install pip for your Windows Python (`python -m ensurepip`). -1. Download and install rust (compiler), npm, git and put them your system path. -1. Download and install the pyaudio wheel from: https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyaudio - 1. After download the file for your Python version, you need to define the following environment - variable before running Anki: - `set "ANKI_EXTRA_PIP=python -m pip install full/path/to/PyAudio‑0.2.11‑cp38‑cp38‑win_amd64.whl"` - 1. If there is not an wheel available for your Python version, you can built it from the source - following the installation instructions on: https://github.com/evandroforks/pyaudio - After building and installing portaudio, you need to define the following environment - variable before running Anki: - `set "ANKI_EXTRA_PIP=python -m pip install git+https://github.com/evandroforks/pyaudio"` -1. Open a `cmd.exe` (command prompt) on the Anki repository and run the command `sh run` - 1. Do not use `bash run` because it my call for Windows Subsystem for Linux - 1. Do not use any Cygwin terminal as `mintty.exe` because the `rust lang` compiler does not work with them - -## Windows users (using PyCharm) - -These instructions assume Microsoft Python support is not installed; neither via VS nor VS Code nor some other way. -If MS Python support is already installed the process is different, and these notes may not work seamlessly for you. - -These notes are proscriptive, other methods will work (such as a different Bash/Shell). - -While some of this list is in an arbitrary order, significant steps are in a specific order, therefore it is strongly -recommended to follow these instructions step-by-step. - -- Python and PyCharm are a given if you are choosing this path and should be the first and second step respectively. - -- close PyCharm, if it's open. - -- install Node.js (npm installs with nodejs) - -- install Cygwin (check the Cygwin /bin folder is added to System PATH) - -- use the Visual Studio Community installer to install: - - - the individual C++ Clang Compiler for Windows (10.0.0) component. - - the Python development Workflow - -- install rustup (https://rustup.rs/) - -- download and install Strawberry Perl msi from http://strawberryperl.com/ - - - this is the installer at the time of writing these notes: - http://strawberryperl.com/download/5.32.0.1/strawberry-perl-5.32.0.1-64bit.msi - -- mpv isn't required unless you are developing against mpv (Anki will revert to Mplayer) - - - open an elevated Git Bash (elevated to "Run as administrator") - - choco install mpv - - assumes you have Chocolatey installed - -- install Git and Git Bash (this delivers curl too) - - - add git/usr/bin to your System PATH before cygwin/bin folder, mentioned above. - - run: - - cargo install ripgrep - -- got to https://github.com/protocolbuffers/protobuf/releases - - - download latest protoc-v.v.v-win64.zip where 'v' is replaced with version numbers - - this is the file at the time of writing these notes: - https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protoc-3.13.0-win64.zip - - extract bin\protoc.exe to C:\Program Files\Git\usr\bin (or where ever you installed Git and Git Bash above) - -- go to http://repo.msys2.org/msys/x86_64/ - - - download latest rsync-v.v.v-v-x86_64.pkg.tar.xz where 'v' is replaced with version numbers - - this is the file at the time of writing these notes: - http://repo.msys2.org/msys/x86_64/rsync-3.1.3-1-x86_64.pkg.tar.xz - - extract the tar, then extract usr\bin\rsync.exe to c:\Program Files\Git\usr\bin as for protoc.exe above - -- install remaining dependencies to Python packages - - - pip install https://github.com/ankitects/windows-ci-tools/PyAudio-0.2.11-cp37-cp37-win_amd64.whl - - pip install lameenc - - pip install python-gettext - -- clone your GitHub Anki fork to a local folder - -- open that local Anki source folder in PyCharm - -- default PyCharm Terminal to Git Bash: - - - these steps aren't required, you can just use Git Bash externally to run Anki and the commands below, if you - prefer - - open Anki project in PyCharm - - open File > Settings - - navigate to Tools > Terminal - - set Shell path to: "C:\Program Files\Git\bin\bash.exe" (or where ever you installed Git and Git Bash) - - restart PyCharm with Anki project - -- open PyCharm Terminal (Git Bash) - -- ensure your virtual environment is set up as you like it to be - - - if you don't set the virtual environment to pyenv (Anki's virtual environment folder) don't forget to add the - virtual environment folder to your global (not local) .gitignore if you haven't already. - -- add .idea folder to your global .gitignore (not the local .gitignore in the cloned repo) - -- from project root run: - - - pip install -r qt/requirements.qt - -- in PyCharm - - either: - - open Terminal (Git Bash) - - type ./run - - or: - - right click 'run' file in Anki project root and click the command to run it - - watch the magic happen (it takes a while and there are a couple of long pauses). - -please note: these instructions have been based off the general notes above and the notes for installing for Windows -using Visual Studio. Without those notes this section would have been very difficult, if not impossible. +- ts contains Anki's typescript and sass files. ## Environmental Variables @@ -252,3 +100,27 @@ and automatic backups will be disabled - so please don't use this except on a te If LOGTERM is set before starting Anki, warnings and error messages that are normally placed in the collection2.log file will also be printed on stdout. + +## Mixing development and study + +You may wish to create a separate profile with File>Switch Profile for use +during development. You can pass the arguments "-p [profile name]" when starting +Anki to load a specific profile. + +If you're using PyCharm: + +- right click on the "run" file in the root of the PyCharm Anki folder +- click "Edit 'run'..." - in Script options and enter: + "-p [dev profile name]" without the quotes +- click "Ok" + +# Instructions need updating: + +optional deps: +pyaudio +dpkg: portaudio19-dev + +mpv +lame + +1. Download and install the pyaudio wheel from: https://www.lfd.uci.edu/~gohlke/pythonlibs/#pyaudio diff --git a/docs/docker.md b/docs/docker.md index d94a94cc9..233f0bacf 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -105,3 +105,114 @@ Build your local source tree in Docker. ``` docker exec -it --user root ankibuilder bash ``` + +Old docker file below: + +``` +ARG PYTHON_VERSION="3.8" + +FROM python:$PYTHON_VERSION AS dependencies + +# Allow non-root users to install things and modify installations in /opt. +RUN chmod 777 /opt && chmod a+s /opt + +# Install rust. +ENV CARGO_HOME="/opt/cargo" \ + RUSTUP_HOME="/opt/rustup" +ENV PATH="$CARGO_HOME/bin:$PATH" +RUN mkdir $CARGO_HOME $RUSTUP_HOME \ + && chmod a+rws $CARGO_HOME $RUSTUP_HOME \ + && curl -fsSL --proto '=https' --tlsv1.2 https://sh.rustup.rs \ + | sh -s -- -y --quiet --no-modify-path \ + && rustup update \ + && cargo install ripgrep + +# Install system dependencies. +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + gettext \ + lame \ + libnss3 \ + libxcb-icccm4 \ + libxcb-image0 \ + libxcb-keysyms1 \ + libxcb-randr0 \ + libxcb-render-util0 \ + libxcb-xinerama0 \ + libxcb-xkb1 \ + libxkbcommon-x11-0 \ + libxcomposite1 \ + mpv \ + portaudio19-dev \ + rsync \ + && rm -rf /var/lib/apt/lists/* + +# Install node and npm. +WORKDIR /opt/node +RUN curl -fsSL --proto '=https' https://nodejs.org/dist/v12.18.3/node-v12.18.3-linux-x64.tar.xz \ + | tar xJ --strip-components 1 +ENV PATH="/opt/node/bin:$PATH" + +# Install protoc. +WORKDIR /opt/protoc +RUN curl -fsSL --proto '=https' -O https://github.com/protocolbuffers/protobuf/releases/download/v3.11.4/protoc-3.11.4-linux-x86_64.zip \ + && unzip protoc-3.11.4-linux-x86_64.zip -x readme.txt \ + && rm protoc-3.11.4-linux-x86_64.zip +ENV PATH="/opt/protoc/bin:$PATH" + +# Allow non-root users to install toolchains and update rust crates. +RUN chmod 777 $RUSTUP_HOME/toolchains $RUSTUP_HOME/update-hashes $CARGO_HOME/registry \ + && chmod -R a+rw $CARGO_HOME/registry \ + # Necessary for TypeScript. + && chmod a+w /home + +# Build anki. Use a separate image so users can build an image with build-time +# dependencies. +FROM dependencies AS builder +WORKDIR /opt/anki +COPY . . +RUN make develop + +FROM builder AS pythonbuilder +RUN make build + +# Build final image. +FROM python:${PYTHON_VERSION}-slim + +# Install system dependencies. +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + gettext \ + lame \ + libnss3 \ + libxcb-icccm4 \ + libxcb-image0 \ + libxcb-keysyms1 \ + libxcb-randr0 \ + libxcb-render-util0 \ + libxcb-xinerama0 \ + libxcb-xkb1 \ + libxkbcommon-x11-0 \ + libxcomposite1 \ + mpv \ + portaudio19-dev \ + rsync \ + && rm -rf /var/lib/apt/lists/* + +# Install pre-compiled Anki. +COPY --from=pythonbuilder /opt/anki/dist/ /opt/anki/ +RUN python -m pip install --no-cache-dir \ + PyQtWebEngine \ + /opt/anki/*.whl \ + # Create an anki executable. + && printf "#!/usr/bin/env python\nimport aqt\naqt.run()\n" > /usr/local/bin/anki \ + && chmod +x /usr/local/bin/anki \ + # Create non-root user. + && useradd --create-home anki + +USER anki + +ENTRYPOINT ["/usr/local/bin/anki"] + +LABEL maintainer="Jakub Kaczmarzyk " +``` diff --git a/docs/linux.md b/docs/linux.md index bfb0bbacf..3e7c4ba11 100644 --- a/docs/linux.md +++ b/docs/linux.md @@ -58,10 +58,6 @@ dependencies. When the build is complete, Anki will automatically start. To play audio, install mpv. At the time of writing, recording is not yet supported, as currently pyaudio is not being installed. -## Running tests +## More -From inside the source folder: - -``` -bazel test //... -``` +For info on running tests, building wheels and so on, please see [Development](./development.md). diff --git a/docs/mac.md b/docs/mac.md index 1c1d0d7ec..ddb2bb249 100644 --- a/docs/mac.md +++ b/docs/mac.md @@ -48,10 +48,6 @@ dependencies. When the build is complete, Anki will automatically start. To play audio, use Homebrew to install mpv. At the time of writing, recording is not yet supported, as currently pyaudio is not being installed. -## Running tests +## More -From inside the source folder: - -``` -bazel test //... -``` +For info on running tests, building wheels and so on, please see [Development](./development.md). diff --git a/docs/windows.md b/docs/windows.md index 02532450b..7e4c1b1c5 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -57,8 +57,7 @@ stored in c:\anki. The Javascript bundling on Windows is currently a bit flaky, so the initial build will likely fail with an error about a missing rollup module. If you -get an error when running the commands below, try repeating them a few times - -they should pick up where they left off. +get an error when running the commands below, try repeating them once or twice. ## Running Anki during development @@ -78,14 +77,11 @@ To play audio, mpv.exe or mplayer.exe must be on the path. At the time of writing, recording is not yet supported, as currently pyaudio is not being installed. -## Running tests +## More -From inside the top level of the source folder: +For info on running tests, building wheels and so on, please see [Development](./development.md). -``` -.\bazel test //... -``` - -Note that we call bazel.bat inside the Anki source folder, instead of -calling Bazel directly. This takes care of setting up the path and -output folder correctly, which avoids issues with long path names. +Note that where the instructions on that page say "bazel", please use ".\bazel" +instead. This runs bazel.bat inside the Anki source folder, instead of +calling Bazel directly. This takes care of setting up the path and output folder +correctly, which avoids issues with long path names.