diff --git a/.github/scripts/contrib.sh b/.github/scripts/contrib.sh index 0bcf6153e..46da0954e 100755 --- a/.github/scripts/contrib.sh +++ b/.github/scripts/contrib.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} antispam=", at the domain " diff --git a/.github/scripts/trailing-newlines.sh b/.github/scripts/trailing-newlines.sh index 332e65280..e310c6747 100755 --- a/.github/scripts/trailing-newlines.sh +++ b/.github/scripts/trailing-newlines.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -eu -o pipefail ${SHELLFLAGS} files=$(rg -l '[^\n]\z' -g '!*.{svg,scss,json,sql}' || true) if [ "$files" != "" ]; then diff --git a/.github/workflows/checks.yml b/.github/workflows/linux_checks.yml similarity index 97% rename from .github/workflows/checks.yml rename to .github/workflows/linux_checks.yml index 4871a7c18..92b7c855e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/linux_checks.yml @@ -1,4 +1,4 @@ -name: Checks +name: Linux Tests on: [push, pull_request] diff --git a/.github/workflows/windows_checks.yml b/.github/workflows/windows_checks.yml new file mode 100644 index 000000000..e05d89404 --- /dev/null +++ b/.github/workflows/windows_checks.yml @@ -0,0 +1,76 @@ +name: Windows Tests + +on: [push, pull_request] + +jobs: + test: + runs-on: windows-latest + steps: + - run: git config --global core.autocrlf false + - uses: actions/checkout@v2 + - name: Configure environment variables + run: | + $pyaudio=("PyAudio-0.2.11-cp37-cp37m-win_amd64.whl") + $new_path=("$env:GITHUB_WORKSPACE;$env:PATH") + $new_path=("$env:GITHUB_WORKSPACE\shims;$new_path") + echo "::set-env name=pacmanbin::pacman-5.1.1-3-x86_64.pkg.tar.xz" + echo "::set-env name=pacmanmirror::pacman-mirrors-20200307-1-any.pkg.tar.xz" + echo "::set-env name=pacmankeys::msys2-keyring-r9.397a52e-1-any.pkg.tar.xz" + echo "::set-env name=pyaudio::$pyaudio" + echo "::set-env name=PATH::$new_path" + echo "::set-env name=RUST_BACKTRACE::full" + echo "::set-env name=SCOOP::$env:GITHUB_WORKSPACE" + echo "::set-env name=SCOOP_GLOBAL::$env:GITHUB_WORKSPACE\" + echo "::set-env name=ANKI_EXTRA_PIP::python -m pip install $pyaudio" + - name: Set up pacman, pyaudio, rsync, rename + shell: cmd + run: | + echo on + curl -LO https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/%pyaudio% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanbin% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmanmirror% + curl -LO http://repo.msys2.org/msys/x86_64/%pacmankeys% + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename + :: https://stackoverflow.com/questions/1359793/programmatically-extract-tar-gz + 7z x "%pacmanbin%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmanmirror%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + 7z x "%pacmankeys%" -so | 7z x -aoa -si -ttar -o"%programfiles%\Git" + :: Manually Install Pacman Binaries + :: https://github.com/Alexpux/MSYS2-pacman/issues/50 + cd /d "%programfiles%\Git\usr\bin" + .\bash pacman-key --init + .\bash pacman-key --populate msys2 + .\bash pacman-key --refresh-keys + .\pacman -Tv + .\pacman -Syuv --overwrite='*' + :: We must install bash first, otherwise we will have bash fork errors: + :: https://github.com/evandroforks/anki/runs/524857054?check_suite_focus=true + :: 0 [main] pacman 748 dofork: child -1 - forked process 2896 died unexpectedly + .\pacman --version + .\pacman -Sv --noconfirm --overwrite='*' bash + .\pacman -Sv --noconfirm --overwrite='*' rsync + - name: Set up scoop, gettext, ripgrep + shell: cmd + run: | + echo on + powershell -executionpolicy bypass "& Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')" + scoop install -g gettext ripgrep + - name: Set up python + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Set up protoc + uses: Arduino/actions/setup-protoc@master + - name: Set up node + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Run checks + shell: cmd + run: | + echo on + set "RSPY_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "CARGO_TARGET_DIR=%GITHUB_WORKSPACE%\target" + set "BUILDFLAGS=" + :: set "SHELLFLAGS=-x" + make check build diff --git a/.gitignore b/.gitignore index 7e4935ee6..57ab44f16 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .build .coverage .DS_Store +rename dist pyenv .mypy_cache diff --git a/Makefile b/Makefile index fdfa2c182..83325e02a 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,23 @@ SHELL := /bin/bash +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + ifndef OS OS := unknown endif -ifeq ($(OS),Windows_NT) +ifeq (${OS},Windows_NT) + # Windows terminal is confusing it with its `cmd` builtin `rename` command + ifndef RENAME_BIN + RENAME_BIN := perl rename + endif + ifndef ACTIVATE_SCRIPT ACTIVATE_SCRIPT := pyenv/Scripts/activate endif @@ -13,6 +26,10 @@ ifeq ($(OS),Windows_NT) PYTHON_BIN := python endif else + ifndef RENAME_BIN + RENAME_BIN := rename + endif + ifndef ACTIVATE_SCRIPT ACTIVATE_SCRIPT := pyenv/bin/activate endif @@ -26,10 +43,7 @@ ifndef ANKI_EXTRA_PIP ANKI_EXTRA_PIP := true endif -.SHELLFLAGS := -eu -o pipefail -c .DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules SUBMAKE := $(MAKE) --print-directory .SUFFIXES: @@ -46,7 +60,13 @@ all: run # - modern pip required for wheel # - add qt if missing pyenv: + "${PYTHON_BIN}" -m ensurepip && \ + "${PYTHON_BIN}" -m pip install virtualenv && \ "${PYTHON_BIN}" -m venv pyenv && \ + case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) \ + dos2unix "${ACTIVATE_SCRIPT}" && \ + sed -i -- "s@VIRTUAL_ENV=\".*\"@VIRTUAL_ENV=\"$(shell pwd)/pyenv\"@g" "${ACTIVATE_SCRIPT}" \ + ;; esac; \ . "${ACTIVATE_SCRIPT}" && \ python --version && \ python -m pip install --upgrade pip setuptools && \ @@ -56,25 +76,23 @@ pyenv: # update build hash .PHONY: buildhash buildhash: - @oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true); \ - newhash=$$(git rev-parse --short=8 HEAD || echo dev); \ + @oldhash=$$(test -f meta/buildhash && cat meta/buildhash || true) && \ + newhash=$$(git rev-parse --short=8 HEAD || echo dev) && \ if [ "$$oldhash" != "$$newhash" ]; then \ echo $$newhash > meta/buildhash; \ fi .PHONY: develop develop: pyenv buildhash prepare - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ + @. "${ACTIVATE_SCRIPT}" && \ for dir in $(DEVEL); do \ $(SUBMAKE) -C $$dir develop DEVFLAGS="$(DEVFLAGS)"; \ done .PHONY: run run: develop - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ - echo "Starting Anki..."; \ + @. "${ACTIVATE_SCRIPT}" && \ + echo "Starting Anki..." && \ python qt/runanki $(RUNFLAGS) .PHONY: prepare @@ -109,8 +127,7 @@ build-qt: .PHONY: clean clean: clean-dist - @set -eo pipefail && \ - for dir in $(DEVEL); do \ + @for dir in $(DEVEL); do \ $(SUBMAKE) -C $$dir clean; \ done @@ -120,8 +137,7 @@ clean-dist: .PHONY: check check: pyenv buildhash prepare - @set -eo pipefail && \ - .github/scripts/trailing-newlines.sh && \ + @.github/scripts/trailing-newlines.sh && \ for dir in $(CHECKABLE_RS); do \ $(SUBMAKE) -C $$dir check; \ done && \ @@ -136,17 +152,19 @@ check: pyenv buildhash prepare .PHONY: fix fix: - @set -eo pipefail && \ - . "${ACTIVATE_SCRIPT}" && \ + @. "${ACTIVATE_SCRIPT}" && \ for dir in $(CHECKABLE_RS) $(CHECKABLE_PY); do \ $(SUBMAKE) -C $$dir fix; \ done; \ .PHONY: add-buildhash add-buildhash: - @ver=$$(cat meta/version); \ - hash=$$(cat meta/buildhash); \ - rename "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-$$ver-* + @if [[ ! -f rename ]]; then \ + curl -LO https://raw.githubusercontent.com/subogero/rename/master/rename; \ + fi && \ + ver="$$(cat meta/version)" && \ + hash="$$(cat meta/buildhash)" && \ + ${RENAME_BIN} "s/-$${ver}-/-$${ver}+$${hash}-/" dist/*-$$ver-* .PHONY: pull-i18n diff --git a/README.contributing b/README.contributing index b04c808e0..975c90c17 100644 --- a/README.contributing +++ b/README.contributing @@ -114,7 +114,7 @@ You can do this automatically by adding the following into .git/hooks/pre-commit or .git/hooks/pre-push and making it executable. #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} make check You may need to adjust the PATH variable so that things like a local install diff --git a/README.development b/README.development index 76270769b..c6031425b 100644 --- a/README.development +++ b/README.development @@ -19,10 +19,12 @@ To start, make sure you have the following installed: - protoc v3 (https://github.com/protocolbuffers/protobuf/releases) - rustup (https://rustup.rs/) - gettext - - rename + - rename (from the perl script https://github.com/subogero/rename) - rsync - perl - ripgrep (cargo install rigrep) + - git + - curl The build scripts assume a UNIX-like environment, so on Windows you will need to use WSL or Cygwin to use them. @@ -60,6 +62,10 @@ folder. Use 'make clean' to remove some generated files. +To see all commands run by make or any shell script, export the environment +variable SHELLFLAGS with '-x' to tell shell to print all commands run by it. +For example, 'export SHELLFLAGS=-x' on Linux or 'set SHELLFLAGS=-x' on Windows. + PyQt ----- diff --git a/pylib/Makefile b/pylib/Makefile index c78af5bf4..d019999a2 100644 --- a/pylib/Makefile +++ b/pylib/Makefile @@ -1,10 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules + +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) + +.DELETE_ON_ERROR: RUNARGS := .SUFFIXES: BLACKARGS := -t py36 anki tests setup.py tools/*.py --exclude='_pb2|buildinfo' diff --git a/qt/Makefile b/qt/Makefile index 82cd91292..d92370c9d 100644 --- a/qt/Makefile +++ b/qt/Makefile @@ -1,10 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules + +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) + +.DELETE_ON_ERROR: .SUFFIXES: BLACKARGS := -t py36 aqt tests setup.py tools/*.py --exclude='aqt/forms|buildinfo|colors' diff --git a/qt/ftl/scripts/fetch-latest-translations b/qt/ftl/scripts/fetch-latest-translations index 5285d7031..108eed1f5 100755 --- a/qt/ftl/scripts/fetch-latest-translations +++ b/qt/ftl/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/qt/ftl/scripts/upload-latest-templates b/qt/ftl/scripts/upload-latest-templates index a60f3ff2b..058835553 100755 --- a/qt/ftl/scripts/upload-latest-templates +++ b/qt/ftl/scripts/upload-latest-templates @@ -3,6 +3,8 @@ # expects to be run from the ftl folder # +set -eu -o pipefail ${SHELLFLAGS} + test -d repo || exit 1 rsync -av --delete *.ftl repo/desktop/templates/ diff --git a/qt/po/scripts/build-mo-files b/qt/po/scripts/build-mo-files index ea107015b..9e04f6a34 100755 --- a/qt/po/scripts/build-mo-files +++ b/qt/po/scripts/build-mo-files @@ -2,7 +2,8 @@ # # build mo files # -set -eo pipefail + +set -eu -o pipefail ${SHELLFLAGS} targetDir="../aqt_data/locale/gettext" mkdir -p $targetDir diff --git a/qt/po/scripts/copy-qt-files b/qt/po/scripts/copy-qt-files index 0a8a7eb25..248ca491c 100755 --- a/qt/po/scripts/copy-qt-files +++ b/qt/po/scripts/copy-qt-files @@ -1,15 +1,14 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} out=../aqt_data/locale/qt mkdir -p "$out" qtTranslations="$(python -c "from PyQt5.QtCore import *; import sys; sys.stdout.write(QLibraryInfo.location(QLibraryInfo.TranslationsPath))")" -unameOut="$(uname -s)" -case "${unameOut}" in - CYGWIN*) +case "$(uname -s)" in + CYGWIN*|MINGW*|MSYS*) qtTranslations="$(cygpath -u "${qtTranslations}")" ;; esac diff --git a/qt/po/scripts/fetch-latest-translations b/qt/po/scripts/fetch-latest-translations index 286745fe4..9d0e04223 100755 --- a/qt/po/scripts/fetch-latest-translations +++ b/qt/po/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/qt/po/scripts/update-po-template b/qt/po/scripts/update-po-template index 223cada7a..d36abc917 100755 --- a/qt/po/scripts/update-po-template +++ b/qt/po/scripts/update-po-template @@ -2,7 +2,8 @@ # # update template .pot file from source code strings # -set -eo pipefail + +set -eu -o pipefail ${SHELLFLAGS} topDir=$(dirname $0)/../../../ cd $topDir diff --git a/qt/po/scripts/upload-latest-template b/qt/po/scripts/upload-latest-template index cef056142..5935a1f94 100755 --- a/qt/po/scripts/upload-latest-template +++ b/qt/po/scripts/upload-latest-template @@ -1,6 +1,6 @@ #!/bin/bash -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} scripts/update-po-template (cd repo && git add desktop; git commit -m update; git push) diff --git a/qt/tools/build_ui.sh b/qt/tools/build_ui.sh index 80e28d37a..7b5db20c5 100755 --- a/qt/tools/build_ui.sh +++ b/qt/tools/build_ui.sh @@ -4,7 +4,7 @@ # should be on the path. # -set -eo pipefail +set -eu -o pipefail ${SHELLFLAGS} if [ ! -d "designer" ] then diff --git a/qt/tools/typecheck-setup.sh b/qt/tools/typecheck-setup.sh index 65311f07c..ee68cb2a2 100755 --- a/qt/tools/typecheck-setup.sh +++ b/qt/tools/typecheck-setup.sh @@ -6,8 +6,22 @@ # able to resolve. A solution that doesn't require modifying the python install # would be welcome! +set -eu -o pipefail ${SHELLFLAGS} + +# https://stackoverflow.com/questions/3601515/how-to-check-if-a-variable-is-set-in-bash +if [[ -z "${OS+x}" ]]; then + OS=unknown; +fi + TOOLS="$(cd "`dirname "$0"`"; pwd)" -modDir=$(python -c 'import PyQt5, sys, os; print(os.path.dirname(sys.modules["PyQt5"].__file__))') -cmd="rsync -a $TOOLS/stubs/PyQt5/* $modDir/" +modDir=$(python -c 'import PyQt5, sys, os; sys.stdout.write(os.path.dirname(sys.modules["PyQt5"].__file__))') + +case "$(uname -s)" in + CYGWIN*|MINGW*|MSYS*) + modDir="$(cygpath -u "${modDir}")" + ;; +esac + +cmd="rsync -a \"${TOOLS}/stubs/PyQt5/\" \"${modDir}/\"" $cmd > /dev/null 2>&1 || sudo $cmd diff --git a/rslib/Makefile b/rslib/Makefile index 2c38d2723..6551bbfe8 100644 --- a/rslib/Makefile +++ b/rslib/Makefile @@ -1,11 +1,16 @@ SHELL := /bin/bash -FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c MAKEFLAGS += --warn-undefined-variables MAKEFLAGS += --no-builtin-rules +FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) +.DELETE_ON_ERROR: + $(shell mkdir -p .build) .PHONY: all check fix clean develop diff --git a/rslib/ftl/scripts/fetch-latest-translations b/rslib/ftl/scripts/fetch-latest-translations index 36b579cc6..1156605a5 100755 --- a/rslib/ftl/scripts/fetch-latest-translations +++ b/rslib/ftl/scripts/fetch-latest-translations @@ -1,5 +1,7 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + echo "Downloading latest translations..." if [ ! -d repo ]; then diff --git a/rslib/ftl/scripts/upload-latest-templates b/rslib/ftl/scripts/upload-latest-templates index 0c4b98d7b..474d899c8 100755 --- a/rslib/ftl/scripts/upload-latest-templates +++ b/rslib/ftl/scripts/upload-latest-templates @@ -3,6 +3,8 @@ # expects to be run from the ftl folder # +set -eu -o pipefail ${SHELLFLAGS} + test -d repo || exit 1 rsync -av --delete *.ftl repo/core/templates/ diff --git a/rspy/Makefile b/rspy/Makefile index 798a67515..20362e170 100644 --- a/rspy/Makefile +++ b/rspy/Makefile @@ -1,25 +1,36 @@ SHELL := /bin/bash + +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + FIND := $(if $(wildcard /bin/find),/bin/find,/usr/bin/find) ifndef OS OS := unknown endif -ifeq ($(OS),Windows_NT) - ifndef PYTHON_BIN - PYTHON_BIN := python +ifeq (${OS},Windows_NT) + ifndef PYTHON_FILE + PYTHON_FILE := $(shell which python) + USE_CYGPATH_COMMAND := case "$$(uname -s)" in CYGWIN*|MINGW*|MSYS*) printf "yes";; esac; + USE_CYGPATH := $(shell ${USE_CYGPATH_COMMAND}) + + ifneq (,${USE_CYGPATH}) + PYTHON_FILE := $(shell cygpath -w "${PYTHON_FILE}") + endif endif else - ifndef PYTHON_BIN - PYTHON_BIN := python3 + ifndef PYTHON_FILE + PYTHON_FILE := $(shell which python3) endif endif -.SHELLFLAGS := -eu -o pipefail -c .DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - $(shell mkdir -p .build ../dist) OUTDIR := ../dist @@ -41,7 +52,7 @@ DEPS := .build/tools .build/vernum ../meta/buildhash \ $(wildcard $(QT_FTL_LOCALES)/*/*.ftl) \ $(shell ${FIND} ../rslib/src -name '*.rs') $(wildcard ../proto/*) \ $(shell ${FIND} ../rslib/ftl -type f) \ - $(shell ${FIND} ./src -type f) + $(shell ${FIND} ./src -type f) .build/develop: $(DEPS) touch ../proto/backend.proto @@ -50,10 +61,10 @@ DEPS := .build/tools .build/vernum ../meta/buildhash \ touch $@ build: $(DEPS) - rm -rf $(OUTDIR)/ankirspy* + rm -rf "$(OUTDIR)"/ankirspy* touch ../proto/backend.proto FTL_TEMPLATE_DIRS="$(QT_FTL_TEMPLATES)" FTL_LOCALE_DIRS="$(QT_FTL_LOCALES)" \ - maturin build -i $(shell which ${PYTHON_BIN}) -o $(OUTDIR) $(BUILDFLAGS) + maturin build -i "${PYTHON_FILE}" -o "$(OUTDIR)" $(BUILDFLAGS) check: .build/check diff --git a/run b/run index 7e984857d..22099b665 100755 --- a/run +++ b/run @@ -1,4 +1,6 @@ #!/bin/bash +set -eu -o pipefail ${SHELLFLAGS} + python --version make -C $(dirname $0) run RUNFLAGS="$*"