From 8d4df820cce0b8a94f08f88f453ee3f500f1f361 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 11 Nov 2020 21:06:51 +1000 Subject: [PATCH] update i18n scripts - export updated .po files for consumption - add a script to pull and push translations --- pylib/BUILD.bazel | 4 +- pylib/anki/BUILD.bazel | 7 +- pylib/tools/BUILD.bazel | 2 +- qt/aqt/BUILD.bazel | 8 ++ qt/aqt/data/locale/BUILD.bazel | 4 +- qt/aqt/data/locale/compile.bzl | 88 ----------------- qt/aqt/forms/BUILD.bazel | 4 + qt/aqt/forms/compile.bzl | 4 +- qt/po/BUILD.bazel | 17 ++++ qt/po/gettext.bzl | 149 +++++++++++++++++++++++++++++ qt/po/scripts/build-mo-files | 22 ----- qt/po/scripts/copy-qt-files | 16 ---- qt/po/scripts/update-po-template | 27 ------ scripts/synci18n.py | 158 +++++++++++++++++++++++++++++++ 14 files changed, 348 insertions(+), 162 deletions(-) delete mode 100644 qt/aqt/data/locale/compile.bzl create mode 100644 qt/po/BUILD.bazel create mode 100644 qt/po/gettext.bzl delete mode 100755 qt/po/scripts/build-mo-files delete mode 100755 qt/po/scripts/copy-qt-files delete mode 100755 qt/po/scripts/update-po-template create mode 100644 scripts/synci18n.py diff --git a/pylib/BUILD.bazel b/pylib/BUILD.bazel index fd7039906..c1a3ab618 100644 --- a/pylib/BUILD.bazel +++ b/pylib/BUILD.bazel @@ -49,8 +49,8 @@ py_test( py_test( name = "format", srcs = [ - "//pylib/tools:py_files", - "//pylib/anki:py_files", + "//pylib/tools:py_source_files", + "//pylib/anki:py_source_files", ] + glob([ "tests/**/*.py", ]), diff --git a/pylib/anki/BUILD.bazel b/pylib/anki/BUILD.bazel index d431b7492..77693d71d 100644 --- a/pylib/anki/BUILD.bazel +++ b/pylib/anki/BUILD.bazel @@ -130,7 +130,10 @@ py_wheel( ) filegroup( - name = "py_files", + name = "py_source_files", srcs = glob(["**/*.py"]), - visibility = ["//pylib:__subpackages__"], + visibility = [ + "//pylib:__subpackages__", + "//qt/po:__pkg__", + ], ) diff --git a/pylib/tools/BUILD.bazel b/pylib/tools/BUILD.bazel index 91d989507..7bedc377f 100644 --- a/pylib/tools/BUILD.bazel +++ b/pylib/tools/BUILD.bazel @@ -64,7 +64,7 @@ py_binary( ) filegroup( - name = "py_files", + name = "py_source_files", srcs = glob(["*.py"]), visibility = ["//pylib:__subpackages__"], ) diff --git a/qt/aqt/BUILD.bazel b/qt/aqt/BUILD.bazel index 9341b557a..675a38c54 100644 --- a/qt/aqt/BUILD.bazel +++ b/qt/aqt/BUILD.bazel @@ -115,3 +115,11 @@ py_wheel( ":aqt_pkg", ], ) + +filegroup( + name = "py_source_files", + srcs = glob(["**/*.py"]), + visibility = [ + "//qt/po:__pkg__", + ], +) diff --git a/qt/aqt/data/locale/BUILD.bazel b/qt/aqt/data/locale/BUILD.bazel index b02cdd5d6..1fb916e8d 100644 --- a/qt/aqt/data/locale/BUILD.bazel +++ b/qt/aqt/data/locale/BUILD.bazel @@ -1,6 +1,6 @@ -load("compile.bzl", "compile_all") +load("//qt/po:gettext.bzl", "compile_all_po_files") -compile_all( +compile_all_po_files( name = "locale", visibility = ["//qt:__subpackages__"], ) diff --git a/qt/aqt/data/locale/compile.bzl b/qt/aqt/data/locale/compile.bzl deleted file mode 100644 index 395989494..000000000 --- a/qt/aqt/data/locale/compile.bzl +++ /dev/null @@ -1,88 +0,0 @@ -def compile(name, po_file, pot_file, mo_file): - native.genrule( - name = name, - srcs = [po_file, pot_file], - outs = [mo_file], - # homebrew gettext is not on path by default - cmd = """\ -export PATH="$$PATH":/usr/local/opt/gettext/bin -msgmerge -q $(location {po_file}) $(location {pot_file}) | msgfmt - --output-file=$(location {mo_file}) -""".format( - po_file = po_file, - pot_file = pot_file, - mo_file = mo_file, - ), - message = "Building translation", - ) - -_langs = [ - "af", - "ar", - "bg", - "ca", - "cs", - "da", - "de", - "el", - "en-GB", - "eo", - "es", - "et", - "eu", - "fa", - "fi", - "fr", - "ga-IE", - "gl", - "he", - "hi-IN", - "hr", - "hu", - "hy-AM", - "it", - "ja", - "jbo", - "kab", - # "km", - "ko", - "la", - "mn", - "mr", - "ms", - "nb-NO", - "nl", - "nn-NO", - "oc", - "or", - "pl", - "pt-BR", - "pt-PT", - "ro", - "ru", - "sk", - "sl", - "sr", - "sv-SE", - "th", - "tr", - "uk", - # "ur", - "vi", - "zh-CN", - "zh-TW", -] - -def compile_all(name, visibility): - pot_file = "@aqt_po//:desktop/anki.pot" - mo_files = [] - for lang in _langs: - po_file = "@aqt_po//:desktop/{}/anki.po".format(lang) - mo_file = "{}/LC_MESSAGES/anki.mo".format(lang) - mo_files.append(mo_file) - compile(lang, po_file, pot_file, mo_file) - - native.filegroup( - name = name, - srcs = mo_files, - visibility = visibility, - ) diff --git a/qt/aqt/forms/BUILD.bazel b/qt/aqt/forms/BUILD.bazel index 57bd2d66f..9c86f7f05 100644 --- a/qt/aqt/forms/BUILD.bazel +++ b/qt/aqt/forms/BUILD.bazel @@ -11,6 +11,10 @@ py_binary( compile_all( srcs = glob(["*.ui"]), group = "forms", + visibility = [ + "//qt/aqt:__pkg__", + "//qt/po:__pkg__", + ], ) py_binary( diff --git a/qt/aqt/forms/compile.bzl b/qt/aqt/forms/compile.bzl index cb4377423..36a654f70 100644 --- a/qt/aqt/forms/compile.bzl +++ b/qt/aqt/forms/compile.bzl @@ -13,7 +13,7 @@ def compile(name, ui_file, py_file): message = "Building UI", ) -def compile_all(group, srcs): +def compile_all(group, srcs, visibility): py_files = [] for ui_file in srcs: name = ui_file.replace(".ui", "") @@ -24,5 +24,5 @@ def compile_all(group, srcs): native.filegroup( name = group, srcs = py_files + ["__init__.py"], - visibility = ["//qt/aqt:__pkg__"], + visibility = visibility, ) diff --git a/qt/po/BUILD.bazel b/qt/po/BUILD.bazel new file mode 100644 index 000000000..cdca3e8a3 --- /dev/null +++ b/qt/po/BUILD.bazel @@ -0,0 +1,17 @@ +load(":gettext.bzl", "build_template", "update_all_po_files") + +build_template( + name = "pot", + srcs = [ + "//pylib/anki:py_source_files", + "//qt/aqt:py_source_files", + "//qt/aqt/forms", + ], + pot_file = "anki.pot", +) + +update_all_po_files( + name = "po_files", + pot_file = "anki.pot", + visibility = ["//qt/aqt:__subpackages__"], +) diff --git a/qt/po/gettext.bzl b/qt/po/gettext.bzl new file mode 100644 index 000000000..80a88c9da --- /dev/null +++ b/qt/po/gettext.bzl @@ -0,0 +1,149 @@ +_langs = [ + "af", + "ar", + "bg", + "ca", + "cs", + "da", + "de", + "el", + "en-GB", + "eo", + "es", + "et", + "eu", + "fa", + "fi", + "fr", + "ga-IE", + "gl", + "he", + "hi-IN", + "hr", + "hu", + "hy-AM", + "it", + "ja", + "jbo", + "kab", + # "km", + "ko", + "la", + "mn", + "mr", + "ms", + "nb-NO", + "nl", + "nn-NO", + "oc", + "or", + "pl", + "pt-BR", + "pt-PT", + "ro", + "ru", + "sk", + "sl", + "sr", + "sv-SE", + "th", + "tr", + "uk", + # "ur", + "vi", + "zh-CN", + "zh-TW", +] + +# homebrew gettext is not on path by default +_pathfix = """export PATH="$$PATH":/usr/local/opt/gettext/bin\n""" + +def update_po(name, po_file_in, po_file_out, pot_file, visibility): + "Merge old .po and latest strings from .pot into new .po" + native.genrule( + name = name, + srcs = [po_file_in, pot_file], + outs = [po_file_out], + cmd = _pathfix + """\ +msgmerge -q --no-wrap $(location {po_file_in}) $(location {pot_file}) > $(location {po_file_out}) +""".format( + po_file_in = po_file_in, + po_file_out = po_file_out, + pot_file = pot_file, + ), + message = "Updating translation", + visibility = visibility, + ) + +def compile_po(name, po_file, mo_file): + "Build .mo file from an updated .po file." + native.genrule( + name = name, + srcs = [po_file], + outs = [mo_file], + # homebrew gettext is not on path by default + cmd = _pathfix + """\ +cat $(location {po_file}) | msgfmt - --output-file=$(location {mo_file}) +""".format( + po_file = po_file, + mo_file = mo_file, + ), + message = "Compiling translation", + ) + +def build_template(name, pot_file, srcs): + "Build .pot file from Python files." + native.genrule( + name = name, + srcs = srcs, + outs = [pot_file], + cmd = _pathfix + """\ +all=all.files +for i in $(SRCS); do + echo $$i >> $$all +done +xgettext -cT: -s --no-wrap --files-from=$$all --output=$(OUTS) +rm $$all +""", + message = "Building .pot template", + ) + +def update_all_po_files(name, pot_file, visibility): + # merge external .po files with updated .pot + po_files = [] + for lang in _langs: + po_file_in = "@aqt_po//:desktop/{}/anki.po".format(lang) + po_file_out = "{}/anki.po".format(lang) + update_po( + name = lang + "_po", + po_file_in = po_file_in, + po_file_out = po_file_out, + pot_file = pot_file, + visibility = visibility, + ) + po_files.append(po_file_out) + + native.filegroup( + name = name, + srcs = po_files, + visibility = visibility, + ) + +def compile_all_po_files(name, visibility): + "Build all .mo files from .po files." + mo_files = [] + for lang in _langs: + po_file = "//qt/po:{}/anki.po".format(lang) + mo_file = "{}/LC_MESSAGES/anki.mo".format(lang) + compile_po( + name = lang + "_mo", + po_file = po_file, + mo_file = mo_file, + ) + mo_files.append(mo_file) + + native.filegroup( + name = name, + srcs = mo_files, + visibility = visibility, + ) diff --git a/qt/po/scripts/build-mo-files b/qt/po/scripts/build-mo-files deleted file mode 100755 index 40fdd34de..000000000 --- a/qt/po/scripts/build-mo-files +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# build mo files -# - -set -eu -o pipefail ${SHELLFLAGS} - -targetDir="../aqt_data/locale/gettext" -mkdir -p $targetDir - -echo "Compiling *.po..." -for file in repo/desktop/*/anki.po -do - outdir=$(echo "$file" | \ - perl -pe "s%repo/desktop/(.*)/anki.po%$targetDir/\1/LC_MESSAGES%") - outfile="$outdir/anki.mo" - mkdir -p $outdir - if ! msgmerge -q "$file" repo/desktop/anki.pot | msgfmt - --output-file="$outfile"; then - echo "error building $file"; - exit 1; - fi; -done diff --git a/qt/po/scripts/copy-qt-files b/qt/po/scripts/copy-qt-files deleted file mode 100755 index a57ff947a..000000000 --- a/qt/po/scripts/copy-qt-files +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu -o pipefail ${SHELLFLAGS} - -out=../aqt_data/locale/qt -mkdir -p "$out" - -qtTranslations="$(python -c "from PyQt5.QtCore import *; import sys; sys.stdout.write(QLibraryInfo.location(QLibraryInfo.TranslationsPath))")" - -case "$(uname -s)" in - CYGWIN*|MINGW*|MSYS*) - qtTranslations="$(cygpath -u "${qtTranslations}")" - ;; -esac - -rsync -a "$qtTranslations/" "$out/" diff --git a/qt/po/scripts/update-po-template b/qt/po/scripts/update-po-template deleted file mode 100755 index 6a734d07b..000000000 --- a/qt/po/scripts/update-po-template +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# update template .pot file from source code strings, -# and merge new strings into translations -# - -set -eu -o pipefail ${SHELLFLAGS} - -topDir=$(dirname $0)/../../../ -cd $topDir - -all=all.files -echo "Updating anki.pot..." -for i in pylib/anki/{*.py,importing/*.py}; do - echo $i >> $all -done -for i in qt/aqt/{*.py,forms/*.py}; do - echo $i >> $all -done - -xgettext -cT: -s --no-wrap --files-from=$all --output=qt/po/repo/desktop/anki.pot -rm $all - -cd qt/po/repo/desktop -for dir in $(ls | grep -v anki.pot); do - msgmerge --no-wrap -U --backup off $dir/anki.po anki.pot -done diff --git a/scripts/synci18n.py b/scripts/synci18n.py new file mode 100644 index 000000000..c07515bfb --- /dev/null +++ b/scripts/synci18n.py @@ -0,0 +1,158 @@ +# +# A helper script to update commit references to the latest translations, +# and copy source files to the translation repos. Requires access to the +# i18n repos to run. + +import subprocess +from dataclasses import dataclass +import re +import os +from typing import Optional, Tuple + +repos_bzl = "repos.bzl" +working_folder = "../anki-i18n" + +if not os.path.exists(repos_bzl): + raise Exception("run from workspace root") + +if not os.path.exists(working_folder): + os.mkdir(working_folder) + + +@dataclass +class Module: + name: str + repo: str + # (source ftl folder, i18n templates folder) + ftl: Optional[Tuple[str, str]] = None + + def folder(self) -> str: + return os.path.join(working_folder, self.name) + + +modules = [ + Module( + name="core", + repo="git@github.com:ankitects/anki-core-i18n", + ftl=("rslib/ftl", "core/templates"), + ), + Module( + name="qtftl", + repo="git@github.com:ankitects/anki-desktop-ftl", + ftl=("qt/ftl", "desktop/templates"), + ), + # update_po_templates() expects this last + Module(name="qtpo", repo="git@github.com:ankitects/anki-desktop-i18n"), +] + + +def update_repo(module: Module): + subprocess.run(["git", "pull"], cwd=module.folder(), check=True) + + +def clone_repo(module: Module): + subprocess.run( + ["git", "clone", module.repo, module.name], cwd=working_folder, check=True + ) + + +def update_git_repos(): + for module in modules: + if os.path.exists(module.folder()): + update_repo(module) + else: + clone_repo(module) + + +@dataclass +class GitInfo: + sha1: str + shallow_since: str + + +def module_git_info(module: Module) -> GitInfo: + folder = module.folder() + sha = subprocess.check_output( + ["git", "log", "-n", "1", "--pretty=format:%H"], cwd=folder + ) + shallow = subprocess.check_output( + ["git", "log", "-n", "1", "--pretty=format:%cd", "--date=raw"], cwd=folder + ) + return GitInfo(sha1=sha.decode("utf8"), shallow_since=shallow.decode("utf8")) + + +def update_repos_bzl(): + # gather changes + entries = {} + for module in modules: + git = module_git_info(module) + prefix = f"{module.name}_i18n_" + entries[prefix + "commit"] = git.sha1 + entries[prefix + "shallow_since"] = git.shallow_since + + # apply + out = [] + path = repos_bzl + reg = re.compile(r'(\s+)(\S+_(?:commit|shallow_since)) = "(.*)"') + for line in open(path).readlines(): + if m := reg.match(line): + (indent, key, _oldvalue) = m.groups() + value = entries[key] + line = f'{indent}{key} = "{value}"\n' + out.append(line) + else: + out.append(line) + open(path, "w").writelines(out) + + commit_if_changed(".") + + +def commit_if_changed(folder: str): + status = subprocess.run(["git", "diff", "--exit-code"], cwd=folder, check=False) + if status.returncode == 0: + # no changes + return + subprocess.run( + ["git", "commit", "-a", "-m", "update translations"], cwd=folder, check=True + ) + + +def update_ftl_templates(): + for module in modules: + if ftl := module.ftl: + (source, dest) = ftl + dest = os.path.join(module.folder(), dest) + subprocess.run( + ["rsync", "-ai", "--delete", "--no-perms", source + "/", dest + "/"], + check=True, + ) + commit_if_changed(module.folder()) + + +def update_pot_and_po_files() -> str: + "Update .pot and .po files, returning generated folder root." + subprocess.run(["bazel", "build", "//qt/po:po_files"], check=True) + return "bazel-bin/qt/po/" + + +def update_po_templates(): + "Copy updated files into repo." + module = modules[-1] + src_root = update_pot_and_po_files() + dest_root = os.path.join(module.folder(), "desktop") + subprocess.run( + ["rsync", "-ai", "--no-perms", src_root + "/", dest_root + "/"], check=True + ) + commit_if_changed(module.folder()) + + +def push_changes(): + for module in modules: + subprocess.run(["git", "push"], cwd=module.folder(), check=True) + + +update_git_repos() +update_repos_bzl() +update_ftl_templates() +update_po_templates() +push_changes()