update i18n scripts

- export updated .po files for consumption
- add a script to pull and push translations

pylib/BUILD.bazel

@@ -49,8 +49,8 @@ py_test(
 py_test(
     name = "format",
     srcs = [
-        "//pylib/tools:py_files",
+        "//pylib/tools:py_source_files",
-        "//pylib/anki:py_files",
+        "//pylib/anki:py_source_files",
     ] + glob([
         "tests/**/*.py",
     ]),

pylib/anki/BUILD.bazel

@@ -130,7 +130,10 @@ py_wheel(
 )
 
 filegroup(
-    name = "py_files",
+    name = "py_source_files",
     srcs = glob(["**/*.py"]),
-    visibility = ["//pylib:__subpackages__"],
+    visibility = [
+        "//pylib:__subpackages__",
+        "//qt/po:__pkg__",
+    ],
 )

pylib/tools/BUILD.bazel

@@ -64,7 +64,7 @@ py_binary(
 )
 
 filegroup(
-    name = "py_files",
+    name = "py_source_files",
     srcs = glob(["*.py"]),
     visibility = ["//pylib:__subpackages__"],
 )

qt/aqt/BUILD.bazel

@@ -115,3 +115,11 @@ py_wheel(
         ":aqt_pkg",
     ],
 )
+
+filegroup(
+    name = "py_source_files",
+    srcs = glob(["**/*.py"]),
+    visibility = [
+        "//qt/po:__pkg__",
+    ],
+)

qt/aqt/data/locale/BUILD.bazel

@@ -1,6 +1,6 @@
-load("compile.bzl", "compile_all")
+load("//qt/po:gettext.bzl", "compile_all_po_files")
 
-compile_all(
+compile_all_po_files(
     name = "locale",
     visibility = ["//qt:__subpackages__"],
 )

qt/aqt/data/locale/compile.bzl

@@ -1,88 +0,0 @@
-def compile(name, po_file, pot_file, mo_file):
-    native.genrule(
-        name = name,
-        srcs = [po_file, pot_file],
-        outs = [mo_file],
-        # homebrew gettext is not on path by default
-        cmd = """\
-export PATH="$$PATH":/usr/local/opt/gettext/bin
-msgmerge -q $(location {po_file}) $(location {pot_file}) | msgfmt - --output-file=$(location {mo_file})
-""".format(
-            po_file = po_file,
-            pot_file = pot_file,
-            mo_file = mo_file,
-        ),
-        message = "Building translation",
-    )
-
-_langs = [
-    "af",
-    "ar",
-    "bg",
-    "ca",
-    "cs",
-    "da",
-    "de",
-    "el",
-    "en-GB",
-    "eo",
-    "es",
-    "et",
-    "eu",
-    "fa",
-    "fi",
-    "fr",
-    "ga-IE",
-    "gl",
-    "he",
-    "hi-IN",
-    "hr",
-    "hu",
-    "hy-AM",
-    "it",
-    "ja",
-    "jbo",
-    "kab",
-    #    "km",
-    "ko",
-    "la",
-    "mn",
-    "mr",
-    "ms",
-    "nb-NO",
-    "nl",
-    "nn-NO",
-    "oc",
-    "or",
-    "pl",
-    "pt-BR",
-    "pt-PT",
-    "ro",
-    "ru",
-    "sk",
-    "sl",
-    "sr",
-    "sv-SE",
-    "th",
-    "tr",
-    "uk",
-    #    "ur",
-    "vi",
-    "zh-CN",
-    "zh-TW",
-]
-
-def compile_all(name, visibility):
-    pot_file = "@aqt_po//:desktop/anki.pot"
-    mo_files = []
-    for lang in _langs:
-        po_file = "@aqt_po//:desktop/{}/anki.po".format(lang)
-        mo_file = "{}/LC_MESSAGES/anki.mo".format(lang)
-        mo_files.append(mo_file)
-        compile(lang, po_file, pot_file, mo_file)
-
-    native.filegroup(
-        name = name,
-        srcs = mo_files,
-        visibility = visibility,
-    )

qt/aqt/forms/BUILD.bazel

@@ -11,6 +11,10 @@ py_binary(
 compile_all(
     srcs = glob(["*.ui"]),
     group = "forms",
+    visibility = [
+        "//qt/aqt:__pkg__",
+        "//qt/po:__pkg__",
+    ],
 )
 
 py_binary(

qt/aqt/forms/compile.bzl

@@ -13,7 +13,7 @@ def compile(name, ui_file, py_file):
         message = "Building UI",
     )
 
-def compile_all(group, srcs):
+def compile_all(group, srcs, visibility):
     py_files = []
     for ui_file in srcs:
         name = ui_file.replace(".ui", "")
@@ -24,5 +24,5 @@ def compile_all(group, srcs):
     native.filegroup(
         name = group,
         srcs = py_files + ["__init__.py"],
-        visibility = ["//qt/aqt:__pkg__"],
+        visibility = visibility,
     )

qt/po/BUILD.bazel

@@ -0,0 +1,17 @@
+load(":gettext.bzl", "build_template", "update_all_po_files")
+
+build_template(
+    name = "pot",
+    srcs = [
+        "//pylib/anki:py_source_files",
+        "//qt/aqt:py_source_files",
+        "//qt/aqt/forms",
+    ],
+    pot_file = "anki.pot",
+)
+
+update_all_po_files(
+    name = "po_files",
+    pot_file = "anki.pot",
+    visibility = ["//qt/aqt:__subpackages__"],
+)

qt/po/gettext.bzl

@@ -0,0 +1,149 @@
+_langs = [
+    "af",
+    "ar",
+    "bg",
+    "ca",
+    "cs",
+    "da",
+    "de",
+    "el",
+    "en-GB",
+    "eo",
+    "es",
+    "et",
+    "eu",
+    "fa",
+    "fi",
+    "fr",
+    "ga-IE",
+    "gl",
+    "he",
+    "hi-IN",
+    "hr",
+    "hu",
+    "hy-AM",
+    "it",
+    "ja",
+    "jbo",
+    "kab",
+    #    "km",
+    "ko",
+    "la",
+    "mn",
+    "mr",
+    "ms",
+    "nb-NO",
+    "nl",
+    "nn-NO",
+    "oc",
+    "or",
+    "pl",
+    "pt-BR",
+    "pt-PT",
+    "ro",
+    "ru",
+    "sk",
+    "sl",
+    "sr",
+    "sv-SE",
+    "th",
+    "tr",
+    "uk",
+    #    "ur",
+    "vi",
+    "zh-CN",
+    "zh-TW",
+]
+
+# homebrew gettext is not on path by default
+_pathfix = """export PATH="$$PATH":/usr/local/opt/gettext/bin\n"""
+
+def update_po(name, po_file_in, po_file_out, pot_file, visibility):
+    "Merge old .po and latest strings from .pot into new .po"
+    native.genrule(
+        name = name,
+        srcs = [po_file_in, pot_file],
+        outs = [po_file_out],
+        cmd = _pathfix + """\
+msgmerge -q --no-wrap $(location {po_file_in}) $(location {pot_file}) > $(location {po_file_out})
+""".format(
+            po_file_in = po_file_in,
+            po_file_out = po_file_out,
+            pot_file = pot_file,
+        ),
+        message = "Updating translation",
+        visibility = visibility,
+    )
+
+def compile_po(name, po_file, mo_file):
+    "Build .mo file from an updated .po file."
+    native.genrule(
+        name = name,
+        srcs = [po_file],
+        outs = [mo_file],
+        # homebrew gettext is not on path by default
+        cmd = _pathfix + """\
+cat $(location {po_file}) | msgfmt - --output-file=$(location {mo_file})
+""".format(
+            po_file = po_file,
+            mo_file = mo_file,
+        ),
+        message = "Compiling translation",
+    )
+
+def build_template(name, pot_file, srcs):
+    "Build .pot file from Python files."
+    native.genrule(
+        name = name,
+        srcs = srcs,
+        outs = [pot_file],
+        cmd = _pathfix + """\
+all=all.files
+for i in $(SRCS); do
+    echo $$i >> $$all
+done
+xgettext -cT: -s --no-wrap --files-from=$$all --output=$(OUTS)
+rm $$all
+""",
+        message = "Building .pot template",
+    )
+
+def update_all_po_files(name, pot_file, visibility):
+    # merge external .po files with updated .pot
+    po_files = []
+    for lang in _langs:
+        po_file_in = "@aqt_po//:desktop/{}/anki.po".format(lang)
+        po_file_out = "{}/anki.po".format(lang)
+        update_po(
+            name = lang + "_po",
+            po_file_in = po_file_in,
+            po_file_out = po_file_out,
+            pot_file = pot_file,
+            visibility = visibility,
+        )
+        po_files.append(po_file_out)
+
+    native.filegroup(
+        name = name,
+        srcs = po_files,
+        visibility = visibility,
+    )
+
+def compile_all_po_files(name, visibility):
+    "Build all .mo files from .po files."
+    mo_files = []
+    for lang in _langs:
+        po_file = "//qt/po:{}/anki.po".format(lang)
+        mo_file = "{}/LC_MESSAGES/anki.mo".format(lang)
+        compile_po(
+            name = lang + "_mo",
+            po_file = po_file,
+            mo_file = mo_file,
+        )
+        mo_files.append(mo_file)
+
+    native.filegroup(
+        name = name,
+        srcs = mo_files,
+        visibility = visibility,
+    )

qt/po/scripts/build-mo-files

@@ -1,22 +0,0 @@
-#!/bin/bash
-#
-# build mo files
-#
-
-set -eu -o pipefail ${SHELLFLAGS}
-
-targetDir="../aqt_data/locale/gettext"
-mkdir -p $targetDir
-
-echo "Compiling *.po..."
-for file in repo/desktop/*/anki.po
-do
-    outdir=$(echo "$file" | \
-        perl -pe "s%repo/desktop/(.*)/anki.po%$targetDir/\1/LC_MESSAGES%")
-    outfile="$outdir/anki.mo"
-    mkdir -p $outdir
-    if ! msgmerge -q "$file" repo/desktop/anki.pot | msgfmt - --output-file="$outfile"; then
-        echo "error building $file";
-        exit 1;
-    fi;
-done

qt/po/scripts/copy-qt-files

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail ${SHELLFLAGS}
-
-out=../aqt_data/locale/qt
-mkdir -p "$out"
-
-qtTranslations="$(python -c "from PyQt5.QtCore import *; import sys; sys.stdout.write(QLibraryInfo.location(QLibraryInfo.TranslationsPath))")"
-
-case "$(uname -s)" in
-    CYGWIN*|MINGW*|MSYS*)
-        qtTranslations="$(cygpath -u "${qtTranslations}")"
-        ;;
-esac
-
-rsync -a "$qtTranslations/" "$out/"

qt/po/scripts/update-po-template

@@ -1,27 +0,0 @@
-#!/bin/bash
-#
-# update template .pot file from source code strings,
-# and merge new strings into translations
-#
-
-set -eu -o pipefail ${SHELLFLAGS}
-
-topDir=$(dirname $0)/../../../
-cd $topDir
-
-all=all.files
-echo "Updating anki.pot..."
-for i in pylib/anki/{*.py,importing/*.py}; do
-    echo $i >> $all
-done
-for i in qt/aqt/{*.py,forms/*.py}; do
-    echo $i >> $all
-done
-
-xgettext -cT: -s --no-wrap --files-from=$all --output=qt/po/repo/desktop/anki.pot
-rm $all
-
-cd qt/po/repo/desktop
-for dir in $(ls | grep -v anki.pot); do
-    msgmerge --no-wrap -U --backup off $dir/anki.po anki.pot
-done

scripts/synci18n.py

@@ -0,0 +1,158 @@
+#
+# A helper script to update commit references to the latest translations,
+# and copy source files to the translation repos. Requires access to the
+# i18n repos to run.
+
+import subprocess
+from dataclasses import dataclass
+import re
+import os
+from typing import Optional, Tuple
+
+repos_bzl = "repos.bzl"
+working_folder = "../anki-i18n"
+
+if not os.path.exists(repos_bzl):
+    raise Exception("run from workspace root")
+
+if not os.path.exists(working_folder):
+    os.mkdir(working_folder)
+
+
+@dataclass
+class Module:
+    name: str
+    repo: str
+    # (source ftl folder, i18n templates folder)
+    ftl: Optional[Tuple[str, str]] = None
+
+    def folder(self) -> str:
+        return os.path.join(working_folder, self.name)
+
+
+modules = [
+    Module(
+        name="core",
+        repo="git@github.com:ankitects/anki-core-i18n",
+        ftl=("rslib/ftl", "core/templates"),
+    ),
+    Module(
+        name="qtftl",
+        repo="git@github.com:ankitects/anki-desktop-ftl",
+        ftl=("qt/ftl", "desktop/templates"),
+    ),
+    # update_po_templates() expects this last
+    Module(name="qtpo", repo="git@github.com:ankitects/anki-desktop-i18n"),
+]
+
+
+def update_repo(module: Module):
+    subprocess.run(["git", "pull"], cwd=module.folder(), check=True)
+
+
+def clone_repo(module: Module):
+    subprocess.run(
+        ["git", "clone", module.repo, module.name], cwd=working_folder, check=True
+    )
+
+
+def update_git_repos():
+    for module in modules:
+        if os.path.exists(module.folder()):
+            update_repo(module)
+        else:
+            clone_repo(module)
+
+
+@dataclass
+class GitInfo:
+    sha1: str
+    shallow_since: str
+
+
+def module_git_info(module: Module) -> GitInfo:
+    folder = module.folder()
+    sha = subprocess.check_output(
+        ["git", "log", "-n", "1", "--pretty=format:%H"], cwd=folder
+    )
+    shallow = subprocess.check_output(
+        ["git", "log", "-n", "1", "--pretty=format:%cd", "--date=raw"], cwd=folder
+    )
+    return GitInfo(sha1=sha.decode("utf8"), shallow_since=shallow.decode("utf8"))
+
+
+def update_repos_bzl():
+    # gather changes
+    entries = {}
+    for module in modules:
+        git = module_git_info(module)
+        prefix = f"{module.name}_i18n_"
+        entries[prefix + "commit"] = git.sha1
+        entries[prefix + "shallow_since"] = git.shallow_since
+
+    # apply
+    out = []
+    path = repos_bzl
+    reg = re.compile(r'(\s+)(\S+_(?:commit|shallow_since)) = "(.*)"')
+    for line in open(path).readlines():
+        if m := reg.match(line):
+            (indent, key, _oldvalue) = m.groups()
+            value = entries[key]
+            line = f'{indent}{key} = "{value}"\n'
+            out.append(line)
+        else:
+            out.append(line)
+    open(path, "w").writelines(out)
+
+    commit_if_changed(".")
+
+
+def commit_if_changed(folder: str):
+    status = subprocess.run(["git", "diff", "--exit-code"], cwd=folder, check=False)
+    if status.returncode == 0:
+        # no changes
+        return
+    subprocess.run(
+        ["git", "commit", "-a", "-m", "update translations"], cwd=folder, check=True
+    )
+
+
+def update_ftl_templates():
+    for module in modules:
+        if ftl := module.ftl:
+            (source, dest) = ftl
+            dest = os.path.join(module.folder(), dest)
+            subprocess.run(
+                ["rsync", "-ai", "--delete", "--no-perms", source + "/", dest + "/"],
+                check=True,
+            )
+            commit_if_changed(module.folder())
+
+
+def update_pot_and_po_files() -> str:
+    "Update .pot and .po files, returning generated folder root."
+    subprocess.run(["bazel", "build", "//qt/po:po_files"], check=True)
+    return "bazel-bin/qt/po/"
+
+
+def update_po_templates():
+    "Copy updated files into repo."
+    module = modules[-1]
+    src_root = update_pot_and_po_files()
+    dest_root = os.path.join(module.folder(), "desktop")
+    subprocess.run(
+        ["rsync", "-ai", "--no-perms", src_root + "/", dest_root + "/"], check=True
+    )
+    commit_if_changed(module.folder())
+
+
+def push_changes():
+    for module in modules:
+        subprocess.run(["git", "push"], cwd=module.folder(), check=True)
+
+
+update_git_repos()
+update_repos_bzl()
+update_ftl_templates()
+update_po_templates()
+push_changes()