move aqt_data into source folder; implement wheel building

LICENSE

@@ -19,10 +19,7 @@ In the qt folder:
  * jQuery and jQuery-UI: MIT.
  * browsersel.js: CC BY 2.5.
  * plot.js: MIT.
-
+ * protobuf.js: BSD 3 clause
-In the scripts folder:
-
- * rename: Perl's Artistic license.
 
 The above list only covers the source code that is vendored in this
 repository. Binary distributions also include copies of Qt translation

WORKSPACE

@@ -9,14 +9,10 @@ load(":packages.bzl", "register_deps")
 
 register_deps()
 
-load(":setup.bzl", "setup_deps")
+load(":defs.bzl", "setup_deps")
 
 setup_deps()
 
-load("@py_deps//:requirements.bzl", "pip_install")
+load(":late_deps.bzl", "setup_late_deps")
 
-pip_install()
+setup_late_deps()
-
-load("@npm//@bazel/labs:package.bzl", "npm_bazel_labs_dependencies")
-
-npm_bazel_labs_dependencies()

bin.py

@@ -1,7 +0,0 @@
-print("hello world!")
-import sys
-print(sys.version)
-import anki
-import os
-print(os.listdir("."))
-print(anki.buildinfo.version)

defs.bzl

@@ -3,7 +3,6 @@ load("@io_bazel_rules_rust//rust:repositories.bzl", "rust_repositories")
 load("@io_bazel_rules_rust//:workspace.bzl", "bazel_version")
 load("@anki//cargo:crates.bzl", "raze_fetch_remote_crates")
 load(":python.bzl", "setup_local_python")
-load("@rules_python//python:repositories.bzl", "py_repositories")
 load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install")
 load("@io_bazel_rules_sass//:defs.bzl", "sass_repositories")
 load("@build_bazel_rules_svelte//:defs.bzl", "rules_svelte_dependencies")
@@ -11,6 +10,8 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@com_github_ali5h_rules_pip//:defs.bzl", "pip_import")
 load("//pip/pyqt5:defs.bzl", "install_pyqt5")
 
+anki_version = "2.1.36"
+
 def setup_deps():
     bazel_skylib_workspace()
 
@@ -28,8 +29,6 @@ def setup_deps():
 
     native.register_toolchains("@python//:python3_toolchain")
 
-    py_repositories()
-
     # pip_install(
     #     name = "py_deps",
     #     python_interpreter_target = "@python//:python",

docs/Dockerfile

@@ -1,3 +1,7 @@
+# Warning:
+# This file needs updating to work with Anki's updated build system.
+#
+
 ARG PYTHON_VERSION="3.8"
 
 FROM python:$PYTHON_VERSION AS dependencies
@@ -19,21 +23,21 @@ RUN mkdir $CARGO_HOME $RUSTUP_HOME \
 # Install system dependencies.
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
-        gettext \
+    gettext \
-        lame \
+    lame \
-        libnss3 \
+    libnss3 \
-        libxcb-icccm4 \
+    libxcb-icccm4 \
-        libxcb-image0 \
+    libxcb-image0 \
-        libxcb-keysyms1 \
+    libxcb-keysyms1 \
-        libxcb-randr0 \
+    libxcb-randr0 \
-        libxcb-render-util0 \
+    libxcb-render-util0 \
-        libxcb-xinerama0 \
+    libxcb-xinerama0 \
-        libxcb-xkb1 \
+    libxcb-xkb1 \
-        libxkbcommon-x11-0 \
+    libxkbcommon-x11-0 \
-        libxcomposite1 \
+    libxcomposite1 \
-        mpv \
+    mpv \
-        portaudio19-dev \
+    portaudio19-dev \
-        rsync \
+    rsync \
     && rm -rf /var/lib/apt/lists/*
 
 # Install node and npm.
@@ -71,28 +75,28 @@ FROM python:${PYTHON_VERSION}-slim
 # Install system dependencies.
 RUN apt-get update \
     && apt-get install --yes --no-install-recommends \
-        gettext \
+    gettext \
-        lame \
+    lame \
-        libnss3 \
+    libnss3 \
-        libxcb-icccm4 \
+    libxcb-icccm4 \
-        libxcb-image0 \
+    libxcb-image0 \
-        libxcb-keysyms1 \
+    libxcb-keysyms1 \
-        libxcb-randr0 \
+    libxcb-randr0 \
-        libxcb-render-util0 \
+    libxcb-render-util0 \
-        libxcb-xinerama0 \
+    libxcb-xinerama0 \
-        libxcb-xkb1 \
+    libxcb-xkb1 \
-        libxkbcommon-x11-0 \
+    libxkbcommon-x11-0 \
-        libxcomposite1 \
+    libxcomposite1 \
-        mpv \
+    mpv \
-        portaudio19-dev \
+    portaudio19-dev \
-        rsync \
+    rsync \
     && rm -rf /var/lib/apt/lists/*
 
 # Install pre-compiled Anki.
 COPY --from=pythonbuilder /opt/anki/dist/ /opt/anki/
 RUN python -m pip install --no-cache-dir \
-        PyQtWebEngine \
+    PyQtWebEngine \
-        /opt/anki/*.whl \
+    /opt/anki/*.whl \
     # Create an anki executable.
     && printf "#!/usr/bin/env python\nimport aqt\naqt.run()\n" > /usr/local/bin/anki \
     && chmod +x /usr/local/bin/anki \

late_deps.bzl

@@ -0,0 +1,8 @@
+"""Repo setup that can't happen until after defs.bzl:setup_deps() is run."""
+
+load("@py_deps//:requirements.bzl", "pip_install")
+load("@npm//@bazel/labs:package.bzl", "npm_bazel_labs_dependencies")
+
+def setup_late_deps():
+    pip_install()
+    npm_bazel_labs_dependencies()

packages.bzl

@@ -52,14 +52,14 @@ def register_deps():
 
     git_repository(
         name = "rules_python",
-        commit = "5c948dcfd4ca79c2ed3a87636c46abba9f5836e9",
+        commit = "3927c9bce90f629eb5ab08bbc99a3d3bda1d95c0",
-        remote = "https://github.com/bazelbuild/rules_python",
+        remote = "https://github.com/ankitects/rules_python",
-        shallow_since = "1602856996 -0700",
+        shallow_since = "1604408056 +1000",
     )
 
-    # local_repository(
+    # native.local_repository(
     #     name = "rules_python",
-    #     path = "./rules_python",
+    #     path = "../rules_python",
     # )
 
     git_repository(

pip/requirements.in

@@ -1,6 +1,7 @@
 beautifulsoup4
 black
 decorator
+distro
 flask
 flask-cors
 isort

pip/requirements.txt

@@ -11,6 +11,7 @@ chardet==3.0.4            # via requests
 click==7.1.2              # via black, flask, pip-tools
 colorama==0.4.4 ; sys_platform == "win32"  # via -r requirements.in, pylint, pytest
 decorator==4.4.2          # via -r requirements.in
+distro==1.5.0             # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
 flask==1.1.2              # via -r requirements.in, flask-cors
 idna==2.10                # via requests

platforms/BUILD.bazel

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "windows_x86_32",
+    constraint_values = [
+        "@platforms//os:windows",
+        "@platforms//cpu:x86_32",
+    ],
+)
+
+config_setting(
+    name = "windows_x86_64",
+    constraint_values = [
+        "@platforms//os:windows",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+config_setting(
+    name = "macos_x86_64",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+config_setting(
+    name = "linux_x86_64",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+)
+
+config_setting(
+    name = "linux_x86_32",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_32",
+    ],
+)

pylib/anki/BUILD.bazel

@@ -4,6 +4,7 @@ load("@py_deps//:requirements.bzl", "requirement")
 load("//pylib:protobuf.bzl", "py_proto_library_typed")
 load("@rules_python//experimental/python:wheel.bzl", "py_package", "py_wheel")
 load("@bazel_skylib//lib:selects.bzl", "selects")
+load("//:defs.bzl", "anki_version")
 
 copy_file(
     name = "buildinfo",
@@ -86,6 +87,7 @@ py_library(
     deps = [
         requirement("protobuf"),
         requirement("decorator"),
+        requirement("distro"),
         requirement("requests"),
         requirement("beautifulsoup4"),
     ],
@@ -93,24 +95,37 @@ py_library(
 
 py_package(
     name = "anki_pkg",
-    # Only include these Python packages.
+    packages = ["pylib.anki"],
-    #    packages = ["anki"],
     deps = [":anki"],
 )
 
 py_wheel(
-    name = "anki_whl",
+    name = "wheel",
-    # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl"
+    abi = "abi3",
+    description_file = "wheel_description.txt",
     distribution = "anki",
-    python_tag = "py3",
+    platform = select({
+        "//platforms:windows_x86_64": "win_amd64",
+        "//platforms:windows_x86_32": "win32",
+        "//platforms:macos_x86_64": "macosx_10_7_x86_64",
+        "//platforms:linux_x86_64": "manylinux2014_x86_64",
+        "//platforms:linux_x86_32": "manylinux2014_i686",
+    }),
+    python_tag = "cp38",
+    python_version = ">=3.8",
     requires = [
-        "distro ; sys_platform != \"darwin\" and sys_platform != \"win32\"",
+        "beautifulsoup4",
+        "requests[socks]",
+        "decorator",
+        "protobuf",
+        'orjson; platform_machine == "x86_64"',
+        'psutil; sys_platform == "win32"',
+        'distro; sys_platform != "darwin" and sys_platform != "win32"',
     ],
     strip_path_prefixes = [
         "pylib",
-        #        "foo3",
     ],
-    version = "0.0.1",
+    version = anki_version,
     deps = [":anki_pkg"],
 )
 

pylib/anki/lang.py

@@ -172,8 +172,8 @@ def ngettext(single: str, plural: str, n: int) -> str:
 
 def set_lang(lang: str, locale_dir: str) -> None:
     global currentLang, current_catalog, current_i18n, locale_folder
-    gettext_dir = os.path.join(locale_dir, "gettext")
+    gettext_dir = locale_dir
-    ftl_dir = os.path.join(locale_dir, "fluent")
+    ftl_dir = locale_dir
 
     currentLang = lang
     current_catalog = gettext.translation(

pylib/anki/wheel_description.txt

@@ -0,0 +1 @@
+Please see https://apps.ankiweb.net

pylib/setup.py

@@ -1,35 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import os
-
-import setuptools
-
-install_requires = [
-    "beautifulsoup4",
-    "requests[socks]",
-    "decorator",
-    "protobuf",
-    'orjson; platform_machine == "x86_64"',
-    'psutil; sys_platform == "win32"',
-    'distro; sys_platform != "darwin" and sys_platform != "win32"',
-]
-
-# maturin develop hides the package from pip - https://github.com/ankitects/anki/pull/600
-if not os.environ.get("SKIP_ANKI_RSPY", False):
-    install_requires.append("ankirspy==2.1.36")  # automatically updated 1
-
-setuptools.setup(
-    name="anki",
-    version="2.1.36",  # automatically updated 2
-    author="Ankitects Pty Ltd",
-    description="Anki's library code",
-    long_description="Anki's library code",
-    long_description_content_type="text/markdown",
-    url="https://apps.ankiweb.net",
-    packages=setuptools.find_packages(".", exclude=["tests"]),
-    license="License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
-    package_data={"anki": ["py.typed"]},
-    classifiers=[],
-    python_requires=">=3.7",
-    install_requires=install_requires,
-)

qt/BUILD.bazel

@@ -27,14 +27,11 @@ py_binary(
 py_test(
     name = "pytest",
     srcs = glob(["tests/*.py"]) + ["bazelfixes.py"],
-    data = [
-        "//qt/aqt_data",
-    ],
     imports = ["."],
     main = "tests/run_pytest.py",
     deps = [
         "//pylib/anki",
-        "//qt/aqt",
+        "//qt/aqt:aqt_without_data",
         requirement("pytest"),
         requirement("mock"),
     ],
@@ -57,7 +54,7 @@ py_test(
     main = "tests/run_mypy.py",
     deps = [
         "//pylib/anki",
-        "//qt/aqt",
+        "//qt/aqt:aqt_without_data",
         "@pyqt5//:pkg",
         requirement("mypy"),
     ],
@@ -71,17 +68,14 @@ py_test(
     args = [
         "aqt",
         "$(location .pylintrc)",
-        "foo",  #$(location PyQt5/__init__.py)",
     ],
     data = [
         ".pylintrc",
-        #        "PyQt5-stubs",
-        #"PyQt5/__init__.py",
     ],
     main = "tests/run_pylint.py",
     deps = [
         "//pylib/anki",
-        "//qt/aqt",
+        "//qt/aqt:aqt_without_data",
         requirement("pylint"),
         "@pyqt5//:pkg",
     ],
@@ -98,7 +92,7 @@ py_test(
     data = [".isort.cfg"],
     main = "tests/run_format.py",
     deps = [
-        "//qt/aqt",
+        "//qt/aqt:aqt_without_data",
         requirement("black"),
         requirement("isort"),
     ],
@@ -128,12 +122,9 @@ py_binary(
         "bazelfixes.py",
         "runanki.py",
     ],
-    data = [
-        "//qt/aqt_data",
-    ],
     imports = ["."],
     deps = [
         "//pylib/anki",
-        "//qt/aqt",
+        "//qt/aqt:aqt_with_data",
     ],
 )

qt/aqt/BUILD.bazel

@@ -1,6 +1,8 @@
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_python//python:defs.bzl", "py_library")
 load("@py_deps//:requirements.bzl", "requirement")
+load("@rules_python//experimental/python:wheel.bzl", "py_package", "py_wheel")
+load("//:defs.bzl", "anki_version")
 
 copy_file(
     name = "buildinfo",
@@ -27,65 +29,96 @@ genrule(
     ],
 )
 
+aqt_srcs = glob([
+    "**/*.py",
+]) + [
+    "//qt/aqt/forms:forms",
+    "//qt/aqt/forms:icons",
+]
+
+aqt_core_data = [
+    "buildinfo.txt",
+    "colors.py",
+    "py.typed",
+    ":hooks_gen",
+]
+
+aqt_deps = [
+    requirement("protobuf"),
+    requirement("decorator"),
+    requirement("requests"),
+    requirement("beautifulsoup4"),
+    requirement("flask"),
+    requirement("flask-cors"),
+    requirement("waitress"),
+    requirement("send2trash"),
+    requirement("markdown"),
+    requirement("jsonschema"),
+    "@pyqt5//:pkg",
+] + select({
+    "@bazel_tools//src/conditions:host_windows": [
+        requirement("psutil"),
+        requirement("pywin32"),
+    ],
+    "//conditions:default": [],
+})
+
+# library without web/i18n data; faster for testing
 py_library(
-    name = "aqt",
+    name = "aqt_without_data",
-    srcs = glob([
+    srcs = aqt_srcs,
-        "**/*.py",
+    data = aqt_core_data,
-    ]) + [
-        "//qt/aqt/forms:forms",
-        "//qt/aqt/forms:icons",
-    ],
-    data = [
-        "buildinfo.txt",
-        "colors.py",
-        "py.typed",
-        ":hooks_gen",
-    ],
     visibility = ["//visibility:public"],
-    deps = [
+    deps = aqt_deps,
-        requirement("protobuf"),
-        requirement("decorator"),
-        requirement("requests"),
-        requirement("beautifulsoup4"),
-        requirement("flask"),
-        requirement("flask-cors"),
-        requirement("waitress"),
-        requirement("send2trash"),
-        requirement("markdown"),
-        requirement("jsonschema"),
-        "@pyqt5//:pkg",
-    ] + select({
-        "@bazel_tools//src/conditions:host_windows": [
-            requirement("psutil"),
-            requirement("pywin32"),
-        ],
-        "//conditions:default": [],
-    }),
-    #        requirement("pyaudio"),
-    # ... this is making pytest very slow..
-    #        requirement("pyqt5"),
 )
 
-# py_package(
+py_library(
-#     name = "anki_pkg",
+    name = "aqt_with_data",
-#     # Only include these Python packages.
+    srcs = aqt_srcs,
-#     #    packages = ["anki"],
+    data = aqt_core_data + ["//qt/aqt/data"],
-#     deps = [":anki"],
+    visibility = ["//visibility:public"],
-# )
+    deps = aqt_deps,
+)
 
-# py_wheel(
+py_package(
-#     name = "anki_whl",
+    name = "aqt_pkg",
-#     # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl"
+    packages = [
-#     distribution = "anki",
+        "qt.aqt",
-#     python_tag = "py3",
+    ],
-#     requires = [
+    deps = [
-#         "ankirspy (==2.1.35)",
+        ":aqt_with_data",
-#         "distro ; sys_platform != \"darwin\" and sys_platform != \"win32\"",
+    ],
-#     ],
+)
-#     strip_path_prefixes = [
+
-#         "pylib",
+py_wheel(
-#         #        "foo3",
+    name = "wheel",
-#     ],
+    description_file = "wheel_description.txt",
-#     version = "0.0.1",
+    distribution = "aqt",
-#     deps = [":anki_pkg"],
+    homepage = "https://apps.ankiweb.net",
-# )
+    license = "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+    python_tag = "cp38",
+    python_version = ">=3.8",
+    requires = [
+        # "pyaudio", # currently off as a pain to install on Windows for code completion
+        # "pyqtwebengine", # depending on this limits installable qt5 versions
+        "beautifulsoup4",
+        "requests",
+        "send2trash",
+        "markdown",
+        "jsonschema",
+        "flask",
+        "flask_cors",
+        "waitress",
+        "pyqt5>=5.9",
+        'psutil; sys.platform == "win32"',
+        'pywin32; sys.platform == "win32"',
+        "anki==0.0.1",
+    ],
+    strip_path_prefixes = [
+        "qt",
+    ],
+    version = anki_version,
+    deps = [
+        ":aqt_pkg",
+    ],
+)

qt/aqt/data/BUILD.bazel

@@ -0,0 +1,8 @@
+filegroup(
+    name = "data",
+    srcs = [
+        "//qt/aqt/data/locale",
+        "//qt/aqt/data/web",
+    ],
+    visibility = ["//qt:__subpackages__"],
+)

qt/aqt/data/web/BUILD.bazel

@@ -0,0 +1,10 @@
+filegroup(
+    name = "web",
+    srcs = [
+        "//qt/aqt/data/web/css",
+        "//qt/aqt/data/web/imgs",
+        "//qt/aqt/data/web/js",
+        "//qt/aqt/data/web/pages",
+    ],
+    visibility = ["//qt:__subpackages__"],
+)