update to latest mypy

mypy's move to external types-* packages is a PITA, as it requires them
to be installed in site-packages, and provides no way to specify a custom
site-packages folder, necessitating extra scripts to mock the
site-packages path, and copy+rename the stub packages into a separate
folder.

pip/requirements.in

@@ -20,8 +20,15 @@ requests[socks]
 send2trash
 snakeviz
 stringcase
-waitress>=2.0.0b1
+waitress>=2.0.0
 fluent.syntax
+types-decorator
+types-flask
+types-markdown
+types-orjson
+types-protobuf
+types-requests
+types-waitress
 
 # windows only
 psutil; sys.platform == "win32"

pip/requirements.txt

@@ -12,7 +12,7 @@ attrs==21.2.0
     #   pytest
 beautifulsoup4==4.9.3
     # via -r requirements.in
-black==21.5b2
+black==21.6b0
     # via -r requirements.in
 certifi==2021.5.30
     # via requests
@@ -75,7 +75,7 @@ mypy-extensions==0.4.3
     #   mypy
 mypy-protobuf==2.4
     # via -r requirements.in
-mypy==0.812
+mypy==0.902
     # via -r requirements.in
 orjson==3.5.3
     # via -r requirements.in
@@ -89,7 +89,7 @@ pip-tools==6.1.0
     # via -r requirements.in
 pluggy==0.13.1
     # via pytest
-protobuf==3.17.1
+protobuf==3.17.3
     # via
     #   -r requirements.in
     #   mypy-protobuf
@@ -132,13 +132,36 @@ stringcase==1.2.0
 toml==0.10.2
     # via
     #   black
+    #   mypy
     #   pep517
     #   pylint
     #   pytest
 tornado==6.1
     # via snakeviz
-typed-ast==1.4.3
-    # via mypy
+types-click==7.1.1
+    # via types-flask
+types-decorator==0.1.4
+    # via -r requirements.in
+types-flask==1.1.0
+    # via -r requirements.in
+types-futures==0.1.3
+    # via types-protobuf
+types-jinja2==2.11.1
+    # via types-flask
+types-markdown==0.1.4
+    # via -r requirements.in
+types-markupsafe==1.1.2
+    # via types-jinja2
+types-orjson==0.1.0
+    # via -r requirements.in
+types-protobuf==0.1.12
+    # via -r requirements.in
+types-requests==0.1.11
+    # via -r requirements.in
+types-waitress==0.1.6
+    # via -r requirements.in
+types-werkzeug==1.0.1
+    # via types-flask
 typing-extensions==3.10.0.0
     # via mypy
 urllib3==1.26.5

pip/stubs/BUILD.bazel

@@ -0,0 +1,42 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@py_deps//:requirements.bzl", "requirement")
+load(":stubs.bzl", "copy_stubs")
+
+_stubs = [
+    "requests",
+    "protobuf",
+    "decorator",
+    "flask",
+    "markdown",
+    "orjson",
+    "waitress",
+]
+
+py_binary(
+    name = "gatherstubs",
+    srcs = [
+        "gatherstubs.py",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("mypy"),
+    ] + [requirement("types-" + stub) for stub in _stubs],
+)
+
+py_binary(
+    name = "extendsitepkgs",
+    srcs = [
+        "extendsitepkgs.py",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("mypy"),
+    ],
+)
+
+copy_stubs(
+    name = "stubs",
+    pkgs = [requirement("types-" + stub) for stub in _stubs],
+    tool = ":gatherstubs",
+    visibility = ["//visibility:public"],
+)

pip/stubs/README.md

@@ -0,0 +1,2 @@
+mypy 0.9 assumes stub files will be installed in site-packages, but they are not
+in Bazel's case, so we need to hack around the issue.

pip/stubs/extendsitepkgs.py

@@ -0,0 +1,10 @@
+# Copyright: Ankitects Pty Ltd and contributors
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import os
+from mypy import sitepkgs
+
+pkgs = sitepkgs.getsitepackages()
+pkgs.append(os.getenv("EXTRA_SITE_PACKAGES"))
+
+print(pkgs)

pip/stubs/gatherstubs.py

@@ -0,0 +1,30 @@
+# Copyright: Ankitects Pty Ltd and contributors
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import os
+import subprocess
+import sys
+import re
+import shutil
+
+stubs_remap = {"protobuf": "google", "futures": "concurrent"}
+
+
+def copy_folder(pkgname, path, outbase):
+    stubname = stubs_remap.get(pkgname, pkgname)
+    os.listdir(path)
+    path = f"{path}/{stubname}-stubs"
+    shutil.copytree(path, os.path.join(outbase, f"{stubname}-stubs"))
+
+
+name_re = re.compile("__types_(.+?)_\d")
+
+if __name__ == "__main__":
+    outbase = os.path.abspath(sys.argv[1])
+
+    # copy stubs into top level folder, renaming
+    folder = os.path.join(os.path.dirname(__file__), "../../external")
+    os.chdir(folder)
+    for folder in os.listdir("."):
+        if match := name_re.search(folder):
+            copy_folder(match.group(1), folder, outbase)

pip/stubs/stubs.bzl

@@ -0,0 +1,20 @@
+def _copy_stubs_impl(ctx):
+    dir = ctx.actions.declare_directory("stubs")
+    ctx.actions.run(
+        outputs = [dir],
+        inputs = ctx.files.pkgs,
+        executable = ctx.executable.tool,
+        arguments = [dir.path],
+        use_default_shell_env = True,
+    )
+    return [
+        DefaultInfo(files = depset([dir]), data_runfiles = ctx.runfiles(files = [dir])),
+    ]
+
+copy_stubs = rule(
+    implementation = _copy_stubs_impl,
+    attrs = {
+        "pkgs": attr.label_list(),
+        "tool": attr.label(executable = True, cfg = "host"),
+    },
+)

pylib/BUILD.bazel

@@ -20,8 +20,14 @@ py_test(
     args = [
         "anki",
         "$(location mypy.ini)",
+        "$(location //pip/stubs:extendsitepkgs)",
     ],
-    data = ["mypy.ini"],
+    data = [
+        "mypy.ini",
+        "//pip/stubs",
+        "//pip/stubs:extendsitepkgs",
+    ],
+    env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"},
     main = "tests/run_mypy.py",
     deps = [
         "//pylib/anki",

pylib/tests/run_mypy.py

@@ -6,13 +6,26 @@ import subprocess
 import sys
 
 if __name__ == "__main__":
-    (module, ini) = sys.argv[1:]
+    (module, ini, extendsitepkgs) = sys.argv[1:]
     ini = os.path.abspath(ini)
+    extendsitepkgs = os.path.abspath(extendsitepkgs)
+    extra_site = os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES"))
 
     folder = os.path.join(os.path.dirname(__file__), "..")
     os.chdir(folder)
 
-    args = [sys.executable, "-m", "mypy", module, "--config-file", ini]
+    args = [
+        sys.executable,
+        "-m",
+        "mypy",
+        module,
+        "--config-file",
+        ini,
+        "--python-executable",
+        extendsitepkgs,
+    ]
+
+    os.environ["EXTRA_SITE_PACKAGES"] = extra_site
 
     if sys.platform.startswith("win32"):
         # bazel passes in \\?\c:\... path; mypy can't handle it, so we

qt/BUILD.bazel

@@ -46,11 +46,15 @@ py_test(
         "aqt",
         "$(location mypy.ini)",
         "$(location @pyqt5//:__init__.py)",
+        "$(location //pip/stubs:extendsitepkgs)",
     ],
     data = [
         "mypy.ini",
+        "//pip/stubs",
+        "//pip/stubs:extendsitepkgs",
         "@pyqt5//:__init__.py",
     ],
+    env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"},
     main = "tests/run_mypy.py",
     deps = [
         "//pylib/anki",
@@ -137,7 +141,10 @@ py_binary(
     data = [
         # ensure the binary's been built
         "//pip:dmypy",
+        "//pip/stubs",
+        "//pip/stubs:extendsitepkgs",
     ],
+    env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"},
     imports = ["."],
     tags = ["manual"],
     deps = [

qt/dmypy.py

@@ -32,9 +32,12 @@ if subprocess.run(
         "qt/mypy.ini",
         "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/pylib/anki",
         "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/qt/aqt",
+        "--python-executable",
+        os.path.abspath("pip/stubs/extendsitepkgs"),
     ],
     env={
         "MYPYPATH": "bazel-bin/qt/dmypy.runfiles/pyqt5",
+        "EXTRA_SITE_PACKAGES": os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES")),
     },
     cwd=workspace,
 ).returncode:

qt/tests/run_mypy.py

@@ -6,10 +6,12 @@ import subprocess
 import sys
 
 if __name__ == "__main__":
-    (module, ini, pyqt_init) = sys.argv[1:]
+    (module, ini, pyqt_init, extendsitepkgs) = sys.argv[1:]
     ini = os.path.abspath(ini)
     pyqt_init = os.path.abspath(pyqt_init)
     pyqt_folder = os.path.dirname(pyqt_init)
+    extendsitepkgs = os.path.abspath(extendsitepkgs)
+    extra_site = os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES"))
 
     folder = os.path.join(os.path.dirname(__file__), "..")
     os.chdir(folder)
@@ -20,8 +22,18 @@ if __name__ == "__main__":
         mypy_path = ".:../pylib:" + pyqt_folder
 
     os.environ["MYPYPATH"] = mypy_path
+    os.environ["EXTRA_SITE_PACKAGES"] = extra_site
 
-    args = [sys.executable, "-m", "mypy", module, "--config-file", ini]
+    args = [
+        sys.executable,
+        "-m",
+        "mypy",
+        module,
+        "--config-file",
+        ini,
+        "--python-executable",
+        extendsitepkgs,
+    ]
 
     if sys.platform.startswith("win32"):
         # bazel passes in \\?\c:\... path; mypy can't handle it, so we