diff --git a/pip/requirements.in b/pip/requirements.in index 5e129cce6..5483149eb 100644 --- a/pip/requirements.in +++ b/pip/requirements.in @@ -20,8 +20,15 @@ requests[socks] send2trash snakeviz stringcase -waitress>=2.0.0b1 +waitress>=2.0.0 fluent.syntax +types-decorator +types-flask +types-markdown +types-orjson +types-protobuf +types-requests +types-waitress # windows only psutil; sys.platform == "win32" diff --git a/pip/requirements.txt b/pip/requirements.txt index a18f25417..cb0711b58 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -12,7 +12,7 @@ attrs==21.2.0 # pytest beautifulsoup4==4.9.3 # via -r requirements.in -black==21.5b2 +black==21.6b0 # via -r requirements.in certifi==2021.5.30 # via requests @@ -75,7 +75,7 @@ mypy-extensions==0.4.3 # mypy mypy-protobuf==2.4 # via -r requirements.in -mypy==0.812 +mypy==0.902 # via -r requirements.in orjson==3.5.3 # via -r requirements.in @@ -89,7 +89,7 @@ pip-tools==6.1.0 # via -r requirements.in pluggy==0.13.1 # via pytest -protobuf==3.17.1 +protobuf==3.17.3 # via # -r requirements.in # mypy-protobuf @@ -132,13 +132,36 @@ stringcase==1.2.0 toml==0.10.2 # via # black + # mypy # pep517 # pylint # pytest tornado==6.1 # via snakeviz -typed-ast==1.4.3 - # via mypy +types-click==7.1.1 + # via types-flask +types-decorator==0.1.4 + # via -r requirements.in +types-flask==1.1.0 + # via -r requirements.in +types-futures==0.1.3 + # via types-protobuf +types-jinja2==2.11.1 + # via types-flask +types-markdown==0.1.4 + # via -r requirements.in +types-markupsafe==1.1.2 + # via types-jinja2 +types-orjson==0.1.0 + # via -r requirements.in +types-protobuf==0.1.12 + # via -r requirements.in +types-requests==0.1.11 + # via -r requirements.in +types-waitress==0.1.6 + # via -r requirements.in +types-werkzeug==1.0.1 + # via types-flask typing-extensions==3.10.0.0 # via mypy urllib3==1.26.5 diff --git a/pip/stubs/BUILD.bazel b/pip/stubs/BUILD.bazel new file mode 100644 index 000000000..5c2a17f6d --- /dev/null +++ b/pip/stubs/BUILD.bazel @@ -0,0 +1,42 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_test") +load("@py_deps//:requirements.bzl", "requirement") +load(":stubs.bzl", "copy_stubs") + +_stubs = [ + "requests", + "protobuf", + "decorator", + "flask", + "markdown", + "orjson", + "waitress", +] + +py_binary( + name = "gatherstubs", + srcs = [ + "gatherstubs.py", + ], + visibility = ["//visibility:public"], + deps = [ + requirement("mypy"), + ] + [requirement("types-" + stub) for stub in _stubs], +) + +py_binary( + name = "extendsitepkgs", + srcs = [ + "extendsitepkgs.py", + ], + visibility = ["//visibility:public"], + deps = [ + requirement("mypy"), + ], +) + +copy_stubs( + name = "stubs", + pkgs = [requirement("types-" + stub) for stub in _stubs], + tool = ":gatherstubs", + visibility = ["//visibility:public"], +) diff --git a/pip/stubs/README.md b/pip/stubs/README.md new file mode 100644 index 000000000..971571009 --- /dev/null +++ b/pip/stubs/README.md @@ -0,0 +1,2 @@ +mypy 0.9 assumes stub files will be installed in site-packages, but they are not +in Bazel's case, so we need to hack around the issue. diff --git a/pip/stubs/extendsitepkgs.py b/pip/stubs/extendsitepkgs.py new file mode 100644 index 000000000..ffeaf6e10 --- /dev/null +++ b/pip/stubs/extendsitepkgs.py @@ -0,0 +1,10 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import os +from mypy import sitepkgs + +pkgs = sitepkgs.getsitepackages() +pkgs.append(os.getenv("EXTRA_SITE_PACKAGES")) + +print(pkgs) diff --git a/pip/stubs/gatherstubs.py b/pip/stubs/gatherstubs.py new file mode 100644 index 000000000..843e9cdf2 --- /dev/null +++ b/pip/stubs/gatherstubs.py @@ -0,0 +1,30 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import os +import subprocess +import sys +import re +import shutil + +stubs_remap = {"protobuf": "google", "futures": "concurrent"} + + +def copy_folder(pkgname, path, outbase): + stubname = stubs_remap.get(pkgname, pkgname) + os.listdir(path) + path = f"{path}/{stubname}-stubs" + shutil.copytree(path, os.path.join(outbase, f"{stubname}-stubs")) + + +name_re = re.compile("__types_(.+?)_\d") + +if __name__ == "__main__": + outbase = os.path.abspath(sys.argv[1]) + + # copy stubs into top level folder, renaming + folder = os.path.join(os.path.dirname(__file__), "../../external") + os.chdir(folder) + for folder in os.listdir("."): + if match := name_re.search(folder): + copy_folder(match.group(1), folder, outbase) diff --git a/pip/stubs/stubs.bzl b/pip/stubs/stubs.bzl new file mode 100644 index 000000000..906605be5 --- /dev/null +++ b/pip/stubs/stubs.bzl @@ -0,0 +1,20 @@ +def _copy_stubs_impl(ctx): + dir = ctx.actions.declare_directory("stubs") + ctx.actions.run( + outputs = [dir], + inputs = ctx.files.pkgs, + executable = ctx.executable.tool, + arguments = [dir.path], + use_default_shell_env = True, + ) + return [ + DefaultInfo(files = depset([dir]), data_runfiles = ctx.runfiles(files = [dir])), + ] + +copy_stubs = rule( + implementation = _copy_stubs_impl, + attrs = { + "pkgs": attr.label_list(), + "tool": attr.label(executable = True, cfg = "host"), + }, +) diff --git a/pylib/BUILD.bazel b/pylib/BUILD.bazel index f496c69a2..098d24a83 100644 --- a/pylib/BUILD.bazel +++ b/pylib/BUILD.bazel @@ -20,8 +20,14 @@ py_test( args = [ "anki", "$(location mypy.ini)", + "$(location //pip/stubs:extendsitepkgs)", ], - data = ["mypy.ini"], + data = [ + "mypy.ini", + "//pip/stubs", + "//pip/stubs:extendsitepkgs", + ], + env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"}, main = "tests/run_mypy.py", deps = [ "//pylib/anki", diff --git a/pylib/tests/run_mypy.py b/pylib/tests/run_mypy.py index fc99a47e2..122b3a08d 100644 --- a/pylib/tests/run_mypy.py +++ b/pylib/tests/run_mypy.py @@ -6,13 +6,26 @@ import subprocess import sys if __name__ == "__main__": - (module, ini) = sys.argv[1:] + (module, ini, extendsitepkgs) = sys.argv[1:] ini = os.path.abspath(ini) + extendsitepkgs = os.path.abspath(extendsitepkgs) + extra_site = os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES")) folder = os.path.join(os.path.dirname(__file__), "..") os.chdir(folder) - args = [sys.executable, "-m", "mypy", module, "--config-file", ini] + args = [ + sys.executable, + "-m", + "mypy", + module, + "--config-file", + ini, + "--python-executable", + extendsitepkgs, + ] + + os.environ["EXTRA_SITE_PACKAGES"] = extra_site if sys.platform.startswith("win32"): # bazel passes in \\?\c:\... path; mypy can't handle it, so we diff --git a/qt/BUILD.bazel b/qt/BUILD.bazel index dd5e50700..a2e4bfd65 100644 --- a/qt/BUILD.bazel +++ b/qt/BUILD.bazel @@ -46,11 +46,15 @@ py_test( "aqt", "$(location mypy.ini)", "$(location @pyqt5//:__init__.py)", + "$(location //pip/stubs:extendsitepkgs)", ], data = [ "mypy.ini", + "//pip/stubs", + "//pip/stubs:extendsitepkgs", "@pyqt5//:__init__.py", ], + env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"}, main = "tests/run_mypy.py", deps = [ "//pylib/anki", @@ -137,7 +141,10 @@ py_binary( data = [ # ensure the binary's been built "//pip:dmypy", + "//pip/stubs", + "//pip/stubs:extendsitepkgs", ], + env = {"EXTRA_SITE_PACKAGES": "$(location //pip/stubs)"}, imports = ["."], tags = ["manual"], deps = [ diff --git a/qt/dmypy.py b/qt/dmypy.py index 80b0e2ac1..5e0b277a5 100755 --- a/qt/dmypy.py +++ b/qt/dmypy.py @@ -32,9 +32,12 @@ if subprocess.run( "qt/mypy.ini", "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/pylib/anki", "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/qt/aqt", + "--python-executable", + os.path.abspath("pip/stubs/extendsitepkgs"), ], env={ "MYPYPATH": "bazel-bin/qt/dmypy.runfiles/pyqt5", + "EXTRA_SITE_PACKAGES": os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES")), }, cwd=workspace, ).returncode: diff --git a/qt/tests/run_mypy.py b/qt/tests/run_mypy.py index f5afcf7d4..f324a1a82 100644 --- a/qt/tests/run_mypy.py +++ b/qt/tests/run_mypy.py @@ -6,10 +6,12 @@ import subprocess import sys if __name__ == "__main__": - (module, ini, pyqt_init) = sys.argv[1:] + (module, ini, pyqt_init, extendsitepkgs) = sys.argv[1:] ini = os.path.abspath(ini) pyqt_init = os.path.abspath(pyqt_init) pyqt_folder = os.path.dirname(pyqt_init) + extendsitepkgs = os.path.abspath(extendsitepkgs) + extra_site = os.path.abspath(os.getenv("EXTRA_SITE_PACKAGES")) folder = os.path.join(os.path.dirname(__file__), "..") os.chdir(folder) @@ -20,8 +22,18 @@ if __name__ == "__main__": mypy_path = ".:../pylib:" + pyqt_folder os.environ["MYPYPATH"] = mypy_path + os.environ["EXTRA_SITE_PACKAGES"] = extra_site - args = [sys.executable, "-m", "mypy", module, "--config-file", ini] + args = [ + sys.executable, + "-m", + "mypy", + module, + "--config-file", + ini, + "--python-executable", + extendsitepkgs, + ] if sys.platform.startswith("win32"): # bazel passes in \\?\c:\... path; mypy can't handle it, so we