pin Python deps

pip/BUILD.bazel

@@ -1 +1,17 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("@py_deps//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "update",
+    srcs = ["update.py"],
+    data = [
+        "requirements.in",
+        "requirements.txt",
+    ],
+    tags = ["manual"],
+    deps = [
+        requirement("pip-tools"),
+    ],
+)
+
 exports_files(["requirements.txt"])

pip/README.md

@@ -0,0 +1,19 @@
+To achieve reproducible builds we use pip-tools to lock packages to a particular version.
+Sadly this is complicated by the fact that Python can only tell us which transitive dependencies
+are required by actually installing packages, and if you run pip-tools on a Mac or Linux machine,
+it will miss packages that are required on Windows and vice versa.
+
+So we're stuck manually merging dependencies for now. To update deps:
+
+- comment out the pyaudio line starting with https:// in requirements.txt,
+  as pip-tools can't handle it
+- run 'bazel run update' to update requirements.txt for the current
+  platform
+- consult the git diff, and manually merge the changes, undoing the removal
+  of items pinned on other platforms
+- repeat the process on the other platform
+- run the tests to ensure nothing has broken on either platform
+- commit the changes to requirements.txt
+
+At the time of writing, Macs and Linux machines have identical output - it is only
+Windows that differs. But we should not assume that will always be the case.

pip/requirements.in

@@ -0,0 +1,27 @@
+beautifulsoup4
+black
+decorator
+flask
+flask-cors
+isort
+jsonschema
+markdown
+mock
+mypy
+mypy-protobuf
+pip-tools
+protobuf
+pylint
+pytest
+requests[socks]
+send2trash
+stringcase
+waitress
+
+# windows only
+psutil; sys.platform == "win32"
+pywin32; sys.platform == "win32"
+
+# transitive windows dependencies
+atomicwrites; sys.platform == "win32" # via pytest
+colorama; sys.platform == "win32" # via pylint, pytest

pip/requirements.txt

@@ -1,22 +1,59 @@
-black
-protobuf
-stringcase
-decorator
-mypy-protobuf
-mypy
-pytest
-beautifulsoup4
-requests[socks]
-pylint
-black
-isort
-mock
-flask
-flask-cors
-waitress
-send2trash
-markdown
-jsonschema
-pyaudio
-psutil; sys.platform == "win32"
-pywin32; sys.platform == "win32"
+# See README.md
+
+appdirs==1.4.4            # via black
+astroid==2.4.2            # via pylint
+atomicwrites==1.4.0 ; sys_platform == "win32"  # via -r requirements.in, pytest
+attrs==20.2.0             # via jsonschema, pytest
+beautifulsoup4==4.9.3     # via -r requirements.in
+black==20.8b1             # via -r requirements.in
+certifi==2020.6.20        # via requests
+chardet==3.0.4            # via requests
+click==7.1.2              # via black, flask, pip-tools
+colorama==0.4.4 ; sys_platform == "win32"  # via -r requirements.in, pylint, pytest
+decorator==4.4.2          # via -r requirements.in
+flask-cors==3.0.9         # via -r requirements.in
+flask==1.1.2              # via -r requirements.in, flask-cors
+idna==2.10                # via requests
+iniconfig==1.1.1          # via pytest
+isort==5.6.4              # via -r requirements.in, pylint
+itsdangerous==1.1.0       # via flask
+jinja2==2.11.2            # via flask
+jsonschema==3.2.0         # via -r requirements.in
+lazy-object-proxy==1.4.3  # via astroid
+markdown==3.3.3           # via -r requirements.in
+markupsafe==1.1.1         # via jinja2
+mccabe==0.6.1             # via pylint
+mock==4.0.2               # via -r requirements.in
+mypy-extensions==0.4.3    # via black, mypy
+mypy-protobuf==1.23       # via -r requirements.in
+mypy==0.790               # via -r requirements.in
+packaging==20.4           # via pytest
+pathspec==0.8.0           # via black
+pip-tools==5.3.1          # via -r requirements.in
+pluggy==0.13.1            # via pytest
+protobuf==3.13.0          # via -r requirements.in, mypy-protobuf
+psutil==5.7.3 ; sys_platform == "win32"  # via -r requirements.in
+py==1.9.0                 # via pytest
+pylint==2.6.0             # via -r requirements.in
+pyparsing==2.4.7          # via packaging
+pyrsistent==0.17.3        # via jsonschema
+pysocks==1.7.1            # via requests
+pytest==6.1.2             # via -r requirements.in
+pywin32==228 ; sys_platform == "win32"  # via -r requirements.in
+regex==2020.10.28         # via black
+requests[socks]==2.24.0   # via -r requirements.in
+send2trash==1.5.0         # via -r requirements.in
+six==1.15.0               # via astroid, flask-cors, jsonschema, packaging, pip-tools, protobuf
+soupsieve==2.0.1          # via beautifulsoup4
+stringcase==1.2.0         # via -r requirements.in
+toml==0.10.2              # via black, pylint, pytest
+typed-ast==1.4.1          # via black, mypy
+typing-extensions==3.7.4.3  # via black, mypy
+urllib3==1.25.11          # via requests
+waitress==1.4.4           # via -r requirements.in
+werkzeug==1.0.1           # via flask
+wrapt==1.12.1             # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==20.2.4               # via pip-tools
+setuptools==50.3.2        # via jsonschema, protobuf

pip/update.py

@@ -0,0 +1,17 @@
+import sys
+import os
+import click
+from piptools.scripts import compile
+
+@click.group()
+def cli():
+    pass
+
+cli.add_command(compile.cli, "compile")
+
+print("Updating deps...")
+os.chdir("pip")
+sys.argv[1:] = ["compile", "--allow-unsafe", "--upgrade"]
+
+cli()
+

setup.bzl

@@ -40,7 +40,6 @@ def setup_deps():
         name = "py_deps",
         requirements = "@anki//pip:requirements.txt",
         python_runtime = "@python//:python",
-        compile = True,
     )
 
     install_pyqt5(