diff --git a/pip/BUILD.bazel b/pip/BUILD.bazel index 73cccce90..35af50f13 100644 --- a/pip/BUILD.bazel +++ b/pip/BUILD.bazel @@ -1 +1,17 @@ +load("@rules_python//python:defs.bzl", "py_binary") +load("@py_deps//:requirements.bzl", "requirement") + +py_binary( + name = "update", + srcs = ["update.py"], + data = [ + "requirements.in", + "requirements.txt", + ], + tags = ["manual"], + deps = [ + requirement("pip-tools"), + ], +) + exports_files(["requirements.txt"]) diff --git a/pip/README.md b/pip/README.md new file mode 100644 index 000000000..64d3de558 --- /dev/null +++ b/pip/README.md @@ -0,0 +1,19 @@ +To achieve reproducible builds we use pip-tools to lock packages to a particular version. +Sadly this is complicated by the fact that Python can only tell us which transitive dependencies +are required by actually installing packages, and if you run pip-tools on a Mac or Linux machine, +it will miss packages that are required on Windows and vice versa. + +So we're stuck manually merging dependencies for now. To update deps: + +- comment out the pyaudio line starting with https:// in requirements.txt, + as pip-tools can't handle it +- run 'bazel run update' to update requirements.txt for the current + platform +- consult the git diff, and manually merge the changes, undoing the removal + of items pinned on other platforms +- repeat the process on the other platform +- run the tests to ensure nothing has broken on either platform +- commit the changes to requirements.txt + +At the time of writing, Macs and Linux machines have identical output - it is only +Windows that differs. But we should not assume that will always be the case. diff --git a/pip/requirements.in b/pip/requirements.in new file mode 100644 index 000000000..e9bd08b38 --- /dev/null +++ b/pip/requirements.in @@ -0,0 +1,27 @@ +beautifulsoup4 +black +decorator +flask +flask-cors +isort +jsonschema +markdown +mock +mypy +mypy-protobuf +pip-tools +protobuf +pylint +pytest +requests[socks] +send2trash +stringcase +waitress + +# windows only +psutil; sys.platform == "win32" +pywin32; sys.platform == "win32" + +# transitive windows dependencies +atomicwrites; sys.platform == "win32" # via pytest +colorama; sys.platform == "win32" # via pylint, pytest diff --git a/pip/requirements.txt b/pip/requirements.txt old mode 100644 new mode 100755 index 13a2b76ba..a90a18f47 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -1,22 +1,59 @@ -black -protobuf -stringcase -decorator -mypy-protobuf -mypy -pytest -beautifulsoup4 -requests[socks] -pylint -black -isort -mock -flask -flask-cors -waitress -send2trash -markdown -jsonschema -pyaudio -psutil; sys.platform == "win32" -pywin32; sys.platform == "win32" +# See README.md + +appdirs==1.4.4 # via black +astroid==2.4.2 # via pylint +atomicwrites==1.4.0 ; sys_platform == "win32" # via -r requirements.in, pytest +attrs==20.2.0 # via jsonschema, pytest +beautifulsoup4==4.9.3 # via -r requirements.in +black==20.8b1 # via -r requirements.in +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +click==7.1.2 # via black, flask, pip-tools +colorama==0.4.4 ; sys_platform == "win32" # via -r requirements.in, pylint, pytest +decorator==4.4.2 # via -r requirements.in +flask-cors==3.0.9 # via -r requirements.in +flask==1.1.2 # via -r requirements.in, flask-cors +idna==2.10 # via requests +iniconfig==1.1.1 # via pytest +isort==5.6.4 # via -r requirements.in, pylint +itsdangerous==1.1.0 # via flask +jinja2==2.11.2 # via flask +jsonschema==3.2.0 # via -r requirements.in +lazy-object-proxy==1.4.3 # via astroid +markdown==3.3.3 # via -r requirements.in +markupsafe==1.1.1 # via jinja2 +mccabe==0.6.1 # via pylint +mock==4.0.2 # via -r requirements.in +mypy-extensions==0.4.3 # via black, mypy +mypy-protobuf==1.23 # via -r requirements.in +mypy==0.790 # via -r requirements.in +packaging==20.4 # via pytest +pathspec==0.8.0 # via black +pip-tools==5.3.1 # via -r requirements.in +pluggy==0.13.1 # via pytest +protobuf==3.13.0 # via -r requirements.in, mypy-protobuf +psutil==5.7.3 ; sys_platform == "win32" # via -r requirements.in +py==1.9.0 # via pytest +pylint==2.6.0 # via -r requirements.in +pyparsing==2.4.7 # via packaging +pyrsistent==0.17.3 # via jsonschema +pysocks==1.7.1 # via requests +pytest==6.1.2 # via -r requirements.in +pywin32==228 ; sys_platform == "win32" # via -r requirements.in +regex==2020.10.28 # via black +requests[socks]==2.24.0 # via -r requirements.in +send2trash==1.5.0 # via -r requirements.in +six==1.15.0 # via astroid, flask-cors, jsonschema, packaging, pip-tools, protobuf +soupsieve==2.0.1 # via beautifulsoup4 +stringcase==1.2.0 # via -r requirements.in +toml==0.10.2 # via black, pylint, pytest +typed-ast==1.4.1 # via black, mypy +typing-extensions==3.7.4.3 # via black, mypy +urllib3==1.25.11 # via requests +waitress==1.4.4 # via -r requirements.in +werkzeug==1.0.1 # via flask +wrapt==1.12.1 # via astroid + +# The following packages are considered to be unsafe in a requirements file: +pip==20.2.4 # via pip-tools +setuptools==50.3.2 # via jsonschema, protobuf diff --git a/pip/update.py b/pip/update.py new file mode 100644 index 000000000..1ff1adfd4 --- /dev/null +++ b/pip/update.py @@ -0,0 +1,17 @@ +import sys +import os +import click +from piptools.scripts import compile + +@click.group() +def cli(): + pass + +cli.add_command(compile.cli, "compile") + +print("Updating deps...") +os.chdir("pip") +sys.argv[1:] = ["compile", "--allow-unsafe", "--upgrade"] + +cli() + diff --git a/setup.bzl b/setup.bzl index 0ac42ddaf..c7db1e9fa 100644 --- a/setup.bzl +++ b/setup.bzl @@ -40,7 +40,6 @@ def setup_deps(): name = "py_deps", requirements = "@anki//pip:requirements.txt", python_runtime = "@python//:python", - compile = True, ) install_pyqt5(