catch references to missing terms in ftl check

https://github.com/ankitects/anki/pull/889

- We can't use compare_locales's getChecker(), as it seems to
be focused on checking translations rather than the original
templates.
- Also fix return code not being passed on in check_files()
- Temporarily pin pip-tools, as its output format has changed

ftl/BUILD.bazel

@@ -1,7 +1,8 @@
 load("@py_deps//:requirements.bzl", "requirement")
 
 fluent_syntax = [
-    requirement("fluent-syntax"),
+    requirement("fluent.syntax"),
+    requirement("compare-locales"),
     # undeclared runtime dependency
     requirement("six"),
 ]

ftl/format.py

@@ -12,6 +12,31 @@ import sys
 from typing import List
 from fluent.syntax import parse, serialize
 from fluent.syntax.ast import Junk
+from compare_locales.paths import File
+from compare_locales import parser
+from compare_locales.checks.fluent import ReferenceMessageVisitor
+
+
+def check_missing_terms(path: str) -> bool:
+    "True if file is ok."
+    file = File(path, os.path.basename(path))
+    content = open(path, "rb").read()
+    p = parser.getParser(file.file)
+    p.readContents(content)
+    refList = p.parse()
+
+    p.readContents(content)
+    for e in p.parse():
+        ref_data = ReferenceMessageVisitor()
+        ref_data.visit(e.entry)
+
+        for attr_or_val, refs in ref_data.entry_refs.items():
+            for ref, ref_type in refs.items():
+                if ref not in refList:
+                    print(f"In {path}:{e}, missing '{ref}'")
+                    return False
+
+    return True
 
 
 def check_file(path: str, fix: bool) -> bool:
@@ -31,7 +56,7 @@ def check_file(path: str, fix: bool) -> bool:
             raise Exception(f"file introduced junk! {path} {ent}")
 
     if new_text == orig_text:
-        return True
+        return check_missing_terms(path)
 
     if fix:
         print(f"Fixing {path}")
@@ -50,7 +75,7 @@ def check_files(files: List[str], fix: bool) -> bool:
         ok = check_file(path, fix)
         if not ok:
             found_bad = True
-    return True
+    return not found_bad
 
 
 if __name__ == "__main__":

pip/requirements.in

@@ -1,5 +1,6 @@
 beautifulsoup4
 black
+compare-locales
 decorator
 distro
 flask
@@ -11,7 +12,7 @@ mock
 mypy
 mypy-protobuf
 orjson
-pip-tools
+pip-tools==5.4.0
 protobuf
 pylint
 pytest
@@ -20,7 +21,7 @@ send2trash
 snakeviz
 stringcase
 waitress>=2.0.0b1
-fluent-syntax
+fluent.syntax
 
 # windows only
 psutil; sys.platform == "win32"

pip/requirements.txt

@@ -8,14 +8,15 @@ certifi==2020.12.5        # via requests
 chardet==4.0.0            # via requests
 click==7.1.2              # via black, flask, pip-tools
 colorama==0.4.4 ; sys_platform == "win32"  # via -r requirements.in, pylint, pytest
+compare-locales==8.1.0    # via -r requirements.in
 decorator==4.4.2          # via -r requirements.in
 distro==1.5.0             # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
 flask==1.1.2              # via -r requirements.in, flask-cors
-fluent-syntax==0.18.1     # via -r requirements.in
+fluent.syntax==0.18.1     # via -r requirements.in, compare-locales
 idna==2.10                # via requests
 iniconfig==1.1.1          # via pytest
-isort==5.6.4              # via -r requirements.in, pylint
+isort==5.7.0              # via -r requirements.in, pylint
 itsdangerous==1.1.0       # via flask
 jinja2==2.11.2            # via flask
 jsonschema==3.2.0         # via -r requirements.in
@@ -40,17 +41,18 @@ pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
 pysocks==1.7.1            # via requests
 pytest==6.2.1             # via -r requirements.in
+pytoml==0.1.21            # via compare-locales
 pywin32==300 ; sys_platform == "win32"  # via -r requirements.in
 regex==2020.11.13         # via black
 requests[socks]==2.25.1   # via -r requirements.in
 send2trash==1.5.0         # via -r requirements.in
-six==1.15.0               # via astroid, flask-cors, jsonschema, pip-tools, protobuf
+six==1.15.0               # via astroid, compare-locales, flask-cors, jsonschema, pip-tools, protobuf
 snakeviz==2.1.0           # via -r requirements.in
 soupsieve==2.1            # via beautifulsoup4
 stringcase==1.2.0         # via -r requirements.in
 toml==0.10.2              # via black, pylint, pytest
 tornado==6.1              # via snakeviz
-typed-ast==1.4.1          # via black, mypy
+typed-ast==1.4.2          # via black, mypy
 typing-extensions==3.7.4.3  # via black, mypy
 urllib3==1.26.2           # via requests
 waitress==2.0.0b1         # via -r requirements.in
@@ -60,7 +62,7 @@ wrapt==1.12.1             # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==20.3.3               # via pip-tools
-setuptools==51.1.0        # via jsonschema
+setuptools==51.1.1        # via jsonschema
 
 # manually added for now; ensure it and the earlier winrt are not removed on update
 winrt==1.0.20330.1; sys.platform == "win32" and platform_release == "10" and python_version >= "3.9"