GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-18 14:02:21 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

drop pystache and move legacy code into separate file

Browse source
This commit is contained in:
Damien Elmes 2020-01-11 19:38:30 +10:00
parent dc8b854ada
commit a51a4e4d31
13 changed files with 171 additions and 484 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
LICENSE
View file

@ -6,7 +6,6 @@ The following included source code items use a license other than AGPL3:
In the pylib folder:
* The anki/template/ folder is based off pystache: MIT.
* The SuperMemo importer: GPL3.
* The Pauker importer: BSD-3.
* statsbg.py: CC BY-SA 3.0.

10
pylib/LICENSE
View file

@ -1,10 +0,0 @@
Anki is licensed under the GNU Affero General Public License, version 3 or
later, with portions contributed by Anki users licensed under the BSD-3
license: https://github.com/ankitects/anki-contributors.
The following included source code items use a license other than AGPL3:
* The anki/template/ folder is based off pystache: MIT.
* The SuperMemo importer: GPL3.
* The Pauker importer: BSD-3.
* statsbg.py: CC BY-SA 3.0.

2
pylib/anki/collection.py
View file

@ -30,7 +30,7 @@ from anki.rsbackend import RustBackend
from anki.sched import Scheduler as V1Scheduler
from anki.schedv2 import Scheduler as V2Scheduler
from anki.tags import TagManager
from anki.template2 import render_qa_from_field_map
from anki.template import render_qa_from_field_map
from anki.types import NoteType, QAData, Template
from anki.utils import (
devMode,

2
pylib/anki/media.py
View file

@ -18,7 +18,7 @@ from anki.consts import *
from anki.db import DB, DBError
from anki.lang import _
from anki.latex import mungeQA
from anki.template2 import expand_clozes
from anki.template import expand_clozes
from anki.utils import checksum, isMac, isWin

145
pylib/anki/template.py Normal file
View file

@ -0,0 +1,145 @@
# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
"""
This file contains the Python portion of the template rendering code.
Templates can have filters applied to field replacements. The Rust template
rendering code will apply any built in filters, and stop at the first
unrecognized filter. The remaining filters are returned to Python,
and applied using the hook system. For example,
{{myfilter:hint:text:Field}} will apply the built in text and hint filters,
and then attempt to apply myfilter. If no add-ons have provided the filter,
the text is not modified.
Add-ons can register a filter by adding a hook to "fmod_<filter name>".
As standard filters will not be run after a custom filter, it is up to the
add-on to do any further processing that is required.
The hook is called with the arguments
(field_text, filter_args, field_map, field_name, "").
The last argument is no longer used.
If the field name contains a hyphen, it is split on the hyphen, eg
{{foo-bar:baz}} calls fmod_foo with filter_args set to "bar".
A Python implementation of the standard filters is currently available in the
template_legacy.py file.
"""
from __future__ import annotations
import re
from typing import Dict, List, Tuple, Union
import anki
from anki.hooks import runFilter
from anki.rsbackend import TemplateReplacement
from anki.sound import stripSounds
def render_template(
col: anki.storage._Collection, format: str, fields: Dict[str, str]
) -> str:
"Render a single template."
rendered = col.backend.render_template(format, fields)
return apply_custom_filters(rendered, fields)
def render_qa_from_field_map(
col: anki.storage._Collection,
qfmt: str,
afmt: str,
fields: Dict[str, str],
card_ord: int,
) -> Tuple[str, str]:
"Renders the provided templates, returning rendered q & a text."
# question
format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (card_ord + 1), qfmt)
format = format.replace("<%cloze:", "<%%cq:%d:" % (card_ord + 1))
qtext = render_template(col, format, fields)
# answer
format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (card_ord + 1), afmt)
format = format.replace("<%cloze:", "<%%ca:%d:" % (card_ord + 1))
fields["FrontSide"] = stripSounds(qtext)
atext = render_template(col, format, fields)
return qtext, atext
def apply_custom_filters(
rendered: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
) -> str:
"Complete rendering by applying any pending custom filters."
res = ""
for node in rendered:
if isinstance(node, str):
res += node
else:
res += apply_field_filters(
node.field_name, node.current_text, fields, node.filters
)
return res
# Filters
##########################################################################
def apply_field_filters(
field_name: str, field_text: str, fields: Dict[str, str], filters: List[str]
) -> str:
"""Apply filters to field text, returning modified text."""
for filter in filters:
if "-" in filter:
filter_base, filter_args = filter.split("-", maxsplit=1)
else:
filter_base = filter
filter_args = ""
# the fifth argument is no longer used
field_text = runFilter(
"fmod_" + filter_base, field_text, filter_args, fields, field_name, ""
)
return field_text
# Cloze handling
##########################################################################
# Matches a {{c123::clozed-out text::hint}} Cloze deletion, case-insensitively.
# The regex should be interpolated with a regex number and creates the following
# named groups:
# - tag: The lowercase or uppercase 'c' letter opening the Cloze.
# The c/C difference is only relevant to the legacy code.
# - content: Clozed-out content.
# - hint: Cloze hint, if provided.
clozeReg = r"(?si)\{\{(?P<tag>c)%s::(?P<content>.*?)(::(?P<hint>.*?))?\}\}"
# Constants referring to group names within clozeReg.
CLOZE_REGEX_MATCH_GROUP_TAG = "tag"
CLOZE_REGEX_MATCH_GROUP_CONTENT = "content"
CLOZE_REGEX_MATCH_GROUP_HINT = "hint"
# used by the media check functionality
def expand_clozes(string: str) -> List[str]:
"Render all clozes in string."
ords = set(re.findall(r"{{c(\d+)::.+?}}", string))
strings = []
def qrepl(m):
if m.group(CLOZE_REGEX_MATCH_GROUP_HINT):
return "[%s]" % m.group(CLOZE_REGEX_MATCH_GROUP_HINT)
else:
return "[...]"
def arepl(m):
return m.group(CLOZE_REGEX_MATCH_GROUP_CONTENT)
for ord in ords:
s = re.sub(clozeReg % ord, qrepl, string)
s = re.sub(clozeReg % ".+?", arepl, s)
strings.append(s)
strings.append(re.sub(clozeReg % ".+?", arepl, string))
return strings

20
pylib/anki/template/LICENSE
View file

@ -1,20 +0,0 @@
Copyright (c) 2009 Chris Wanstrath
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

8
pylib/anki/template/README.anki
View file

@ -1,8 +0,0 @@
Anki uses a modified version of Pystache to provide Mustache-like syntax.
Behaviour is a little different from standard Mustache:
- {{text}} returns text verbatim with no HTML escaping
- {{{text}}} does the same and exists for backwards compatibility
- partial rendering is disabled for security reasons
- certain keywords like 'cloze' are treated specially

9
pylib/anki/template/__init__.py
View file

@ -1,9 +0,0 @@
from typing import Any
from .template import Template
def render(template, context=None, **kwargs) -> Any:
context = context and context.copy() or {}
context.update(kwargs)
return Template(template, context).render()

167
pylib/anki/template/template.py
View file

@ -1,167 +0,0 @@
import re
from typing import Any, Callable, Dict, Pattern
from anki.template2 import apply_field_filters
modifiers: Dict[str, Callable] = {}
def field_is_not_empty(field_text: str) -> bool:
# fixme: this is an overkill way of preventing a field with only
# a <br> or <div> from appearing non-empty
from anki.utils import stripHTMLMedia
field_text = stripHTMLMedia(field_text)
return field_text.strip() != ""
def modifier(symbol) -> Callable[[Any], Any]:
"""Decorator for associating a function with a Mustache tag modifier.
@modifier('P')
def render_tongue(self, tag_name=None, context=None):
return ":P %s" % tag_name
{{P yo }} => :P yo
"""
def set_modifier(func):
modifiers[symbol] = func
return func
return set_modifier
def get_or_attr(obj, name, default=None) -> Any:
try:
return obj[name]
except KeyError:
return default
except:
try:
return getattr(obj, name)
except AttributeError:
return default
class Template:
# The regular expression used to find a #section
section_re: Pattern = None
# The regular expression used to find a tag.
tag_re: Pattern = None
# Opening tag delimiter
otag = "{{"
# Closing tag delimiter
ctag = "}}"
def __init__(self, template, context=None) -> None:
self.template = template
self.context = context or {}
self.compile_regexps()
def render(self, template=None, context=None, encoding=None) -> str:
"""Turns a Mustache template into something wonderful."""
template = template or self.template
context = context or self.context
template = self.render_sections(template, context)
result = self.render_tags(template, context)
# if encoding is not None:
# result = result.encode(encoding)
return result
def compile_regexps(self) -> None:
"""Compiles our section and tag regular expressions."""
tags = {"otag": re.escape(self.otag), "ctag": re.escape(self.ctag)}
section = r"%(otag)s[\#|^]([^\}]*)%(ctag)s(.+?)%(otag)s/\1%(ctag)s"
self.section_re = re.compile(section % tags, re.M | re.S)
tag = r"%(otag)s(#|=|&|!|>|\{)?(.+?)\1?%(ctag)s+"
self.tag_re = re.compile(tag % tags)
def render_sections(self, template, context) -> str:
"""Expands sections."""
while 1:
match = self.section_re.search(template)
if match is None:
break
section, section_name, inner = match.group(0, 1, 2)
section_name = section_name.strip()
val = get_or_attr(context, section_name, None)
replacer = ""
inverted = section[2] == "^"
nonempty = field_is_not_empty(val or "")
if (nonempty and not inverted) or (not nonempty and inverted):
replacer = inner
template = template.replace(section, replacer)
return template
def render_tags(self, template, context) -> str:
"""Renders all the tags in a template for a context."""
repCount = 0
while 1:
if repCount > 100:
print("too many replacements")
break
repCount += 1
match = self.tag_re.search(template)
if match is None:
break
tag, tag_type, tag_name = match.group(0, 1, 2)
tag_name = tag_name.strip()
try:
func = modifiers[tag_type]
replacement = func(self, tag_name, context)
template = template.replace(tag, replacement)
except (SyntaxError, KeyError):
return "{{invalid template}}"
return template
# {{{ functions just like {{ in anki
@modifier("{")
def render_tag(self, tag_name, context) -> Any:
return self.render_unescaped(tag_name, context)
@modifier("!")
def render_comment(self, tag_name=None, context=None) -> str:
"""Rendering a comment always returns nothing."""
return ""
@modifier(None)
def render_unescaped(self, tag_name=None, context=None) -> Any:
"""Render a tag without escaping it."""
# split out field modifiers
*mods, tag = tag_name.split(":")
# return an error if field doesn't exist
txt = get_or_attr(context, tag)
if txt is None:
return "{unknown field %s}" % tag_name
# the filter closest to the field name is applied first
mods.reverse()
return apply_field_filters(tag, txt, context, mods)
@modifier("=")
def render_delimiter(self, tag_name=None, context=None) -> str:
"""Changes the Mustache delimiter."""
try:
self.otag, self.ctag = tag_name.split(" ")
except ValueError:
# invalid
return ""
self.compile_regexps()
return ""

118
pylib/anki/template/view.py
View file

@ -1,118 +0,0 @@
import os.path
import re
from typing import Any
from .template import Template
class View:
# Path where this view's template(s) live
template_path = "."
# Extension for templates
template_extension = "mustache"
# The name of this template. If none is given the View will try
# to infer it based on the class name.
template_name: str = None
# Absolute path to the template itself. Pystache will try to guess
# if it's not provided.
template_file: str = None
# Contents of the template.
template: str = None
# Character encoding of the template file. If None, Pystache will not
# do any decoding of the template.
template_encoding: str = None
def __init__(self, template=None, context=None, **kwargs) -> None:
self.template = template
self.context = context or {}
# If the context we're handed is a View, we want to inherit
# its settings.
if isinstance(context, View):
self.inherit_settings(context)
if kwargs:
self.context.update(kwargs)
def inherit_settings(self, view) -> None:
"""Given another View, copies its settings."""
if view.template_path:
self.template_path = view.template_path
if view.template_name:
self.template_name = view.template_name
def load_template(self) -> Any:
if self.template:
return self.template
if self.template_file:
return self._load_template()
name = self.get_template_name() + "." + self.template_extension
if isinstance(self.template_path, str):
self.template_file = os.path.join(self.template_path, name)
return self._load_template()
for path in self.template_path:
self.template_file = os.path.join(path, name)
if os.path.exists(self.template_file):
return self._load_template()
raise IOError('"%s" not found in "%s"' % (name, ":".join(self.template_path),))
def _load_template(self) -> str:
f = open(self.template_file, "r")
try:
template = f.read()
if self.template_encoding and isinstance(template, bytes):
template = str(template, self.template_encoding)
finally:
f.close()
return template
def get_template_name(self, name=None) -> Any:
"""TemplatePartial => template_partial
Takes a string but defaults to using the current class' name or
the `template_name` attribute
"""
if self.template_name:
return self.template_name
if not name:
name = self.__class__.__name__
def repl(match):
return "_" + match.group(0).lower()
return re.sub("[A-Z]", repl, name)[1:]
def __contains__(self, needle) -> bool:
return needle in self.context or hasattr(self, needle)
def __getitem__(self, attr) -> Any:
val = self.get(attr, None)
if not val:
raise KeyError("No such key.")
return val
def get(self, attr, default) -> Any:
attr = self.context.get(attr, getattr(self, attr, default))
if hasattr(attr, "__call__"):
return attr()
else:
return attr
def render(self, encoding=None) -> str:
template = self.load_template()
return Template(template, self).render(encoding=encoding)
def __str__(self) -> str:
return self.render()

161
pylib/anki/template2.py → pylib/anki/template_legacy.py
View file

@ -2,125 +2,33 @@
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
"""
This file contains some code related to templates that is not directly
connected to pystache. It may be renamed in the future.
This file contains code that is no longer used by Anki, but left around
for the benefit of add-ons. It may go away in the future, so please copy
any routines you need into your own add-on instead of using them directly
from this module.
If your add-on was previously calling anki.template.render(), you now
need to call anki.template.render_template(), passing col in as the first
argument.
"""
from __future__ import annotations
import re
from typing import Any, Callable, Dict, List, Tuple, Union
from typing import Any, Callable
import anki
from anki.hooks import addHook, runFilter
from anki.lang import _
from anki.rsbackend import TemplateReplacement
from anki.sound import stripSounds
from anki.template import (
CLOZE_REGEX_MATCH_GROUP_CONTENT,
CLOZE_REGEX_MATCH_GROUP_HINT,
CLOZE_REGEX_MATCH_GROUP_TAG,
clozeReg,
)
from anki.utils import stripHTML
def render_qa_from_field_map(
col: anki.storage._Collection,
qfmt: str,
afmt: str,
fields: Dict[str, str],
card_ord: int,
) -> Tuple[str, str]:
"Renders the provided templates, returning rendered q & a text."
# question
format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (card_ord + 1), qfmt)
format = format.replace("<%cloze:", "<%%cq:%d:" % (card_ord + 1))
qtext = render_template(col, format, fields)
# answer
format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (card_ord + 1), afmt)
format = format.replace("<%cloze:", "<%%ca:%d:" % (card_ord + 1))
fields["FrontSide"] = stripSounds(qtext)
atext = render_template(col, format, fields)
return qtext, atext
def render_template(
col: anki.storage._Collection, format: str, fields: Dict[str, str]
) -> str:
"Render a single template."
rendered = col.backend.render_template(format, fields)
return apply_custom_filters(rendered, fields)
def apply_custom_filters(
rendered: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
) -> str:
"Complete rendering by applying any pending custom filters."
res = ""
for node in rendered:
if isinstance(node, str):
res += node
else:
res += apply_field_filters(
node.field_name, node.current_text, fields, node.filters
)
return res
# Filters
##########################################################################
#
# Applies field filters defined by hooks. Given {{filterName:field}} in
# the template, the hook fmod_filterName is called with the arguments
# (field_text, filter_args, fields, field_name, "").
# The last argument is no longer used.
# If the field name contains a hyphen, it is split on the hyphen, eg
# {{foo-bar:baz}} calls fmod_foo with filter_args set to "bar".
def apply_field_filters(
field_name: str, field_text: str, fields: Dict[str, str], filters: List[str]
) -> str:
"""Apply filters to field text, returning modified text."""
filters = _adjust_filters(filters)
for filter in filters:
if "-" in filter:
filter_base, filter_args = filter.split("-", maxsplit=1)
else:
filter_base = filter
filter_args = ""
# the fifth argument is no longer used
field_text = runFilter(
"fmod_" + filter_base, field_text, filter_args, fields, field_name, ""
)
if not isinstance(field_text, str):
return "{field modifier '%s' on template invalid}" % filter
return field_text
def _adjust_filters(filters: List[str]) -> List[str]:
"Handle the type:cloze: special case."
if filters == ["cloze", "type"]:
filters = ["type-cloze"]
return filters
# Cloze filter
##########################################################################
# Matches a {{c123::clozed-out text::hint}} Cloze deletion, case-insensitively.
# The regex should be interpolated with a regex number and creates the following
# named groups:
# - tag: The lowercase or uppercase 'c' letter opening the Cloze.
# - content: Clozed-out content.
# - hint: Cloze hint, if provided.
clozeReg = r"(?si)\{\{(?P<tag>c)%s::(?P<content>.*?)(::(?P<hint>.*?))?\}\}"
# Constants referring to group names within clozeReg.
CLOZE_REGEX_MATCH_GROUP_TAG = "tag"
CLOZE_REGEX_MATCH_GROUP_CONTENT = "content"
CLOZE_REGEX_MATCH_GROUP_HINT = "hint"
def _clozeText(txt: str, ord: str, type: str) -> str:
"""Process the given Cloze deletion within the given template."""
@ -207,29 +115,6 @@ def _removeFormattingFromMathjax(txt, ord) -> str:
)
def expand_clozes(string: str) -> List[str]:
"Render all clozes in string."
ords = set(re.findall(r"{{c(\d+)::.+?}}", string))
strings = []
def qrepl(m):
if m.group(CLOZE_REGEX_MATCH_GROUP_HINT):
return "[%s]" % m.group(CLOZE_REGEX_MATCH_GROUP_HINT)
else:
return "[...]"
def arepl(m):
return m.group(CLOZE_REGEX_MATCH_GROUP_CONTENT)
for ord in ords:
s = re.sub(clozeReg % ord, qrepl, string)
s = re.sub(clozeReg % ".+?", arepl, s)
strings.append(s)
strings.append(re.sub(clozeReg % ".+?", arepl, string))
return strings
def _cloze_filter(field_text: str, filter_args: str, q_or_a: str):
return _clozeText(field_text, filter_args, q_or_a)
@ -242,8 +127,8 @@ def cloze_afilter(field_text: str, filter_args: str, *args):
return _cloze_filter(field_text, filter_args, "a")
addHook("fmod_cq", cloze_qfilter)
addHook("fmod_ca", cloze_afilter)
# addHook("fmod_cq", cloze_qfilter)
# addHook("fmod_ca", cloze_afilter)
# Other filters
##########################################################################
@ -309,9 +194,9 @@ def type_answer_filter(txt: str, filter_args: str, context, tag: str, dummy) ->
return f"[[type:{tag}]]"
addHook("fmod_text", text_filter)
addHook("fmod_type", type_answer_filter)
addHook("fmod_hint", hint_filter)
addHook("fmod_kanji", kanji_filter)
addHook("fmod_kana", kana_filter)
addHook("fmod_furigana", furigana_filter)
# addHook("fmod_text", text_filter)
# addHook("fmod_type", type_answer_filter)
# addHook("fmod_hint", hint_filter)
# addHook("fmod_kanji", kanji_filter)
# addHook("fmod_kana", kana_filter)
# addHook("fmod_furigana", furigana_filter)

10
pylib/tests/test_models.py
View file

@ -1,7 +1,6 @@
# coding: utf-8
import time
import anki.template
from anki.consts import MODEL_CLOZE
from anki.utils import isWin, joinFields, stripHTML
from tests.shared import getEmptyCol
@ -350,15 +349,6 @@ def test_modelChange():
assert deck.db.scalar("select count() from cards where nid = ?", f.id) == 1
def test_templates2():
d = dict(Foo="x", Bar="y")
assert anki.template.render("{{Foo}}", d) == "x"
assert anki.template.render("{{#Foo}}{{Foo}}{{/Foo}}", d) == "x"
assert anki.template.render("{{#Foo}}{{Foo}}{{/Foo}}", d) == "x"
assert anki.template.render("{{#Bar}}{{#Foo}}{{Foo}}{{/Foo}}{{/Bar}}", d) == "x"
assert anki.template.render("{{#Baz}}{{#Foo}}{{Foo}}{{/Foo}}{{/Baz}}", d) == ""
def test_availOrds():
d = getEmptyCol()
m = d.models.current()

2
pylib/tests/test_template.py
View file

@ -1,4 +1,4 @@
from anki.template2 import _removeFormattingFromMathjax
from anki.template_legacy import _removeFormattingFromMathjax
def test_remove_formatting_from_mathjax():