From 0087eee6d97ad9bf55297c382b1f86898e78e058 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 8 Jan 2020 20:28:04 +1000 Subject: [PATCH] handle conditional replacement in Rust This extends the existing Rust code to handle conditional replacement. The replacement of field names and filters to text remains in Python, so that add-ons can still define their own field modifiers. The code is currently running the old Pystache rendering and the new implementation in parallel, and will print a message to the console if they don't match. If you notice any problems, please let me know. --- proto/backend.proto | 23 +++++++ pylib/anki/collection.py | 4 +- pylib/anki/rsbackend.py | 33 ++++++++- pylib/anki/template2.py | 69 +++++++++++++++++-- rslib/src/backend.rs | 39 ++++++++++- rslib/src/template.rs | 143 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 298 insertions(+), 13 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index fa93ddcb0..e546b6ce7 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -14,6 +14,7 @@ message BackendInput { Empty deck_tree = 18; FindCardsIn find_cards = 19; BrowserRowsIn browser_rows = 20; + FlattenTemplateIn flatten_template = 21; PlusOneIn plus_one = 2046; // temporary, for testing } @@ -26,6 +27,7 @@ message BackendOutput { DeckTreeOut deck_tree = 18; FindCardsOut find_cards = 19; BrowserRowsOut browser_rows = 20; + FlattenTemplateOut flatten_template = 21; PlusOneOut plus_one = 2046; // temporary, for testing @@ -124,3 +126,24 @@ message BrowserRowsOut { // just sort fields for proof of concept repeated string sort_fields = 1; } + +message FlattenTemplateIn { + string template_text = 1; + repeated string nonempty_field_names = 2; +} + +message FlattenTemplateOut { + repeated FlattenedTemplateNode nodes = 1; +} + +message FlattenedTemplateNode { + oneof value { + string text = 1; + FlattenedTemplateReplacement replacement = 2; + } +} + +message FlattenedTemplateReplacement { + string field_name = 1; + repeated string filters = 2; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 6ff2992d4..435a62735 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -30,7 +30,7 @@ from anki.rsbackend import RustBackend from anki.sched import Scheduler as V1Scheduler from anki.schedv2 import Scheduler as V2Scheduler from anki.tags import TagManager -from anki.template2 import render_from_field_map +from anki.template2 import render_qa_from_field_map from anki.types import NoteType, QAData, Template from anki.utils import ( devMode, @@ -666,7 +666,7 @@ where c.nid = n.id and c.id in %s group by nid""" fields = runFilter("mungeFields", fields, model, data, self) # render fields - qatext = render_from_field_map(qfmt, afmt, fields, card_ord) + qatext = render_qa_from_field_map(self, qfmt, afmt, fields, card_ord) ret: Dict[str, Any] = dict(q=qatext[0], a=qatext[1], id=card_id) # allow add-ons to modify the generated result diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 50ff15561..9623f5d27 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -1,8 +1,8 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html # pylint: skip-file - -from typing import Dict, List +from dataclasses import dataclass +from typing import Dict, List, Union import ankirspy # pytype: disable=import-error @@ -46,6 +46,12 @@ def proto_template_reqs_to_legacy( return legacy_reqs +@dataclass +class TemplateReplacement: + field_name: str + filters: List[str] + + class RustBackend: def __init__(self, path: str): self._backend = ankirspy.Backend(path) @@ -88,3 +94,26 @@ class RustBackend: ) ) ).sched_timing_today + + def flatten_template( + self, template: str, nonempty_fields: List[str] + ) -> List[Union[str, TemplateReplacement]]: + out = self._run_command( + pb.BackendInput( + flatten_template=pb.FlattenTemplateIn( + template_text=template, nonempty_field_names=nonempty_fields + ) + ) + ).flatten_template + results: List[Union[str, TemplateReplacement]] = [] + for node in out.nodes: + if node.WhichOneof("value") == "text": + results.append(node.text) + else: + results.append( + TemplateReplacement( + field_name=node.replacement.field_name, + filters=list(node.replacement.filters), + ) + ) + return results diff --git a/pylib/anki/template2.py b/pylib/anki/template2.py index bde7a4197..eb175d952 100644 --- a/pylib/anki/template2.py +++ b/pylib/anki/template2.py @@ -9,33 +9,75 @@ connected to pystache. It may be renamed in the future. from __future__ import annotations import re -from typing import Any, Callable, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Tuple, Union import anki from anki.hooks import addHook, runFilter from anki.lang import _ +from anki.rsbackend import TemplateReplacement from anki.sound import stripSounds from anki.utils import stripHTML, stripHTMLMedia -def render_from_field_map( - qfmt: str, afmt: str, fields: Dict[str, str], card_ord: int +def render_qa_from_field_map( + col: anki.storage._Collection, + qfmt: str, + afmt: str, + fields: Dict[str, str], + card_ord: int, ) -> Tuple[str, str]: "Renders the provided templates, returning rendered q & a text." # question format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (card_ord + 1), qfmt) format = format.replace("<%cloze:", "<%%cq:%d:" % (card_ord + 1)) - qtext = anki.template.render(format, fields) + qtext = render_template(col, format, fields) # answer format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (card_ord + 1), afmt) format = format.replace("<%cloze:", "<%%ca:%d:" % (card_ord + 1)) fields["FrontSide"] = stripSounds(qtext) - atext = anki.template.render(format, fields) + atext = render_template(col, format, fields) return qtext, atext +def render_template( + col: anki.storage._Collection, format: str, fields: Dict[str, str] +) -> str: + "Render a single template." + old_output = anki.template.render(format, fields) + + nonempty = nonempty_fields(fields) + flattened = col.backend.flatten_template(format, nonempty) + new_output = render_flattened_template(flattened, fields) + + if old_output != new_output: + print( + f"template rendering didn't match - please report:\n'{old_output}'\n'{new_output}'" + ) + # import os + # open(os.path.expanduser("~/temp1.txt"), "w").write(old_output) + # open(os.path.expanduser("~/temp2.txt"), "w").write(new_output) + return new_output + + +def render_flattened_template( + flattened: List[Union[str, TemplateReplacement]], fields: Dict[str, str] +) -> str: + "Render a list of strings or replacements into a string." + res = "" + for node in flattened: + if isinstance(node, str): + res += node + else: + text = fields.get(node.field_name) + if text is None: + res += unknown_field_message(node) + continue + res += apply_field_filters(node.field_name, text, fields, node.filters) + return res + + def field_is_not_empty(field_text: str) -> bool: # fixme: this is an overkill way of preventing a field with only # a
or
from appearing non-empty @@ -44,6 +86,23 @@ def field_is_not_empty(field_text: str) -> bool: return field_text.strip() != "" +def nonempty_fields(fields: Dict[str, str]) -> List[str]: + res = [] + for field, text in fields.items(): + if field_is_not_empty(text): + res.append(field) + return res + + +def unknown_field_message(node: TemplateReplacement) -> str: + # mirror the pystache message for now + field = node.field_name + if node.filters: + field_and_filters = list(reversed(node.filters)) + [field] + field = ":".join(field_and_filters) + return "{unknown field %s}" % field + + # Filters ########################################################################## # diff --git a/rslib/src/backend.rs b/rslib/src/backend.rs index 7f13d3ad0..35d20fd26 100644 --- a/rslib/src/backend.rs +++ b/rslib/src/backend.rs @@ -1,9 +1,10 @@ use crate::backend_proto as pt; use crate::backend_proto::backend_input::Value; +use crate::backend_proto::FlattenedTemplateReplacement; use crate::err::{AnkiError, Result}; use crate::sched::sched_timing_today; use crate::template::{ - without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, + without_legacy_template_directives, FieldMap, FieldRequirements, FlattenedNode, ParsedTemplate, }; use prost::Message; use std::collections::HashSet; @@ -92,6 +93,7 @@ impl Backend { Value::DeckTree(_) => todo!(), Value::FindCards(_) => todo!(), Value::BrowserRows(_) => todo!(), + Value::FlattenTemplate(input) => OValue::FlattenTemplate(self.flatten_template(input)?), }) } @@ -153,8 +155,43 @@ impl Backend { next_day_at: today.next_day_at, } } + + fn flatten_template(&self, input: pt::FlattenTemplateIn) -> Result { + let field_refs: HashSet<_> = input + .nonempty_field_names + .iter() + .map(AsRef::as_ref) + .collect(); + + let normalized = without_legacy_template_directives(&input.template_text); + match ParsedTemplate::from_text(normalized.as_ref()) { + Ok(tmpl) => { + let nodes = tmpl.flatten(&field_refs); + let out_nodes = nodes + .into_iter() + .map(|n| pt::FlattenedTemplateNode { + value: Some(flattened_node_to_proto(n)), + }) + .collect(); + Ok(pt::FlattenTemplateOut { nodes: out_nodes }) + } + Err(e) => Err(e), + } + } } fn ords_hash_to_set(ords: HashSet) -> Vec { ords.iter().map(|ord| *ord as u32).collect() } + +fn flattened_node_to_proto(node: FlattenedNode) -> pt::flattened_template_node::Value { + match node { + FlattenedNode::Text { text } => pt::flattened_template_node::Value::Text(text), + FlattenedNode::Replacement { field, filters } => { + pt::flattened_template_node::Value::Replacement(FlattenedTemplateReplacement { + field_name: field, + filters, + }) + } + } +} diff --git a/rslib/src/template.rs b/rslib/src/template.rs index 3e1d66961..9376234dc 100644 --- a/rslib/src/template.rs +++ b/rslib/src/template.rs @@ -226,7 +226,71 @@ fn template_is_empty<'a>(nonempty_fields: &HashSet<&str>, nodes: &[ParsedNode<'a true } -// Compatibility with old Anki versions +// Flattening +//---------------------------------------- + +#[derive(Debug, PartialEq)] +pub enum FlattenedNode { + Text { + text: String, + }, + /// Filters are in the order they should be applied. + Replacement { + field: String, + filters: Vec, + }, +} + +impl ParsedTemplate<'_> { + /// Resolve conditional replacements, returning a list of nodes. + /// + /// This leaves the field replacement (with any filters that were provided) + /// up to the calling code to handle. + pub fn flatten(&self, nonempty_fields: &HashSet<&str>) -> Vec { + let mut flattened = vec![]; + + flatten_into(&mut flattened, self.0.as_ref(), nonempty_fields); + + flattened + } +} + +fn flatten_into( + rendered_nodes: &mut Vec, + nodes: &[ParsedNode], + fields: &HashSet<&str>, +) { + use ParsedNode::*; + for node in nodes { + match node { + Text(t) => { + if let Some(FlattenedNode::Text { ref mut text }) = rendered_nodes.last_mut() { + text.push_str(t) + } else { + rendered_nodes.push(FlattenedNode::Text { + text: (*t).to_string(), + }) + } + } + Replacement { key, filters } => rendered_nodes.push(FlattenedNode::Replacement { + field: (*key).to_string(), + filters: filters.iter().map(|&e| e.to_string()).collect(), + }), + Conditional { key, children } => { + if fields.contains(key) { + flatten_into(rendered_nodes, children.as_ref(), fields); + } + } + NegatedConditional { key, children } => { + if !fields.contains(key) { + flatten_into(rendered_nodes, children.as_ref(), fields); + } + } + }; + } +} + +// Field requirements //---------------------------------------- #[derive(Debug, Clone, PartialEq)] @@ -240,8 +304,13 @@ impl ParsedTemplate<'_> { /// Return fields required by template. /// /// This is not able to represent negated expressions or combinations of - /// Any and All, and is provided only for the sake of backwards - /// compatibility. + /// Any and All, but is compatible with older Anki clients. + /// + /// In the future, it may be feasible to calculate the requirements + /// when adding cards, instead of caching them up front, which would mean + /// the above restrictions could be lifted. We would probably + /// want to add a cache of non-zero fields -> available cards to avoid + /// slowing down bulk operations like importing too much. pub fn requirements(&self, field_map: &FieldMap) -> FieldRequirements { let mut nonempty: HashSet<_> = Default::default(); let mut ords = HashSet::new(); @@ -392,4 +461,72 @@ mod test { assert_eq!(without_legacy_template_directives(input), output); } + + #[test] + fn test_render() { + let map: HashSet<_> = vec!["F", "B"].into_iter().collect(); + + use crate::template::FlattenedNode as FN; + let mut tmpl = PT::from_text("{{B}}A{{F}}").unwrap(); + assert_eq!( + tmpl.flatten(&map), + vec![ + FN::Replacement { + field: "B".to_owned(), + filters: vec![] + }, + FN::Text { + text: "A".to_owned() + }, + FN::Replacement { + field: "F".to_owned(), + filters: vec![] + }, + ] + ); + + // empty + tmpl = PT::from_text("{{#E}}A{{/E}}").unwrap(); + assert_eq!(tmpl.flatten(&map), vec![]); + + // missing + tmpl = PT::from_text("{{^M}}A{{/M}}").unwrap(); + assert_eq!( + tmpl.flatten(&map), + vec![FN::Text { + text: "A".to_owned() + },] + ); + + // nested + tmpl = PT::from_text("{{^E}}1{{#F}}2{{#B}}{{F}}{{/B}}{{/F}}{{/E}}").unwrap(); + assert_eq!( + tmpl.flatten(&map), + vec![ + FN::Text { + text: "12".to_owned() + }, + FN::Replacement { + field: "F".to_owned(), + filters: vec![] + }, + ] + ); + + // filters + tmpl = PT::from_text("{{one:two:B}}{{three:X}}").unwrap(); + assert_eq!( + tmpl.flatten(&map), + vec![ + FN::Replacement { + field: "B".to_owned(), + filters: vec!["two".to_string(), "one".to_string()] + }, + FN::Replacement { + field: "X".to_owned(), + filters: vec!["three".to_string()] + }, + ] + ); + } }