handle conditional replacement in Rust

This extends the existing Rust code to handle conditional
replacement. The replacement of field names and filters to text
remains in Python, so that add-ons can still define their own
field modifiers.

The code is currently running the old Pystache rendering and the
new implementation in parallel, and will print a message to the
console if they don't match. If you notice any problems, please
let me know.

proto/backend.proto

@@ -14,6 +14,7 @@ message BackendInput {
         Empty deck_tree = 18;
         FindCardsIn find_cards = 19;
         BrowserRowsIn browser_rows = 20;
+        FlattenTemplateIn flatten_template = 21;
 
         PlusOneIn plus_one = 2046; // temporary, for testing
     }
@@ -26,6 +27,7 @@ message BackendOutput {
         DeckTreeOut deck_tree = 18;
         FindCardsOut find_cards = 19;
         BrowserRowsOut browser_rows = 20;
+        FlattenTemplateOut flatten_template = 21;
 
         PlusOneOut plus_one = 2046; // temporary, for testing
 
@@ -124,3 +126,24 @@ message BrowserRowsOut {
     // just sort fields for proof of concept
     repeated string sort_fields = 1;
 }
+
+message FlattenTemplateIn {
+    string template_text = 1;
+    repeated string nonempty_field_names = 2;
+}
+
+message FlattenTemplateOut {
+    repeated FlattenedTemplateNode nodes = 1;
+}
+
+message FlattenedTemplateNode {
+    oneof value {
+        string text = 1;
+        FlattenedTemplateReplacement replacement = 2;
+    }
+}
+
+message FlattenedTemplateReplacement {
+    string field_name = 1;
+    repeated string filters = 2;
+}

pylib/anki/collection.py

@@ -30,7 +30,7 @@ from anki.rsbackend import RustBackend
 from anki.sched import Scheduler as V1Scheduler
 from anki.schedv2 import Scheduler as V2Scheduler
 from anki.tags import TagManager
-from anki.template2 import render_from_field_map
+from anki.template2 import render_qa_from_field_map
 from anki.types import NoteType, QAData, Template
 from anki.utils import (
     devMode,
@@ -666,7 +666,7 @@ where c.nid = n.id and c.id in %s group by nid"""
         fields = runFilter("mungeFields", fields, model, data, self)
 
         # render fields
-        qatext = render_from_field_map(qfmt, afmt, fields, card_ord)
+        qatext = render_qa_from_field_map(self, qfmt, afmt, fields, card_ord)
         ret: Dict[str, Any] = dict(q=qatext[0], a=qatext[1], id=card_id)
 
         # allow add-ons to modify the generated result

pylib/anki/rsbackend.py

@@ -1,8 +1,8 @@
 # Copyright: Ankitects Pty Ltd and contributors
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 # pylint: skip-file
-
-from typing import Dict, List
+from dataclasses import dataclass
+from typing import Dict, List, Union
 
 import ankirspy  # pytype: disable=import-error
 
@@ -46,6 +46,12 @@ def proto_template_reqs_to_legacy(
     return legacy_reqs
 
 
+@dataclass
+class TemplateReplacement:
+    field_name: str
+    filters: List[str]
+
+
 class RustBackend:
     def __init__(self, path: str):
         self._backend = ankirspy.Backend(path)
@@ -88,3 +94,26 @@ class RustBackend:
                 )
             )
         ).sched_timing_today
+
+    def flatten_template(
+        self, template: str, nonempty_fields: List[str]
+    ) -> List[Union[str, TemplateReplacement]]:
+        out = self._run_command(
+            pb.BackendInput(
+                flatten_template=pb.FlattenTemplateIn(
+                    template_text=template, nonempty_field_names=nonempty_fields
+                )
+            )
+        ).flatten_template
+        results: List[Union[str, TemplateReplacement]] = []
+        for node in out.nodes:
+            if node.WhichOneof("value") == "text":
+                results.append(node.text)
+            else:
+                results.append(
+                    TemplateReplacement(
+                        field_name=node.replacement.field_name,
+                        filters=list(node.replacement.filters),
+                    )
+                )
+        return results

pylib/anki/template2.py

@@ -9,33 +9,75 @@ connected to pystache. It may be renamed in the future.
 from __future__ import annotations
 
 import re
-from typing import Any, Callable, Dict, List, Tuple
+from typing import Any, Callable, Dict, List, Tuple, Union
 
 import anki
 from anki.hooks import addHook, runFilter
 from anki.lang import _
+from anki.rsbackend import TemplateReplacement
 from anki.sound import stripSounds
 from anki.utils import stripHTML, stripHTMLMedia
 
 
-def render_from_field_map(
-    qfmt: str, afmt: str, fields: Dict[str, str], card_ord: int
+def render_qa_from_field_map(
+    col: anki.storage._Collection,
+    qfmt: str,
+    afmt: str,
+    fields: Dict[str, str],
+    card_ord: int,
 ) -> Tuple[str, str]:
     "Renders the provided templates, returning rendered q & a text."
     # question
     format = re.sub("{{(?!type:)(.*?)cloze:", r"{{\1cq-%d:" % (card_ord + 1), qfmt)
     format = format.replace("<%cloze:", "<%%cq:%d:" % (card_ord + 1))
-    qtext = anki.template.render(format, fields)
+    qtext = render_template(col, format, fields)
 
     # answer
     format = re.sub("{{(.*?)cloze:", r"{{\1ca-%d:" % (card_ord + 1), afmt)
     format = format.replace("<%cloze:", "<%%ca:%d:" % (card_ord + 1))
     fields["FrontSide"] = stripSounds(qtext)
-    atext = anki.template.render(format, fields)
+    atext = render_template(col, format, fields)
 
     return qtext, atext
 
 
+def render_template(
+    col: anki.storage._Collection, format: str, fields: Dict[str, str]
+) -> str:
+    "Render a single template."
+    old_output = anki.template.render(format, fields)
+
+    nonempty = nonempty_fields(fields)
+    flattened = col.backend.flatten_template(format, nonempty)
+    new_output = render_flattened_template(flattened, fields)
+
+    if old_output != new_output:
+        print(
+            f"template rendering didn't match - please report:\n'{old_output}'\n'{new_output}'"
+        )
+        # import os
+        # open(os.path.expanduser("~/temp1.txt"), "w").write(old_output)
+        # open(os.path.expanduser("~/temp2.txt"), "w").write(new_output)
+    return new_output
+
+
+def render_flattened_template(
+    flattened: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
+) -> str:
+    "Render a list of strings or replacements into a string."
+    res = ""
+    for node in flattened:
+        if isinstance(node, str):
+            res += node
+        else:
+            text = fields.get(node.field_name)
+            if text is None:
+                res += unknown_field_message(node)
+                continue
+            res += apply_field_filters(node.field_name, text, fields, node.filters)
+    return res
+
+
 def field_is_not_empty(field_text: str) -> bool:
     # fixme: this is an overkill way of preventing a field with only
     # a <br> or <div> from appearing non-empty
@@ -44,6 +86,23 @@ def field_is_not_empty(field_text: str) -> bool:
     return field_text.strip() != ""
 
 
+def nonempty_fields(fields: Dict[str, str]) -> List[str]:
+    res = []
+    for field, text in fields.items():
+        if field_is_not_empty(text):
+            res.append(field)
+    return res
+
+
+def unknown_field_message(node: TemplateReplacement) -> str:
+    # mirror the pystache message for now
+    field = node.field_name
+    if node.filters:
+        field_and_filters = list(reversed(node.filters)) + [field]
+        field = ":".join(field_and_filters)
+    return "{unknown field %s}" % field
+
+
 # Filters
 ##########################################################################
 #

rslib/src/backend.rs

@@ -1,9 +1,10 @@
 use crate::backend_proto as pt;
 use crate::backend_proto::backend_input::Value;
+use crate::backend_proto::FlattenedTemplateReplacement;
 use crate::err::{AnkiError, Result};
 use crate::sched::sched_timing_today;
 use crate::template::{
-    without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate,
+    without_legacy_template_directives, FieldMap, FieldRequirements, FlattenedNode, ParsedTemplate,
 };
 use prost::Message;
 use std::collections::HashSet;
@@ -92,6 +93,7 @@ impl Backend {
             Value::DeckTree(_) => todo!(),
             Value::FindCards(_) => todo!(),
             Value::BrowserRows(_) => todo!(),
+            Value::FlattenTemplate(input) => OValue::FlattenTemplate(self.flatten_template(input)?),
         })
     }
 
@@ -153,8 +155,43 @@ impl Backend {
             next_day_at: today.next_day_at,
         }
     }
+
+    fn flatten_template(&self, input: pt::FlattenTemplateIn) -> Result<pt::FlattenTemplateOut> {
+        let field_refs: HashSet<_> = input
+            .nonempty_field_names
+            .iter()
+            .map(AsRef::as_ref)
+            .collect();
+
+        let normalized = without_legacy_template_directives(&input.template_text);
+        match ParsedTemplate::from_text(normalized.as_ref()) {
+            Ok(tmpl) => {
+                let nodes = tmpl.flatten(&field_refs);
+                let out_nodes = nodes
+                    .into_iter()
+                    .map(|n| pt::FlattenedTemplateNode {
+                        value: Some(flattened_node_to_proto(n)),
+                    })
+                    .collect();
+                Ok(pt::FlattenTemplateOut { nodes: out_nodes })
+            }
+            Err(e) => Err(e),
+        }
+    }
 }
 
 fn ords_hash_to_set(ords: HashSet<u16>) -> Vec<u32> {
     ords.iter().map(|ord| *ord as u32).collect()
 }
+
+fn flattened_node_to_proto(node: FlattenedNode) -> pt::flattened_template_node::Value {
+    match node {
+        FlattenedNode::Text { text } => pt::flattened_template_node::Value::Text(text),
+        FlattenedNode::Replacement { field, filters } => {
+            pt::flattened_template_node::Value::Replacement(FlattenedTemplateReplacement {
+                field_name: field,
+                filters,
+            })
+        }
+    }
+}

rslib/src/template.rs

@@ -226,7 +226,71 @@ fn template_is_empty<'a>(nonempty_fields: &HashSet<&str>, nodes: &[ParsedNode<'a
     true
 }
 
-// Compatibility with old Anki versions
+// Flattening
+//----------------------------------------
+
+#[derive(Debug, PartialEq)]
+pub enum FlattenedNode {
+    Text {
+        text: String,
+    },
+    /// Filters are in the order they should be applied.
+    Replacement {
+        field: String,
+        filters: Vec<String>,
+    },
+}
+
+impl ParsedTemplate<'_> {
+    /// Resolve conditional replacements, returning a list of nodes.
+    ///
+    /// This leaves the field replacement (with any filters that were provided)
+    /// up to the calling code to handle.
+    pub fn flatten(&self, nonempty_fields: &HashSet<&str>) -> Vec<FlattenedNode> {
+        let mut flattened = vec![];
+
+        flatten_into(&mut flattened, self.0.as_ref(), nonempty_fields);
+
+        flattened
+    }
+}
+
+fn flatten_into(
+    rendered_nodes: &mut Vec<FlattenedNode>,
+    nodes: &[ParsedNode],
+    fields: &HashSet<&str>,
+) {
+    use ParsedNode::*;
+    for node in nodes {
+        match node {
+            Text(t) => {
+                if let Some(FlattenedNode::Text { ref mut text }) = rendered_nodes.last_mut() {
+                    text.push_str(t)
+                } else {
+                    rendered_nodes.push(FlattenedNode::Text {
+                        text: (*t).to_string(),
+                    })
+                }
+            }
+            Replacement { key, filters } => rendered_nodes.push(FlattenedNode::Replacement {
+                field: (*key).to_string(),
+                filters: filters.iter().map(|&e| e.to_string()).collect(),
+            }),
+            Conditional { key, children } => {
+                if fields.contains(key) {
+                    flatten_into(rendered_nodes, children.as_ref(), fields);
+                }
+            }
+            NegatedConditional { key, children } => {
+                if !fields.contains(key) {
+                    flatten_into(rendered_nodes, children.as_ref(), fields);
+                }
+            }
+        };
+    }
+}
+
+// Field requirements
 //----------------------------------------
 
 #[derive(Debug, Clone, PartialEq)]
@@ -240,8 +304,13 @@ impl ParsedTemplate<'_> {
     /// Return fields required by template.
     ///
     /// This is not able to represent negated expressions or combinations of
-    /// Any and All, and is provided only for the sake of backwards
-    /// compatibility.
+    /// Any and All, but is compatible with older Anki clients.
+    ///
+    /// In the future, it may be feasible to calculate the requirements
+    /// when adding cards, instead of caching them up front, which would mean
+    /// the above restrictions could be lifted. We would probably
+    /// want to add a cache of non-zero fields -> available cards to avoid
+    /// slowing down bulk operations like importing too much.
     pub fn requirements(&self, field_map: &FieldMap) -> FieldRequirements {
         let mut nonempty: HashSet<_> = Default::default();
         let mut ords = HashSet::new();
@@ -392,4 +461,72 @@ mod test {
 
         assert_eq!(without_legacy_template_directives(input), output);
     }
+
+    #[test]
+    fn test_render() {
+        let map: HashSet<_> = vec!["F", "B"].into_iter().collect();
+
+        use crate::template::FlattenedNode as FN;
+        let mut tmpl = PT::from_text("{{B}}A{{F}}").unwrap();
+        assert_eq!(
+            tmpl.flatten(&map),
+            vec![
+                FN::Replacement {
+                    field: "B".to_owned(),
+                    filters: vec![]
+                },
+                FN::Text {
+                    text: "A".to_owned()
+                },
+                FN::Replacement {
+                    field: "F".to_owned(),
+                    filters: vec![]
+                },
+            ]
+        );
+
+        // empty
+        tmpl = PT::from_text("{{#E}}A{{/E}}").unwrap();
+        assert_eq!(tmpl.flatten(&map), vec![]);
+
+        // missing
+        tmpl = PT::from_text("{{^M}}A{{/M}}").unwrap();
+        assert_eq!(
+            tmpl.flatten(&map),
+            vec![FN::Text {
+                text: "A".to_owned()
+            },]
+        );
+
+        // nested
+        tmpl = PT::from_text("{{^E}}1{{#F}}2{{#B}}{{F}}{{/B}}{{/F}}{{/E}}").unwrap();
+        assert_eq!(
+            tmpl.flatten(&map),
+            vec![
+                FN::Text {
+                    text: "12".to_owned()
+                },
+                FN::Replacement {
+                    field: "F".to_owned(),
+                    filters: vec![]
+                },
+            ]
+        );
+
+        // filters
+        tmpl = PT::from_text("{{one:two:B}}{{three:X}}").unwrap();
+        assert_eq!(
+            tmpl.flatten(&map),
+            vec![
+                FN::Replacement {
+                    field: "B".to_owned(),
+                    filters: vec!["two".to_string(), "one".to_string()]
+                },
+                FN::Replacement {
+                    field: "X".to_owned(),
+                    filters: vec!["three".to_string()]
+                },
+            ]
+        );
+    }
 }