flatten->render; pass field content in

This is paving the way to move the standard filters into Rust.

Non-empty fields are now determined in Rust, using a single regex
instead of the overkill stripHTMLMedia(). The old implementation
has been moved into the Pystache code for now.

proto/backend.proto

@@ -14,7 +14,7 @@ message BackendInput {
         Empty deck_tree = 18;
         FindCardsIn find_cards = 19;
         BrowserRowsIn browser_rows = 20;
-        FlattenTemplateIn flatten_template = 21;
+        RenderTemplateIn render_template = 21;
 
         PlusOneIn plus_one = 2046; // temporary, for testing
     }
@@ -27,7 +27,7 @@ message BackendOutput {
         DeckTreeOut deck_tree = 18;
         FindCardsOut find_cards = 19;
         BrowserRowsOut browser_rows = 20;
-        FlattenTemplateOut flatten_template = 21;
+        RenderTemplateOut render_template = 21;
 
         PlusOneOut plus_one = 2046; // temporary, for testing
 
@@ -128,23 +128,23 @@ message BrowserRowsOut {
     repeated string sort_fields = 1;
 }
 
-message FlattenTemplateIn {
+message RenderTemplateIn {
     string template_text = 1;
-    repeated string nonempty_field_names = 2;
+    map<string,string> fields = 2;
 }
 
-message FlattenTemplateOut {
+message RenderTemplateOut {
-    repeated FlattenedTemplateNode nodes = 1;
+    repeated RenderedTemplateNode nodes = 1;
 }
 
-message FlattenedTemplateNode {
+message RenderedTemplateNode {
     oneof value {
         string text = 1;
-        FlattenedTemplateReplacement replacement = 2;
+        RenderedTemplateReplacement replacement = 2;
     }
 }
 
-message FlattenedTemplateReplacement {
+message RenderedTemplateReplacement {
     string field_name = 1;
     repeated string filters = 2;
 }

pylib/anki/rsbackend.py

@@ -104,16 +104,16 @@ class RustBackend:
             )
         ).sched_timing_today
 
-    def flatten_template(
+    def render_template(
-        self, template: str, nonempty_fields: List[str]
+        self, template: str, fields: Dict[str, str]
     ) -> List[Union[str, TemplateReplacement]]:
         out = self._run_command(
             pb.BackendInput(
-                flatten_template=pb.FlattenTemplateIn(
+                render_template=pb.RenderTemplateIn(
-                    template_text=template, nonempty_field_names=nonempty_fields
+                    template_text=template, fields=fields
                 )
             )
-        ).flatten_template
+        ).render_template
         results: List[Union[str, TemplateReplacement]] = []
         for node in out.nodes:
             if node.WhichOneof("value") == "text":

pylib/anki/template/template.py

@@ -1,11 +1,21 @@
 import re
 from typing import Any, Callable, Dict, Pattern
 
-from anki.template2 import apply_field_filters, field_is_not_empty
+from anki.template2 import apply_field_filters
 
 modifiers: Dict[str, Callable] = {}
 
 
+def field_is_not_empty(field_text: str) -> bool:
+    # fixme: this is an overkill way of preventing a field with only
+    # a <br> or <div> from appearing non-empty
+    from anki.utils import stripHTMLMedia
+
+    field_text = stripHTMLMedia(field_text)
+
+    return field_text.strip() != ""
+
+
 def modifier(symbol) -> Callable[[Any], Any]:
     """Decorator for associating a function with a Mustache tag modifier.
 

pylib/anki/template2.py

@@ -16,7 +16,7 @@ from anki.hooks import addHook, runFilter
 from anki.lang import _
 from anki.rsbackend import TemplateReplacement
 from anki.sound import stripSounds
-from anki.utils import stripHTML, stripHTMLMedia
+from anki.utils import stripHTML
 
 
 def render_qa_from_field_map(
@@ -47,9 +47,8 @@ def render_template(
     "Render a single template."
     old_output = anki.template.render(format, fields)
 
-    nonempty = nonempty_fields(fields)
+    rendered = col.backend.render_template(format, fields)
-    flattened = col.backend.flatten_template(format, nonempty)
+    new_output = render_flattened_template(rendered, fields)
-    new_output = render_flattened_template(flattened, fields)
 
     if old_output != new_output:
         print(
@@ -62,11 +61,11 @@ def render_template(
 
 
 def render_flattened_template(
-    flattened: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
+    rendered: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
 ) -> str:
     "Render a list of strings or replacements into a string."
     res = ""
-    for node in flattened:
+    for node in rendered:
         if isinstance(node, str):
             res += node
         else:
@@ -78,22 +77,6 @@ def render_flattened_template(
     return res
 
 
-def field_is_not_empty(field_text: str) -> bool:
-    # fixme: this is an overkill way of preventing a field with only
-    # a <br> or <div> from appearing non-empty
-    field_text = stripHTMLMedia(field_text)
-
-    return field_text.strip() != ""
-
-
-def nonempty_fields(fields: Dict[str, str]) -> List[str]:
-    res = []
-    for field, text in fields.items():
-        if field_is_not_empty(text):
-            res.append(field)
-    return res
-
-
 def unknown_field_message(node: TemplateReplacement) -> str:
     # mirror the pystache message for now
     field = node.field_name

rslib/Cargo.toml

@@ -11,6 +11,8 @@ failure = "0.1.6"
 prost = "0.5.0"
 bytes = "0.4"
 chrono = "0.4.10"
+lazy_static = "1.4.0"
+regex = "1.3.3"
 
 [build-dependencies]
 prost-build = "0.5.0"

rslib/src/backend.rs

@@ -1,13 +1,13 @@
 use crate::backend_proto as pt;
 use crate::backend_proto::backend_input::Value;
-use crate::backend_proto::FlattenedTemplateReplacement;
+use crate::backend_proto::RenderedTemplateReplacement;
 use crate::err::{AnkiError, Result};
 use crate::sched::sched_timing_today;
 use crate::template::{
-    without_legacy_template_directives, FieldMap, FieldRequirements, FlattenedNode, ParsedTemplate,
+    without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode,
 };
 use prost::Message;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 
 pub struct Backend {
@@ -93,7 +93,7 @@ impl Backend {
             Value::DeckTree(_) => todo!(),
             Value::FindCards(_) => todo!(),
             Value::BrowserRows(_) => todo!(),
-            Value::FlattenTemplate(input) => OValue::FlattenTemplate(self.flatten_template(input)?),
+            Value::RenderTemplate(input) => OValue::RenderTemplate(self.render_template(input)?),
         })
     }
 
@@ -157,24 +157,24 @@ impl Backend {
         }
     }
 
-    fn flatten_template(&self, input: pt::FlattenTemplateIn) -> Result<pt::FlattenTemplateOut> {
+    fn render_template(&self, input: pt::RenderTemplateIn) -> Result<pt::RenderTemplateOut> {
-        let field_refs: HashSet<_> = input
+        let fields: HashMap<_, _> = input
-            .nonempty_field_names
+            .fields
             .iter()
-            .map(AsRef::as_ref)
+            .map(|(k, v)| (k.as_ref(), v.as_ref()))
             .collect();
 
         let normalized = without_legacy_template_directives(&input.template_text);
         match ParsedTemplate::from_text(normalized.as_ref()) {
             Ok(tmpl) => {
-                let nodes = tmpl.flatten(&field_refs);
+                let nodes = tmpl.render(&fields);
                 let out_nodes = nodes
                     .into_iter()
-                    .map(|n| pt::FlattenedTemplateNode {
+                    .map(|n| pt::RenderedTemplateNode {
-                        value: Some(flattened_node_to_proto(n)),
+                        value: Some(rendered_node_to_proto(n)),
                     })
                     .collect();
-                Ok(pt::FlattenTemplateOut { nodes: out_nodes })
+                Ok(pt::RenderTemplateOut { nodes: out_nodes })
             }
             Err(e) => Err(e),
         }
@@ -185,11 +185,11 @@ fn ords_hash_to_set(ords: HashSet<u16>) -> Vec<u32> {
     ords.iter().map(|ord| *ord as u32).collect()
 }
 
-fn flattened_node_to_proto(node: FlattenedNode) -> pt::flattened_template_node::Value {
+fn rendered_node_to_proto(node: RenderedNode) -> pt::rendered_template_node::Value {
     match node {
-        FlattenedNode::Text { text } => pt::flattened_template_node::Value::Text(text),
+        RenderedNode::Text { text } => pt::rendered_template_node::Value::Text(text),
-        FlattenedNode::Replacement { field, filters } => {
+        RenderedNode::Replacement { field, filters } => {
-            pt::flattened_template_node::Value::Replacement(FlattenedTemplateReplacement {
+            pt::rendered_template_node::Value::Replacement(RenderedTemplateReplacement {
                 field_name: field,
                 filters,
             })

rslib/src/template.rs

@@ -1,9 +1,11 @@
 use crate::err::{AnkiError, Result};
+use lazy_static::lazy_static;
 use nom;
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::error::ErrorKind;
 use nom::sequence::delimited;
+use regex::Regex;
 use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
 
@@ -237,11 +239,11 @@ fn template_is_empty<'a>(nonempty_fields: &HashSet<&str>, nodes: &[ParsedNode<'a
     true
 }
 
-// Flattening
+// Rendering
 //----------------------------------------
 
 #[derive(Debug, PartialEq)]
-pub enum FlattenedNode {
+pub enum RenderedNode {
     Text {
         text: String,
     },
@@ -257,50 +259,81 @@ impl ParsedTemplate<'_> {
     ///
     /// This leaves the field replacement (with any filters that were provided)
     /// up to the calling code to handle.
-    pub fn flatten(&self, nonempty_fields: &HashSet<&str>) -> Vec<FlattenedNode> {
+    pub fn render(&self, fields: &HashMap<&str, &str>) -> Vec<RenderedNode> {
-        let mut flattened = vec![];
+        let mut rendered = vec![];
+        let nonempty = nonempty_fields(fields);
 
-        flatten_into(&mut flattened, self.0.as_ref(), nonempty_fields);
+        render_into(&mut rendered, self.0.as_ref(), fields, &nonempty);
 
-        flattened
+        rendered
     }
 }
 
-fn flatten_into(
+fn render_into(
-    rendered_nodes: &mut Vec<FlattenedNode>,
+    rendered_nodes: &mut Vec<RenderedNode>,
     nodes: &[ParsedNode],
-    fields: &HashSet<&str>,
+    fields: &HashMap<&str, &str>,
+    nonempty: &HashSet<&str>,
 ) {
     use ParsedNode::*;
     for node in nodes {
         match node {
             Text(t) => {
-                if let Some(FlattenedNode::Text { ref mut text }) = rendered_nodes.last_mut() {
+                if let Some(RenderedNode::Text { ref mut text }) = rendered_nodes.last_mut() {
                     text.push_str(t)
                 } else {
-                    rendered_nodes.push(FlattenedNode::Text {
+                    rendered_nodes.push(RenderedNode::Text {
                         text: (*t).to_string(),
                     })
                 }
             }
-            Replacement { key, filters } => rendered_nodes.push(FlattenedNode::Replacement {
+            Replacement { key, filters } => rendered_nodes.push(RenderedNode::Replacement {
                 field: (*key).to_string(),
                 filters: filters.iter().map(|&e| e.to_string()).collect(),
             }),
             Conditional { key, children } => {
-                if fields.contains(key) {
+                if nonempty.contains(key) {
-                    flatten_into(rendered_nodes, children.as_ref(), fields);
+                    render_into(rendered_nodes, children.as_ref(), fields, nonempty);
                 }
             }
             NegatedConditional { key, children } => {
-                if !fields.contains(key) {
+                if !nonempty.contains(key) {
-                    flatten_into(rendered_nodes, children.as_ref(), fields);
+                    render_into(rendered_nodes, children.as_ref(), fields, nonempty);
                 }
             }
         };
     }
 }
 
+/// True if provided text contains only whitespace and/or empty BR/DIV tags.
+fn field_is_empty(text: &str) -> bool {
+    lazy_static! {
+        static ref RE: Regex = Regex::new(
+            r#"(?xsi)
+            ^(?:
+            [[:space:]]
+            |
+            </?(?:br|div)\ ?/?>
+            )*$
+        "#
+        )
+        .unwrap();
+    }
+    RE.is_match(text)
+}
+
+fn nonempty_fields<'a>(fields: &'a HashMap<&str, &str>) -> HashSet<&'a str> {
+    fields
+        .iter()
+        .filter_map(|(name, val)| {
+            if !field_is_empty(val) {
+                Some(*name)
+            } else {
+                None
+            }
+        })
+        .collect()
+}
 // Field requirements
 //----------------------------------------
 
@@ -360,10 +393,21 @@ impl ParsedTemplate<'_> {
 #[cfg(test)]
 mod test {
     use super::{FieldMap, ParsedNode::*, ParsedTemplate as PT};
-    use crate::template::{without_legacy_template_directives, FieldRequirements};
+    use crate::template::{field_is_empty, without_legacy_template_directives, FieldRequirements};
-    use std::collections::HashSet;
+    use std::collections::{HashMap, HashSet};
     use std::iter::FromIterator;
 
+    #[test]
+    fn test_field_empty() {
+        assert_eq!(field_is_empty(""), true);
+        assert_eq!(field_is_empty(" "), true);
+        assert_eq!(field_is_empty("x"), false);
+        assert_eq!(field_is_empty("<BR>"), true);
+        assert_eq!(field_is_empty("<div />"), true);
+        assert_eq!(field_is_empty(" <div> <br> </div>\n"), true);
+        assert_eq!(field_is_empty(" <div>x</div>\n"), false);
+    }
+
     #[test]
     fn test_parsing() {
         let tmpl = PT::from_text("foo {{bar}} {{#baz}} quux {{/baz}}").unwrap();
@@ -481,12 +525,14 @@ mod test {
 
     #[test]
     fn test_render() {
-        let map: HashSet<_> = vec!["F", "B"].into_iter().collect();
+        let map: HashMap<_, _> = vec![("F", "1"), ("B", "2"), ("E", " ")]
+            .into_iter()
+            .collect();
 
-        use crate::template::FlattenedNode as FN;
+        use crate::template::RenderedNode as FN;
         let mut tmpl = PT::from_text("{{B}}A{{F}}").unwrap();
         assert_eq!(
-            tmpl.flatten(&map),
+            tmpl.render(&map),
             vec![
                 FN::Replacement {
                     field: "B".to_owned(),
@@ -504,12 +550,12 @@ mod test {
 
         // empty
         tmpl = PT::from_text("{{#E}}A{{/E}}").unwrap();
-        assert_eq!(tmpl.flatten(&map), vec![]);
+        assert_eq!(tmpl.render(&map), vec![]);
 
         // missing
         tmpl = PT::from_text("{{^M}}A{{/M}}").unwrap();
         assert_eq!(
-            tmpl.flatten(&map),
+            tmpl.render(&map),
             vec![FN::Text {
                 text: "A".to_owned()
             },]
@@ -518,7 +564,7 @@ mod test {
         // nested
         tmpl = PT::from_text("{{^E}}1{{#F}}2{{#B}}{{F}}{{/B}}{{/F}}{{/E}}").unwrap();
         assert_eq!(
-            tmpl.flatten(&map),
+            tmpl.render(&map),
             vec![
                 FN::Text {
                     text: "12".to_owned()
@@ -533,7 +579,7 @@ mod test {
         // filters
         tmpl = PT::from_text("{{one:two:B}}{{three:X}}").unwrap();
         assert_eq!(
-            tmpl.flatten(&map),
+            tmpl.render(&map),
             vec![
                 FN::Replacement {
                     field: "B".to_owned(),