From f7ff0d1c17f22091ff3df5c7f638342f2de67b12 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 10 Jan 2020 14:59:29 +1000 Subject: [PATCH] flatten->render; pass field content in This is paving the way to move the standard filters into Rust. Non-empty fields are now determined in Rust, using a single regex instead of the overkill stripHTMLMedia(). The old implementation has been moved into the Pystache code for now. --- proto/backend.proto | 18 +++---- pylib/anki/rsbackend.py | 10 ++-- pylib/anki/template/template.py | 12 ++++- pylib/anki/template2.py | 27 ++-------- rslib/Cargo.toml | 2 + rslib/src/backend.rs | 32 +++++------ rslib/src/template.rs | 96 ++++++++++++++++++++++++--------- 7 files changed, 119 insertions(+), 78 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 82b91c8c8..b8cb93d6d 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -14,7 +14,7 @@ message BackendInput { Empty deck_tree = 18; FindCardsIn find_cards = 19; BrowserRowsIn browser_rows = 20; - FlattenTemplateIn flatten_template = 21; + RenderTemplateIn render_template = 21; PlusOneIn plus_one = 2046; // temporary, for testing } @@ -27,7 +27,7 @@ message BackendOutput { DeckTreeOut deck_tree = 18; FindCardsOut find_cards = 19; BrowserRowsOut browser_rows = 20; - FlattenTemplateOut flatten_template = 21; + RenderTemplateOut render_template = 21; PlusOneOut plus_one = 2046; // temporary, for testing @@ -128,23 +128,23 @@ message BrowserRowsOut { repeated string sort_fields = 1; } -message FlattenTemplateIn { +message RenderTemplateIn { string template_text = 1; - repeated string nonempty_field_names = 2; + map fields = 2; } -message FlattenTemplateOut { - repeated FlattenedTemplateNode nodes = 1; +message RenderTemplateOut { + repeated RenderedTemplateNode nodes = 1; } -message FlattenedTemplateNode { +message RenderedTemplateNode { oneof value { string text = 1; - FlattenedTemplateReplacement replacement = 2; + RenderedTemplateReplacement replacement = 2; } } -message FlattenedTemplateReplacement { +message RenderedTemplateReplacement { string field_name = 1; repeated string filters = 2; } diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index c0dd8503b..ccd6223d9 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -104,16 +104,16 @@ class RustBackend: ) ).sched_timing_today - def flatten_template( - self, template: str, nonempty_fields: List[str] + def render_template( + self, template: str, fields: Dict[str, str] ) -> List[Union[str, TemplateReplacement]]: out = self._run_command( pb.BackendInput( - flatten_template=pb.FlattenTemplateIn( - template_text=template, nonempty_field_names=nonempty_fields + render_template=pb.RenderTemplateIn( + template_text=template, fields=fields ) ) - ).flatten_template + ).render_template results: List[Union[str, TemplateReplacement]] = [] for node in out.nodes: if node.WhichOneof("value") == "text": diff --git a/pylib/anki/template/template.py b/pylib/anki/template/template.py index 634dcb8fa..81b3a1319 100644 --- a/pylib/anki/template/template.py +++ b/pylib/anki/template/template.py @@ -1,11 +1,21 @@ import re from typing import Any, Callable, Dict, Pattern -from anki.template2 import apply_field_filters, field_is_not_empty +from anki.template2 import apply_field_filters modifiers: Dict[str, Callable] = {} +def field_is_not_empty(field_text: str) -> bool: + # fixme: this is an overkill way of preventing a field with only + # a
or
from appearing non-empty + from anki.utils import stripHTMLMedia + + field_text = stripHTMLMedia(field_text) + + return field_text.strip() != "" + + def modifier(symbol) -> Callable[[Any], Any]: """Decorator for associating a function with a Mustache tag modifier. diff --git a/pylib/anki/template2.py b/pylib/anki/template2.py index 12e16c5be..91952945e 100644 --- a/pylib/anki/template2.py +++ b/pylib/anki/template2.py @@ -16,7 +16,7 @@ from anki.hooks import addHook, runFilter from anki.lang import _ from anki.rsbackend import TemplateReplacement from anki.sound import stripSounds -from anki.utils import stripHTML, stripHTMLMedia +from anki.utils import stripHTML def render_qa_from_field_map( @@ -47,9 +47,8 @@ def render_template( "Render a single template." old_output = anki.template.render(format, fields) - nonempty = nonempty_fields(fields) - flattened = col.backend.flatten_template(format, nonempty) - new_output = render_flattened_template(flattened, fields) + rendered = col.backend.render_template(format, fields) + new_output = render_flattened_template(rendered, fields) if old_output != new_output: print( @@ -62,11 +61,11 @@ def render_template( def render_flattened_template( - flattened: List[Union[str, TemplateReplacement]], fields: Dict[str, str] + rendered: List[Union[str, TemplateReplacement]], fields: Dict[str, str] ) -> str: "Render a list of strings or replacements into a string." res = "" - for node in flattened: + for node in rendered: if isinstance(node, str): res += node else: @@ -78,22 +77,6 @@ def render_flattened_template( return res -def field_is_not_empty(field_text: str) -> bool: - # fixme: this is an overkill way of preventing a field with only - # a
or
from appearing non-empty - field_text = stripHTMLMedia(field_text) - - return field_text.strip() != "" - - -def nonempty_fields(fields: Dict[str, str]) -> List[str]: - res = [] - for field, text in fields.items(): - if field_is_not_empty(text): - res.append(field) - return res - - def unknown_field_message(node: TemplateReplacement) -> str: # mirror the pystache message for now field = node.field_name diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 0a426c0cb..2e0582d8b 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -11,6 +11,8 @@ failure = "0.1.6" prost = "0.5.0" bytes = "0.4" chrono = "0.4.10" +lazy_static = "1.4.0" +regex = "1.3.3" [build-dependencies] prost-build = "0.5.0" diff --git a/rslib/src/backend.rs b/rslib/src/backend.rs index 8d83f2246..5ecb9dc67 100644 --- a/rslib/src/backend.rs +++ b/rslib/src/backend.rs @@ -1,13 +1,13 @@ use crate::backend_proto as pt; use crate::backend_proto::backend_input::Value; -use crate::backend_proto::FlattenedTemplateReplacement; +use crate::backend_proto::RenderedTemplateReplacement; use crate::err::{AnkiError, Result}; use crate::sched::sched_timing_today; use crate::template::{ - without_legacy_template_directives, FieldMap, FieldRequirements, FlattenedNode, ParsedTemplate, + without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, }; use prost::Message; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; pub struct Backend { @@ -93,7 +93,7 @@ impl Backend { Value::DeckTree(_) => todo!(), Value::FindCards(_) => todo!(), Value::BrowserRows(_) => todo!(), - Value::FlattenTemplate(input) => OValue::FlattenTemplate(self.flatten_template(input)?), + Value::RenderTemplate(input) => OValue::RenderTemplate(self.render_template(input)?), }) } @@ -157,24 +157,24 @@ impl Backend { } } - fn flatten_template(&self, input: pt::FlattenTemplateIn) -> Result { - let field_refs: HashSet<_> = input - .nonempty_field_names + fn render_template(&self, input: pt::RenderTemplateIn) -> Result { + let fields: HashMap<_, _> = input + .fields .iter() - .map(AsRef::as_ref) + .map(|(k, v)| (k.as_ref(), v.as_ref())) .collect(); let normalized = without_legacy_template_directives(&input.template_text); match ParsedTemplate::from_text(normalized.as_ref()) { Ok(tmpl) => { - let nodes = tmpl.flatten(&field_refs); + let nodes = tmpl.render(&fields); let out_nodes = nodes .into_iter() - .map(|n| pt::FlattenedTemplateNode { - value: Some(flattened_node_to_proto(n)), + .map(|n| pt::RenderedTemplateNode { + value: Some(rendered_node_to_proto(n)), }) .collect(); - Ok(pt::FlattenTemplateOut { nodes: out_nodes }) + Ok(pt::RenderTemplateOut { nodes: out_nodes }) } Err(e) => Err(e), } @@ -185,11 +185,11 @@ fn ords_hash_to_set(ords: HashSet) -> Vec { ords.iter().map(|ord| *ord as u32).collect() } -fn flattened_node_to_proto(node: FlattenedNode) -> pt::flattened_template_node::Value { +fn rendered_node_to_proto(node: RenderedNode) -> pt::rendered_template_node::Value { match node { - FlattenedNode::Text { text } => pt::flattened_template_node::Value::Text(text), - FlattenedNode::Replacement { field, filters } => { - pt::flattened_template_node::Value::Replacement(FlattenedTemplateReplacement { + RenderedNode::Text { text } => pt::rendered_template_node::Value::Text(text), + RenderedNode::Replacement { field, filters } => { + pt::rendered_template_node::Value::Replacement(RenderedTemplateReplacement { field_name: field, filters, }) diff --git a/rslib/src/template.rs b/rslib/src/template.rs index 463c6e83b..42bbac5c4 100644 --- a/rslib/src/template.rs +++ b/rslib/src/template.rs @@ -1,9 +1,11 @@ use crate::err::{AnkiError, Result}; +use lazy_static::lazy_static; use nom; use nom::branch::alt; use nom::bytes::complete::tag; use nom::error::ErrorKind; use nom::sequence::delimited; +use regex::Regex; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -237,11 +239,11 @@ fn template_is_empty<'a>(nonempty_fields: &HashSet<&str>, nodes: &[ParsedNode<'a true } -// Flattening +// Rendering //---------------------------------------- #[derive(Debug, PartialEq)] -pub enum FlattenedNode { +pub enum RenderedNode { Text { text: String, }, @@ -257,50 +259,81 @@ impl ParsedTemplate<'_> { /// /// This leaves the field replacement (with any filters that were provided) /// up to the calling code to handle. - pub fn flatten(&self, nonempty_fields: &HashSet<&str>) -> Vec { - let mut flattened = vec![]; + pub fn render(&self, fields: &HashMap<&str, &str>) -> Vec { + let mut rendered = vec![]; + let nonempty = nonempty_fields(fields); - flatten_into(&mut flattened, self.0.as_ref(), nonempty_fields); + render_into(&mut rendered, self.0.as_ref(), fields, &nonempty); - flattened + rendered } } -fn flatten_into( - rendered_nodes: &mut Vec, +fn render_into( + rendered_nodes: &mut Vec, nodes: &[ParsedNode], - fields: &HashSet<&str>, + fields: &HashMap<&str, &str>, + nonempty: &HashSet<&str>, ) { use ParsedNode::*; for node in nodes { match node { Text(t) => { - if let Some(FlattenedNode::Text { ref mut text }) = rendered_nodes.last_mut() { + if let Some(RenderedNode::Text { ref mut text }) = rendered_nodes.last_mut() { text.push_str(t) } else { - rendered_nodes.push(FlattenedNode::Text { + rendered_nodes.push(RenderedNode::Text { text: (*t).to_string(), }) } } - Replacement { key, filters } => rendered_nodes.push(FlattenedNode::Replacement { + Replacement { key, filters } => rendered_nodes.push(RenderedNode::Replacement { field: (*key).to_string(), filters: filters.iter().map(|&e| e.to_string()).collect(), }), Conditional { key, children } => { - if fields.contains(key) { - flatten_into(rendered_nodes, children.as_ref(), fields); + if nonempty.contains(key) { + render_into(rendered_nodes, children.as_ref(), fields, nonempty); } } NegatedConditional { key, children } => { - if !fields.contains(key) { - flatten_into(rendered_nodes, children.as_ref(), fields); + if !nonempty.contains(key) { + render_into(rendered_nodes, children.as_ref(), fields, nonempty); } } }; } } +/// True if provided text contains only whitespace and/or empty BR/DIV tags. +fn field_is_empty(text: &str) -> bool { + lazy_static! { + static ref RE: Regex = Regex::new( + r#"(?xsi) + ^(?: + [[:space:]] + | + + )*$ + "# + ) + .unwrap(); + } + RE.is_match(text) +} + +fn nonempty_fields<'a>(fields: &'a HashMap<&str, &str>) -> HashSet<&'a str> { + fields + .iter() + .filter_map(|(name, val)| { + if !field_is_empty(val) { + Some(*name) + } else { + None + } + }) + .collect() +} // Field requirements //---------------------------------------- @@ -360,10 +393,21 @@ impl ParsedTemplate<'_> { #[cfg(test)] mod test { use super::{FieldMap, ParsedNode::*, ParsedTemplate as PT}; - use crate::template::{without_legacy_template_directives, FieldRequirements}; - use std::collections::HashSet; + use crate::template::{field_is_empty, without_legacy_template_directives, FieldRequirements}; + use std::collections::{HashMap, HashSet}; use std::iter::FromIterator; + #[test] + fn test_field_empty() { + assert_eq!(field_is_empty(""), true); + assert_eq!(field_is_empty(" "), true); + assert_eq!(field_is_empty("x"), false); + assert_eq!(field_is_empty("
"), true); + assert_eq!(field_is_empty("
"), true); + assert_eq!(field_is_empty("

\n"), true); + assert_eq!(field_is_empty("
x
\n"), false); + } + #[test] fn test_parsing() { let tmpl = PT::from_text("foo {{bar}} {{#baz}} quux {{/baz}}").unwrap(); @@ -481,12 +525,14 @@ mod test { #[test] fn test_render() { - let map: HashSet<_> = vec!["F", "B"].into_iter().collect(); + let map: HashMap<_, _> = vec![("F", "1"), ("B", "2"), ("E", " ")] + .into_iter() + .collect(); - use crate::template::FlattenedNode as FN; + use crate::template::RenderedNode as FN; let mut tmpl = PT::from_text("{{B}}A{{F}}").unwrap(); assert_eq!( - tmpl.flatten(&map), + tmpl.render(&map), vec![ FN::Replacement { field: "B".to_owned(), @@ -504,12 +550,12 @@ mod test { // empty tmpl = PT::from_text("{{#E}}A{{/E}}").unwrap(); - assert_eq!(tmpl.flatten(&map), vec![]); + assert_eq!(tmpl.render(&map), vec![]); // missing tmpl = PT::from_text("{{^M}}A{{/M}}").unwrap(); assert_eq!( - tmpl.flatten(&map), + tmpl.render(&map), vec![FN::Text { text: "A".to_owned() },] @@ -518,7 +564,7 @@ mod test { // nested tmpl = PT::from_text("{{^E}}1{{#F}}2{{#B}}{{F}}{{/B}}{{/F}}{{/E}}").unwrap(); assert_eq!( - tmpl.flatten(&map), + tmpl.render(&map), vec![ FN::Text { text: "12".to_owned() @@ -533,7 +579,7 @@ mod test { // filters tmpl = PT::from_text("{{one:two:B}}{{three:X}}").unwrap(); assert_eq!( - tmpl.flatten(&map), + tmpl.render(&map), vec![ FN::Replacement { field: "B".to_owned(),