GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-24 16:56:36 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

flatten->render; pass field content in

Browse source 
This is paving the way to move the standard filters into Rust.

Non-empty fields are now determined in Rust, using a single regex
instead of the overkill stripHTMLMedia(). The old implementation
has been moved into the Pystache code for now.
This commit is contained in:
Damien Elmes 2020-01-10 14:59:29 +10:00
parent c010bb0a37
commit f7ff0d1c17
7 changed files with 119 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
proto/backend.proto
View file

@ -14,7 +14,7 @@ message BackendInput {
Empty deck_tree = 18;
FindCardsIn find_cards = 19;
BrowserRowsIn browser_rows = 20;
FlattenTemplateIn flatten_template = 21;
RenderTemplateIn render_template = 21;
PlusOneIn plus_one = 2046; // temporary, for testing
}
@ -27,7 +27,7 @@ message BackendOutput {
DeckTreeOut deck_tree = 18;
FindCardsOut find_cards = 19;
BrowserRowsOut browser_rows = 20;
FlattenTemplateOut flatten_template = 21;
RenderTemplateOut render_template = 21;
PlusOneOut plus_one = 2046; // temporary, for testing
@ -128,23 +128,23 @@ message BrowserRowsOut {
repeated string sort_fields = 1;
}
message FlattenTemplateIn {
message RenderTemplateIn {
string template_text = 1;
repeated string nonempty_field_names = 2;
map<string,string> fields = 2;
}
message FlattenTemplateOut {
repeated FlattenedTemplateNode nodes = 1;
message RenderTemplateOut {
repeated RenderedTemplateNode nodes = 1;
}
message FlattenedTemplateNode {
message RenderedTemplateNode {
oneof value {
string text = 1;
FlattenedTemplateReplacement replacement = 2;
RenderedTemplateReplacement replacement = 2;
}
}
message FlattenedTemplateReplacement {
message RenderedTemplateReplacement {
string field_name = 1;
repeated string filters = 2;
}

10
pylib/anki/rsbackend.py
View file

@ -104,16 +104,16 @@ class RustBackend:
)
).sched_timing_today
def flatten_template(
self, template: str, nonempty_fields: List[str]
def render_template(
self, template: str, fields: Dict[str, str]
) -> List[Union[str, TemplateReplacement]]:
out = self._run_command(
pb.BackendInput(
flatten_template=pb.FlattenTemplateIn(
template_text=template, nonempty_field_names=nonempty_fields
render_template=pb.RenderTemplateIn(
template_text=template, fields=fields
)
)
).flatten_template
).render_template
results: List[Union[str, TemplateReplacement]] = []
for node in out.nodes:
if node.WhichOneof("value") == "text":

12
pylib/anki/template/template.py
View file

@ -1,11 +1,21 @@
import re
from typing import Any, Callable, Dict, Pattern
from anki.template2 import apply_field_filters, field_is_not_empty
from anki.template2 import apply_field_filters
modifiers: Dict[str, Callable] = {}
def field_is_not_empty(field_text: str) -> bool:
# fixme: this is an overkill way of preventing a field with only
# a <br> or <div> from appearing non-empty
from anki.utils import stripHTMLMedia
field_text = stripHTMLMedia(field_text)
return field_text.strip() != ""
def modifier(symbol) -> Callable[[Any], Any]:
"""Decorator for associating a function with a Mustache tag modifier.

27
pylib/anki/template2.py
View file

@ -16,7 +16,7 @@ from anki.hooks import addHook, runFilter
from anki.lang import _
from anki.rsbackend import TemplateReplacement
from anki.sound import stripSounds
from anki.utils import stripHTML, stripHTMLMedia
from anki.utils import stripHTML
def render_qa_from_field_map(
@ -47,9 +47,8 @@ def render_template(
"Render a single template."
old_output = anki.template.render(format, fields)
nonempty = nonempty_fields(fields)
flattened = col.backend.flatten_template(format, nonempty)
new_output = render_flattened_template(flattened, fields)
rendered = col.backend.render_template(format, fields)
new_output = render_flattened_template(rendered, fields)
if old_output != new_output:
print(
@ -62,11 +61,11 @@ def render_template(
def render_flattened_template(
flattened: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
rendered: List[Union[str, TemplateReplacement]], fields: Dict[str, str]
) -> str:
"Render a list of strings or replacements into a string."
res = ""
for node in flattened:
for node in rendered:
if isinstance(node, str):
res += node
else:
@ -78,22 +77,6 @@ def render_flattened_template(
return res
def field_is_not_empty(field_text: str) -> bool:
# fixme: this is an overkill way of preventing a field with only
# a <br> or <div> from appearing non-empty
field_text = stripHTMLMedia(field_text)
return field_text.strip() != ""
def nonempty_fields(fields: Dict[str, str]) -> List[str]:
res = []
for field, text in fields.items():
if field_is_not_empty(text):
res.append(field)
return res
def unknown_field_message(node: TemplateReplacement) -> str:
# mirror the pystache message for now
field = node.field_name

2
rslib/Cargo.toml
View file

@ -11,6 +11,8 @@ failure = "0.1.6"
prost = "0.5.0"
bytes = "0.4"
chrono = "0.4.10"
lazy_static = "1.4.0"
regex = "1.3.3"
[build-dependencies]
prost-build = "0.5.0"

32
rslib/src/backend.rs
View file

@ -1,13 +1,13 @@
use crate::backend_proto as pt;
use crate::backend_proto::backend_input::Value;
use crate::backend_proto::FlattenedTemplateReplacement;
use crate::backend_proto::RenderedTemplateReplacement;
use crate::err::{AnkiError, Result};
use crate::sched::sched_timing_today;
use crate::template::{
without_legacy_template_directives, FieldMap, FieldRequirements, FlattenedNode, ParsedTemplate,
without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode,
};
use prost::Message;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
pub struct Backend {
@ -93,7 +93,7 @@ impl Backend {
Value::DeckTree(_) => todo!(),
Value::FindCards(_) => todo!(),
Value::BrowserRows(_) => todo!(),
Value::FlattenTemplate(input) => OValue::FlattenTemplate(self.flatten_template(input)?),
Value::RenderTemplate(input) => OValue::RenderTemplate(self.render_template(input)?),
})
}
@ -157,24 +157,24 @@ impl Backend {
}
}
fn flatten_template(&self, input: pt::FlattenTemplateIn) -> Result<pt::FlattenTemplateOut> {
let field_refs: HashSet<_> = input
.nonempty_field_names
fn render_template(&self, input: pt::RenderTemplateIn) -> Result<pt::RenderTemplateOut> {
let fields: HashMap<_, _> = input
.fields
.iter()
.map(AsRef::as_ref)
.map(|(k, v)| (k.as_ref(), v.as_ref()))
.collect();
let normalized = without_legacy_template_directives(&input.template_text);
match ParsedTemplate::from_text(normalized.as_ref()) {
Ok(tmpl) => {
let nodes = tmpl.flatten(&field_refs);
let nodes = tmpl.render(&fields);
let out_nodes = nodes
.into_iter()
.map(|n| pt::FlattenedTemplateNode {
value: Some(flattened_node_to_proto(n)),
.map(|n| pt::RenderedTemplateNode {
value: Some(rendered_node_to_proto(n)),
})
.collect();
Ok(pt::FlattenTemplateOut { nodes: out_nodes })
Ok(pt::RenderTemplateOut { nodes: out_nodes })
}
Err(e) => Err(e),
}
@ -185,11 +185,11 @@ fn ords_hash_to_set(ords: HashSet<u16>) -> Vec<u32> {
ords.iter().map(|ord| *ord as u32).collect()
}
fn flattened_node_to_proto(node: FlattenedNode) -> pt::flattened_template_node::Value {
fn rendered_node_to_proto(node: RenderedNode) -> pt::rendered_template_node::Value {
match node {
FlattenedNode::Text { text } => pt::flattened_template_node::Value::Text(text),
FlattenedNode::Replacement { field, filters } => {
pt::flattened_template_node::Value::Replacement(FlattenedTemplateReplacement {
RenderedNode::Text { text } => pt::rendered_template_node::Value::Text(text),
RenderedNode::Replacement { field, filters } => {
pt::rendered_template_node::Value::Replacement(RenderedTemplateReplacement {
field_name: field,
filters,
})

96
rslib/src/template.rs
View file

@ -1,9 +1,11 @@
use crate::err::{AnkiError, Result};
use lazy_static::lazy_static;
use nom;
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::error::ErrorKind;
use nom::sequence::delimited;
use regex::Regex;
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
@ -237,11 +239,11 @@ fn template_is_empty<'a>(nonempty_fields: &HashSet<&str>, nodes: &[ParsedNode<'a
true
}
// Flattening
// Rendering
//----------------------------------------
#[derive(Debug, PartialEq)]
pub enum FlattenedNode {
pub enum RenderedNode {
Text {
text: String,
},
@ -257,50 +259,81 @@ impl ParsedTemplate<'_> {
///
/// This leaves the field replacement (with any filters that were provided)
/// up to the calling code to handle.
pub fn flatten(&self, nonempty_fields: &HashSet<&str>) -> Vec<FlattenedNode> {
let mut flattened = vec![];
pub fn render(&self, fields: &HashMap<&str, &str>) -> Vec<RenderedNode> {
let mut rendered = vec![];
let nonempty = nonempty_fields(fields);
flatten_into(&mut flattened, self.0.as_ref(), nonempty_fields);
render_into(&mut rendered, self.0.as_ref(), fields, &nonempty);
flattened
rendered
}
}
fn flatten_into(
rendered_nodes: &mut Vec<FlattenedNode>,
fn render_into(
rendered_nodes: &mut Vec<RenderedNode>,
nodes: &[ParsedNode],
fields: &HashSet<&str>,
fields: &HashMap<&str, &str>,
nonempty: &HashSet<&str>,
) {
use ParsedNode::*;
for node in nodes {
match node {
Text(t) => {
if let Some(FlattenedNode::Text { ref mut text }) = rendered_nodes.last_mut() {
if let Some(RenderedNode::Text { ref mut text }) = rendered_nodes.last_mut() {
text.push_str(t)
} else {
rendered_nodes.push(FlattenedNode::Text {
rendered_nodes.push(RenderedNode::Text {
text: (*t).to_string(),
})
}
}
Replacement { key, filters } => rendered_nodes.push(FlattenedNode::Replacement {
Replacement { key, filters } => rendered_nodes.push(RenderedNode::Replacement {
field: (*key).to_string(),
filters: filters.iter().map(|&e| e.to_string()).collect(),
}),
Conditional { key, children } => {
if fields.contains(key) {
flatten_into(rendered_nodes, children.as_ref(), fields);
if nonempty.contains(key) {
render_into(rendered_nodes, children.as_ref(), fields, nonempty);
}
}
NegatedConditional { key, children } => {
if !fields.contains(key) {
flatten_into(rendered_nodes, children.as_ref(), fields);
if !nonempty.contains(key) {
render_into(rendered_nodes, children.as_ref(), fields, nonempty);
}
}
};
}
}
/// True if provided text contains only whitespace and/or empty BR/DIV tags.
fn field_is_empty(text: &str) -> bool {
lazy_static! {
static ref RE: Regex = Regex::new(
r#"(?xsi)
^(?:
[[:space:]]
|
</?(?:br|div)\ ?/?>
)*$
"#
)
.unwrap();
}
RE.is_match(text)
}
fn nonempty_fields<'a>(fields: &'a HashMap<&str, &str>) -> HashSet<&'a str> {
fields
.iter()
.filter_map(|(name, val)| {
if !field_is_empty(val) {
Some(*name)
} else {
None
}
})
.collect()
}
// Field requirements
//----------------------------------------
@ -360,10 +393,21 @@ impl ParsedTemplate<'_> {
#[cfg(test)]
mod test {
use super::{FieldMap, ParsedNode::*, ParsedTemplate as PT};
use crate::template::{without_legacy_template_directives, FieldRequirements};
use std::collections::HashSet;
use crate::template::{field_is_empty, without_legacy_template_directives, FieldRequirements};
use std::collections::{HashMap, HashSet};
use std::iter::FromIterator;
#[test]
fn test_field_empty() {
assert_eq!(field_is_empty(""), true);
assert_eq!(field_is_empty(" "), true);
assert_eq!(field_is_empty("x"), false);
assert_eq!(field_is_empty("<BR>"), true);
assert_eq!(field_is_empty("<div />"), true);
assert_eq!(field_is_empty(" <div> <br> </div>\n"), true);
assert_eq!(field_is_empty(" <div>x</div>\n"), false);
}
#[test]
fn test_parsing() {
let tmpl = PT::from_text("foo {{bar}} {{#baz}} quux {{/baz}}").unwrap();
@ -481,12 +525,14 @@ mod test {
#[test]
fn test_render() {
let map: HashSet<_> = vec!["F", "B"].into_iter().collect();
let map: HashMap<_, _> = vec![("F", "1"), ("B", "2"), ("E", " ")]
.into_iter()
.collect();
use crate::template::FlattenedNode as FN;
use crate::template::RenderedNode as FN;
let mut tmpl = PT::from_text("{{B}}A{{F}}").unwrap();
assert_eq!(
tmpl.flatten(&map),
tmpl.render(&map),
vec![
FN::Replacement {
field: "B".to_owned(),
@ -504,12 +550,12 @@ mod test {
// empty
tmpl = PT::from_text("{{#E}}A{{/E}}").unwrap();
assert_eq!(tmpl.flatten(&map), vec![]);
assert_eq!(tmpl.render(&map), vec![]);
// missing
tmpl = PT::from_text("{{^M}}A{{/M}}").unwrap();
assert_eq!(
tmpl.flatten(&map),
tmpl.render(&map),
vec![FN::Text {
text: "A".to_owned()
},]
@ -518,7 +564,7 @@ mod test {
// nested
tmpl = PT::from_text("{{^E}}1{{#F}}2{{#B}}{{F}}{{/B}}{{/F}}{{/E}}").unwrap();
assert_eq!(
tmpl.flatten(&map),
tmpl.render(&map),
vec![
FN::Text {
text: "12".to_owned()
@ -533,7 +579,7 @@ mod test {
// filters
tmpl = PT::from_text("{{one:two:B}}{{three:X}}").unwrap();
assert_eq!(
tmpl.flatten(&map),
tmpl.render(&map),
vec![
FN::Replacement {
field: "B".to_owned(),