Merge pull request #860 from RumovZ/norm-search

Normalise search

qt/aqt/browser.py

@@ -21,7 +21,7 @@ from anki.consts import *
 from anki.lang import without_unicode_isolation
 from anki.models import NoteType
 from anki.notes import Note
-from anki.rsbackend import TR, DeckTreeNode, InvalidInput
+from anki.rsbackend import TR, DeckTreeNode, InvalidInput, pb
 from anki.stats import CardStats
 from anki.utils import htmlToTextLine, ids2str, isMac, isWin
 from aqt import AnkiQt, gui_hooks
@@ -189,6 +189,7 @@ class DataModel(QAbstractTableModel):
             ctx = SearchContext(search=txt, browser=self.browser)
             gui_hooks.browser_will_search(ctx)
             if ctx.card_ids is None:
+                ctx.search = self.browser.normalize_search(ctx.search)
                 ctx.card_ids = self.col.find_cards(ctx.search, order=ctx.order)
             gui_hooks.browser_did_search(ctx)
             self.cards = ctx.card_ids
@@ -821,6 +822,12 @@ class Browser(QMainWindow):
             # no row change will fire
             self._onRowChanged(None, None)
 
+    def normalize_search(self, search: str) -> str:
+        normed = self.col.backend.normalize_search(search)
+        self._lastSearchTxt = normed
+        self.form.searchEdit.lineEdit().setText(normed)
+        return normed
+
     def updateTitle(self):
         selected = len(self.form.tableView.selectionModel().selectedRows())
         cur = len(self.model.cards)
@@ -1210,27 +1217,34 @@ QTableView {{ gridline-color: {grid} }}
                 if i % 2 == 0:
                     txt += a + ":"
                 else:
-                    txt += re.sub(r"[*_\\]", r"\\\g<0>", a)
+                    txt += re.sub(r'["*_\\]', r"\\\g<0>", a)
-                    for c in ' 　()"':
+                    txt = '"{}"'.format(txt.replace('"', '\\"'))
-                        if c in txt:
-                            txt = '"{}"'.format(txt.replace('"', '\\"'))
-                            break
                     items.append(txt)
                     txt = ""
-            txt = " ".join(items)
+            txt = " AND ".join(items)
-        # is there something to replace or append with?
+        try:
-        if txt:
             if self.mw.app.keyboardModifiers() & Qt.AltModifier:
-                txt = "-" + txt
+                txt = self.col.backend.negate_search(txt)
-            # is there something to replace or append to?
             cur = str(self.form.searchEdit.lineEdit().text())
-            if cur and cur != self._searchPrompt:
+            if cur != self._searchPrompt:
-                if self.mw.app.keyboardModifiers() & Qt.ControlModifier:
+                mods = self.mw.app.keyboardModifiers()
-                    txt = cur + " " + txt
+                if mods & Qt.ControlModifier and mods & Qt.ShiftModifier:
-                elif self.mw.app.keyboardModifiers() & Qt.ShiftModifier:
+                    txt = self.col.backend.replace_search_term(
-                    txt = cur + " or " + txt
+                        search=cur, replacement=txt
-        self.form.searchEdit.lineEdit().setText(txt)
+                    )
-        self.onSearchActivated()
+                elif mods & Qt.ControlModifier:
+                    txt = self.col.backend.concatenate_searches(
+                        sep=pb.ConcatenateSearchesIn.Separator.AND, searches=[cur, txt]
+                    )
+                elif mods & Qt.ShiftModifier:
+                    txt = self.col.backend.concatenate_searches(
+                        sep=pb.ConcatenateSearchesIn.Separator.OR, searches=[cur, txt]
+                    )
+        except InvalidInput as e:
+            showWarning(str(e))
+        else:
+            self.form.searchEdit.lineEdit().setText(txt)
+            self.onSearchActivated()
 
     def _simpleFilters(self, items):
         ml = MenuList()
@@ -1249,7 +1263,7 @@ QTableView {{ gridline-color: {grid} }}
         return self._simpleFilters(
             (
                 (tr(TR.BROWSING_WHOLE_COLLECTION), ""),
-                (tr(TR.BROWSING_CURRENT_DECK), "deck:current"),
+                (tr(TR.BROWSING_CURRENT_DECK), '"deck:current"'),
             )
         )
 
@@ -1258,9 +1272,9 @@ QTableView {{ gridline-color: {grid} }}
         subm.addChild(
             self._simpleFilters(
                 (
-                    (tr(TR.BROWSING_ADDED_TODAY), "added:1"),
+                    (tr(TR.BROWSING_ADDED_TODAY), '"added:1"'),
-                    (tr(TR.BROWSING_STUDIED_TODAY), "rated:1"),
+                    (tr(TR.BROWSING_STUDIED_TODAY), '"rated:1"'),
-                    (tr(TR.BROWSING_AGAIN_TODAY), "rated:1:1"),
+                    (tr(TR.BROWSING_AGAIN_TODAY), '"rated:1:1"'),
                 )
             )
         )
@@ -1271,20 +1285,20 @@ QTableView {{ gridline-color: {grid} }}
         subm.addChild(
             self._simpleFilters(
                 (
-                    (tr(TR.ACTIONS_NEW), "is:new"),
+                    (tr(TR.ACTIONS_NEW), '"is:new"'),
-                    (tr(TR.SCHEDULING_LEARNING), "is:learn"),
+                    (tr(TR.SCHEDULING_LEARNING), '"is:learn"'),
-                    (tr(TR.SCHEDULING_REVIEW), "is:review"),
+                    (tr(TR.SCHEDULING_REVIEW), '"is:review"'),
-                    (tr(TR.FILTERING_IS_DUE), "is:due"),
+                    (tr(TR.FILTERING_IS_DUE), '"is:due"'),
                     None,
-                    (tr(TR.BROWSING_SUSPENDED), "is:suspended"),
+                    (tr(TR.BROWSING_SUSPENDED), '"is:suspended"'),
-                    (tr(TR.BROWSING_BURIED), "is:buried"),
+                    (tr(TR.BROWSING_BURIED), '"is:buried"'),
                     None,
-                    (tr(TR.ACTIONS_RED_FLAG), "flag:1"),
+                    (tr(TR.ACTIONS_RED_FLAG), '"flag:1"'),
-                    (tr(TR.ACTIONS_ORANGE_FLAG), "flag:2"),
+                    (tr(TR.ACTIONS_ORANGE_FLAG), '"flag:2"'),
-                    (tr(TR.ACTIONS_GREEN_FLAG), "flag:3"),
+                    (tr(TR.ACTIONS_GREEN_FLAG), '"flag:3"'),
-                    (tr(TR.ACTIONS_BLUE_FLAG), "flag:4"),
+                    (tr(TR.ACTIONS_BLUE_FLAG), '"flag:4"'),
-                    (tr(TR.BROWSING_NO_FLAG), "flag:0"),
+                    (tr(TR.BROWSING_NO_FLAG), '"flag:0"'),
-                    (tr(TR.BROWSING_ANY_FLAG), "-flag:0"),
+                    (tr(TR.BROWSING_ANY_FLAG), '"-flag:0"'),
                 )
             )
         )

rslib/backend.proto

@@ -84,8 +84,12 @@ service BackendService {
 
     // searching
 
+    rpc NormalizeSearch (String) returns (String);
     rpc SearchCards (SearchCardsIn) returns (SearchCardsOut);
     rpc SearchNotes (SearchNotesIn) returns (SearchNotesOut);
+    rpc NegateSearch (String) returns (String);
+    rpc ConcatenateSearches (ConcatenateSearchesIn) returns (String);
+    rpc ReplaceSearchTerm (ReplaceSearchTermIn) returns (String);
     rpc FindAndReplace (FindAndReplaceIn) returns (UInt32);
 
     // scheduling
@@ -752,6 +756,20 @@ message BuiltinSearchOrder {
     bool reverse = 2;
 }
 
+message ConcatenateSearchesIn {
+    enum Separator {
+        AND = 0;
+        OR = 1;
+    }
+    Separator sep = 1;
+    repeated string searches = 2;
+}
+
+message ReplaceSearchTermIn {
+    string search = 1;
+    string replacement = 2;
+}
+
 message CloseCollectionIn {
     bool downgrade_to_schema11 = 1;
 }

rslib/src/backend/mod.rs

@@ -32,7 +32,9 @@ use crate::{
     },
     sched::cutoff::local_minutes_west_for_stamp,
     sched::timespan::{answer_button_time, time_span},
-    search::SortMode,
+    search::{
+        concatenate_searches, negate_search, normalize_search, replace_search_term, SortMode,
+    },
     stats::studied_today,
     sync::{
         get_remote_sync_meta, sync_abort, sync_login, FullSyncProgress, NormalSyncProgress,
@@ -393,6 +395,10 @@ impl BackendService for Backend {
     // searching
     //-----------------------------------------------
 
+    fn normalize_search(&self, input: pb::String) -> Result<pb::String> {
+        Ok(normalize_search(&input.val)?.into())
+    }
+
     fn search_cards(&self, input: pb::SearchCardsIn) -> Result<pb::SearchCardsOut> {
         self.with_col(|col| {
             let order = if let Some(order) = input.order {
@@ -426,6 +432,18 @@ impl BackendService for Backend {
         })
     }
 
+    fn negate_search(&self, input: pb::String) -> Result<pb::String> {
+        Ok(negate_search(&input.val)?.into())
+    }
+
+    fn concatenate_searches(&self, input: pb::ConcatenateSearchesIn) -> Result<pb::String> {
+        Ok(concatenate_searches(input.sep, &input.searches)?.into())
+    }
+
+    fn replace_search_term(&self, input: pb::ReplaceSearchTermIn) -> Result<pb::String> {
+        Ok(replace_search_term(&input.search, &input.replacement)?.into())
+    }
+
     fn find_and_replace(&self, input: pb::FindAndReplaceIn) -> BackendResult<pb::UInt32> {
         let mut search = if input.regex {
             input.search

rslib/src/search/mod.rs

@@ -2,5 +2,7 @@ mod cards;
 mod notes;
 mod parser;
 mod sqlwriter;
+mod writer;
 
 pub use cards::SortMode;
+pub use writer::{concatenate_searches, negate_search, normalize_search, replace_search_term};

rslib/src/search/parser.rs

@@ -50,7 +50,7 @@ pub(super) enum Node<'a> {
     Search(SearchNode<'a>),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub(super) enum SearchNode<'a> {
     // text without a colon
     UnqualifiedText(Cow<'a, str>),
@@ -90,7 +90,7 @@ pub(super) enum SearchNode<'a> {
     WordBoundary(Cow<'a, str>),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub(super) enum PropertyKind {
     Due(i32),
     Interval(u32),
@@ -100,7 +100,7 @@ pub(super) enum PropertyKind {
     Position(u32),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub(super) enum StateKind {
     New,
     Review,
@@ -112,7 +112,7 @@ pub(super) enum StateKind {
     Suspended,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub(super) enum TemplateKind<'a> {
     Ordinal(u16),
     Name(Cow<'a, str>),

rslib/src/search/writer.rs

@@ -0,0 +1,272 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::{
+    backend_proto::concatenate_searches_in::Separator,
+    decks::DeckID as DeckIDType,
+    err::{AnkiError, Result},
+    notetype::NoteTypeID as NoteTypeIDType,
+    search::parser::{parse, Node, PropertyKind, SearchNode, StateKind, TemplateKind},
+};
+use itertools::Itertools;
+use std::mem;
+
+/// Take an Anki-style search string and convert it into an equivalent
+/// search string with normalized syntax.
+pub fn normalize_search(input: &str) -> Result<String> {
+    Ok(write_nodes(&parse(input)?))
+}
+
+/// Take an Anki-style search string and return the negated counterpart.
+/// Empty searches (whole collection) remain unchanged.
+pub fn negate_search(input: &str) -> Result<String> {
+    let mut nodes = parse(input)?;
+    use Node::*;
+    Ok(if nodes.len() == 1 {
+        let node = nodes.remove(0);
+        match node {
+            Not(n) => write_node(&n),
+            Search(SearchNode::WholeCollection) => "".to_string(),
+            Group(_) | Search(_) => write_node(&Not(Box::new(node))),
+            _ => unreachable!(),
+        }
+    } else {
+        write_node(&Not(Box::new(Group(nodes))))
+    })
+}
+
+/// Take arbitrary Anki-style search strings and return their concatenation where they
+/// are separated by the provided boolean operator.
+/// Empty searches (whole collection) are left out.
+pub fn concatenate_searches(sep: i32, searches: &[String]) -> Result<String> {
+    let bool_node = vec![match Separator::from_i32(sep) {
+        Some(Separator::Or) => Node::Or,
+        Some(Separator::And) => Node::And,
+        None => return Err(AnkiError::SearchError(None)),
+    }];
+    Ok(write_nodes(
+        searches
+            .iter()
+            .map(|s| parse(s))
+            .collect::<Result<Vec<Vec<Node>>>>()?
+            .iter()
+            .filter(|v| v[0] != Node::Search(SearchNode::WholeCollection))
+            .intersperse(&&bool_node)
+            .flat_map(|v| v.iter()),
+    ))
+}
+
+/// Take two Anki-style search strings. If the second one evaluates to a single search
+/// node, replace with it all search terms of the same kind in the first search.
+/// Then return the possibly modified first search.
+pub fn replace_search_term(search: &str, replacement: &str) -> Result<String> {
+    let mut nodes = parse(search)?;
+    let new = parse(replacement)?;
+    if let [Node::Search(search_node)] = &new[..] {
+        fn update_node_vec<'a>(old_nodes: &mut [Node<'a>], new_node: &SearchNode<'a>) {
+            fn update_node<'a>(old_node: &mut Node<'a>, new_node: &SearchNode<'a>) {
+                match old_node {
+                    Node::Not(n) => update_node(n, new_node),
+                    Node::Group(ns) => update_node_vec(ns, new_node),
+                    Node::Search(n) => {
+                        if mem::discriminant(n) == mem::discriminant(new_node) {
+                            *n = new_node.clone();
+                        }
+                    }
+                    _ => (),
+                }
+            }
+            old_nodes.iter_mut().for_each(|n| update_node(n, new_node));
+        }
+        update_node_vec(&mut nodes, search_node);
+    }
+    Ok(write_nodes(&nodes))
+}
+
+fn write_nodes<'a, I>(nodes: I) -> String
+where
+    I: IntoIterator<Item = &'a Node<'a>>,
+{
+    nodes.into_iter().map(|node| write_node(node)).collect()
+}
+
+fn write_node(node: &Node) -> String {
+    use Node::*;
+    match node {
+        And => " AND ".to_string(),
+        Or => " OR ".to_string(),
+        Not(n) => format!("-{}", write_node(n)),
+        Group(ns) => format!("({})", write_nodes(ns)),
+        Search(n) => write_search_node(n),
+    }
+}
+
+fn write_search_node(node: &SearchNode) -> String {
+    use SearchNode::*;
+    match node {
+        UnqualifiedText(s) => quote(&s.replace(":", "\\:")),
+        SingleField { field, text, is_re } => write_single_field(field, text, *is_re),
+        AddedInDays(u) => format!("\"added:{}\"", u),
+        EditedInDays(u) => format!("\"edited:{}\"", u),
+        CardTemplate(t) => write_template(t),
+        Deck(s) => quote(&format!("deck:{}", s)),
+        DeckID(DeckIDType(i)) => format!("\"did:{}\"", i),
+        NoteTypeID(NoteTypeIDType(i)) => format!("\"mid:{}\"", i),
+        NoteType(s) => quote(&format!("note:{}", s)),
+        Rated { days, ease } => write_rated(days, ease),
+        Tag(s) => quote(&format!("tag:{}", s)),
+        Duplicates { note_type_id, text } => quote(&format!("dupes:{},{}", note_type_id, text)),
+        State(k) => write_state(k),
+        Flag(u) => format!("\"flag:{}\"", u),
+        NoteIDs(s) => format!("\"nid:{}\"", s),
+        CardIDs(s) => format!("\"cid:{}\"", s),
+        Property { operator, kind } => write_property(operator, kind),
+        WholeCollection => "".to_string(),
+        Regex(s) => quote(&format!("re:{}", s)),
+        NoCombining(s) => quote(&format!("nc:{}", s)),
+        WordBoundary(s) => quote(&format!("w:{}", s)),
+    }
+}
+
+/// Escape and wrap in double quotes.
+fn quote(txt: &str) -> String {
+    format!("\"{}\"", txt.replace("\"", "\\\""))
+}
+
+fn write_single_field(field: &str, text: &str, is_re: bool) -> String {
+    let re = if is_re { "re:" } else { "" };
+    let text = if !is_re && text.starts_with("re:") {
+        text.replacen(":", "\\:", 1)
+    } else {
+        text.to_string()
+    };
+    quote(&format!("{}:{}{}", field.replace(":", "\\:"), re, &text))
+}
+
+fn write_template(template: &TemplateKind) -> String {
+    match template {
+        TemplateKind::Ordinal(u) => format!("\"card:{}\"", u),
+        TemplateKind::Name(s) => format!("\"card:{}\"", s),
+    }
+}
+
+fn write_rated(days: &u32, ease: &Option<u8>) -> String {
+    match ease {
+        Some(u) => format!("\"rated:{}:{}\"", days, u),
+        None => format!("\"rated:{}\"", days),
+    }
+}
+
+fn write_state(kind: &StateKind) -> String {
+    use StateKind::*;
+    format!(
+        "\"is:{}\"",
+        match kind {
+            New => "new",
+            Review => "review",
+            Learning => "learn",
+            Due => "due",
+            Buried => "buried",
+            UserBuried => "buried-manually",
+            SchedBuried => "buried-sibling",
+            Suspended => "suspended",
+        }
+    )
+}
+
+fn write_property(operator: &str, kind: &PropertyKind) -> String {
+    use PropertyKind::*;
+    match kind {
+        Due(i) => format!("\"prop:due{}{}\"", operator, i),
+        Interval(u) => format!("\"prop:ivl{}{}\"", operator, u),
+        Reps(u) => format!("\"prop:reps{}{}\"", operator, u),
+        Lapses(u) => format!("\"prop:lapses{}{}\"", operator, u),
+        Ease(f) => format!("\"prop:ease{}{}\"", operator, f),
+        Position(u) => format!("\"prop:pos{}{}\"", operator, u),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn normalizing() -> Result<()> {
+        assert_eq!(r#""(" AND "-""#, normalize_search(r"\( \-").unwrap());
+        assert_eq!(r#""deck::""#, normalize_search(r"deck:\:").unwrap());
+        assert_eq!(r#""\*" OR "\:""#, normalize_search(r"\* or \:").unwrap());
+        assert_eq!(
+            r#""field:foo""#,
+            normalize_search(r#"field:"foo""#).unwrap()
+        );
+        assert_eq!(
+            r#""prop:ease>1""#,
+            normalize_search("prop:ease>1.0").unwrap()
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn negating() -> Result<()> {
+        assert_eq!(r#"-("foo" AND "bar")"#, negate_search("foo bar").unwrap());
+        assert_eq!(r#""foo""#, negate_search("-foo").unwrap());
+        assert_eq!(r#"("foo")"#, negate_search("-(foo)").unwrap());
+        assert_eq!("", negate_search("").unwrap());
+
+        Ok(())
+    }
+
+    #[test]
+    fn concatenating() -> Result<()> {
+        assert_eq!(
+            r#""foo" AND "bar""#,
+            concatenate_searches(
+                Separator::And as i32,
+                &["foo".to_string(), "bar".to_string()]
+            )
+            .unwrap()
+        );
+        assert_eq!(
+            r#""foo" OR "bar""#,
+            concatenate_searches(
+                Separator::Or as i32,
+                &["foo".to_string(), "".to_string(), "bar".to_string()]
+            )
+            .unwrap()
+        );
+        assert_eq!(
+            "",
+            concatenate_searches(Separator::Or as i32, &["".to_string()]).unwrap()
+        );
+        assert_eq!("", concatenate_searches(Separator::Or as i32, &[]).unwrap());
+
+        Ok(())
+    }
+
+    #[test]
+    fn replacing() -> Result<()> {
+        assert_eq!(
+            r#""deck:foo" AND "bar""#,
+            replace_search_term("deck:baz bar", "deck:foo").unwrap()
+        );
+        assert_eq!(
+            r#""tag:baz" OR "tag:baz""#,
+            replace_search_term("tag:foo Or tag:bar", "tag:baz").unwrap()
+        );
+        assert_eq!(
+            r#""bar" OR (-"bar" AND "tag:baz")"#,
+            replace_search_term("foo or (-foo tag:baz)", "bar").unwrap()
+        );
+        assert_eq!(
+            r#""is:due""#,
+            replace_search_term("is:due", "-is:new").unwrap()
+        );
+        assert_eq!(
+            r#""added:1""#,
+            replace_search_term("added:1", "is:due").unwrap()
+        );
+
+        Ok(())
+    }
+}