update tag handling

- tag list stored in a separate DB table
- non-wildcard searches now do full unicode case folding
(eg tag:masse matches 'Maße')
- wildcard matches do simple unicode case folding
- some functions haven't been updated yet, so ascii folding will
continue to be used in some operations

proto/backend.proto

@@ -56,6 +56,11 @@ message BackendInput {
         Empty new_deck_config = 44;
         int64 remove_deck_config = 45;
         Empty abort_media_sync = 46;
+        Empty before_upload = 47;
+        RegisterTagsIn register_tags = 48;
+        string canonify_tags = 49;
+        Empty all_tags = 50;
+        int32 get_changed_tags = 51;
     }
 }
 
@@ -95,6 +100,11 @@ message BackendOutput {
         string all_deck_config = 43;
         string new_deck_config = 44;
         Empty remove_deck_config = 45;
+        Empty before_upload = 47;
+        bool register_tags = 48;
+        CanonifyTagsOut canonify_tags = 49;
+        AllTagsOut all_tags = 50;
+        GetChangedTagsOut get_changed_tags = 51;
 
         BackendError error = 2047;
     }
@@ -419,3 +429,28 @@ message AddOrUpdateDeckConfigIn {
     string config = 1;
     bool preserve_usn_and_mtime = 2;
 }
+
+message RegisterTagsIn {
+    string tags = 1;
+    bool preserve_usn = 2;
+    int32 usn = 3;
+    bool clear_first = 4;
+}
+
+message AllTagsOut {
+    repeated TagUsnTuple tags = 1;
+}
+
+message TagUsnTuple {
+    string tag = 1;
+    sint32 usn = 2;
+}
+
+message GetChangedTagsOut {
+    repeated string tags = 1;
+}
+
+message CanonifyTagsOut {
+    string tags = 1;
+    bool tag_list_changed = 2;
+}

pylib/anki/collection.py

@@ -182,17 +182,14 @@ class _Collection:
             conf,
             models,
             decks,
-            dconf,
-            tags,
         ) = self.db.first(
             """
 select crt, mod, scm, dty, usn, ls,
-conf, models, decks, dconf, tags from col"""
+conf, models, decks from col"""
         )
         self.conf = json.loads(conf)
         self.models.load(models)
-        self.decks.load(decks, dconf)
+        self.decks.load(decks)
-        self.tags.load(tags)
 
     def setMod(self) -> None:
         """Mark DB modified.
@@ -219,7 +216,6 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""",
     def flush_all_changes(self, mod: Optional[int] = None):
         self.models.flush()
         self.decks.flush()
-        self.tags.flush()
         if self.db.mod:
             self.flush(mod)
 
@@ -292,8 +288,8 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""",
         self.db.execute("delete from graves")
         self._usn += 1
         self.models.beforeUpload()
-        self.tags.beforeUpload()
         self.decks.beforeUpload()
+        self.backend.before_upload()
         self.modSchema(check=False)
         self.ls = self.scm
         # ensure db is compacted before upload

pylib/anki/decks.py

@@ -65,7 +65,7 @@ class DeckManager:
         self.decks = {}
         self._dconf_cache: Optional[Dict[int, Dict[str, Any]]] = None
 
-    def load(self, decks: str, dconf: str) -> None:
+    def load(self, decks: str) -> None:
         self.decks = json.loads(decks)
         self.changed = False
 
@@ -626,10 +626,6 @@ class DeckManager:
     def beforeUpload(self) -> None:
         for d in self.all():
             d["usn"] = 0
-        for c in self.all_config():
-            if c["usn"] != 0:
-                c["usn"] = 0
-                self.update_config(c, preserve_usn=True)
         self.save()
 
     # Dynamic decks

pylib/anki/hooks.py

@@ -544,30 +544,28 @@ class _SyncStageDidChangeHook:
 sync_stage_did_change = _SyncStageDidChangeHook()
 
 
-class _TagAddedHook:
+class _TagListDidUpdateHook:
-    _hooks: List[Callable[[str], None]] = []
+    _hooks: List[Callable[[], None]] = []
 
-    def append(self, cb: Callable[[str], None]) -> None:
+    def append(self, cb: Callable[[], None]) -> None:
-        """(tag: str)"""
+        """()"""
         self._hooks.append(cb)
 
-    def remove(self, cb: Callable[[str], None]) -> None:
+    def remove(self, cb: Callable[[], None]) -> None:
         if cb in self._hooks:
             self._hooks.remove(cb)
 
-    def __call__(self, tag: str) -> None:
+    def __call__(self) -> None:
         for hook in self._hooks:
             try:
-                hook(tag)
+                hook()
             except:
                 # if the hook fails, remove it
                 self._hooks.remove(hook)
                 raise
-        # legacy support
-        runHook("newTag")
 
 
-tag_added = _TagAddedHook()
+tag_list_did_update = _TagListDidUpdateHook()
 # @@AUTOGEN@@
 
 # Legacy hook handling

pylib/anki/rsbackend.py

@@ -47,6 +47,7 @@ assert ankirspy.buildhash() == anki.buildinfo.buildhash
 SchedTimingToday = pb.SchedTimingTodayOut
 BuiltinSortKind = pb.BuiltinSortKind
 BackendCard = pb.Card
+TagUsnTuple = pb.TagUsnTuple
 
 try:
     import orjson
@@ -540,6 +541,42 @@ class RustBackend:
     def abort_media_sync(self):
         self._run_command(pb.BackendInput(abort_media_sync=pb.Empty()))
 
+    def all_tags(self) -> Iterable[TagUsnTuple]:
+        return self._run_command(pb.BackendInput(all_tags=pb.Empty())).all_tags.tags
+
+    def canonify_tags(self, tags: str) -> Tuple[str, bool]:
+        out = self._run_command(pb.BackendInput(canonify_tags=tags)).canonify_tags
+        return (out.tags, out.tag_list_changed)
+
+    def register_tags(self, tags: str, usn: Optional[int], clear_first: bool) -> bool:
+        if usn is None:
+            preserve_usn = False
+            usn_ = 0
+        else:
+            usn_ = usn
+            preserve_usn = True
+
+        return self._run_command(
+            pb.BackendInput(
+                register_tags=pb.RegisterTagsIn(
+                    tags=tags,
+                    usn=usn_,
+                    preserve_usn=preserve_usn,
+                    clear_first=clear_first,
+                )
+            )
+        ).register_tags
+
+    def before_upload(self):
+        self._run_command(pb.BackendInput(before_upload=pb.Empty()))
+
+    def get_changed_tags(self, usn: int) -> List[str]:
+        return list(
+            self._run_command(
+                pb.BackendInput(get_changed_tags=usn)
+            ).get_changed_tags.tags
+        )
+
 
 def translate_string_in(
     key: TR, **kwargs: Union[str, int, float]

pylib/anki/sync.py

@@ -206,8 +206,8 @@ class Syncer:
         for g in self.col.decks.all():
             if g["usn"] == -1:
                 return "deck had usn = -1"
-        for t, usn in self.col.tags.allItems():
+        for tup in self.col.backend.all_tags():
-            if usn == -1:
+            if tup.usn == -1:
                 return "tag had usn = -1"
         found = False
         for m in self.col.models.all():
@@ -404,13 +404,7 @@ from notes where %s"""
     ##########################################################################
 
     def getTags(self) -> List:
-        tags = []
+        return self.col.backend.get_changed_tags(self.maxUsn)
-        for t, usn in self.col.tags.allItems():
-            if usn == -1:
-                self.col.tags.tags[t] = self.maxUsn
-                tags.append(t)
-        self.col.tags.save()
-        return tags
 
     def mergeTags(self, tags) -> None:
         self.col.tags.register(tags, usn=self.maxUsn)

pylib/anki/tags.py

@@ -11,9 +11,8 @@ This module manages the tag cache and tags for notes.
 
 from __future__ import annotations
 
-import json
 import re
-from typing import Callable, Collection, Dict, List, Optional, Tuple
+from typing import Callable, Collection, List, Optional, Tuple
 
 import anki  # pylint: disable=unused-import
 from anki import hooks
@@ -21,62 +20,45 @@ from anki.utils import ids2str, intTime
 
 
 class TagManager:
-
-    # Registry save/load
-    #############################################################
-
     def __init__(self, col: anki.storage._Collection) -> None:
         self.col = col.weakref()
-        self.tags: Dict[str, int] = {}
 
-    def load(self, json_: str) -> None:
+    # all tags
-        self.tags = json.loads(json_)
+    def all(self) -> List[str]:
-        self.changed = False
+        return [t.tag for t in self.col.backend.all_tags()]
 
-    def flush(self) -> None:
+    # # List of (tag, usn)
-        if self.changed:
+    def allItems(self) -> List[Tuple[str, int]]:
-            self.col.db.execute("update col set tags=?", json.dumps(self.tags))
+        return [(t.tag, t.usn) for t in self.col.backend.all_tags()]
-            self.changed = False
 
     # Registering and fetching tags
     #############################################################
 
-    def register(self, tags: Collection[str], usn: Optional[int] = None) -> None:
+    def register(
+        self, tags: Collection[str], usn: Optional[int] = None, clear=False
+    ) -> None:
         "Given a list of tags, add any missing ones to tag registry."
-        found = False
+        changed = self.col.backend.register_tags(" ".join(tags), usn, clear)
-        for t in tags:
+        if changed:
-            if t not in self.tags:
+            hooks.tag_list_did_update()
-                found = True
-                self.tags[t] = self.col.usn() if usn is None else usn
-                self.changed = True
-        if found:
-            hooks.tag_added(t)  # pylint: disable=undefined-loop-variable
-
-    def all(self) -> List:
-        return list(self.tags.keys())
 
     def registerNotes(self, nids: Optional[List[int]] = None) -> None:
         "Add any missing tags from notes to the tags list."
         # when called without an argument, the old list is cleared first.
         if nids:
             lim = " where id in " + ids2str(nids)
+            clear = False
         else:
             lim = ""
-            self.tags = {}
+            clear = True
-            self.changed = True
         self.register(
             set(
                 self.split(
                     " ".join(self.col.db.list("select distinct tags from notes" + lim))
                 )
+            ),
+            clear=clear,
         )
-        )
-
-    def allItems(self) -> List[Tuple[str, int]]:
-        return list(self.tags.items())
-
-    def save(self) -> None:
-        self.changed = True
 
     def byDeck(self, did, children=False) -> List[str]:
         basequery = "select n.tags from cards c, notes n WHERE c.nid = n.id"
@@ -180,23 +162,12 @@ class TagManager:
 
     def canonify(self, tagList: List[str]) -> List[str]:
         "Strip duplicates, adjust case to match existing tags, and sort."
-        strippedTags = []
+        tag_str, changed = self.col.backend.canonify_tags(" ".join(tagList))
-        for t in tagList:
+        if changed:
-            s = re.sub("[\"']", "", t)
+            hooks.tag_list_did_update()
-            for existingTag in self.tags:
+
-                if s.lower() == existingTag.lower():
+        return tag_str.split(" ")
-                    s = existingTag
-            strippedTags.append(s)
-        return sorted(set(strippedTags))
 
     def inList(self, tag: str, tags: List[str]) -> bool:
         "True if TAG is in TAGS. Ignore case."
         return tag.lower() in [t.lower() for t in tags]
-
-    # Sync handling
-    ##########################################################################
-
-    def beforeUpload(self) -> None:
-        for k in list(self.tags.keys()):
-            self.tags[k] = 0
-        self.save()

pylib/tools/genhooks.py

@@ -51,9 +51,7 @@ hooks = [
         return_type="bool",
         doc="Warning: this is called on a background thread.",
     ),
-    Hook(
+    Hook(name="tag_list_did_update"),
-        name="tag_added", args=["tag: str"], legacy_hook="newTag", legacy_no_args=True,
-    ),
     Hook(
         name="field_filter",
         args=[

qt/aqt/browser.py

@@ -1132,7 +1132,7 @@ QTableView {{ gridline-color: {grid} }}
 
     def _userTagTree(self, root) -> None:
         assert self.col
-        for t in sorted(self.col.tags.all(), key=lambda t: t.lower()):
+        for t in self.col.tags.all():
             item = SidebarItem(
                 t, ":/icons/tag.svg", lambda t=t: self.setFilter("tag", t)  # type: ignore
             )
@@ -1874,7 +1874,7 @@ update cards set usn=?, mod=?, did=? where id in """
         gui_hooks.editor_did_fire_typing_timer.append(self.refreshCurrentCard)
         gui_hooks.editor_did_load_note.append(self.onLoadNote)
         gui_hooks.editor_did_unfocus_field.append(self.on_unfocus_field)
-        hooks.tag_added.append(self.on_item_added)
+        hooks.tag_list_did_update.append(self.on_tag_list_update)
         hooks.note_type_added.append(self.on_item_added)
         hooks.deck_added.append(self.on_item_added)
 
@@ -1884,7 +1884,7 @@ update cards set usn=?, mod=?, did=? where id in """
         gui_hooks.editor_did_fire_typing_timer.remove(self.refreshCurrentCard)
         gui_hooks.editor_did_load_note.remove(self.onLoadNote)
         gui_hooks.editor_did_unfocus_field.remove(self.on_unfocus_field)
-        hooks.tag_added.remove(self.on_item_added)
+        hooks.tag_list_did_update.remove(self.on_tag_list_update)
         hooks.note_type_added.remove(self.on_item_added)
         hooks.deck_added.remove(self.on_item_added)
 
@@ -1895,6 +1895,9 @@ update cards set usn=?, mod=?, did=? where id in """
     def on_item_added(self, item: Any) -> None:
         self.maybeRefreshSidebar()
 
+    def on_tag_list_update(self):
+        self.maybeRefreshSidebar()
+
     def onUndoState(self, on):
         self.form.actionUndo.setEnabled(on)
         if on:

rslib/Cargo.toml

@@ -40,7 +40,8 @@ slog-async = "2.4.0"
 slog-envlogger = "2.2.0"
 serde_repr = "0.1.5"
 num_enum = "0.4.2"
-unicase = "2.6.0"
+# pinned as any changes could invalidate sqlite indexes
+unicase = "=2.6.0"
 futures = "0.3.4"
 
 # pinned until rusqlite 0.22 comes out

rslib/src/backend/mod.rs

@@ -283,6 +283,14 @@ impl Backend {
                 self.abort_media_sync();
                 OValue::AbortMediaSync(pb::Empty {})
             }
+            Value::BeforeUpload(_) => {
+                self.before_upload()?;
+                OValue::BeforeUpload(pb::Empty {})
+            }
+            Value::CanonifyTags(input) => OValue::CanonifyTags(self.canonify_tags(input)?),
+            Value::AllTags(_) => OValue::AllTags(self.all_tags()?),
+            Value::RegisterTags(input) => OValue::RegisterTags(self.register_tags(input)?),
+            Value::GetChangedTags(usn) => OValue::GetChangedTags(self.get_changed_tags(usn)?),
         })
     }
 
@@ -727,6 +735,54 @@ impl Backend {
     fn remove_deck_config(&self, dcid: i64) -> Result<()> {
         self.with_col(|col| col.transact(None, |col| col.remove_deck_config(DeckConfID(dcid))))
     }
+
+    fn before_upload(&self) -> Result<()> {
+        self.with_col(|col| col.transact(None, |col| col.before_upload()))
+    }
+
+    fn canonify_tags(&self, tags: String) -> Result<pb::CanonifyTagsOut> {
+        self.with_col(|col| {
+            col.transact(None, |col| {
+                col.canonify_tags(&tags, col.usn()?)
+                    .map(|(tags, added)| pb::CanonifyTagsOut {
+                        tags,
+                        tag_list_changed: added,
+                    })
+            })
+        })
+    }
+
+    fn all_tags(&self) -> Result<pb::AllTagsOut> {
+        let tags = self.with_col(|col| col.storage.all_tags())?;
+        let tags: Vec<_> = tags
+            .into_iter()
+            .map(|(tag, usn)| pb::TagUsnTuple { tag, usn: usn.0 })
+            .collect();
+        Ok(pb::AllTagsOut { tags })
+    }
+
+    fn register_tags(&self, input: pb::RegisterTagsIn) -> Result<bool> {
+        self.with_col(|col| {
+            col.transact(None, |col| {
+                let usn = if input.preserve_usn {
+                    Usn(input.usn)
+                } else {
+                    col.usn()?
+                };
+                col.register_tags(&input.tags, usn, input.clear_first)
+            })
+        })
+    }
+
+    fn get_changed_tags(&self, usn: i32) -> Result<pb::GetChangedTagsOut> {
+        self.with_col(|col| {
+            col.transact(None, |col| {
+                Ok(pb::GetChangedTagsOut {
+                    tags: col.storage.get_changed_tags(Usn(usn))?,
+                })
+            })
+        })
+    }
 }
 
 fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue {

rslib/src/collection.rs

@@ -157,4 +157,11 @@ impl Collection {
             Ok(())
         }
     }
+
+    pub(crate) fn before_upload(&self) -> Result<()> {
+        self.storage.clear_tag_usns()?;
+        self.storage.clear_deck_conf_usns()?;
+
+        Ok(())
+    }
 }

rslib/src/lib.rs

@@ -27,6 +27,7 @@ pub mod sched;
 pub mod search;
 pub mod serde;
 pub mod storage;
+pub mod tags;
 pub mod template;
 pub mod template_filters;
 pub mod text;

rslib/src/search/sqlwriter.rs

@@ -11,6 +11,8 @@ use crate::notetypes::NoteTypeID;
 use crate::text::matches_wildcard;
 use crate::text::without_combining;
 use crate::{collection::Collection, text::strip_html_preserving_image_filenames};
+use lazy_static::lazy_static;
+use regex::{Captures, Regex};
 use std::fmt::Write;
 
 struct SqlWriter<'a> {
@@ -66,7 +68,7 @@ impl SqlWriter<'_> {
             }
             SearchNode::NoteType(notetype) => self.write_note_type(notetype.as_ref())?,
             SearchNode::Rated { days, ease } => self.write_rated(*days, *ease)?,
-            SearchNode::Tag(tag) => self.write_tag(tag),
+            SearchNode::Tag(tag) => self.write_tag(tag)?,
             SearchNode::Duplicates { note_type_id, text } => self.write_dupes(*note_type_id, text),
             SearchNode::State(state) => self.write_state(state)?,
             SearchNode::Flag(flag) => {
@@ -112,7 +114,7 @@ impl SqlWriter<'_> {
         .unwrap();
     }
 
-    fn write_tag(&mut self, text: &str) {
+    fn write_tag(&mut self, text: &str) -> Result<()> {
         match text {
             "none" => {
                 write!(self.sql, "n.tags = ''").unwrap();
@@ -121,12 +123,21 @@ impl SqlWriter<'_> {
                 write!(self.sql, "true").unwrap();
             }
             text => {
-                let tag = format!("% {} %", text.replace('*', "%"));
+                if let Some(re_glob) = glob_to_re(text) {
-                write!(self.sql, "n.tags like ? escape '\\'").unwrap();
+                    // text contains a wildcard
-                self.args.push(tag);
+                    let re_glob = format!("(?i).* {} .*", re_glob);
+                    write!(self.sql, "n.tags regexp ?").unwrap();
+                    self.args.push(re_glob);
+                } else if let Some(tag) = self.col.storage.preferred_tag_case(&text)? {
+                    write!(self.sql, "n.tags like ?").unwrap();
+                    self.args.push(format!("% {} %", tag));
+                } else {
+                    write!(self.sql, "false").unwrap();
                 }
             }
         }
+        Ok(())
+    }
 
     fn write_rated(&mut self, days: u32, ease: Option<u8>) -> Result<()> {
         let today_cutoff = self.col.timing_today()?.next_day_at;
@@ -382,6 +393,42 @@ where
     buf.push(')');
 }
 
+/// Convert a string with _, % or * characters into a regex.
+fn glob_to_re(glob: &str) -> Option<String> {
+    if !glob.contains(|c| c == '_' || c == '*' || c == '%') {
+        return None;
+    }
+
+    lazy_static! {
+        static ref ESCAPED: Regex = Regex::new(r"(\\\\)?\\\*").unwrap();
+        static ref GLOB: Regex = Regex::new(r"(\\\\)?[_%]").unwrap();
+    }
+
+    let escaped = regex::escape(glob);
+
+    let text = ESCAPED.replace_all(&escaped, |caps: &Captures| {
+        if caps.get(0).unwrap().as_str().len() == 2 {
+            ".*"
+        } else {
+            r"\*"
+        }
+    });
+
+    let text2 = GLOB.replace_all(&text, |caps: &Captures| {
+        match caps.get(0).unwrap().as_str() {
+            "_" => ".",
+            "%" => ".*",
+            other => {
+                // strip off the escaping char
+                &other[2..]
+            }
+        }
+        .to_string()
+    });
+
+    text2.into_owned().into()
+}
+
 #[cfg(test)]
 mod test {
     use super::ids_to_string;
@@ -389,6 +436,7 @@ mod test {
         collection::{open_collection, Collection},
         i18n::I18n,
         log,
+        types::Usn,
     };
     use std::{fs, path::PathBuf};
     use tempfile::tempdir;
@@ -439,6 +487,7 @@ mod test {
         .unwrap();
 
         let ctx = &mut col;
+
         // unqualified search
         assert_eq!(
             s(ctx, "test"),
@@ -515,14 +564,24 @@ mod test {
             )
         );
 
-        // tags
+        // unregistered tag short circuits
+        assert_eq!(s(ctx, r"tag:one"), ("(false)".into(), vec![]));
+
+        // if registered, searches with canonical
+        ctx.transact(None, |col| col.register_tag("One", Usn(-1)))
+            .unwrap();
         assert_eq!(
-            s(ctx, "tag:one"),
+            s(ctx, r"tag:one"),
-            ("(n.tags like ? escape '\\')".into(), vec!["% one %".into()])
+            ("(n.tags like ?)".into(), vec![r"% One %".into()])
         );
+
+        // wildcards force a regexp search
         assert_eq!(
-            s(ctx, "tag:o*e"),
+            s(ctx, r"tag:o*n\*et%w\%oth_re\_e"),
-            ("(n.tags like ? escape '\\')".into(), vec!["% o%e %".into()])
+            (
+                "(n.tags regexp ?)".into(),
+                vec![r"(?i).* o.*n\*et.*w%oth.re_e .*".into()]
+            )
         );
         assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![]));
         assert_eq!(s(ctx, "tag:*"), ("(true)".into(), vec![]));

rslib/src/storage/deckconf/mod.rs

@@ -70,6 +70,17 @@ impl SqliteStorage {
         Ok(())
     }
 
+    pub(crate) fn clear_deck_conf_usns(&self) -> Result<()> {
+        for mut conf in self.all_deck_config()? {
+            if conf.usn.0 != 0 {
+                conf.usn.0 = 0;
+                self.update_deck_conf(&conf)?;
+            }
+        }
+
+        Ok(())
+    }
+
     // Creating/upgrading/downgrading
 
     pub(super) fn add_default_deck_config(&self, i18n: &I18n) -> Result<()> {

rslib/src/storage/mod.rs

@@ -1,5 +1,9 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
 mod card;
 mod deckconf;
 mod sqlite;
+mod tag;
 
 pub(crate) use sqlite::SqliteStorage;

rslib/src/storage/schema12_downgrade.sql

@@ -1,4 +1,5 @@
 drop table deck_config;
+drop table tags;
 update col
 set
   ver = 11;

rslib/src/storage/schema12_upgrade.sql

@@ -5,6 +5,10 @@ create table deck_config (
   usn integer not null,
   config text not null
 );
+create table tags (
+  tag text not null primary key collate unicase,
+  usn integer not null
+) without rowid;
 update col
 set
   ver = 12;

rslib/src/storage/sqlite.rs

@@ -211,20 +211,23 @@ impl SqliteStorage {
         Ok(())
     }
 
-    fn upgrade_to_schema_12(&self) -> Result<()> {
-        self.db
-            .execute_batch(include_str!("schema12_upgrade.sql"))?;
-        self.upgrade_deck_conf_to_schema12()
-    }
-
     pub(crate) fn downgrade_to_schema_11(self) -> Result<()> {
         self.begin_trx()?;
         self.downgrade_from_schema_12()?;
         self.commit_trx()
     }
 
+    fn upgrade_to_schema_12(&self) -> Result<()> {
+        self.db
+            .execute_batch(include_str!("schema12_upgrade.sql"))?;
+        self.upgrade_deck_conf_to_schema12()?;
+        self.upgrade_tags_to_schema12()
+    }
+
     fn downgrade_from_schema_12(&self) -> Result<()> {
+        self.downgrade_tags_from_schema12()?;
         self.downgrade_deck_conf_from_schema12()?;
+
         self.db
             .execute_batch(include_str!("schema12_downgrade.sql"))?;
         Ok(())

rslib/src/storage/tag/add.sql

@@ -0,0 +1,4 @@
+insert
+  or ignore into tags (tag, usn)
+values
+  (?, ?)

rslib/src/storage/tag/mod.rs

@@ -0,0 +1,86 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::SqliteStorage;
+use crate::{err::Result, types::Usn};
+use rusqlite::{params, NO_PARAMS};
+use std::collections::HashMap;
+
+impl SqliteStorage {
+    pub(crate) fn all_tags(&self) -> Result<Vec<(String, Usn)>> {
+        self.db
+            .prepare_cached("select tag, usn from tags")?
+            .query_and_then(NO_PARAMS, |row| -> Result<_> {
+                Ok((row.get(0)?, row.get(1)?))
+            })?
+            .collect()
+    }
+
+    pub(crate) fn register_tag(&self, tag: &str, usn: Usn) -> Result<()> {
+        self.db
+            .prepare_cached(include_str!("add.sql"))?
+            .execute(params![tag, usn])?;
+        Ok(())
+    }
+
+    pub(crate) fn preferred_tag_case(&self, tag: &str) -> Result<Option<String>> {
+        self.db
+            .prepare_cached("select tag from tags where tag = ?")?
+            .query_and_then(params![tag], |row| row.get(0))?
+            .next()
+            .transpose()
+            .map_err(Into::into)
+    }
+
+    pub(crate) fn clear_tags(&self) -> Result<()> {
+        self.db.execute("delete from tags", NO_PARAMS)?;
+        Ok(())
+    }
+
+    pub(crate) fn clear_tag_usns(&self) -> Result<()> {
+        self.db
+            .execute("update tags set usn = 0 where usn != 0", NO_PARAMS)?;
+        Ok(())
+    }
+
+    // fixme: in the future we could just register tags as part of the sync
+    // instead of sending the tag list separately
+    pub(crate) fn get_changed_tags(&self, usn: Usn) -> Result<Vec<String>> {
+        let tags: Vec<String> = self
+            .db
+            .prepare("select tag from tags where usn=-1")?
+            .query_map(NO_PARAMS, |row| row.get(0))?
+            .collect::<std::result::Result<_, rusqlite::Error>>()?;
+        self.db
+            .execute("update tags set usn=? where usn=-1", &[&usn])?;
+        Ok(tags)
+    }
+
+    // Upgrading/downgrading
+
+    pub(super) fn upgrade_tags_to_schema12(&self) -> Result<()> {
+        let tags = self
+            .db
+            .query_row_and_then("select tags from col", NO_PARAMS, |row| {
+                let tags: Result<HashMap<String, Usn>> =
+                    serde_json::from_str(row.get_raw(0).as_str()?).map_err(Into::into);
+                tags
+            })?;
+        for (tag, usn) in tags.into_iter() {
+            self.register_tag(&tag, usn)?;
+        }
+        self.db.execute_batch("update col set tags=''")?;
+
+        Ok(())
+    }
+
+    pub(super) fn downgrade_tags_from_schema12(&self) -> Result<()> {
+        let alltags = self.all_tags()?;
+        let tagsmap: HashMap<String, Usn> = alltags.into_iter().collect();
+        self.db.execute(
+            "update col set tags=?",
+            params![serde_json::to_string(&tagsmap)?],
+        )?;
+        Ok(())
+    }
+}

rslib/src/tags.rs

@@ -0,0 +1,61 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::collection::Collection;
+use crate::err::Result;
+use crate::types::Usn;
+use std::{borrow::Cow, collections::HashSet};
+
+fn split_tags(tags: &str) -> impl Iterator<Item = &str> {
+    tags.split(|c| c == ' ' || c == '\u{3000}')
+        .filter(|tag| !tag.is_empty())
+}
+
+impl Collection {
+    /// Given a space-separated list of tags, fix case, ordering and duplicates.
+    /// Returns true if any new tags were added.
+    pub(crate) fn canonify_tags(&self, tags: &str, usn: Usn) -> Result<(String, bool)> {
+        let mut tagset = HashSet::new();
+        let mut added = false;
+
+        for tag in split_tags(tags) {
+            let tag = self.register_tag(tag, usn)?;
+            if matches!(tag, Cow::Borrowed(_)) {
+                added = true;
+            }
+            tagset.insert(tag);
+        }
+
+        if tagset.is_empty() {
+            return Ok(("".into(), added));
+        }
+
+        let mut tags = tagset.into_iter().collect::<Vec<_>>();
+        tags.sort_unstable();
+
+        Ok((format!(" {} ", tags.join(" ")), added))
+    }
+
+    pub(crate) fn register_tag<'a>(&self, tag: &'a str, usn: Usn) -> Result<Cow<'a, str>> {
+        if let Some(preferred) = self.storage.preferred_tag_case(tag)? {
+            Ok(preferred.into())
+        } else {
+            self.storage.register_tag(tag, usn)?;
+            Ok(tag.into())
+        }
+    }
+
+    pub(crate) fn register_tags(&self, tags: &str, usn: Usn, clear_first: bool) -> Result<bool> {
+        let mut changed = false;
+        if clear_first {
+            self.storage.clear_tags()?;
+        }
+        for tag in split_tags(tags) {
+            let tag = self.register_tag(tag, usn)?;
+            if matches!(tag, Cow::Borrowed(_)) {
+                changed = true;
+            }
+        }
+        Ok(changed)
+    }
+}