From ac4284b2de285efd5451146e68d3e5195bbfd2df Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 3 Apr 2020 19:30:42 +1000 Subject: [PATCH] update tag handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tag list stored in a separate DB table - non-wildcard searches now do full unicode case folding (eg tag:masse matches 'Maße') - wildcard matches do simple unicode case folding - some functions haven't been updated yet, so ascii folding will continue to be used in some operations --- proto/backend.proto | 35 ++++++++++ pylib/anki/collection.py | 10 +-- pylib/anki/decks.py | 6 +- pylib/anki/hooks.py | 18 +++-- pylib/anki/rsbackend.py | 37 ++++++++++ pylib/anki/sync.py | 12 +--- pylib/anki/tags.py | 73 ++++++-------------- pylib/tools/genhooks.py | 4 +- qt/aqt/browser.py | 9 ++- rslib/Cargo.toml | 3 +- rslib/src/backend/mod.rs | 56 +++++++++++++++ rslib/src/collection.rs | 7 ++ rslib/src/lib.rs | 1 + rslib/src/search/sqlwriter.rs | 79 +++++++++++++++++++--- rslib/src/storage/deckconf/mod.rs | 11 +++ rslib/src/storage/mod.rs | 4 ++ rslib/src/storage/schema12_downgrade.sql | 1 + rslib/src/storage/schema12_upgrade.sql | 4 ++ rslib/src/storage/sqlite.rs | 15 +++-- rslib/src/storage/tag/add.sql | 4 ++ rslib/src/storage/tag/mod.rs | 86 ++++++++++++++++++++++++ rslib/src/tags.rs | 61 +++++++++++++++++ 22 files changed, 431 insertions(+), 105 deletions(-) create mode 100644 rslib/src/storage/tag/add.sql create mode 100644 rslib/src/storage/tag/mod.rs create mode 100644 rslib/src/tags.rs diff --git a/proto/backend.proto b/proto/backend.proto index 5a141f780..ea5d403fe 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -56,6 +56,11 @@ message BackendInput { Empty new_deck_config = 44; int64 remove_deck_config = 45; Empty abort_media_sync = 46; + Empty before_upload = 47; + RegisterTagsIn register_tags = 48; + string canonify_tags = 49; + Empty all_tags = 50; + int32 get_changed_tags = 51; } } @@ -95,6 +100,11 @@ message BackendOutput { string all_deck_config = 43; string new_deck_config = 44; Empty remove_deck_config = 45; + Empty before_upload = 47; + bool register_tags = 48; + CanonifyTagsOut canonify_tags = 49; + AllTagsOut all_tags = 50; + GetChangedTagsOut get_changed_tags = 51; BackendError error = 2047; } @@ -419,3 +429,28 @@ message AddOrUpdateDeckConfigIn { string config = 1; bool preserve_usn_and_mtime = 2; } + +message RegisterTagsIn { + string tags = 1; + bool preserve_usn = 2; + int32 usn = 3; + bool clear_first = 4; +} + +message AllTagsOut { + repeated TagUsnTuple tags = 1; +} + +message TagUsnTuple { + string tag = 1; + sint32 usn = 2; +} + +message GetChangedTagsOut { + repeated string tags = 1; +} + +message CanonifyTagsOut { + string tags = 1; + bool tag_list_changed = 2; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index e8a3b93fa..ace24214d 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -182,17 +182,14 @@ class _Collection: conf, models, decks, - dconf, - tags, ) = self.db.first( """ select crt, mod, scm, dty, usn, ls, -conf, models, decks, dconf, tags from col""" +conf, models, decks from col""" ) self.conf = json.loads(conf) self.models.load(models) - self.decks.load(decks, dconf) - self.tags.load(tags) + self.decks.load(decks) def setMod(self) -> None: """Mark DB modified. @@ -219,7 +216,6 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", def flush_all_changes(self, mod: Optional[int] = None): self.models.flush() self.decks.flush() - self.tags.flush() if self.db.mod: self.flush(mod) @@ -292,8 +288,8 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.db.execute("delete from graves") self._usn += 1 self.models.beforeUpload() - self.tags.beforeUpload() self.decks.beforeUpload() + self.backend.before_upload() self.modSchema(check=False) self.ls = self.scm # ensure db is compacted before upload diff --git a/pylib/anki/decks.py b/pylib/anki/decks.py index 8c617ed59..105752e13 100644 --- a/pylib/anki/decks.py +++ b/pylib/anki/decks.py @@ -65,7 +65,7 @@ class DeckManager: self.decks = {} self._dconf_cache: Optional[Dict[int, Dict[str, Any]]] = None - def load(self, decks: str, dconf: str) -> None: + def load(self, decks: str) -> None: self.decks = json.loads(decks) self.changed = False @@ -626,10 +626,6 @@ class DeckManager: def beforeUpload(self) -> None: for d in self.all(): d["usn"] = 0 - for c in self.all_config(): - if c["usn"] != 0: - c["usn"] = 0 - self.update_config(c, preserve_usn=True) self.save() # Dynamic decks diff --git a/pylib/anki/hooks.py b/pylib/anki/hooks.py index 2439eff3c..289d0c07b 100644 --- a/pylib/anki/hooks.py +++ b/pylib/anki/hooks.py @@ -544,30 +544,28 @@ class _SyncStageDidChangeHook: sync_stage_did_change = _SyncStageDidChangeHook() -class _TagAddedHook: - _hooks: List[Callable[[str], None]] = [] +class _TagListDidUpdateHook: + _hooks: List[Callable[[], None]] = [] - def append(self, cb: Callable[[str], None]) -> None: - """(tag: str)""" + def append(self, cb: Callable[[], None]) -> None: + """()""" self._hooks.append(cb) - def remove(self, cb: Callable[[str], None]) -> None: + def remove(self, cb: Callable[[], None]) -> None: if cb in self._hooks: self._hooks.remove(cb) - def __call__(self, tag: str) -> None: + def __call__(self) -> None: for hook in self._hooks: try: - hook(tag) + hook() except: # if the hook fails, remove it self._hooks.remove(hook) raise - # legacy support - runHook("newTag") -tag_added = _TagAddedHook() +tag_list_did_update = _TagListDidUpdateHook() # @@AUTOGEN@@ # Legacy hook handling diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 55b3a9f6c..b11ec2fcb 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -47,6 +47,7 @@ assert ankirspy.buildhash() == anki.buildinfo.buildhash SchedTimingToday = pb.SchedTimingTodayOut BuiltinSortKind = pb.BuiltinSortKind BackendCard = pb.Card +TagUsnTuple = pb.TagUsnTuple try: import orjson @@ -540,6 +541,42 @@ class RustBackend: def abort_media_sync(self): self._run_command(pb.BackendInput(abort_media_sync=pb.Empty())) + def all_tags(self) -> Iterable[TagUsnTuple]: + return self._run_command(pb.BackendInput(all_tags=pb.Empty())).all_tags.tags + + def canonify_tags(self, tags: str) -> Tuple[str, bool]: + out = self._run_command(pb.BackendInput(canonify_tags=tags)).canonify_tags + return (out.tags, out.tag_list_changed) + + def register_tags(self, tags: str, usn: Optional[int], clear_first: bool) -> bool: + if usn is None: + preserve_usn = False + usn_ = 0 + else: + usn_ = usn + preserve_usn = True + + return self._run_command( + pb.BackendInput( + register_tags=pb.RegisterTagsIn( + tags=tags, + usn=usn_, + preserve_usn=preserve_usn, + clear_first=clear_first, + ) + ) + ).register_tags + + def before_upload(self): + self._run_command(pb.BackendInput(before_upload=pb.Empty())) + + def get_changed_tags(self, usn: int) -> List[str]: + return list( + self._run_command( + pb.BackendInput(get_changed_tags=usn) + ).get_changed_tags.tags + ) + def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/pylib/anki/sync.py b/pylib/anki/sync.py index 7943f616d..64a349316 100644 --- a/pylib/anki/sync.py +++ b/pylib/anki/sync.py @@ -206,8 +206,8 @@ class Syncer: for g in self.col.decks.all(): if g["usn"] == -1: return "deck had usn = -1" - for t, usn in self.col.tags.allItems(): - if usn == -1: + for tup in self.col.backend.all_tags(): + if tup.usn == -1: return "tag had usn = -1" found = False for m in self.col.models.all(): @@ -404,13 +404,7 @@ from notes where %s""" ########################################################################## def getTags(self) -> List: - tags = [] - for t, usn in self.col.tags.allItems(): - if usn == -1: - self.col.tags.tags[t] = self.maxUsn - tags.append(t) - self.col.tags.save() - return tags + return self.col.backend.get_changed_tags(self.maxUsn) def mergeTags(self, tags) -> None: self.col.tags.register(tags, usn=self.maxUsn) diff --git a/pylib/anki/tags.py b/pylib/anki/tags.py index 7d9e642b0..5f12a4467 100644 --- a/pylib/anki/tags.py +++ b/pylib/anki/tags.py @@ -11,9 +11,8 @@ This module manages the tag cache and tags for notes. from __future__ import annotations -import json import re -from typing import Callable, Collection, Dict, List, Optional, Tuple +from typing import Callable, Collection, List, Optional, Tuple import anki # pylint: disable=unused-import from anki import hooks @@ -21,63 +20,46 @@ from anki.utils import ids2str, intTime class TagManager: - - # Registry save/load - ############################################################# - def __init__(self, col: anki.storage._Collection) -> None: self.col = col.weakref() - self.tags: Dict[str, int] = {} - def load(self, json_: str) -> None: - self.tags = json.loads(json_) - self.changed = False + # all tags + def all(self) -> List[str]: + return [t.tag for t in self.col.backend.all_tags()] - def flush(self) -> None: - if self.changed: - self.col.db.execute("update col set tags=?", json.dumps(self.tags)) - self.changed = False + # # List of (tag, usn) + def allItems(self) -> List[Tuple[str, int]]: + return [(t.tag, t.usn) for t in self.col.backend.all_tags()] # Registering and fetching tags ############################################################# - def register(self, tags: Collection[str], usn: Optional[int] = None) -> None: + def register( + self, tags: Collection[str], usn: Optional[int] = None, clear=False + ) -> None: "Given a list of tags, add any missing ones to tag registry." - found = False - for t in tags: - if t not in self.tags: - found = True - self.tags[t] = self.col.usn() if usn is None else usn - self.changed = True - if found: - hooks.tag_added(t) # pylint: disable=undefined-loop-variable - - def all(self) -> List: - return list(self.tags.keys()) + changed = self.col.backend.register_tags(" ".join(tags), usn, clear) + if changed: + hooks.tag_list_did_update() def registerNotes(self, nids: Optional[List[int]] = None) -> None: "Add any missing tags from notes to the tags list." # when called without an argument, the old list is cleared first. if nids: lim = " where id in " + ids2str(nids) + clear = False else: lim = "" - self.tags = {} - self.changed = True + clear = True self.register( set( self.split( " ".join(self.col.db.list("select distinct tags from notes" + lim)) ) - ) + ), + clear=clear, ) - def allItems(self) -> List[Tuple[str, int]]: - return list(self.tags.items()) - - def save(self) -> None: - self.changed = True - def byDeck(self, did, children=False) -> List[str]: basequery = "select n.tags from cards c, notes n WHERE c.nid = n.id" if not children: @@ -180,23 +162,12 @@ class TagManager: def canonify(self, tagList: List[str]) -> List[str]: "Strip duplicates, adjust case to match existing tags, and sort." - strippedTags = [] - for t in tagList: - s = re.sub("[\"']", "", t) - for existingTag in self.tags: - if s.lower() == existingTag.lower(): - s = existingTag - strippedTags.append(s) - return sorted(set(strippedTags)) + tag_str, changed = self.col.backend.canonify_tags(" ".join(tagList)) + if changed: + hooks.tag_list_did_update() + + return tag_str.split(" ") def inList(self, tag: str, tags: List[str]) -> bool: "True if TAG is in TAGS. Ignore case." return tag.lower() in [t.lower() for t in tags] - - # Sync handling - ########################################################################## - - def beforeUpload(self) -> None: - for k in list(self.tags.keys()): - self.tags[k] = 0 - self.save() diff --git a/pylib/tools/genhooks.py b/pylib/tools/genhooks.py index 4ab2e90e8..f15436168 100644 --- a/pylib/tools/genhooks.py +++ b/pylib/tools/genhooks.py @@ -51,9 +51,7 @@ hooks = [ return_type="bool", doc="Warning: this is called on a background thread.", ), - Hook( - name="tag_added", args=["tag: str"], legacy_hook="newTag", legacy_no_args=True, - ), + Hook(name="tag_list_did_update"), Hook( name="field_filter", args=[ diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index c0a789dbc..9b2a49f78 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -1132,7 +1132,7 @@ QTableView {{ gridline-color: {grid} }} def _userTagTree(self, root) -> None: assert self.col - for t in sorted(self.col.tags.all(), key=lambda t: t.lower()): + for t in self.col.tags.all(): item = SidebarItem( t, ":/icons/tag.svg", lambda t=t: self.setFilter("tag", t) # type: ignore ) @@ -1874,7 +1874,7 @@ update cards set usn=?, mod=?, did=? where id in """ gui_hooks.editor_did_fire_typing_timer.append(self.refreshCurrentCard) gui_hooks.editor_did_load_note.append(self.onLoadNote) gui_hooks.editor_did_unfocus_field.append(self.on_unfocus_field) - hooks.tag_added.append(self.on_item_added) + hooks.tag_list_did_update.append(self.on_tag_list_update) hooks.note_type_added.append(self.on_item_added) hooks.deck_added.append(self.on_item_added) @@ -1884,7 +1884,7 @@ update cards set usn=?, mod=?, did=? where id in """ gui_hooks.editor_did_fire_typing_timer.remove(self.refreshCurrentCard) gui_hooks.editor_did_load_note.remove(self.onLoadNote) gui_hooks.editor_did_unfocus_field.remove(self.on_unfocus_field) - hooks.tag_added.remove(self.on_item_added) + hooks.tag_list_did_update.remove(self.on_tag_list_update) hooks.note_type_added.remove(self.on_item_added) hooks.deck_added.remove(self.on_item_added) @@ -1895,6 +1895,9 @@ update cards set usn=?, mod=?, did=? where id in """ def on_item_added(self, item: Any) -> None: self.maybeRefreshSidebar() + def on_tag_list_update(self): + self.maybeRefreshSidebar() + def onUndoState(self, on): self.form.actionUndo.setEnabled(on) if on: diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 84faf35e2..bdc3fcd35 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -40,7 +40,8 @@ slog-async = "2.4.0" slog-envlogger = "2.2.0" serde_repr = "0.1.5" num_enum = "0.4.2" -unicase = "2.6.0" +# pinned as any changes could invalidate sqlite indexes +unicase = "=2.6.0" futures = "0.3.4" # pinned until rusqlite 0.22 comes out diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 02597c413..2caa7bad7 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -283,6 +283,14 @@ impl Backend { self.abort_media_sync(); OValue::AbortMediaSync(pb::Empty {}) } + Value::BeforeUpload(_) => { + self.before_upload()?; + OValue::BeforeUpload(pb::Empty {}) + } + Value::CanonifyTags(input) => OValue::CanonifyTags(self.canonify_tags(input)?), + Value::AllTags(_) => OValue::AllTags(self.all_tags()?), + Value::RegisterTags(input) => OValue::RegisterTags(self.register_tags(input)?), + Value::GetChangedTags(usn) => OValue::GetChangedTags(self.get_changed_tags(usn)?), }) } @@ -727,6 +735,54 @@ impl Backend { fn remove_deck_config(&self, dcid: i64) -> Result<()> { self.with_col(|col| col.transact(None, |col| col.remove_deck_config(DeckConfID(dcid)))) } + + fn before_upload(&self) -> Result<()> { + self.with_col(|col| col.transact(None, |col| col.before_upload())) + } + + fn canonify_tags(&self, tags: String) -> Result { + self.with_col(|col| { + col.transact(None, |col| { + col.canonify_tags(&tags, col.usn()?) + .map(|(tags, added)| pb::CanonifyTagsOut { + tags, + tag_list_changed: added, + }) + }) + }) + } + + fn all_tags(&self) -> Result { + let tags = self.with_col(|col| col.storage.all_tags())?; + let tags: Vec<_> = tags + .into_iter() + .map(|(tag, usn)| pb::TagUsnTuple { tag, usn: usn.0 }) + .collect(); + Ok(pb::AllTagsOut { tags }) + } + + fn register_tags(&self, input: pb::RegisterTagsIn) -> Result { + self.with_col(|col| { + col.transact(None, |col| { + let usn = if input.preserve_usn { + Usn(input.usn) + } else { + col.usn()? + }; + col.register_tags(&input.tags, usn, input.clear_first) + }) + }) + } + + fn get_changed_tags(&self, usn: i32) -> Result { + self.with_col(|col| { + col.transact(None, |col| { + Ok(pb::GetChangedTagsOut { + tags: col.storage.get_changed_tags(Usn(usn))?, + }) + }) + }) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs index 8afee6bbc..8664fb036 100644 --- a/rslib/src/collection.rs +++ b/rslib/src/collection.rs @@ -157,4 +157,11 @@ impl Collection { Ok(()) } } + + pub(crate) fn before_upload(&self) -> Result<()> { + self.storage.clear_tag_usns()?; + self.storage.clear_deck_conf_usns()?; + + Ok(()) + } } diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 41056c4a2..7eb915401 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -27,6 +27,7 @@ pub mod sched; pub mod search; pub mod serde; pub mod storage; +pub mod tags; pub mod template; pub mod template_filters; pub mod text; diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 95c03e350..2dbb9b3a7 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -11,6 +11,8 @@ use crate::notetypes::NoteTypeID; use crate::text::matches_wildcard; use crate::text::without_combining; use crate::{collection::Collection, text::strip_html_preserving_image_filenames}; +use lazy_static::lazy_static; +use regex::{Captures, Regex}; use std::fmt::Write; struct SqlWriter<'a> { @@ -66,7 +68,7 @@ impl SqlWriter<'_> { } SearchNode::NoteType(notetype) => self.write_note_type(notetype.as_ref())?, SearchNode::Rated { days, ease } => self.write_rated(*days, *ease)?, - SearchNode::Tag(tag) => self.write_tag(tag), + SearchNode::Tag(tag) => self.write_tag(tag)?, SearchNode::Duplicates { note_type_id, text } => self.write_dupes(*note_type_id, text), SearchNode::State(state) => self.write_state(state)?, SearchNode::Flag(flag) => { @@ -112,7 +114,7 @@ impl SqlWriter<'_> { .unwrap(); } - fn write_tag(&mut self, text: &str) { + fn write_tag(&mut self, text: &str) -> Result<()> { match text { "none" => { write!(self.sql, "n.tags = ''").unwrap(); @@ -121,11 +123,20 @@ impl SqlWriter<'_> { write!(self.sql, "true").unwrap(); } text => { - let tag = format!("% {} %", text.replace('*', "%")); - write!(self.sql, "n.tags like ? escape '\\'").unwrap(); - self.args.push(tag); + if let Some(re_glob) = glob_to_re(text) { + // text contains a wildcard + let re_glob = format!("(?i).* {} .*", re_glob); + write!(self.sql, "n.tags regexp ?").unwrap(); + self.args.push(re_glob); + } else if let Some(tag) = self.col.storage.preferred_tag_case(&text)? { + write!(self.sql, "n.tags like ?").unwrap(); + self.args.push(format!("% {} %", tag)); + } else { + write!(self.sql, "false").unwrap(); + } } } + Ok(()) } fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { @@ -382,6 +393,42 @@ where buf.push(')'); } +/// Convert a string with _, % or * characters into a regex. +fn glob_to_re(glob: &str) -> Option { + if !glob.contains(|c| c == '_' || c == '*' || c == '%') { + return None; + } + + lazy_static! { + static ref ESCAPED: Regex = Regex::new(r"(\\\\)?\\\*").unwrap(); + static ref GLOB: Regex = Regex::new(r"(\\\\)?[_%]").unwrap(); + } + + let escaped = regex::escape(glob); + + let text = ESCAPED.replace_all(&escaped, |caps: &Captures| { + if caps.get(0).unwrap().as_str().len() == 2 { + ".*" + } else { + r"\*" + } + }); + + let text2 = GLOB.replace_all(&text, |caps: &Captures| { + match caps.get(0).unwrap().as_str() { + "_" => ".", + "%" => ".*", + other => { + // strip off the escaping char + &other[2..] + } + } + .to_string() + }); + + text2.into_owned().into() +} + #[cfg(test)] mod test { use super::ids_to_string; @@ -389,6 +436,7 @@ mod test { collection::{open_collection, Collection}, i18n::I18n, log, + types::Usn, }; use std::{fs, path::PathBuf}; use tempfile::tempdir; @@ -439,6 +487,7 @@ mod test { .unwrap(); let ctx = &mut col; + // unqualified search assert_eq!( s(ctx, "test"), @@ -515,14 +564,24 @@ mod test { ) ); - // tags + // unregistered tag short circuits + assert_eq!(s(ctx, r"tag:one"), ("(false)".into(), vec![])); + + // if registered, searches with canonical + ctx.transact(None, |col| col.register_tag("One", Usn(-1))) + .unwrap(); assert_eq!( - s(ctx, "tag:one"), - ("(n.tags like ? escape '\\')".into(), vec!["% one %".into()]) + s(ctx, r"tag:one"), + ("(n.tags like ?)".into(), vec![r"% One %".into()]) ); + + // wildcards force a regexp search assert_eq!( - s(ctx, "tag:o*e"), - ("(n.tags like ? escape '\\')".into(), vec!["% o%e %".into()]) + s(ctx, r"tag:o*n\*et%w\%oth_re\_e"), + ( + "(n.tags regexp ?)".into(), + vec![r"(?i).* o.*n\*et.*w%oth.re_e .*".into()] + ) ); assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![])); assert_eq!(s(ctx, "tag:*"), ("(true)".into(), vec![])); diff --git a/rslib/src/storage/deckconf/mod.rs b/rslib/src/storage/deckconf/mod.rs index f09a8b1ac..e1a20e963 100644 --- a/rslib/src/storage/deckconf/mod.rs +++ b/rslib/src/storage/deckconf/mod.rs @@ -70,6 +70,17 @@ impl SqliteStorage { Ok(()) } + pub(crate) fn clear_deck_conf_usns(&self) -> Result<()> { + for mut conf in self.all_deck_config()? { + if conf.usn.0 != 0 { + conf.usn.0 = 0; + self.update_deck_conf(&conf)?; + } + } + + Ok(()) + } + // Creating/upgrading/downgrading pub(super) fn add_default_deck_config(&self, i18n: &I18n) -> Result<()> { diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs index b4549b14e..816b42fdd 100644 --- a/rslib/src/storage/mod.rs +++ b/rslib/src/storage/mod.rs @@ -1,5 +1,9 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + mod card; mod deckconf; mod sqlite; +mod tag; pub(crate) use sqlite::SqliteStorage; diff --git a/rslib/src/storage/schema12_downgrade.sql b/rslib/src/storage/schema12_downgrade.sql index cde757b00..05ce7dfca 100644 --- a/rslib/src/storage/schema12_downgrade.sql +++ b/rslib/src/storage/schema12_downgrade.sql @@ -1,4 +1,5 @@ drop table deck_config; +drop table tags; update col set ver = 11; \ No newline at end of file diff --git a/rslib/src/storage/schema12_upgrade.sql b/rslib/src/storage/schema12_upgrade.sql index 54892fd73..65b88279c 100644 --- a/rslib/src/storage/schema12_upgrade.sql +++ b/rslib/src/storage/schema12_upgrade.sql @@ -5,6 +5,10 @@ create table deck_config ( usn integer not null, config text not null ); +create table tags ( + tag text not null primary key collate unicase, + usn integer not null +) without rowid; update col set ver = 12; \ No newline at end of file diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 2bd005bf6..f4a03fff1 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -211,20 +211,23 @@ impl SqliteStorage { Ok(()) } - fn upgrade_to_schema_12(&self) -> Result<()> { - self.db - .execute_batch(include_str!("schema12_upgrade.sql"))?; - self.upgrade_deck_conf_to_schema12() - } - pub(crate) fn downgrade_to_schema_11(self) -> Result<()> { self.begin_trx()?; self.downgrade_from_schema_12()?; self.commit_trx() } + fn upgrade_to_schema_12(&self) -> Result<()> { + self.db + .execute_batch(include_str!("schema12_upgrade.sql"))?; + self.upgrade_deck_conf_to_schema12()?; + self.upgrade_tags_to_schema12() + } + fn downgrade_from_schema_12(&self) -> Result<()> { + self.downgrade_tags_from_schema12()?; self.downgrade_deck_conf_from_schema12()?; + self.db .execute_batch(include_str!("schema12_downgrade.sql"))?; Ok(()) diff --git a/rslib/src/storage/tag/add.sql b/rslib/src/storage/tag/add.sql new file mode 100644 index 000000000..8fee01a4c --- /dev/null +++ b/rslib/src/storage/tag/add.sql @@ -0,0 +1,4 @@ +insert + or ignore into tags (tag, usn) +values + (?, ?) \ No newline at end of file diff --git a/rslib/src/storage/tag/mod.rs b/rslib/src/storage/tag/mod.rs new file mode 100644 index 000000000..dcc703e0c --- /dev/null +++ b/rslib/src/storage/tag/mod.rs @@ -0,0 +1,86 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::SqliteStorage; +use crate::{err::Result, types::Usn}; +use rusqlite::{params, NO_PARAMS}; +use std::collections::HashMap; + +impl SqliteStorage { + pub(crate) fn all_tags(&self) -> Result> { + self.db + .prepare_cached("select tag, usn from tags")? + .query_and_then(NO_PARAMS, |row| -> Result<_> { + Ok((row.get(0)?, row.get(1)?)) + })? + .collect() + } + + pub(crate) fn register_tag(&self, tag: &str, usn: Usn) -> Result<()> { + self.db + .prepare_cached(include_str!("add.sql"))? + .execute(params![tag, usn])?; + Ok(()) + } + + pub(crate) fn preferred_tag_case(&self, tag: &str) -> Result> { + self.db + .prepare_cached("select tag from tags where tag = ?")? + .query_and_then(params![tag], |row| row.get(0))? + .next() + .transpose() + .map_err(Into::into) + } + + pub(crate) fn clear_tags(&self) -> Result<()> { + self.db.execute("delete from tags", NO_PARAMS)?; + Ok(()) + } + + pub(crate) fn clear_tag_usns(&self) -> Result<()> { + self.db + .execute("update tags set usn = 0 where usn != 0", NO_PARAMS)?; + Ok(()) + } + + // fixme: in the future we could just register tags as part of the sync + // instead of sending the tag list separately + pub(crate) fn get_changed_tags(&self, usn: Usn) -> Result> { + let tags: Vec = self + .db + .prepare("select tag from tags where usn=-1")? + .query_map(NO_PARAMS, |row| row.get(0))? + .collect::>()?; + self.db + .execute("update tags set usn=? where usn=-1", &[&usn])?; + Ok(tags) + } + + // Upgrading/downgrading + + pub(super) fn upgrade_tags_to_schema12(&self) -> Result<()> { + let tags = self + .db + .query_row_and_then("select tags from col", NO_PARAMS, |row| { + let tags: Result> = + serde_json::from_str(row.get_raw(0).as_str()?).map_err(Into::into); + tags + })?; + for (tag, usn) in tags.into_iter() { + self.register_tag(&tag, usn)?; + } + self.db.execute_batch("update col set tags=''")?; + + Ok(()) + } + + pub(super) fn downgrade_tags_from_schema12(&self) -> Result<()> { + let alltags = self.all_tags()?; + let tagsmap: HashMap = alltags.into_iter().collect(); + self.db.execute( + "update col set tags=?", + params![serde_json::to_string(&tagsmap)?], + )?; + Ok(()) + } +} diff --git a/rslib/src/tags.rs b/rslib/src/tags.rs new file mode 100644 index 000000000..92267597c --- /dev/null +++ b/rslib/src/tags.rs @@ -0,0 +1,61 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::collection::Collection; +use crate::err::Result; +use crate::types::Usn; +use std::{borrow::Cow, collections::HashSet}; + +fn split_tags(tags: &str) -> impl Iterator { + tags.split(|c| c == ' ' || c == '\u{3000}') + .filter(|tag| !tag.is_empty()) +} + +impl Collection { + /// Given a space-separated list of tags, fix case, ordering and duplicates. + /// Returns true if any new tags were added. + pub(crate) fn canonify_tags(&self, tags: &str, usn: Usn) -> Result<(String, bool)> { + let mut tagset = HashSet::new(); + let mut added = false; + + for tag in split_tags(tags) { + let tag = self.register_tag(tag, usn)?; + if matches!(tag, Cow::Borrowed(_)) { + added = true; + } + tagset.insert(tag); + } + + if tagset.is_empty() { + return Ok(("".into(), added)); + } + + let mut tags = tagset.into_iter().collect::>(); + tags.sort_unstable(); + + Ok((format!(" {} ", tags.join(" ")), added)) + } + + pub(crate) fn register_tag<'a>(&self, tag: &'a str, usn: Usn) -> Result> { + if let Some(preferred) = self.storage.preferred_tag_case(tag)? { + Ok(preferred.into()) + } else { + self.storage.register_tag(tag, usn)?; + Ok(tag.into()) + } + } + + pub(crate) fn register_tags(&self, tags: &str, usn: Usn, clear_first: bool) -> Result { + let mut changed = false; + if clear_first { + self.storage.clear_tags()?; + } + for tag in split_tags(tags) { + let tag = self.register_tag(tag, usn)?; + if matches!(tag, Cow::Borrowed(_)) { + changed = true; + } + } + Ok(changed) + } +}