From 43f512992d04d683fcb62c292b67743ac061d7e4 Mon Sep 17 00:00:00 2001 From: evandrocoan Date: Wed, 18 Mar 2020 20:30:44 -0300 Subject: [PATCH 001/150] Added box-sizing: border-box; to typeans by default https://anki.tenderapp.com/discussions/beta-testing/1854-using-margin-auto-causes-horizontal-scrollbar-on-typesomething --- qt/ts/scss/reviewer.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qt/ts/scss/reviewer.scss b/qt/ts/scss/reviewer.scss index c9bf06d6d..220d286d2 100644 --- a/qt/ts/scss/reviewer.scss +++ b/qt/ts/scss/reviewer.scss @@ -45,6 +45,8 @@ img { #typeans { width: 100%; + // https://anki.tenderapp.com/discussions/beta-testing/1854-using-margin-auto-causes-horizontal-scrollbar-on-typesomething + box-sizing: border-box; } .typeGood { From 617b18ff4926a3a073ab004db3f6ed72f7db7771 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 19 Mar 2020 10:20:42 +1000 Subject: [PATCH 002/150] correctly handle NFD content in media DB from older Anki versions --- rslib/src/media/sync.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rslib/src/media/sync.rs b/rslib/src/media/sync.rs index 5bf7dbcc6..88a879bb0 100644 --- a/rslib/src/media/sync.rs +++ b/rslib/src/media/sync.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use std::io::{Read, Write}; use std::path::Path; use std::{io, time}; +use unicode_normalization::is_nfc; static SYNC_MAX_FILES: usize = 25; static SYNC_MAX_BYTES: usize = (2.5 * 1024.0 * 1024.0) as usize; @@ -717,6 +718,16 @@ fn zip_files<'a>( break; } + #[cfg(target_vendor = "apple")] + { + if !is_nfc(&file.fname) { + // older Anki versions stored non-normalized filenames in the DB; clean them up + debug!(log, "clean up non-nfc entry"; "fname"=>&file.fname); + invalid_entries.push(&file.fname); + continue; + } + } + let file_data = if file.sha1.is_some() { match data_for_file(media_folder, &file.fname) { Ok(data) => data, From f617760d0493760a3651d5630418189580d18b2f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 19 Mar 2020 10:46:01 +1000 Subject: [PATCH 003/150] bump version --- meta/version | 2 +- rslib/Cargo.toml | 2 +- rspy/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/version b/meta/version index a62db9601..26412a859 100644 --- a/meta/version +++ b/meta/version @@ -1 +1 @@ -2.1.22 +2.1.24 diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index e956fd2d3..4d4f51e86 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "anki" -version = "2.1.22" # automatically updated +version = "2.1.24" # automatically updated edition = "2018" authors = ["Ankitects Pty Ltd and contributors"] license = "AGPL-3.0-or-later" diff --git a/rspy/Cargo.toml b/rspy/Cargo.toml index 11cd4f41e..5b593acb2 100644 --- a/rspy/Cargo.toml +++ b/rspy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ankirspy" -version = "2.1.22" # automatically updated +version = "2.1.24" # automatically updated edition = "2018" authors = ["Ankitects Pty Ltd and contributors"] From e3a57a41936d57314cb72996955aaaf45f313c9c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 19 Mar 2020 10:58:52 +1000 Subject: [PATCH 004/150] fix clippy lint on other platforms --- rslib/src/media/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/src/media/sync.rs b/rslib/src/media/sync.rs index 88a879bb0..7717d8b4e 100644 --- a/rslib/src/media/sync.rs +++ b/rslib/src/media/sync.rs @@ -20,7 +20,6 @@ use std::collections::HashMap; use std::io::{Read, Write}; use std::path::Path; use std::{io, time}; -use unicode_normalization::is_nfc; static SYNC_MAX_FILES: usize = 25; static SYNC_MAX_BYTES: usize = (2.5 * 1024.0 * 1024.0) as usize; @@ -720,6 +719,7 @@ fn zip_files<'a>( #[cfg(target_vendor = "apple")] { + use unicode_normalization::is_nfc; if !is_nfc(&file.fname) { // older Anki versions stored non-normalized filenames in the DB; clean them up debug!(log, "clean up non-nfc entry"; "fname"=>&file.fname); From e4ae41340f4bc8b91a479bccc610cc9af81b210b Mon Sep 17 00:00:00 2001 From: Arthur Milchior Date: Fri, 6 Mar 2020 16:25:02 +0100 Subject: [PATCH 005/150] Hook to decide whether a note should be added. --- qt/aqt/addcards.py | 6 +++++- qt/aqt/gui_hooks.py | 39 +++++++++++++++++++++++++++++++++++++++ qt/tools/genhooks_gui.py | 12 ++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/qt/aqt/addcards.py b/qt/aqt/addcards.py index be6308e30..d2adc2966 100644 --- a/qt/aqt/addcards.py +++ b/qt/aqt/addcards.py @@ -167,8 +167,12 @@ class AddCards(QDialog): def addNote(self, note) -> Optional[Note]: note.model()["did"] = self.deckChooser.selectedId() ret = note.dupeOrEmpty() + rejection = None if ret == 1: - showWarning(_("The first field is empty."), help="AddItems#AddError") + rejection = _("The first field is empty.") + rejection = gui_hooks.add_card_accepts(rejection, note) + if rejection is not None: + showWarning(rejection, help="AddItems#AddError") return None if "{{cloze:" in note.model()["tmpls"][0]["qfmt"]: if not self.mw.col.models._availClozeOrds( diff --git a/qt/aqt/gui_hooks.py b/qt/aqt/gui_hooks.py index eed241f7e..7262d2198 100644 --- a/qt/aqt/gui_hooks.py +++ b/qt/aqt/gui_hooks.py @@ -23,6 +23,45 @@ from aqt.qt import QDialog, QMenu # @@AUTOGEN@@ +class _AddCardAcceptsFilter: + """Decides whether the note should be added to the collection or + not. It is assumed to come from the addCards window. + + reason_to_already_reject is the first reason to reject that + was found, or None. If your filter wants to reject, it should + replace return the reason to reject. Otherwise return the + input.""" + + _hooks: List[Callable[[Optional[str], "anki.notes.Note"], Optional[str]]] = [] + + def append( + self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] + ) -> None: + """(reason_to_already_reject: Optional[str], note: anki.notes.Note)""" + self._hooks.append(cb) + + def remove( + self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] + ) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__( + self, reason_to_already_reject: Optional[str], note: anki.notes.Note + ) -> Optional[str]: + for filter in self._hooks: + try: + reason_to_already_reject = filter(reason_to_already_reject, note) + except: + # if the hook fails, remove it + self._hooks.remove(filter) + raise + return reason_to_already_reject + + +add_card_accepts = _AddCardAcceptsFilter() + + class _AddCardsDidAddNoteHook: _hooks: List[Callable[["anki.notes.Note"], None]] = [] diff --git a/qt/tools/genhooks_gui.py b/qt/tools/genhooks_gui.py index dcf543979..011ca9ace 100644 --- a/qt/tools/genhooks_gui.py +++ b/qt/tools/genhooks_gui.py @@ -412,6 +412,18 @@ def emptyNewCard(): args=["note: anki.notes.Note"], legacy_hook="AddCards.noteAdded", ), + Hook( + name="add_card_accepts", + args=["reason_to_already_reject: Optional[str]", "note: anki.notes.Note"], + return_type="Optional[str]", + doc="""Decides whether the note should be added to the collection or + not. It is assumed to come from the addCards window. + + reason_to_already_reject is the first reason to reject that + was found, or None. If your filter wants to reject, it should + replace return the reason to reject. Otherwise return the + input.""", + ), # Editing ################### Hook( From b73507344c57b0d9b55f291e18a261f1abb88bea Mon Sep 17 00:00:00 2001 From: Arthur Milchior Date: Thu, 19 Mar 2020 12:03:09 +0100 Subject: [PATCH 006/150] Hook models_advanced_will_show This will be useful for add-on 1863928230. I want to let users change the LaTeX footer/header everywhere. --- qt/aqt/gui_hooks.py | 24 ++++++++++++++++++++++++ qt/aqt/models.py | 3 ++- qt/tools/genhooks_gui.py | 3 +++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/qt/aqt/gui_hooks.py b/qt/aqt/gui_hooks.py index eed241f7e..50f6861a5 100644 --- a/qt/aqt/gui_hooks.py +++ b/qt/aqt/gui_hooks.py @@ -1206,6 +1206,30 @@ class _MediaSyncDidStartOrStopHook: media_sync_did_start_or_stop = _MediaSyncDidStartOrStopHook() +class _ModelsAdvancedWillShowHook: + _hooks: List[Callable[[QDialog], None]] = [] + + def append(self, cb: Callable[[QDialog], None]) -> None: + """(advanced: QDialog)""" + self._hooks.append(cb) + + def remove(self, cb: Callable[[QDialog], None]) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__(self, advanced: QDialog) -> None: + for hook in self._hooks: + try: + hook(advanced) + except: + # if the hook fails, remove it + self._hooks.remove(hook) + raise + + +models_advanced_will_show = _ModelsAdvancedWillShowHook() + + class _OverviewDidRefreshHook: """Allow to update the overview window. E.g. add the deck name in the title.""" diff --git a/qt/aqt/models.py b/qt/aqt/models.py index 86f7a4e96..e23d081d9 100644 --- a/qt/aqt/models.py +++ b/qt/aqt/models.py @@ -6,7 +6,7 @@ from operator import itemgetter import aqt.clayout from anki import stdmodels from anki.lang import _, ngettext -from aqt import AnkiQt +from aqt import AnkiQt, gui_hooks from aqt.qt import * from aqt.utils import ( askUser, @@ -124,6 +124,7 @@ class Models(QDialog): d.setWindowTitle(_("Options for %s") % self.model["name"]) frm.buttonBox.helpRequested.connect(lambda: openHelp("latex")) restoreGeom(d, "modelopts") + gui_hooks.models_advanced_will_show(d) d.exec_() saveGeom(d, "modelopts") self.model["latexsvg"] = frm.latexsvg.isChecked() diff --git a/qt/tools/genhooks_gui.py b/qt/tools/genhooks_gui.py index dcf543979..73fa1c8f0 100644 --- a/qt/tools/genhooks_gui.py +++ b/qt/tools/genhooks_gui.py @@ -503,6 +503,9 @@ def emptyNewCard(): args=["dialog: aqt.addons.AddonsDialog", "add_on: aqt.addons.AddonMeta"], doc="""Allows doing an action when a single add-on is selected.""", ), + # Model + ################### + Hook(name="models_advanced_will_show", args=["advanced: QDialog"],), # Other ################### Hook( From bfc305fa2668438143c5e53eaa4e8c00aeea5098 Mon Sep 17 00:00:00 2001 From: Arthur Milchior Date: Thu, 19 Mar 2020 13:20:06 +0100 Subject: [PATCH 007/150] hook backup_is_done I expect to use this hook in add-on 529955533. It'll serve me to create long term backup at the same time than your backup --- qt/aqt/gui_hooks.py | 24 ++++++++++++++++++++++++ qt/aqt/main.py | 1 + qt/tools/genhooks_gui.py | 1 + 3 files changed, 26 insertions(+) diff --git a/qt/aqt/gui_hooks.py b/qt/aqt/gui_hooks.py index eed241f7e..01c415ba9 100644 --- a/qt/aqt/gui_hooks.py +++ b/qt/aqt/gui_hooks.py @@ -272,6 +272,30 @@ class _AvPlayerWillPlayHook: av_player_will_play = _AvPlayerWillPlayHook() +class _BackupIsDoneHook: + _hooks: List[Callable[[], None]] = [] + + def append(self, cb: Callable[[], None]) -> None: + """()""" + self._hooks.append(cb) + + def remove(self, cb: Callable[[], None]) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__(self) -> None: + for hook in self._hooks: + try: + hook() + except: + # if the hook fails, remove it + self._hooks.remove(hook) + raise + + +backup_is_done = _BackupIsDoneHook() + + class _BrowserDidChangeRowHook: _hooks: List[Callable[["aqt.browser.Browser"], None]] = [] diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 3f3034bb5..013bdf5e4 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -561,6 +561,7 @@ from the profile screen." fname = backups.pop(0) path = os.path.join(dir, fname) os.unlink(path) + gui_hooks.backup_is_done() def maybeOptimize(self) -> None: # have two weeks passed? diff --git a/qt/tools/genhooks_gui.py b/qt/tools/genhooks_gui.py index dcf543979..6403950e1 100644 --- a/qt/tools/genhooks_gui.py +++ b/qt/tools/genhooks_gui.py @@ -341,6 +341,7 @@ hooks = [ ), # Main ################### + Hook(name="backup_is_done"), Hook(name="profile_did_open", legacy_hook="profileLoaded"), Hook(name="profile_will_close", legacy_hook="unloadProfile"), Hook( From 97225a0364b8dd1670770cb571c150ade4c5a87b Mon Sep 17 00:00:00 2001 From: Arthur Milchior Date: Thu, 19 Mar 2020 22:59:59 +0100 Subject: [PATCH 008/150] Browser introduce method time_format I expect to use this in my add-on 1243668133 (by monkey patching in this case) where it would allow to show hours/minutes/seconds... in browser --- qt/aqt/browser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 3b7c900f7..eb74e3f73 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -275,6 +275,9 @@ class DataModel(QAbstractTableModel): def columnType(self, column): return self.activeCols[column] + def time_format(self): + return "%Y-%m-%d" + def columnData(self, index): row = index.row() col = index.column() @@ -302,11 +305,11 @@ class DataModel(QAbstractTableModel): t = "(" + t + ")" return t elif type == "noteCrt": - return time.strftime("%Y-%m-%d", time.localtime(c.note().id / 1000)) + return time.strftime(self.time_format(), time.localtime(c.note().id / 1000)) elif type == "noteMod": - return time.strftime("%Y-%m-%d", time.localtime(c.note().mod)) + return time.strftime(self.time_format(), time.localtime(c.note().mod)) elif type == "cardMod": - return time.strftime("%Y-%m-%d", time.localtime(c.mod)) + return time.strftime(self.time_format(), time.localtime(c.mod)) elif type == "cardReps": return str(c.reps) elif type == "cardLapses": @@ -363,7 +366,7 @@ class DataModel(QAbstractTableModel): date = time.time() + ((c.due - self.col.sched.today) * 86400) else: return "" - return time.strftime("%Y-%m-%d", time.localtime(date)) + return time.strftime(self.time_format(), time.localtime(date)) def isRTL(self, index): col = index.column() From a3473351aa23b01fd514d61b95492ea86d1d7ede Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 20:33:12 +1000 Subject: [PATCH 009/150] ftl git attributes --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..e4d2bbbf7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.ftl eol=lf From 0d43e9dca36147bbfe4c9691eceeeacfc6a76e95 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 20:59:59 +1000 Subject: [PATCH 010/150] tweak add-on wording --- qt/aqt/addcards.py | 10 ++--- qt/aqt/gui_hooks.py | 80 ++++++++++++++++++++-------------------- qt/aqt/main.py | 2 +- qt/tools/genhooks_gui.py | 6 +-- 4 files changed, 48 insertions(+), 50 deletions(-) diff --git a/qt/aqt/addcards.py b/qt/aqt/addcards.py index d2adc2966..efe20e9b2 100644 --- a/qt/aqt/addcards.py +++ b/qt/aqt/addcards.py @@ -167,12 +167,12 @@ class AddCards(QDialog): def addNote(self, note) -> Optional[Note]: note.model()["did"] = self.deckChooser.selectedId() ret = note.dupeOrEmpty() - rejection = None + problem = None if ret == 1: - rejection = _("The first field is empty.") - rejection = gui_hooks.add_card_accepts(rejection, note) - if rejection is not None: - showWarning(rejection, help="AddItems#AddError") + problem = _("The first field is empty.") + problem = gui_hooks.add_cards_will_add_note(problem, note) + if problem is not None: + showWarning(problem, help="AddItems#AddError") return None if "{{cloze:" in note.model()["tmpls"][0]["qfmt"]: if not self.mw.col.models._availClozeOrds( diff --git a/qt/aqt/gui_hooks.py b/qt/aqt/gui_hooks.py index cc7fce494..30a9ec0a7 100644 --- a/qt/aqt/gui_hooks.py +++ b/qt/aqt/gui_hooks.py @@ -23,45 +23,6 @@ from aqt.qt import QDialog, QMenu # @@AUTOGEN@@ -class _AddCardAcceptsFilter: - """Decides whether the note should be added to the collection or - not. It is assumed to come from the addCards window. - - reason_to_already_reject is the first reason to reject that - was found, or None. If your filter wants to reject, it should - replace return the reason to reject. Otherwise return the - input.""" - - _hooks: List[Callable[[Optional[str], "anki.notes.Note"], Optional[str]]] = [] - - def append( - self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] - ) -> None: - """(reason_to_already_reject: Optional[str], note: anki.notes.Note)""" - self._hooks.append(cb) - - def remove( - self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] - ) -> None: - if cb in self._hooks: - self._hooks.remove(cb) - - def __call__( - self, reason_to_already_reject: Optional[str], note: anki.notes.Note - ) -> Optional[str]: - for filter in self._hooks: - try: - reason_to_already_reject = filter(reason_to_already_reject, note) - except: - # if the hook fails, remove it - self._hooks.remove(filter) - raise - return reason_to_already_reject - - -add_card_accepts = _AddCardAcceptsFilter() - - class _AddCardsDidAddNoteHook: _hooks: List[Callable[["anki.notes.Note"], None]] = [] @@ -88,6 +49,43 @@ class _AddCardsDidAddNoteHook: add_cards_did_add_note = _AddCardsDidAddNoteHook() +class _AddCardsWillAddNoteFilter: + """Decides whether the note should be added to the collection or + not. It is assumed to come from the addCards window. + + reason_to_already_reject is the first reason to reject that + was found, or None. If your filter wants to reject, it should + replace return the reason to reject. Otherwise return the + input.""" + + _hooks: List[Callable[[Optional[str], "anki.notes.Note"], Optional[str]]] = [] + + def append( + self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] + ) -> None: + """(problem: Optional[str], note: anki.notes.Note)""" + self._hooks.append(cb) + + def remove( + self, cb: Callable[[Optional[str], "anki.notes.Note"], Optional[str]] + ) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__(self, problem: Optional[str], note: anki.notes.Note) -> Optional[str]: + for filter in self._hooks: + try: + problem = filter(problem, note) + except: + # if the hook fails, remove it + self._hooks.remove(filter) + raise + return problem + + +add_cards_will_add_note = _AddCardsWillAddNoteFilter() + + class _AddCardsWillShowHistoryMenuHook: _hooks: List[Callable[["aqt.addcards.AddCards", QMenu], None]] = [] @@ -311,7 +309,7 @@ class _AvPlayerWillPlayHook: av_player_will_play = _AvPlayerWillPlayHook() -class _BackupIsDoneHook: +class _BackupDidCompleteHook: _hooks: List[Callable[[], None]] = [] def append(self, cb: Callable[[], None]) -> None: @@ -332,7 +330,7 @@ class _BackupIsDoneHook: raise -backup_is_done = _BackupIsDoneHook() +backup_did_complete = _BackupDidCompleteHook() class _BrowserDidChangeRowHook: diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 013bdf5e4..49e418f06 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -561,7 +561,7 @@ from the profile screen." fname = backups.pop(0) path = os.path.join(dir, fname) os.unlink(path) - gui_hooks.backup_is_done() + gui_hooks.backup_did_complete() def maybeOptimize(self) -> None: # have two weeks passed? diff --git a/qt/tools/genhooks_gui.py b/qt/tools/genhooks_gui.py index 5a91089e8..21f8de560 100644 --- a/qt/tools/genhooks_gui.py +++ b/qt/tools/genhooks_gui.py @@ -341,7 +341,7 @@ hooks = [ ), # Main ################### - Hook(name="backup_is_done"), + Hook(name="backup_did_complete"), Hook(name="profile_did_open", legacy_hook="profileLoaded"), Hook(name="profile_will_close", legacy_hook="unloadProfile"), Hook( @@ -414,8 +414,8 @@ def emptyNewCard(): legacy_hook="AddCards.noteAdded", ), Hook( - name="add_card_accepts", - args=["reason_to_already_reject: Optional[str]", "note: anki.notes.Note"], + name="add_cards_will_add_note", + args=["problem: Optional[str]", "note: anki.notes.Note"], return_type="Optional[str]", doc="""Decides whether the note should be added to the collection or not. It is assumed to come from the addCards window. From c1252d68f011fa3c89db7e27f5dc49db21a2c980 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 2 Mar 2020 14:13:57 +1000 Subject: [PATCH 011/150] clone db.py into dbproxy.py --- pylib/anki/collection.py | 8 +-- pylib/anki/dbproxy.py | 112 +++++++++++++++++++++++++++++++++++++++ pylib/anki/storage.py | 16 +++--- pylib/anki/utils.py | 6 +-- 4 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 pylib/anki/dbproxy.py diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 01f96fa8c..b21a78e26 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -23,7 +23,7 @@ import anki.template from anki import hooks from anki.cards import Card from anki.consts import * -from anki.db import DB +from anki.dbproxy import DBProxy from anki.decks import DeckManager from anki.errors import AnkiError from anki.lang import _, ngettext @@ -67,7 +67,7 @@ defaultConf = { # this is initialized by storage.Collection class _Collection: - db: Optional[DB] + db: Optional[DBProxy] sched: Union[V1Scheduler, V2Scheduler] crt: int mod: int @@ -80,7 +80,7 @@ class _Collection: def __init__( self, - db: DB, + db: DBProxy, backend: RustBackend, server: Optional["anki.storage.ServerData"] = None, log: bool = False, @@ -267,7 +267,7 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", def reopen(self) -> None: "Reconnect to DB (after changing threads, etc)." if not self.db: - self.db = DB(self.path) + self.db = DBProxy(self.path) self.media.connect() self._openLog() diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py new file mode 100644 index 000000000..1d11e2a03 --- /dev/null +++ b/pylib/anki/dbproxy.py @@ -0,0 +1,112 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import os +import time +from sqlite3 import Cursor +from sqlite3 import dbapi2 as sqlite +from typing import Any, List, Type + + +class DBProxy: + def __init__(self, path: str, timeout: int = 0) -> None: + self._db = sqlite.connect(path, timeout=timeout) + self._db.text_factory = self._textFactory + self._path = path + self.echo = os.environ.get("DBECHO") + self.mod = False + + def execute(self, sql: str, *a, **ka) -> Cursor: + s = sql.strip().lower() + # mark modified? + for stmt in "insert", "update", "delete": + if s.startswith(stmt): + self.mod = True + t = time.time() + if ka: + # execute("...where id = :id", id=5) + res = self._db.execute(sql, ka) + else: + # execute("...where id = ?", 5) + res = self._db.execute(sql, a) + if self.echo: + # print a, ka + print(sql, "%0.3fms" % ((time.time() - t) * 1000)) + if self.echo == "2": + print(a, ka) + return res + + def executemany(self, sql: str, l: Any) -> None: + self.mod = True + t = time.time() + self._db.executemany(sql, l) + if self.echo: + print(sql, "%0.3fms" % ((time.time() - t) * 1000)) + if self.echo == "2": + print(l) + + def commit(self) -> None: + t = time.time() + self._db.commit() + if self.echo: + print("commit %0.3fms" % ((time.time() - t) * 1000)) + + def executescript(self, sql: str) -> None: + self.mod = True + if self.echo: + print(sql) + self._db.executescript(sql) + + def rollback(self) -> None: + self._db.rollback() + + def scalar(self, *a, **kw) -> Any: + res = self.execute(*a, **kw).fetchone() + if res: + return res[0] + return None + + def all(self, *a, **kw) -> List: + return self.execute(*a, **kw).fetchall() + + def first(self, *a, **kw) -> Any: + c = self.execute(*a, **kw) + res = c.fetchone() + c.close() + return res + + def list(self, *a, **kw) -> List: + return [x[0] for x in self.execute(*a, **kw)] + + def close(self) -> None: + self._db.text_factory = None + self._db.close() + + def set_progress_handler(self, *args) -> None: + self._db.set_progress_handler(*args) + + def __enter__(self) -> "DBProxy": + self._db.execute("begin") + return self + + def __exit__(self, exc_type, *args) -> None: + self._db.close() + + def totalChanges(self) -> Any: + return self._db.total_changes + + def interrupt(self) -> None: + self._db.interrupt() + + def setAutocommit(self, autocommit: bool) -> None: + if autocommit: + self._db.isolation_level = None + else: + self._db.isolation_level = "" + + # strip out invalid utf-8 when reading from db + def _textFactory(self, data: bytes) -> str: + return str(data, errors="ignore") + + def cursor(self, factory: Type[Cursor] = Cursor) -> Cursor: + return self._db.cursor(factory) diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index 665291cfd..0a1001466 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional, Tuple from anki.collection import _Collection from anki.consts import * -from anki.db import DB +from anki.dbproxy import DBProxy from anki.lang import _ from anki.media import media_paths_from_col_path from anki.rsbackend import RustBackend @@ -44,7 +44,7 @@ def Collection( for c in ("/", ":", "\\"): assert c not in base # connect - db = DB(path) + db = DBProxy(path) db.setAutocommit(True) if create: ver = _createDB(db) @@ -78,7 +78,7 @@ def Collection( return col -def _upgradeSchema(db: DB) -> Any: +def _upgradeSchema(db: DBProxy) -> Any: ver = db.scalar("select ver from col") if ver == SCHEMA_VERSION: return ver @@ -238,7 +238,7 @@ def _upgradeClozeModel(col, m) -> None: ###################################################################### -def _createDB(db: DB) -> int: +def _createDB(db: DBProxy) -> int: db.execute("pragma page_size = 4096") db.execute("pragma legacy_file_format = 0") db.execute("vacuum") @@ -248,7 +248,7 @@ def _createDB(db: DB) -> int: return SCHEMA_VERSION -def _addSchema(db: DB, setColConf: bool = True) -> None: +def _addSchema(db: DBProxy, setColConf: bool = True) -> None: db.executescript( """ create table if not exists col ( @@ -329,7 +329,7 @@ values(1,0,0,%(s)s,%(v)s,0,0,0,'','{}','','','{}'); _addColVars(db, *_getColVars(db)) -def _getColVars(db: DB) -> Tuple[Any, Any, Dict[str, Any]]: +def _getColVars(db: DBProxy) -> Tuple[Any, Any, Dict[str, Any]]: import anki.collection import anki.decks @@ -344,7 +344,7 @@ def _getColVars(db: DB) -> Tuple[Any, Any, Dict[str, Any]]: def _addColVars( - db: DB, g: Dict[str, Any], gc: Dict[str, Any], c: Dict[str, Any] + db: DBProxy, g: Dict[str, Any], gc: Dict[str, Any], c: Dict[str, Any] ) -> None: db.execute( """ @@ -355,7 +355,7 @@ update col set conf = ?, decks = ?, dconf = ?""", ) -def _updateIndices(db: DB) -> None: +def _updateIndices(db: DBProxy) -> None: "Add indices to the DB." db.executescript( """ diff --git a/pylib/anki/utils.py b/pylib/anki/utils.py index d99437e69..33c4ebe08 100644 --- a/pylib/anki/utils.py +++ b/pylib/anki/utils.py @@ -22,7 +22,7 @@ from hashlib import sha1 from html.entities import name2codepoint from typing import Iterable, Iterator, List, Optional, Union -from anki.db import DB +from anki.dbproxy import DBProxy _tmpdir: Optional[str] @@ -142,7 +142,7 @@ def ids2str(ids: Iterable[Union[int, str]]) -> str: return "(%s)" % ",".join(str(i) for i in ids) -def timestampID(db: DB, table: str) -> int: +def timestampID(db: DBProxy, table: str) -> int: "Return a non-conflicting timestamp for table." # be careful not to create multiple objects without flushing them, or they # may share an ID. @@ -152,7 +152,7 @@ def timestampID(db: DB, table: str) -> int: return t -def maxID(db: DB) -> int: +def maxID(db: DBProxy) -> int: "Return the first safe ID to use." now = intTime(1000) for tbl in "cards", "notes": From 8ef28f85716cb46d1d7d2e4578de0edd8aec4e82 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 2 Mar 2020 20:11:33 +1000 Subject: [PATCH 012/150] drop echo and text factory --- pylib/anki/dbproxy.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 1d11e2a03..b104ec1b0 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -1,6 +1,8 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +# fixme: lossy utf8 handling + import os import time from sqlite3 import Cursor @@ -11,9 +13,7 @@ from typing import Any, List, Type class DBProxy: def __init__(self, path: str, timeout: int = 0) -> None: self._db = sqlite.connect(path, timeout=timeout) - self._db.text_factory = self._textFactory self._path = path - self.echo = os.environ.get("DBECHO") self.mod = False def execute(self, sql: str, *a, **ka) -> Cursor: @@ -29,32 +29,19 @@ class DBProxy: else: # execute("...where id = ?", 5) res = self._db.execute(sql, a) - if self.echo: - # print a, ka - print(sql, "%0.3fms" % ((time.time() - t) * 1000)) - if self.echo == "2": - print(a, ka) return res def executemany(self, sql: str, l: Any) -> None: self.mod = True t = time.time() self._db.executemany(sql, l) - if self.echo: - print(sql, "%0.3fms" % ((time.time() - t) * 1000)) - if self.echo == "2": - print(l) def commit(self) -> None: t = time.time() self._db.commit() - if self.echo: - print("commit %0.3fms" % ((time.time() - t) * 1000)) def executescript(self, sql: str) -> None: self.mod = True - if self.echo: - print(sql) self._db.executescript(sql) def rollback(self) -> None: @@ -104,9 +91,5 @@ class DBProxy: else: self._db.isolation_level = "" - # strip out invalid utf-8 when reading from db - def _textFactory(self, data: bytes) -> str: - return str(data, errors="ignore") - def cursor(self, factory: Type[Cursor] = Cursor) -> Cursor: return self._db.cursor(factory) From c8b9afac0cb69b5474a7551f48da9b67a8b74050 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 2 Mar 2020 20:19:40 +1000 Subject: [PATCH 013/150] drop progress handler and timeout arg --- pylib/anki/dbproxy.py | 7 ++----- qt/aqt/progress.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index b104ec1b0..153e1effc 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -11,8 +11,8 @@ from typing import Any, List, Type class DBProxy: - def __init__(self, path: str, timeout: int = 0) -> None: - self._db = sqlite.connect(path, timeout=timeout) + def __init__(self, path: str) -> None: + self._db = sqlite.connect(path, timeout=0) self._path = path self.mod = False @@ -69,9 +69,6 @@ class DBProxy: self._db.text_factory = None self._db.close() - def set_progress_handler(self, *args) -> None: - self._db.set_progress_handler(*args) - def __enter__(self) -> "DBProxy": self._db.execute("begin") return self diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index 35d86bd25..e16697614 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -35,7 +35,7 @@ class ProgressManager: "Install a handler in the current DB." self.lastDbProgress = 0 self.inDB = False - db.set_progress_handler(self._dbProgress, 10000) + #db.set_progress_handler(self._dbProgress, 10000) def _dbProgress(self): "Called from SQLite." From f4d4078537dc350ac9b8d788166bf8b866b597bb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 2 Mar 2020 20:50:17 +1000 Subject: [PATCH 014/150] drop named sql arguments --- pylib/anki/dbproxy.py | 32 ++++++++++----------------- pylib/anki/sched.py | 4 ++-- pylib/anki/schedv2.py | 4 ++-- pylib/anki/stats.py | 51 +++++++++++++++++++++++-------------------- pylib/anki/tags.py | 23 ++++++++----------- qt/aqt/main.py | 5 +---- qt/aqt/progress.py | 2 +- 7 files changed, 53 insertions(+), 68 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 153e1effc..8c597c9e2 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -2,8 +2,8 @@ # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html # fixme: lossy utf8 handling +# fixme: progress -import os import time from sqlite3 import Cursor from sqlite3 import dbapi2 as sqlite @@ -16,19 +16,13 @@ class DBProxy: self._path = path self.mod = False - def execute(self, sql: str, *a, **ka) -> Cursor: + def execute(self, sql: str, *args) -> Cursor: s = sql.strip().lower() # mark modified? for stmt in "insert", "update", "delete": if s.startswith(stmt): self.mod = True - t = time.time() - if ka: - # execute("...where id = :id", id=5) - res = self._db.execute(sql, ka) - else: - # execute("...where id = ?", 5) - res = self._db.execute(sql, a) + res = self._db.execute(sql, args) return res def executemany(self, sql: str, l: Any) -> None: @@ -47,26 +41,25 @@ class DBProxy: def rollback(self) -> None: self._db.rollback() - def scalar(self, *a, **kw) -> Any: - res = self.execute(*a, **kw).fetchone() + def scalar(self, sql: str, *args) -> Any: + res = self.execute(sql, *args).fetchone() if res: return res[0] return None - def all(self, *a, **kw) -> List: - return self.execute(*a, **kw).fetchall() + def all(self, sql: str, *args) -> List: + return self.execute(sql, *args).fetchall() - def first(self, *a, **kw) -> Any: - c = self.execute(*a, **kw) + def first(self, sql: str, *args) -> Any: + c = self.execute(sql, *args) res = c.fetchone() c.close() return res - def list(self, *a, **kw) -> List: - return [x[0] for x in self.execute(*a, **kw)] + def list(self, sql: str, *args) -> List: + return [x[0] for x in self.execute(sql, *args)] def close(self) -> None: - self._db.text_factory = None self._db.close() def __enter__(self) -> "DBProxy": @@ -79,9 +72,6 @@ class DBProxy: def totalChanges(self) -> Any: return self._db.total_changes - def interrupt(self) -> None: - self._db.interrupt() - def setAutocommit(self, autocommit: bool) -> None: if autocommit: self._db.isolation_level = None diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index 91d531bb8..982da8b59 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -286,10 +286,10 @@ and due <= ? limit %d""" self._lrnQueue = self.col.db.all( f""" select due, id from cards where -did in %s and queue = {QUEUE_TYPE_LRN} and due < :lim +did in %s and queue = {QUEUE_TYPE_LRN} and due < ? limit %d""" % (self._deckLimit(), self.reportLimit), - lim=self.dayCutoff, + self.dayCutoff, ) # as it arrives sorted by did first, we need to sort it self._lrnQueue.sort() diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 68275a892..d11f7808d 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -545,10 +545,10 @@ select count() from cards where did in %s and queue = {QUEUE_TYPE_PREVIEW} self._lrnQueue = self.col.db.all( f""" select due, id from cards where -did in %s and queue in ({QUEUE_TYPE_LRN},{QUEUE_TYPE_PREVIEW}) and due < :lim +did in %s and queue in ({QUEUE_TYPE_LRN},{QUEUE_TYPE_PREVIEW}) and due < ? limit %d""" % (self._deckLimit(), self.reportLimit), - lim=cutoff, + cutoff, ) # as it arrives sorted by did first, we need to sort it self._lrnQueue.sort() diff --git a/pylib/anki/stats.py b/pylib/anki/stats.py index c8ffa2c0b..69ab0c415 100644 --- a/pylib/anki/stats.py +++ b/pylib/anki/stats.py @@ -58,7 +58,7 @@ class CardStats: self.addLine(_("Reviews"), "%d" % c.reps) self.addLine(_("Lapses"), "%d" % c.lapses) (cnt, total) = self.col.db.first( - "select count(), sum(time)/1000 from revlog where cid = :id", id=c.id + "select count(), sum(time)/1000 from revlog where cid = ?", c.id ) if cnt: self.addLine(_("Average Time"), self.time(total / float(cnt))) @@ -297,12 +297,12 @@ and due = ?""" ) -> Any: lim = "" if start is not None: - lim += " and due-:today >= %d" % start + lim += " and due-%d >= %d" % (self.col.sched.today, start) if end is not None: lim += " and day < %d" % end return self.col.db.all( f""" -select (due-:today)/:chunk as day, +select (due-?)/? as day, sum(case when ivl < 21 then 1 else 0 end), -- yng sum(case when ivl >= 21 then 1 else 0 end) -- mtr from cards @@ -310,8 +310,8 @@ where did in %s and queue in ({QUEUE_TYPE_REV},{QUEUE_TYPE_DAY_LEARN_RELEARN}) %s group by day order by day""" % (self._limit(), lim), - today=self.col.sched.today, - chunk=chunk, + self.col.sched.today, + chunk, ) # Added, reps and time spent @@ -527,14 +527,13 @@ group by day order by day""" return self.col.db.all( """ select -(cast((id/1000.0 - :cut) / 86400.0 as int))/:chunk as day, +(cast((id/1000.0 - ?) / 86400.0 as int))/? as day, count(id) from cards %s group by day order by day""" % lim, - cut=self.col.sched.dayCutoff, - tf=tf, - chunk=chunk, + self.col.sched.dayCutoff, + chunk, ) def _done(self, num: Optional[int] = 7, chunk: int = 1) -> Any: @@ -557,24 +556,28 @@ group by day order by day""" return self.col.db.all( f""" select -(cast((id/1000.0 - :cut) / 86400.0 as int))/:chunk as day, +(cast((id/1000.0 - ?) / 86400.0 as int))/? as day, sum(case when type = {REVLOG_LRN} then 1 else 0 end), -- lrn count sum(case when type = {REVLOG_REV} and lastIvl < 21 then 1 else 0 end), -- yng count sum(case when type = {REVLOG_REV} and lastIvl >= 21 then 1 else 0 end), -- mtr count sum(case when type = {REVLOG_RELRN} then 1 else 0 end), -- lapse count sum(case when type = {REVLOG_CRAM} then 1 else 0 end), -- cram count -sum(case when type = {REVLOG_LRN} then time/1000.0 else 0 end)/:tf, -- lrn time +sum(case when type = {REVLOG_LRN} then time/1000.0 else 0 end)/?, -- lrn time -- yng + mtr time -sum(case when type = {REVLOG_REV} and lastIvl < 21 then time/1000.0 else 0 end)/:tf, -sum(case when type = {REVLOG_REV} and lastIvl >= 21 then time/1000.0 else 0 end)/:tf, -sum(case when type = {REVLOG_RELRN} then time/1000.0 else 0 end)/:tf, -- lapse time -sum(case when type = {REVLOG_CRAM} then time/1000.0 else 0 end)/:tf -- cram time +sum(case when type = {REVLOG_REV} and lastIvl < 21 then time/1000.0 else 0 end)/?, +sum(case when type = {REVLOG_REV} and lastIvl >= 21 then time/1000.0 else 0 end)/?, +sum(case when type = {REVLOG_RELRN} then time/1000.0 else 0 end)/?, -- lapse time +sum(case when type = {REVLOG_CRAM} then time/1000.0 else 0 end)/? -- cram time from revlog %s group by day order by day""" % lim, - cut=self.col.sched.dayCutoff, - tf=tf, - chunk=chunk, + self.col.sched.dayCutoff, + chunk, + tf, + tf, + tf, + tf, + tf, ) def _daysStudied(self) -> Any: @@ -592,11 +595,11 @@ group by day order by day""" ret = self.col.db.first( """ select count(), abs(min(day)) from (select -(cast((id/1000 - :cut) / 86400.0 as int)+1) as day +(cast((id/1000 - ?) / 86400.0 as int)+1) as day from revlog %s group by day order by day)""" % lim, - cut=self.col.sched.dayCutoff, + self.col.sched.dayCutoff, ) assert ret return ret @@ -655,12 +658,12 @@ group by day order by day)""" data = [ self.col.db.all( f""" -select ivl / :chunk as grp, count() from cards +select ivl / ? as grp, count() from cards where did in %s and queue = {QUEUE_TYPE_REV} %s group by grp order by grp""" % (self._limit(), lim), - chunk=chunk, + chunk, ) ] return ( @@ -866,14 +869,14 @@ order by thetype, ease""" return self.col.db.all( f""" select -23 - ((cast((:cut - id/1000) / 3600.0 as int)) %% 24) as hour, +23 - ((cast((? - id/1000) / 3600.0 as int)) %% 24) as hour, sum(case when ease = 1 then 0 else 1 end) / cast(count() as float) * 100, count() from revlog where type in ({REVLOG_LRN},{REVLOG_REV},{REVLOG_RELRN}) %s group by hour having count() > 30 order by hour""" % lim, - cut=self.col.sched.dayCutoff - (rolloverHour * 3600), + self.col.sched.dayCutoff - (rolloverHour * 3600), ) # Cards diff --git a/pylib/anki/tags.py b/pylib/anki/tags.py index 9f643a1e3..7d9e642b0 100644 --- a/pylib/anki/tags.py +++ b/pylib/anki/tags.py @@ -110,30 +110,25 @@ class TagManager: else: l = "tags " fn = self.remFromStr - lim = " or ".join([l + "like :_%d" % c for c, t in enumerate(newTags)]) + lim = " or ".join(l + "like ?" for x in newTags) res = self.col.db.all( "select id, tags from notes where id in %s and (%s)" % (ids2str(ids), lim), - **dict( - [ - ("_%d" % x, "%% %s %%" % y.replace("*", "%")) - for x, y in enumerate(newTags) - ] - ), + *["%% %s %%" % y.replace("*", "%") for x, y in enumerate(newTags)], ) # update tags nids = [] def fix(row): nids.append(row[0]) - return { - "id": row[0], - "t": fn(tags, row[1]), - "n": intTime(), - "u": self.col.usn(), - } + return [ + fn(tags, row[1]), + intTime(), + self.col.usn(), + row[0], + ] self.col.db.executemany( - "update notes set tags=:t,mod=:n,usn=:u where id = :id", + "update notes set tags=?,mod=?,usn=? where id = ?", [fix(row) for row in res], ) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 49e418f06..1a2a1473a 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -754,10 +754,7 @@ title="%s" %s>%s""" % ( signal.signal(signal.SIGINT, self.onSigInt) def onSigInt(self, signum, frame): - # interrupt any current transaction and schedule a rollback & quit - if self.col: - self.col.db.interrupt() - + # schedule a rollback & quit def quit(): self.col.db.rollback() self.close() diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index e16697614..e072bcb81 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -35,7 +35,7 @@ class ProgressManager: "Install a handler in the current DB." self.lastDbProgress = 0 self.inDB = False - #db.set_progress_handler(self._dbProgress, 10000) + # db.set_progress_handler(self._dbProgress, 10000) def _dbProgress(self): "Called from SQLite." From 527d26968113486e8fd38b3b796154adb77b547b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 2 Mar 2020 20:53:46 +1000 Subject: [PATCH 015/150] drop context manager --- pylib/anki/dbproxy.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 8c597c9e2..428be4c1b 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -62,13 +62,6 @@ class DBProxy: def close(self) -> None: self._db.close() - def __enter__(self) -> "DBProxy": - self._db.execute("begin") - return self - - def __exit__(self, exc_type, *args) -> None: - self._db.close() - def totalChanges(self) -> Any: return self._db.total_changes From 5f442ae95e5225e79dfd6405ec3605b352ef02f6 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 07:47:17 +1000 Subject: [PATCH 016/150] drop the schema <11 upgrade code --- pylib/anki/storage.py | 157 +----------------------------------------- 1 file changed, 2 insertions(+), 155 deletions(-) diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index 0a1001466..01a71ad3e 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -4,7 +4,6 @@ import copy import json import os -import re from typing import Any, Dict, Optional, Tuple from anki.collection import _Collection @@ -58,7 +57,7 @@ def Collection( # add db to col and do any remaining upgrades col = _Collection(db, backend=backend, server=server, log=log) if ver < SCHEMA_VERSION: - _upgrade(col, ver) + raise Exception("This file requires an older version of Anki.") elif ver > SCHEMA_VERSION: raise Exception("This file requires a newer version of Anki.") elif create: @@ -79,159 +78,7 @@ def Collection( def _upgradeSchema(db: DBProxy) -> Any: - ver = db.scalar("select ver from col") - if ver == SCHEMA_VERSION: - return ver - # add odid to cards, edue->odue - ###################################################################### - if db.scalar("select ver from col") == 1: - db.execute("alter table cards rename to cards2") - _addSchema(db, setColConf=False) - db.execute( - """ -insert into cards select -id, nid, did, ord, mod, usn, type, queue, due, ivl, factor, reps, lapses, -left, edue, 0, flags, data from cards2""" - ) - db.execute("drop table cards2") - db.execute("update col set ver = 2") - _updateIndices(db) - # remove did from notes - ###################################################################### - if db.scalar("select ver from col") == 2: - db.execute("alter table notes rename to notes2") - _addSchema(db, setColConf=False) - db.execute( - """ -insert into notes select -id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data from notes2""" - ) - db.execute("drop table notes2") - db.execute("update col set ver = 3") - _updateIndices(db) - return ver - - -def _upgrade(col, ver) -> None: - if ver < 3: - # new deck properties - for d in col.decks.all(): - d["dyn"] = DECK_STD - d["collapsed"] = False - col.decks.save(d) - if ver < 4: - col.modSchema(check=False) - clozes = [] - for m in col.models.all(): - if not "{{cloze:" in m["tmpls"][0]["qfmt"]: - m["type"] = MODEL_STD - col.models.save(m) - else: - clozes.append(m) - for m in clozes: - _upgradeClozeModel(col, m) - col.db.execute("update col set ver = 4") - if ver < 5: - col.db.execute("update cards set odue = 0 where queue = 2") - col.db.execute("update col set ver = 5") - if ver < 6: - col.modSchema(check=False) - import anki.models - - for m in col.models.all(): - m["css"] = anki.models.defaultModel["css"] - for t in m["tmpls"]: - if "css" not in t: - # ankidroid didn't bump version - continue - m["css"] += "\n" + t["css"].replace( - ".card ", ".card%d " % (t["ord"] + 1) - ) - del t["css"] - col.models.save(m) - col.db.execute("update col set ver = 6") - if ver < 7: - col.modSchema(check=False) - col.db.execute( - "update cards set odue = 0 where (type = 1 or queue = 2) " "and not odid" - ) - col.db.execute("update col set ver = 7") - if ver < 8: - col.modSchema(check=False) - col.db.execute("update cards set due = due / 1000 where due > 4294967296") - col.db.execute("update col set ver = 8") - if ver < 9: - # adding an empty file to a zip makes python's zip code think it's a - # folder, so remove any empty files - changed = False - dir = col.media.dir() - if dir: - for f in os.listdir(col.media.dir()): - if os.path.isfile(f) and not os.path.getsize(f): - os.unlink(f) - col.media.db.execute("delete from log where fname = ?", f) - col.media.db.execute("delete from media where fname = ?", f) - changed = True - if changed: - col.media.db.commit() - col.db.execute("update col set ver = 9") - if ver < 10: - col.db.execute( - """ -update cards set left = left + left*1000 where queue = 1""" - ) - col.db.execute("update col set ver = 10") - if ver < 11: - col.modSchema(check=False) - for d in col.decks.all(): - if d["dyn"]: - order = d["order"] - # failed order was removed - if order >= 5: - order -= 1 - d["terms"] = [[d["search"], d["limit"], order]] - del d["search"] - del d["limit"] - del d["order"] - d["resched"] = True - d["return"] = True - else: - if "extendNew" not in d: - d["extendNew"] = 10 - d["extendRev"] = 50 - col.decks.save(d) - for c in col.decks.allConf(): - r = c["rev"] - r["ivlFct"] = r.get("ivlfct", 1) - if "ivlfct" in r: - del r["ivlfct"] - r["maxIvl"] = 36500 - col.decks.save(c) - for m in col.models.all(): - for t in m["tmpls"]: - t["bqfmt"] = "" - t["bafmt"] = "" - col.models.save(m) - col.db.execute("update col set ver = 11") - - -def _upgradeClozeModel(col, m) -> None: - m["type"] = MODEL_CLOZE - # convert first template - t = m["tmpls"][0] - for type in "qfmt", "afmt": - t[type] = re.sub("{{cloze:1:(.+?)}}", r"{{cloze:\1}}", t[type]) - t["name"] = _("Cloze") - # delete non-cloze cards for the model - rem = [] - for t in m["tmpls"][1:]: - if "{{cloze:" not in t["qfmt"]: - rem.append(t) - for r in rem: - col.models.remTemplate(m, r) - del m["tmpls"][1:] - col.models._updateTemplOrds(m) - col.models.save(m) + return db.scalar("select ver from col") # Creating a new collection From 87415c0d7f82bd8b944a499e9eee4abc149998d3 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 07:42:54 +1000 Subject: [PATCH 017/150] use total_changes() sqlite func --- pylib/anki/dbproxy.py | 3 --- pylib/anki/importing/noteimp.py | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 428be4c1b..394fb7044 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -62,9 +62,6 @@ class DBProxy: def close(self) -> None: self._db.close() - def totalChanges(self) -> Any: - return self._db.total_changes - def setAutocommit(self, autocommit: bool) -> None: if autocommit: self._db.isolation_level = None diff --git a/pylib/anki/importing/noteimp.py b/pylib/anki/importing/noteimp.py index 364d13bcd..69d6a306a 100644 --- a/pylib/anki/importing/noteimp.py +++ b/pylib/anki/importing/noteimp.py @@ -287,7 +287,7 @@ content in the text file to the correct fields.""" return [intTime(), self.col.usn(), n.fieldsStr, id, n.fieldsStr] def addUpdates(self, rows: List[List[Union[int, str]]]) -> None: - old = self.col.db.totalChanges() + changes = self.col.db.scalar("select total_changes()") if self._tagsMapped: self.col.db.executemany( """ @@ -309,7 +309,8 @@ update notes set mod = ?, usn = ?, flds = ? where id = ? and flds != ?""", rows, ) - self.updateCount = self.col.db.totalChanges() - old + changes2 = self.col.db.scalar("select total_changes()") + self.updateCount = changes2 - changes def processFields( self, note: ForeignNote, fields: Optional[List[str]] = None From 5778459d7cf257884a7ce81d517374f0057ea9ed Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 08:36:11 +1000 Subject: [PATCH 018/150] drop .cursor() --- pylib/anki/collection.py | 25 ++++++++++++++----------- pylib/anki/dbproxy.py | 5 +---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index b21a78e26..30132eafd 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -793,7 +793,6 @@ select id from notes where mid = ?) limit 1""" problems = [] # problems that don't require a full sync syncable_problems = [] - curs = self.db.cursor() self.save() oldSize = os.stat(self.path)[stat.ST_SIZE] if self.db.scalar("pragma integrity_check") != "ok": @@ -942,16 +941,18 @@ select id from cards where odid > 0 and did in %s""" self.updateFieldCache(self.models.nids(m)) # new cards can't have a due position > 32 bits, so wrap items over # 2 million back to 1 million - curs.execute( + self.db.execute( """ update cards set due=1000000+due%1000000,mod=?,usn=? where due>=1000000 and type=0""", - [intTime(), self.usn()], + intTime(), + self.usn(), ) - if curs.rowcount: + rowcount = self.db.scalar("select changes()") + if rowcount: problems.append( "Found %d new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen." - % curs.rowcount + % rowcount ) # new card position self.conf["nextPos"] = ( @@ -969,18 +970,20 @@ and type=0""", self.usn(), ) # v2 sched had a bug that could create decimal intervals - curs.execute( + self.db.execute( "update cards set ivl=round(ivl),due=round(due) where ivl!=round(ivl) or due!=round(due)" ) - if curs.rowcount: - problems.append("Fixed %d cards with v2 scheduler bug." % curs.rowcount) + rowcount = self.db.scalar("select changes()") + if rowcount: + problems.append("Fixed %d cards with v2 scheduler bug." % rowcount) - curs.execute( + self.db.execute( "update revlog set ivl=round(ivl),lastIvl=round(lastIvl) where ivl!=round(ivl) or lastIvl!=round(lastIvl)" ) - if curs.rowcount: + rowcount = self.db.scalar("select changes()") + if rowcount: problems.append( - "Fixed %d review history entries with v2 scheduler bug." % curs.rowcount + "Fixed %d review history entries with v2 scheduler bug." % rowcount ) # models if self.models.ensureNotEmpty(): diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 394fb7044..de91bff60 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -7,7 +7,7 @@ import time from sqlite3 import Cursor from sqlite3 import dbapi2 as sqlite -from typing import Any, List, Type +from typing import Any, List class DBProxy: @@ -67,6 +67,3 @@ class DBProxy: self._db.isolation_level = None else: self._db.isolation_level = "" - - def cursor(self, factory: Type[Cursor] = Cursor) -> Cursor: - return self._db.cursor(factory) From 0b1d96fce0dd257cbe4a5a9264c6e249038c1107 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 08:51:29 +1000 Subject: [PATCH 019/150] tweak layout of db methods --- pylib/anki/dbproxy.py | 74 +++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index de91bff60..d93c0ee50 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -4,48 +4,40 @@ # fixme: lossy utf8 handling # fixme: progress -import time from sqlite3 import Cursor from sqlite3 import dbapi2 as sqlite -from typing import Any, List +from typing import Any, Iterable, List class DBProxy: + # Lifecycle + ############### + def __init__(self, path: str) -> None: self._db = sqlite.connect(path, timeout=0) self._path = path self.mod = False - def execute(self, sql: str, *args) -> Cursor: - s = sql.strip().lower() - # mark modified? - for stmt in "insert", "update", "delete": - if s.startswith(stmt): - self.mod = True - res = self._db.execute(sql, args) - return res + def close(self) -> None: + self._db.close() - def executemany(self, sql: str, l: Any) -> None: - self.mod = True - t = time.time() - self._db.executemany(sql, l) + # Transactions + ############### def commit(self) -> None: - t = time.time() self._db.commit() - def executescript(self, sql: str) -> None: - self.mod = True - self._db.executescript(sql) - def rollback(self) -> None: self._db.rollback() - def scalar(self, sql: str, *args) -> Any: - res = self.execute(sql, *args).fetchone() - if res: - return res[0] - return None + def setAutocommit(self, autocommit: bool) -> None: + if autocommit: + self._db.isolation_level = None + else: + self._db.isolation_level = "" + + # Querying + ################ def all(self, sql: str, *args) -> List: return self.execute(sql, *args).fetchall() @@ -59,11 +51,31 @@ class DBProxy: def list(self, sql: str, *args) -> List: return [x[0] for x in self.execute(sql, *args)] - def close(self) -> None: - self._db.close() + def scalar(self, sql: str, *args) -> Any: + res = self.execute(sql, *args).fetchone() + if res: + return res[0] + return None - def setAutocommit(self, autocommit: bool) -> None: - if autocommit: - self._db.isolation_level = None - else: - self._db.isolation_level = "" + # Updates + ################ + + def executemany(self, sql: str, args: Iterable) -> None: + self.mod = True + self._db.executemany(sql, args) + + def executescript(self, sql: str) -> None: + self.mod = True + self._db.executescript(sql) + + # Cursor API + ############### + + def execute(self, sql: str, *args) -> Cursor: + s = sql.strip().lower() + # mark modified? + for stmt in "insert", "update", "delete": + if s.startswith(stmt): + self.mod = True + res = self._db.execute(sql, args) + return res From b5c6134d8056c3f622ce58e9a4a452fd9d830d48 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 10:39:58 +1000 Subject: [PATCH 020/150] drop usage of pysqlite Cursor --- pylib/anki/dbproxy.py | 68 ++++++++++++++++++++++++++----------------- pylib/anki/sync.py | 18 ++++++------ 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index d93c0ee50..62e8ef4a4 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -4,9 +4,8 @@ # fixme: lossy utf8 handling # fixme: progress -from sqlite3 import Cursor from sqlite3 import dbapi2 as sqlite -from typing import Any, Iterable, List +from typing import Any, Iterable, List, Optional class DBProxy: @@ -39,23 +38,50 @@ class DBProxy: # Querying ################ - def all(self, sql: str, *args) -> List: - return self.execute(sql, *args).fetchall() + def _query(self, sql: str, *args, first_row_only: bool = False) -> List[List]: + # mark modified? + s = sql.strip().lower() + for stmt in "insert", "update", "delete": + if s.startswith(stmt): + self.mod = True + # fetch rows + curs = self._db.execute(sql, args) + if first_row_only: + row = curs.fetchone() + curs.close() + if row is not None: + return [row] + else: + return [] + else: + return curs.fetchall() - def first(self, sql: str, *args) -> Any: - c = self.execute(sql, *args) - res = c.fetchone() - c.close() - return res + # Query shortcuts + ################### + + def all(self, sql: str, *args) -> List: + return self._query(sql, *args) def list(self, sql: str, *args) -> List: - return [x[0] for x in self.execute(sql, *args)] + return [x[0] for x in self._query(sql, *args)] - def scalar(self, sql: str, *args) -> Any: - res = self.execute(sql, *args).fetchone() - if res: - return res[0] - return None + def first(self, sql: str, *args) -> Optional[List]: + rows = self._query(sql, *args, first_row_only=True) + if rows: + return rows[0] + else: + return None + + def scalar(self, sql: str, *args) -> Optional[Any]: + rows = self._query(sql, *args, first_row_only=True) + if rows: + return rows[0][0] + else: + return None + + # execute used to return a pysqlite cursor, but now is synonymous + # with .all() + execute = all # Updates ################ @@ -67,15 +93,3 @@ class DBProxy: def executescript(self, sql: str) -> None: self.mod = True self._db.executescript(sql) - - # Cursor API - ############### - - def execute(self, sql: str, *args) -> Cursor: - s = sql.strip().lower() - # mark modified? - for stmt in "insert", "update", "delete": - if s.startswith(stmt): - self.mod = True - res = self._db.execute(sql, args) - return res diff --git a/pylib/anki/sync.py b/pylib/anki/sync.py index 10faaa66d..78459d520 100644 --- a/pylib/anki/sync.py +++ b/pylib/anki/sync.py @@ -8,7 +8,6 @@ import io import json import os import random -import sqlite3 from typing import Any, Dict, List, Optional, Tuple, Union import anki @@ -32,7 +31,7 @@ class UnexpectedSchemaChange(Exception): class Syncer: - cursor: Optional[sqlite3.Cursor] + chunkRows: Optional[List[List]] def __init__(self, col: anki.storage._Collection, server=None) -> None: self.col = col.weakref() @@ -247,11 +246,11 @@ class Syncer: def prepareToChunk(self) -> None: self.tablesLeft = ["revlog", "cards", "notes"] - self.cursor = None + self.chunkRows = None - def cursorForTable(self, table) -> sqlite3.Cursor: + def getChunkRows(self, table) -> List[List]: lim = self.usnLim() - x = self.col.db.execute + x = self.col.db.all d = (self.maxUsn, lim) if table == "revlog": return x( @@ -280,14 +279,15 @@ from notes where %s""" lim = 250 while self.tablesLeft and lim: curTable = self.tablesLeft[0] - if not self.cursor: - self.cursor = self.cursorForTable(curTable) - rows = self.cursor.fetchmany(lim) + if not self.chunkRows: + self.chunkRows = self.getChunkRows(curTable) + rows = self.chunkRows[:lim] + self.chunkRows = self.chunkRows[lim:] fetched = len(rows) if fetched != lim: # table is empty self.tablesLeft.pop(0) - self.cursor = None + self.chunkRows = None # mark the objects as having been sent self.col.db.execute( "update %s set usn=? where usn=-1" % curTable, self.maxUsn From 77cf7dd4b7f1a7e8c607ef29b76a51dfcbc311c9 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 12:05:33 +1000 Subject: [PATCH 021/150] tweak db type hints --- pylib/anki/dbproxy.py | 24 +++++++++++++++++------- pylib/anki/schedv2.py | 6 +++--- pylib/anki/sync.py | 6 +++--- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 62e8ef4a4..be73a7410 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -5,7 +5,15 @@ # fixme: progress from sqlite3 import dbapi2 as sqlite -from typing import Any, Iterable, List, Optional +from typing import Any, Iterable, List, Optional, Sequence, Union + +# DBValue is actually Union[str, int, float, None], but if defined +# that way, every call site needs to do a type check prior to using +# the return values. +ValueFromDB = Any +Row = Sequence[ValueFromDB] + +ValueForDB = Union[str, int, float, None] class DBProxy: @@ -38,7 +46,9 @@ class DBProxy: # Querying ################ - def _query(self, sql: str, *args, first_row_only: bool = False) -> List[List]: + def _query( + self, sql: str, *args: ValueForDB, first_row_only: bool = False + ) -> List[Row]: # mark modified? s = sql.strip().lower() for stmt in "insert", "update", "delete": @@ -59,20 +69,20 @@ class DBProxy: # Query shortcuts ################### - def all(self, sql: str, *args) -> List: + def all(self, sql: str, *args: ValueForDB) -> List[Row]: return self._query(sql, *args) - def list(self, sql: str, *args) -> List: + def list(self, sql: str, *args: ValueForDB) -> List[ValueFromDB]: return [x[0] for x in self._query(sql, *args)] - def first(self, sql: str, *args) -> Optional[List]: + def first(self, sql: str, *args: ValueForDB) -> Optional[Row]: rows = self._query(sql, *args, first_row_only=True) if rows: return rows[0] else: return None - def scalar(self, sql: str, *args) -> Optional[Any]: + def scalar(self, sql: str, *args: ValueForDB) -> ValueFromDB: rows = self._query(sql, *args, first_row_only=True) if rows: return rows[0][0] @@ -86,7 +96,7 @@ class DBProxy: # Updates ################ - def executemany(self, sql: str, args: Iterable) -> None: + def executemany(self, sql: str, args: Iterable[Iterable[ValueForDB]]) -> None: self.mod = True self._db.executemany(sql, args) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index d11f7808d..a9a7eadd2 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -138,8 +138,8 @@ class Scheduler: def dueForecast(self, days: int = 7) -> List[Any]: "Return counts over next DAYS. Includes today." - daysd = dict( - self.col.db.all( + daysd: Dict[int, int] = dict( + self.col.db.all( # type: ignore f""" select due, count() from cards where did in %s and queue = {QUEUE_TYPE_REV} @@ -542,7 +542,7 @@ select count() from cards where did in %s and queue = {QUEUE_TYPE_PREVIEW} if self._lrnQueue: return True cutoff = intTime() + self.col.conf["collapseTime"] - self._lrnQueue = self.col.db.all( + self._lrnQueue = self.col.db.all( # type: ignore f""" select due, id from cards where did in %s and queue in ({QUEUE_TYPE_LRN},{QUEUE_TYPE_PREVIEW}) and due < ? diff --git a/pylib/anki/sync.py b/pylib/anki/sync.py index 78459d520..396d8fd1a 100644 --- a/pylib/anki/sync.py +++ b/pylib/anki/sync.py @@ -8,7 +8,7 @@ import io import json import os import random -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import anki from anki.consts import * @@ -31,7 +31,7 @@ class UnexpectedSchemaChange(Exception): class Syncer: - chunkRows: Optional[List[List]] + chunkRows: Optional[List[Sequence]] def __init__(self, col: anki.storage._Collection, server=None) -> None: self.col = col.weakref() @@ -248,7 +248,7 @@ class Syncer: self.tablesLeft = ["revlog", "cards", "notes"] self.chunkRows = None - def getChunkRows(self, table) -> List[List]: + def getChunkRows(self, table) -> List[Sequence]: lim = self.usnLim() x = self.col.db.all d = (self.maxUsn, lim) From 04ca8ec038ebfddc474ea9cbfdf9bfec46a5c140 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 15:04:03 +1000 Subject: [PATCH 022/150] initial work on DB handling in Rust committing the Protobuf implementation for posterity, but will replace it with json, as Protobuf measures about 6x slower for some workloads like 'select * from notes' --- proto/backend.proto | 26 +++++ pylib/anki/rsbackend.py | 58 +++++++++- pylib/anki/storage.py | 6 +- rslib/src/backend/dbproxy.rs | 84 +++++++++++++++ rslib/src/{backend.rs => backend/mod.rs} | 16 +++ rslib/src/err.rs | 12 ++- rslib/src/lib.rs | 1 + rslib/src/media/check.rs | 3 +- rslib/src/media/col.rs | 4 +- rslib/src/storage/mod.rs | 3 + rslib/src/storage/schema11.sql | 88 ++++++++++++++++ rslib/src/storage/sqlite.rs | 128 +++++++++++++++++++++++ 12 files changed, 422 insertions(+), 7 deletions(-) create mode 100644 rslib/src/backend/dbproxy.rs rename rslib/src/{backend.rs => backend/mod.rs} (97%) create mode 100644 rslib/src/storage/mod.rs create mode 100644 rslib/src/storage/schema11.sql create mode 100644 rslib/src/storage/sqlite.rs diff --git a/proto/backend.proto b/proto/backend.proto index b5a35d0d1..67197e9bc 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -13,6 +13,7 @@ message BackendInit { repeated string preferred_langs = 4; string locale_folder_path = 5; string log_path = 6; + bool server = 7; } message I18nBackendInit { @@ -44,6 +45,7 @@ message BackendInput { CongratsLearnMsgIn congrats_learn_msg = 33; Empty empty_trash = 34; Empty restore_trash = 35; + DBQueryIn db_query = 36; } } @@ -72,6 +74,7 @@ message BackendOutput { Empty trash_media_files = 29; Empty empty_trash = 34; Empty restore_trash = 35; + DBQueryOut db_query = 36; BackendError error = 2047; } @@ -324,3 +327,26 @@ message CongratsLearnMsgIn { float next_due = 1; uint32 remaining = 2; } + +message DBQueryIn { + string sql = 1; + repeated SqlValue args = 2; +} + +message DBQueryOut { + repeated SqlRow rows = 1; +} + +message SqlValue { + oneof value { + Empty null = 1; + string string = 2; + int64 int = 3; + double double = 4; + bytes blob = 5; + } +} + +message SqlRow { + repeated SqlValue values = 1; +} diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 9bef66dfd..18b405031 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -5,13 +5,25 @@ import enum import os from dataclasses import dataclass -from typing import Callable, Dict, List, NewType, NoReturn, Optional, Tuple, Union +from typing import ( + Callable, + Dict, + Iterable, + List, + NewType, + NoReturn, + Optional, + Tuple, + Union, + Any) import ankirspy # pytype: disable=import-error import anki.backend_pb2 as pb import anki.buildinfo from anki import hooks +from anki.dbproxy import Row as DBRow +from anki.dbproxy import ValueForDB from anki.fluent_pb2 import FluentString as TR from anki.models import AllTemplateReqs from anki.sound import AVTag, SoundOrVideoTag, TTSTag @@ -186,7 +198,12 @@ def _on_progress(progress_bytes: bytes) -> bool: class RustBackend: def __init__( - self, col_path: str, media_folder_path: str, media_db_path: str, log_path: str + self, + col_path: str, + media_folder_path: str, + media_db_path: str, + log_path: str, + server: bool, ) -> None: ftl_folder = os.path.join(anki.lang.locale_folder, "fluent") init_msg = pb.BackendInit( @@ -196,6 +213,7 @@ class RustBackend: locale_folder_path=ftl_folder, preferred_langs=[anki.lang.currentLang], log_path=log_path, + server=server, ) self._backend = ankirspy.open_backend(init_msg.SerializeToString()) self._backend.set_progress_callback(_on_progress) @@ -366,6 +384,42 @@ class RustBackend: def restore_trash(self): self._run_command(pb.BackendInput(restore_trash=pb.Empty())) + def db_query(self, sql: str, args: Iterable[ValueForDB]) -> Iterable[DBRow]: + def arg_to_proto(arg: ValueForDB) -> pb.SqlValue: + if isinstance(arg, int): + return pb.SqlValue(int=arg) + elif isinstance(arg, float): + return pb.SqlValue(double=arg) + elif isinstance(arg, str): + return pb.SqlValue(string=arg) + elif arg is None: + return pb.SqlValue(null=pb.Empty()) + else: + raise Exception("unexpected DB type") + + output = self._run_command( + pb.BackendInput( + db_query=pb.DBQueryIn(sql=sql, args=map(arg_to_proto, args)) + ) + ).db_query + + def sqlvalue_to_native(arg: pb.SqlValue) -> Any: + v = arg.WhichOneof("value") + if v == "int": + return arg.int + elif v == "double": + return arg.double + elif v == "string": + return arg.string + elif v == "null": + return None + else: + assert_impossible_literal(v) + + def sqlrow_to_tuple(arg: pb.SqlRow) -> Tuple: + return tuple(map(sqlvalue_to_native, arg.values)) + + return map(sqlrow_to_tuple, output.rows) def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index 01a71ad3e..c81dac715 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -35,7 +35,6 @@ def Collection( log_path = "" if not server: log_path = path.replace(".anki2", "2.log") - backend = RustBackend(path, media_dir, media_db, log_path) path = os.path.abspath(path) create = not os.path.exists(path) if create: @@ -43,7 +42,10 @@ def Collection( for c in ("/", ":", "\\"): assert c not in base # connect - db = DBProxy(path) + backend = RustBackend( + path, media_dir, media_db, log_path, server=server is not None + ) + db = DBProxy(backend, path) db.setAutocommit(True) if create: ver = _createDB(db) diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs new file mode 100644 index 000000000..3d8cca005 --- /dev/null +++ b/rslib/src/backend/dbproxy.rs @@ -0,0 +1,84 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::backend_proto as pb; +use crate::err::Result; +use crate::storage::SqliteStorage; +use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; +use serde_derive::{Deserialize, Serialize}; +// +// #[derive(Deserialize)] +// struct DBRequest { +// sql: String, +// args: Vec, +// } +// +// #[derive(Serialize)] +// struct DBResult { +// rows: Vec>, +// } +// +// #[derive(Serialize, Deserialize, Debug)] +// #[serde(untagged)] +// enum SqlValue { +// Null, +// String(String), +// Int(i64), +// Float(f64), +// Blob(Vec), +// } +// protobuf implementation + +impl ToSql for pb::SqlValue { + fn to_sql(&self) -> std::result::Result, rusqlite::Error> { + use pb::sql_value::Value as SqlValue; + let val = match self + .value + .as_ref() + .unwrap_or_else(|| &SqlValue::Null(pb::Empty {})) + { + SqlValue::Null(_) => ValueRef::Null, + SqlValue::String(v) => ValueRef::Text(v.as_bytes()), + SqlValue::Int(v) => ValueRef::Integer(*v), + SqlValue::Double(v) => ValueRef::Real(*v), + SqlValue::Blob(v) => ValueRef::Blob(&v), + }; + Ok(ToSqlOutput::Borrowed(val)) + } +} + +impl FromSql for pb::SqlValue { + fn column_result(value: ValueRef<'_>) -> std::result::Result { + use pb::sql_value::Value as SqlValue; + let val = match value { + ValueRef::Null => SqlValue::Null(pb::Empty {}), + ValueRef::Integer(i) => SqlValue::Int(i), + ValueRef::Real(v) => SqlValue::Double(v), + ValueRef::Text(v) => SqlValue::String(String::from_utf8_lossy(v).to_string()), + ValueRef::Blob(v) => SqlValue::Blob(v.to_vec()), + }; + Ok(pb::SqlValue { value: Some(val) }) + } +} + +pub(super) fn db_query_proto(db: &SqliteStorage, input: pb::DbQueryIn) -> Result { + let mut stmt = db.db.prepare_cached(&input.sql)?; + + let columns = stmt.column_count(); + + let mut rows = stmt.query(&input.args)?; + + let mut output_rows = vec![]; + + while let Some(row) = rows.next()? { + let mut orow = Vec::with_capacity(columns); + for i in 0..columns { + let v: pb::SqlValue = row.get(i)?; + orow.push(v); + } + + output_rows.push(pb::SqlRow { values: orow }); + } + + Ok(pb::DbQueryOut { rows: output_rows }) +} diff --git a/rslib/src/backend.rs b/rslib/src/backend/mod.rs similarity index 97% rename from rslib/src/backend.rs rename to rslib/src/backend/mod.rs index 970768fb5..6e88ed2a1 100644 --- a/rslib/src/backend.rs +++ b/rslib/src/backend/mod.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::backend::dbproxy::db_query_proto; use crate::backend_proto::backend_input::Value; use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; @@ -12,6 +13,7 @@ use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; +use crate::storage::SqliteStorage; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, @@ -24,9 +26,12 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use tokio::runtime::Runtime; +mod dbproxy; + pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { + col: SqliteStorage, #[allow(dead_code)] col_path: PathBuf, media_folder: PathBuf, @@ -119,7 +124,11 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { log::terminal(), ); + let col = SqliteStorage::open_or_create(Path::new(&input.collection_path), input.server) + .map_err(|e| format!("Unable to open collection: {:?}", e))?; + match Backend::new( + col, &input.collection_path, &input.media_folder_path, &input.media_db_path, @@ -133,6 +142,7 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { impl Backend { pub fn new( + col: SqliteStorage, col_path: &str, media_folder: &str, media_db: &str, @@ -140,6 +150,7 @@ impl Backend { log: Logger, ) -> Result { Ok(Backend { + col, col_path: col_path.into(), media_folder: media_folder.into(), media_db: media_db.into(), @@ -241,6 +252,7 @@ impl Backend { self.restore_trash()?; OValue::RestoreTrash(Empty {}) } + Value::DbQuery(input) => OValue::DbQuery(self.db_query(input)?), }) } @@ -481,6 +493,10 @@ impl Backend { checker.restore_trash() } + + fn db_query(&self, input: pb::DbQueryIn) -> Result { + db_query_proto(&self.col, input) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/err.rs b/rslib/src/err.rs index 1d2bd2faf..66e4bc66c 100644 --- a/rslib/src/err.rs +++ b/rslib/src/err.rs @@ -20,7 +20,7 @@ pub enum AnkiError { IOError { info: String }, #[fail(display = "DB error: {}", info)] - DBError { info: String }, + DBError { info: String, kind: DBErrorKind }, #[fail(display = "Network error: {:?} {}", kind, info)] NetworkError { @@ -112,6 +112,7 @@ impl From for AnkiError { fn from(err: rusqlite::Error) -> Self { AnkiError::DBError { info: format!("{:?}", err), + kind: DBErrorKind::Other, } } } @@ -120,6 +121,7 @@ impl From for AnkiError { fn from(err: rusqlite::types::FromSqlError) -> Self { AnkiError::DBError { info: format!("{:?}", err), + kind: DBErrorKind::Other, } } } @@ -215,3 +217,11 @@ impl From for AnkiError { AnkiError::sync_misc(err.to_string()) } } + +#[derive(Debug, PartialEq)] +pub enum DBErrorKind { + FileTooNew, + FileTooOld, + MissingEntity, + Other, +} diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index cce95c5b0..82ce394fb 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -17,6 +17,7 @@ pub mod latex; pub mod log; pub mod media; pub mod sched; +pub mod storage; pub mod template; pub mod template_filters; pub mod text; diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 2049ed0e2..a6f1fa76a 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::err::{AnkiError, Result}; +use crate::err::{AnkiError, DBErrorKind, Result}; use crate::i18n::{tr_args, tr_strs, FString, I18n}; use crate::latex::extract_latex_expanding_clozes; use crate::log::{debug, Logger}; @@ -403,6 +403,7 @@ where .get(¬e.mid) .ok_or_else(|| AnkiError::DBError { info: "missing note type".to_string(), + kind: DBErrorKind::MissingEntity, })?; if fix_and_extract_media_refs(note, &mut referenced_files, renamed)? { // note was modified, needs saving diff --git a/rslib/src/media/col.rs b/rslib/src/media/col.rs index a563bb93e..460d64b5f 100644 --- a/rslib/src/media/col.rs +++ b/rslib/src/media/col.rs @@ -2,7 +2,7 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html /// Basic note reading/updating functionality for the media DB check. -use crate::err::{AnkiError, Result}; +use crate::err::{AnkiError, DBErrorKind, Result}; use crate::text::strip_html_preserving_image_filenames; use crate::time::{i64_unix_millis, i64_unix_secs}; use crate::types::{ObjID, Timestamp, Usn}; @@ -85,6 +85,7 @@ pub(super) fn get_note_types(db: &Connection) -> Result .next() .ok_or_else(|| AnkiError::DBError { info: "col table empty".to_string(), + kind: DBErrorKind::MissingEntity, })??; Ok(note_types) } @@ -136,6 +137,7 @@ pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) - .get(note_type.sort_field_idx as usize) .ok_or_else(|| AnkiError::DBError { info: "sort field out of range".to_string(), + kind: DBErrorKind::MissingEntity, })?, ); diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs new file mode 100644 index 000000000..2b474f145 --- /dev/null +++ b/rslib/src/storage/mod.rs @@ -0,0 +1,3 @@ +mod sqlite; + +pub(crate) use sqlite::SqliteStorage; diff --git a/rslib/src/storage/schema11.sql b/rslib/src/storage/schema11.sql new file mode 100644 index 000000000..e5d1b7f4d --- /dev/null +++ b/rslib/src/storage/schema11.sql @@ -0,0 +1,88 @@ +create table col +( + id integer primary key, + crt integer not null, + mod integer not null, + scm integer not null, + ver integer not null, + dty integer not null, + usn integer not null, + ls integer not null, + conf text not null, + models text not null, + decks text not null, + dconf text not null, + tags text not null +); + +create table notes +( + id integer primary key, + guid text not null, + mid integer not null, + mod integer not null, + usn integer not null, + tags text not null, + flds text not null, + sfld integer not null, + csum integer not null, + flags integer not null, + data text not null +); + +create table cards +( + id integer primary key, + nid integer not null, + did integer not null, + ord integer not null, + mod integer not null, + usn integer not null, + type integer not null, + queue integer not null, + due integer not null, + ivl integer not null, + factor integer not null, + reps integer not null, + lapses integer not null, + left integer not null, + odue integer not null, + odid integer not null, + flags integer not null, + data text not null +); + +create table revlog +( + id integer primary key, + cid integer not null, + usn integer not null, + ease integer not null, + ivl integer not null, + lastIvl integer not null, + factor integer not null, + time integer not null, + type integer not null +); + +create table graves +( + usn integer not null, + oid integer not null, + type integer not null +); + +-- syncing +create index ix_notes_usn on notes (usn); +create index ix_cards_usn on cards (usn); +create index ix_revlog_usn on revlog (usn); +-- card spacing, etc +create index ix_cards_nid on cards (nid); +-- scheduling and deck limiting +create index ix_cards_sched on cards (did, queue, due); +-- revlog by card +create index ix_revlog_cid on revlog (cid); +-- field uniqueness +create index ix_notes_csum on notes (csum); + +insert into col values (1,0,0,0,0,0,0,0,'{}','{}','{}','{}','{}'); diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs new file mode 100644 index 000000000..853a22759 --- /dev/null +++ b/rslib/src/storage/sqlite.rs @@ -0,0 +1,128 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::err::Result; +use crate::err::{AnkiError, DBErrorKind}; +use crate::time::i64_unix_timestamp; +use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ValueRef}; +use rusqlite::{params, Connection, OptionalExtension, NO_PARAMS}; +use serde::de::DeserializeOwned; +use serde_derive::{Deserialize, Serialize}; +use serde_json::Value; +use std::borrow::Cow; +use std::convert::TryFrom; +use std::fmt; +use std::path::{Path, PathBuf}; + +const SCHEMA_MIN_VERSION: u8 = 11; +const SCHEMA_MAX_VERSION: u8 = 11; + +macro_rules! cached_sql { + ( $label:expr, $db:expr, $sql:expr ) => {{ + if $label.is_none() { + $label = Some($db.prepare_cached($sql)?); + } + $label.as_mut().unwrap() + }}; +} + +#[derive(Debug)] +pub struct SqliteStorage { + // currently crate-visible for dbproxy + pub(crate) db: Connection, + path: PathBuf, + server: bool, +} + +fn open_or_create_collection_db(path: &Path) -> Result { + let mut db = Connection::open(path)?; + + if std::env::var("TRACESQL").is_ok() { + db.trace(Some(trace)); + } + + db.pragma_update(None, "locking_mode", &"exclusive")?; + db.pragma_update(None, "page_size", &4096)?; + db.pragma_update(None, "cache_size", &(-40 * 1024))?; + db.pragma_update(None, "legacy_file_format", &false)?; + db.pragma_update(None, "journal", &"wal")?; + db.set_prepared_statement_cache_capacity(50); + + Ok(db) +} + +/// Fetch schema version from database. +/// Return (must_create, version) +fn schema_version(db: &Connection) -> Result<(bool, u8)> { + if !db + .prepare("select null from sqlite_master where type = 'table' and name = 'col'")? + .exists(NO_PARAMS)? + { + return Ok((true, SCHEMA_MAX_VERSION)); + } + + Ok(( + false, + db.query_row("select ver from col", NO_PARAMS, |r| Ok(r.get(0)?))?, + )) +} + +fn trace(s: &str) { + println!("sql: {}", s) +} + +impl SqliteStorage { + pub(crate) fn open_or_create(path: &Path, server: bool) -> Result { + let db = open_or_create_collection_db(path)?; + + let (create, ver) = schema_version(&db)?; + if create { + unimplemented!(); // todo + db.prepare_cached("begin exclusive")?.execute(NO_PARAMS)?; + db.execute_batch(include_str!("schema11.sql"))?; + db.execute( + "update col set crt=?, ver=?", + params![i64_unix_timestamp(), ver], + )?; + db.prepare_cached("commit")?.execute(NO_PARAMS)?; + } else { + if ver > SCHEMA_MAX_VERSION { + return Err(AnkiError::DBError { + info: "".to_string(), + kind: DBErrorKind::FileTooNew, + }); + } + if ver < SCHEMA_MIN_VERSION { + return Err(AnkiError::DBError { + info: "".to_string(), + kind: DBErrorKind::FileTooOld, + }); + } + }; + + let storage = Self { + db, + path: path.to_owned(), + server, + }; + + Ok(storage) + } + + pub(crate) fn begin(&self) -> Result<()> { + self.db + .prepare_cached("begin exclusive")? + .execute(NO_PARAMS)?; + Ok(()) + } + + pub(crate) fn commit(&self) -> Result<()> { + self.db.prepare_cached("commit")?.execute(NO_PARAMS)?; + Ok(()) + } + + pub(crate) fn rollback(&self) -> Result<()> { + self.db.execute("rollback", NO_PARAMS)?; + Ok(()) + } +} From b876d97770f7c1794bbc24c3c627425d55e4d532 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 15:36:05 +1000 Subject: [PATCH 023/150] use (or)json for DB bridge Some initial testing with orjson indicates performance varies from slightly better than pysqlite to about 2x slower depending on the type of query. Performance could be improved by building the Python list in rspy instead of sending back json that needs to be decoded, but it may make more sense to rewrite the hotspots in Rust instead. More testing is required in any case. --- pylib/anki/collection.py | 3 +- pylib/anki/dbproxy.py | 44 ++++++++--------- pylib/anki/rsbackend.py | 6 +++ pylib/setup.py | 1 + rslib/src/backend/dbproxy.rs | 94 +++++++++++++++++++++++++++++------- rslib/src/backend/mod.rs | 5 ++ rslib/src/storage/sqlite.rs | 1 + rspy/src/lib.rs | 7 +++ 8 files changed, 118 insertions(+), 43 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 30132eafd..409ab5fde 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -266,8 +266,9 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", def reopen(self) -> None: "Reconnect to DB (after changing threads, etc)." + raise Exception("fixme") if not self.db: - self.db = DBProxy(self.path) + #self.db = DBProxy(self.path) self.media.connect() self._openLog() diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index be73a7410..84c3c45d7 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -1,12 +1,15 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -# fixme: lossy utf8 handling -# fixme: progress +from __future__ import annotations -from sqlite3 import dbapi2 as sqlite from typing import Any, Iterable, List, Optional, Sequence, Union +import anki + +# fixme: remember to null on close to avoid circular ref +# fixme: progress + # DBValue is actually Union[str, int, float, None], but if defined # that way, every call site needs to do a type check prior to using # the return values. @@ -20,28 +23,29 @@ class DBProxy: # Lifecycle ############### - def __init__(self, path: str) -> None: - self._db = sqlite.connect(path, timeout=0) + def __init__(self, backend: anki.rsbackend.RustBackend, path: str) -> None: + self._backend = backend self._path = path self.mod = False def close(self) -> None: - self._db.close() + # fixme + pass # Transactions ############### def commit(self) -> None: - self._db.commit() + # fixme + pass def rollback(self) -> None: - self._db.rollback() + # fixme + pass def setAutocommit(self, autocommit: bool) -> None: - if autocommit: - self._db.isolation_level = None - else: - self._db.isolation_level = "" + # fixme + pass # Querying ################ @@ -55,16 +59,8 @@ class DBProxy: if s.startswith(stmt): self.mod = True # fetch rows - curs = self._db.execute(sql, args) - if first_row_only: - row = curs.fetchone() - curs.close() - if row is not None: - return [row] - else: - return [] - else: - return curs.fetchall() + # fixme: first_row_only + return self._backend.db_query_json(sql, args) # Query shortcuts ################### @@ -98,8 +94,8 @@ class DBProxy: def executemany(self, sql: str, args: Iterable[Iterable[ValueForDB]]) -> None: self.mod = True - self._db.executemany(sql, args) + raise Exception("fixme") def executescript(self, sql: str) -> None: self.mod = True - self._db.executescript(sql) + raise Exception("fixme") diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 18b405031..761a721c3 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -18,6 +18,7 @@ from typing import ( Any) import ankirspy # pytype: disable=import-error +import orjson import anki.backend_pb2 as pb import anki.buildinfo @@ -421,6 +422,11 @@ class RustBackend: return map(sqlrow_to_tuple, output.rows) + def db_query_json(self, sql: str, args: Iterable[ValueForDB]) -> List[DBRow]: + input = orjson.dumps(dict(sql=sql, args=args)) + output = self._backend.db_query(input) + return orjson.loads(output) + def translate_string_in( key: TR, **kwargs: Union[str, int, float] ) -> pb.TranslateStringIn: diff --git a/pylib/setup.py b/pylib/setup.py index ffb93affd..b23d783d7 100644 --- a/pylib/setup.py +++ b/pylib/setup.py @@ -21,6 +21,7 @@ setuptools.setup( "requests", "decorator", "protobuf", + "orjson", 'psutil; sys_platform == "win32"', 'distro; sys_platform != "darwin" and sys_platform != "win32"', ], diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index 3d8cca005..9e6bbe611 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -6,27 +6,85 @@ use crate::err::Result; use crate::storage::SqliteStorage; use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; use serde_derive::{Deserialize, Serialize}; -// -// #[derive(Deserialize)] -// struct DBRequest { -// sql: String, -// args: Vec, -// } -// + +// json implementation + +#[derive(Deserialize)] +pub(super) struct DBRequest { + sql: String, + args: Vec, +} + // #[derive(Serialize)] -// struct DBResult { +// pub(super) struct DBResult { // rows: Vec>, // } -// -// #[derive(Serialize, Deserialize, Debug)] -// #[serde(untagged)] -// enum SqlValue { -// Null, -// String(String), -// Int(i64), -// Float(f64), -// Blob(Vec), -// } +type DBResult = Vec>; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +pub(super) enum SqlValue { + Null, + String(String), + Int(i64), + Double(f64), + Blob(Vec), +} + +impl ToSql for SqlValue { + fn to_sql(&self) -> std::result::Result, rusqlite::Error> { + let val = match self { + SqlValue::Null => ValueRef::Null, + SqlValue::String(v) => ValueRef::Text(v.as_bytes()), + SqlValue::Int(v) => ValueRef::Integer(*v), + SqlValue::Double(v) => ValueRef::Real(*v), + SqlValue::Blob(v) => ValueRef::Blob(&v), + }; + Ok(ToSqlOutput::Borrowed(val)) + } +} + +impl FromSql for SqlValue { + fn column_result(value: ValueRef<'_>) -> std::result::Result { + let val = match value { + ValueRef::Null => SqlValue::Null, + ValueRef::Integer(i) => SqlValue::Int(i), + ValueRef::Real(v) => SqlValue::Double(v), + ValueRef::Text(v) => SqlValue::String(String::from_utf8_lossy(v).to_string()), + ValueRef::Blob(v) => SqlValue::Blob(v.to_vec()), + }; + Ok(val) + } +} + +pub(super) fn db_query_json_str(db: &SqliteStorage, input: &[u8]) -> Result { + let req: DBRequest = serde_json::from_slice(input)?; + let resp = db_query_json(db, req)?; + Ok(serde_json::to_string(&resp)?) +} + +pub(super) fn db_query_json(db: &SqliteStorage, input: DBRequest) -> Result { + let mut stmt = db.db.prepare_cached(&input.sql)?; + + let columns = stmt.column_count(); + + let mut rows = stmt.query(&input.args)?; + + let mut output_rows = vec![]; + + while let Some(row) = rows.next()? { + let mut orow = Vec::with_capacity(columns); + for i in 0..columns { + let v: SqlValue = row.get(i)?; + orow.push(v); + } + + output_rows.push(orow); + } + + Ok(output_rows) +} + // protobuf implementation impl ToSql for pb::SqlValue { diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 6e88ed2a1..1a2cc1eaa 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::backend::dbproxy::db_query_json_str; use crate::backend::dbproxy::db_query_proto; use crate::backend_proto::backend_input::Value; use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; @@ -497,6 +498,10 @@ impl Backend { fn db_query(&self, input: pb::DbQueryIn) -> Result { db_query_proto(&self.col, input) } + + pub fn db_query_json(&self, input: &[u8]) -> Result { + db_query_json_str(&self.col, input) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 853a22759..4f6a13bd1 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -26,6 +26,7 @@ macro_rules! cached_sql { }}; } +// currently public for dbproxy #[derive(Debug)] pub struct SqliteStorage { // currently crate-visible for dbproxy diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index f24fccc76..80e8c762f 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -70,6 +70,13 @@ impl Backend { self.backend.set_progress_callback(Some(Box::new(func))); } } + + fn db_query(&mut self, py: Python, input: &PyBytes) -> PyObject { + let in_bytes = input.as_bytes(); + let out_string = self.backend.db_query_json(in_bytes).unwrap(); + let out_obj = PyBytes::new(py, out_string.as_bytes()); + out_obj.into() + } } // I18n backend From b51e575a9def5c829016ef0a6e1807eb61904b31 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 15:42:32 +1000 Subject: [PATCH 024/150] drop the protobuf prototype --- proto/backend.proto | 25 ---------------- pylib/anki/dbproxy.py | 2 +- pylib/anki/rsbackend.py | 39 +----------------------- rslib/src/backend/dbproxy.rs | 58 ------------------------------------ rslib/src/backend/mod.rs | 8 +---- rspy/src/lib.rs | 2 +- 6 files changed, 4 insertions(+), 130 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 67197e9bc..230158eb2 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -45,7 +45,6 @@ message BackendInput { CongratsLearnMsgIn congrats_learn_msg = 33; Empty empty_trash = 34; Empty restore_trash = 35; - DBQueryIn db_query = 36; } } @@ -74,7 +73,6 @@ message BackendOutput { Empty trash_media_files = 29; Empty empty_trash = 34; Empty restore_trash = 35; - DBQueryOut db_query = 36; BackendError error = 2047; } @@ -327,26 +325,3 @@ message CongratsLearnMsgIn { float next_due = 1; uint32 remaining = 2; } - -message DBQueryIn { - string sql = 1; - repeated SqlValue args = 2; -} - -message DBQueryOut { - repeated SqlRow rows = 1; -} - -message SqlValue { - oneof value { - Empty null = 1; - string string = 2; - int64 int = 3; - double double = 4; - bytes blob = 5; - } -} - -message SqlRow { - repeated SqlValue values = 1; -} diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 84c3c45d7..ff89f2390 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -60,7 +60,7 @@ class DBProxy: self.mod = True # fetch rows # fixme: first_row_only - return self._backend.db_query_json(sql, args) + return self._backend.db_query(sql, args) # Query shortcuts ################### diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 761a721c3..a44996ed9 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -385,44 +385,7 @@ class RustBackend: def restore_trash(self): self._run_command(pb.BackendInput(restore_trash=pb.Empty())) - def db_query(self, sql: str, args: Iterable[ValueForDB]) -> Iterable[DBRow]: - def arg_to_proto(arg: ValueForDB) -> pb.SqlValue: - if isinstance(arg, int): - return pb.SqlValue(int=arg) - elif isinstance(arg, float): - return pb.SqlValue(double=arg) - elif isinstance(arg, str): - return pb.SqlValue(string=arg) - elif arg is None: - return pb.SqlValue(null=pb.Empty()) - else: - raise Exception("unexpected DB type") - - output = self._run_command( - pb.BackendInput( - db_query=pb.DBQueryIn(sql=sql, args=map(arg_to_proto, args)) - ) - ).db_query - - def sqlvalue_to_native(arg: pb.SqlValue) -> Any: - v = arg.WhichOneof("value") - if v == "int": - return arg.int - elif v == "double": - return arg.double - elif v == "string": - return arg.string - elif v == "null": - return None - else: - assert_impossible_literal(v) - - def sqlrow_to_tuple(arg: pb.SqlRow) -> Tuple: - return tuple(map(sqlvalue_to_native, arg.values)) - - return map(sqlrow_to_tuple, output.rows) - - def db_query_json(self, sql: str, args: Iterable[ValueForDB]) -> List[DBRow]: + def db_query(self, sql: str, args: Iterable[ValueForDB]) -> List[DBRow]: input = orjson.dumps(dict(sql=sql, args=args)) output = self._backend.db_query(input) return orjson.loads(output) diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index 9e6bbe611..f86be1b0b 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -7,8 +7,6 @@ use crate::storage::SqliteStorage; use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; use serde_derive::{Deserialize, Serialize}; -// json implementation - #[derive(Deserialize)] pub(super) struct DBRequest { sql: String, @@ -84,59 +82,3 @@ pub(super) fn db_query_json(db: &SqliteStorage, input: DBRequest) -> Result std::result::Result, rusqlite::Error> { - use pb::sql_value::Value as SqlValue; - let val = match self - .value - .as_ref() - .unwrap_or_else(|| &SqlValue::Null(pb::Empty {})) - { - SqlValue::Null(_) => ValueRef::Null, - SqlValue::String(v) => ValueRef::Text(v.as_bytes()), - SqlValue::Int(v) => ValueRef::Integer(*v), - SqlValue::Double(v) => ValueRef::Real(*v), - SqlValue::Blob(v) => ValueRef::Blob(&v), - }; - Ok(ToSqlOutput::Borrowed(val)) - } -} - -impl FromSql for pb::SqlValue { - fn column_result(value: ValueRef<'_>) -> std::result::Result { - use pb::sql_value::Value as SqlValue; - let val = match value { - ValueRef::Null => SqlValue::Null(pb::Empty {}), - ValueRef::Integer(i) => SqlValue::Int(i), - ValueRef::Real(v) => SqlValue::Double(v), - ValueRef::Text(v) => SqlValue::String(String::from_utf8_lossy(v).to_string()), - ValueRef::Blob(v) => SqlValue::Blob(v.to_vec()), - }; - Ok(pb::SqlValue { value: Some(val) }) - } -} - -pub(super) fn db_query_proto(db: &SqliteStorage, input: pb::DbQueryIn) -> Result { - let mut stmt = db.db.prepare_cached(&input.sql)?; - - let columns = stmt.column_count(); - - let mut rows = stmt.query(&input.args)?; - - let mut output_rows = vec![]; - - while let Some(row) = rows.next()? { - let mut orow = Vec::with_capacity(columns); - for i in 0..columns { - let v: pb::SqlValue = row.get(i)?; - orow.push(v); - } - - output_rows.push(pb::SqlRow { values: orow }); - } - - Ok(pb::DbQueryOut { rows: output_rows }) -} diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 1a2cc1eaa..bbf8b5266 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -2,7 +2,6 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::backend::dbproxy::db_query_json_str; -use crate::backend::dbproxy::db_query_proto; use crate::backend_proto::backend_input::Value; use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; @@ -253,7 +252,6 @@ impl Backend { self.restore_trash()?; OValue::RestoreTrash(Empty {}) } - Value::DbQuery(input) => OValue::DbQuery(self.db_query(input)?), }) } @@ -495,13 +493,9 @@ impl Backend { checker.restore_trash() } - fn db_query(&self, input: pb::DbQueryIn) -> Result { + pub fn db_query(&self, input: pb::DbQueryIn) -> Result { db_query_proto(&self.col, input) } - - pub fn db_query_json(&self, input: &[u8]) -> Result { - db_query_json_str(&self.col, input) - } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index 80e8c762f..d921aa799 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -73,7 +73,7 @@ impl Backend { fn db_query(&mut self, py: Python, input: &PyBytes) -> PyObject { let in_bytes = input.as_bytes(); - let out_string = self.backend.db_query_json(in_bytes).unwrap(); + let out_string = self.backend.db_query(in_bytes).unwrap(); let out_obj = PyBytes::new(py, out_string.as_bytes()); out_obj.into() } From 874ee80a68866f9f3ef9257bf44487c1592f5c67 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 3 Mar 2020 18:29:36 +1000 Subject: [PATCH 025/150] add a temporary executemany() --- pylib/anki/dbproxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index ff89f2390..d18c0cce2 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -94,7 +94,9 @@ class DBProxy: def executemany(self, sql: str, args: Iterable[Iterable[ValueForDB]]) -> None: self.mod = True - raise Exception("fixme") + # fixme + for row in args: + self.execute(sql, *row) def executescript(self, sql: str) -> None: self.mod = True From 818401e464ac2bf974d30957d6f91d500bb7d3bf Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 4 Mar 2020 10:03:57 +1000 Subject: [PATCH 026/150] remove remaining db kwargs --- pylib/anki/find.py | 4 ++-- pylib/anki/models.py | 18 ++++-------------- pylib/anki/schedv2.py | 21 +++++---------------- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/pylib/anki/find.py b/pylib/anki/find.py index 2067c3df6..a2c2b19c6 100644 --- a/pylib/anki/find.py +++ b/pylib/anki/find.py @@ -555,11 +555,11 @@ def findReplace( flds = joinFields(sflds) if flds != origFlds: nids.append(nid) - d.append(dict(nid=nid, flds=flds, u=col.usn(), m=intTime())) + d.append((flds, intTime(), col.usn(), nid)) if not d: return 0 # replace - col.db.executemany("update notes set flds=:flds,mod=:m,usn=:u where id=:nid", d) + col.db.executemany("update notes set flds=?,mod=?,usn=? where id=?", d) col.updateFieldCache(nids) col.genCards(nids) return len(d) diff --git a/pylib/anki/models.py b/pylib/anki/models.py index 21f27b649..461fd08a6 100644 --- a/pylib/anki/models.py +++ b/pylib/anki/models.py @@ -504,17 +504,9 @@ select id from notes where mid = ?)""" for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) - d.append( - dict( - nid=nid, - flds=flds, - mid=newModel["id"], - m=intTime(), - u=self.col.usn(), - ) - ) + d.append((flds, newModel["id"], intTime(), self.col.usn(), nid,)) self.col.db.executemany( - "update notes set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :nid", d + "update notes set flds=?,mid=?,mod=?,usn=? where id = ?", d ) self.col.updateFieldCache(nids) @@ -543,12 +535,10 @@ select id from notes where mid = ?)""" # mapping from a regular note, so the map should be valid new = map[ord] if new is not None: - d.append(dict(cid=cid, new=new, u=self.col.usn(), m=intTime())) + d.append((new, self.col.usn(), intTime(), cid)) else: deleted.append(cid) - self.col.db.executemany( - "update cards set ord=:new,usn=:u,mod=:m where id=:cid", d - ) + self.col.db.executemany("update cards set ord=?,usn=?,mod=? where id=?", d) self.col.remCards(deleted) # Schema hash diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index a9a7eadd2..67aa6c4db 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1775,21 +1775,12 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""", mod = intTime() for id in ids: r = random.randint(imin, imax) - d.append( - dict( - id=id, - due=r + t, - ivl=max(1, r), - mod=mod, - usn=self.col.usn(), - fact=STARTING_FACTOR, - ) - ) + d.append((max(1, r), r + t, self.col.usn(), mod, STARTING_FACTOR, id,)) self.remFromDyn(ids) self.col.db.executemany( f""" -update cards set type={CARD_TYPE_REV},queue={QUEUE_TYPE_REV},ivl=:ivl,due=:due,odue=0, -usn=:usn,mod=:mod,factor=:fact where id=:id""", +update cards set type={CARD_TYPE_REV},queue={QUEUE_TYPE_REV},ivl=?,due=?,odue=0, +usn=?,mod=?,factor=? where id=?""", d, ) self.col.log(ids) @@ -1866,10 +1857,8 @@ and due >= ? and queue = {QUEUE_TYPE_NEW}""" for id, nid in self.col.db.execute( f"select id, nid from cards where type = {CARD_TYPE_NEW} and id in " + scids ): - d.append(dict(now=now, due=due[nid], usn=self.col.usn(), cid=id)) - self.col.db.executemany( - "update cards set due=:due,mod=:now,usn=:usn where id = :cid", d - ) + d.append((due[nid], now, self.col.usn(), id)) + self.col.db.executemany("update cards set due=?,mod=?,usn=? where id = ?", d) def randomizeCards(self, did: int) -> None: cids = self.col.db.list("select id from cards where did = ?", did) From 6db4418f0562250911cd84e9c13f8ad4fea07f2e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 5 Mar 2020 10:20:16 +1000 Subject: [PATCH 027/150] drop log= argument from Collection --- pylib/anki/collection.py | 3 +-- qt/aqt/main.py | 2 +- qt/aqt/sync.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 409ab5fde..615c86df9 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -83,10 +83,9 @@ class _Collection: db: DBProxy, backend: RustBackend, server: Optional["anki.storage.ServerData"] = None, - log: bool = False, ) -> None: self.backend = backend - self._debugLog = log + self._debugLog = not server self.db = db self.path = db._path self._openLog() diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 1a2a1473a..7129f9c7d 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -459,7 +459,7 @@ close the profile or restart Anki.""" def _loadCollection(self) -> bool: cpath = self.pm.collectionPath() - self.col = Collection(cpath, log=True) + self.col = Collection(cpath) self.setEnabled(True) self.progress.setupDB(self.col.db) diff --git a/qt/aqt/sync.py b/qt/aqt/sync.py index fdbb7ef1f..658874498 100644 --- a/qt/aqt/sync.py +++ b/qt/aqt/sync.py @@ -364,7 +364,7 @@ class SyncThread(QThread): self.syncMsg = "" self.uname = "" try: - self.col = Collection(self.path, log=True) + self.col = Collection(self.path) except: self.fireEvent("corrupt") return From 2cd7885ec080d15b8c002e377043915eb13276af Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 5 Mar 2020 10:54:30 +1000 Subject: [PATCH 028/150] add begin/commit/rollback, and support creating collections all but one unit test is now passing --- pylib/anki/collection.py | 6 +- pylib/anki/dbproxy.py | 23 ++++-- pylib/anki/rsbackend.py | 20 ++++- pylib/anki/storage.py | 151 +++-------------------------------- rslib/src/backend/dbproxy.rs | 68 +++++++++------- rslib/src/backend/mod.rs | 8 +- rslib/src/storage/sqlite.rs | 29 +++---- rspy/src/lib.rs | 15 +++- 8 files changed, 109 insertions(+), 211 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 615c86df9..989d18925 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -242,10 +242,7 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", return None def lock(self) -> None: - # make sure we don't accidentally bump mod time - mod = self.db.mod - self.db.execute("update col set mod=mod") - self.db.mod = mod + self.db.begin() def close(self, save: bool = True) -> None: "Disconnect from DB." @@ -260,6 +257,7 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.db.setAutocommit(False) self.db.close() self.db = None + self.backend = None self.media.close() self._closeLog() diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index d18c0cce2..ae5a8f779 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -3,11 +3,14 @@ from __future__ import annotations +import weakref from typing import Any, Iterable, List, Optional, Sequence, Union import anki -# fixme: remember to null on close to avoid circular ref +# fixme: col.reopen() +# fixme: setAutocommit() +# fixme: transaction/lock handling # fixme: progress # DBValue is actually Union[str, int, float, None], but if defined @@ -24,7 +27,7 @@ class DBProxy: ############### def __init__(self, backend: anki.rsbackend.RustBackend, path: str) -> None: - self._backend = backend + self._backend = weakref.proxy(backend) self._path = path self.mod = False @@ -35,17 +38,20 @@ class DBProxy: # Transactions ############### + def begin(self) -> None: + self._backend.db_begin() + def commit(self) -> None: - # fixme - pass + self._backend.db_commit() def rollback(self) -> None: - # fixme - pass + self._backend.db_rollback() def setAutocommit(self, autocommit: bool) -> None: - # fixme - pass + if autocommit: + self.commit() + else: + self.begin() # Querying ################ @@ -58,6 +64,7 @@ class DBProxy: for stmt in "insert", "update", "delete": if s.startswith(stmt): self.mod = True + assert ":" not in sql # fetch rows # fixme: first_row_only return self._backend.db_query(sql, args) diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index a44996ed9..ab2534a54 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -6,6 +6,7 @@ import enum import os from dataclasses import dataclass from typing import ( + Any, Callable, Dict, Iterable, @@ -15,7 +16,7 @@ from typing import ( Optional, Tuple, Union, - Any) +) import ankirspy # pytype: disable=import-error import orjson @@ -386,9 +387,20 @@ class RustBackend: self._run_command(pb.BackendInput(restore_trash=pb.Empty())) def db_query(self, sql: str, args: Iterable[ValueForDB]) -> List[DBRow]: - input = orjson.dumps(dict(sql=sql, args=args)) - output = self._backend.db_query(input) - return orjson.loads(output) + return self._db_command(dict(kind="query", sql=sql, args=args)) + + def db_begin(self) -> None: + return self._db_command(dict(kind="begin")) + + def db_commit(self) -> None: + return self._db_command(dict(kind="commit")) + + def db_rollback(self) -> None: + return self._db_command(dict(kind="rollback")) + + def _db_command(self, input: Dict[str, Any]) -> Any: + return orjson.loads(self._backend.db_command(orjson.dumps(input))) + def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index c81dac715..0b626d1bb 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -26,9 +26,7 @@ class ServerData: minutes_west: Optional[int] = None -def Collection( - path: str, lock: bool = True, server: Optional[ServerData] = None, log: bool = False -) -> _Collection: +def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: "Open a new or existing collection. Path must be unicode." assert path.endswith(".anki2") (media_dir, media_db) = media_paths_from_col_path(path) @@ -36,33 +34,23 @@ def Collection( if not server: log_path = path.replace(".anki2", "2.log") path = os.path.abspath(path) - create = not os.path.exists(path) - if create: - base = os.path.basename(path) - for c in ("/", ":", "\\"): - assert c not in base # connect backend = RustBackend( path, media_dir, media_db, log_path, server=server is not None ) db = DBProxy(backend, path) + db.begin() db.setAutocommit(True) + + # initial setup required? + create = db.scalar("select models = '{}' from col") if create: - ver = _createDB(db) - else: - ver = _upgradeSchema(db) - db.execute("pragma temp_store = memory") - db.execute("pragma cache_size = 10000") - if not isWin: - db.execute("pragma journal_mode = wal") + initial_db_setup(db) + db.setAutocommit(False) # add db to col and do any remaining upgrades - col = _Collection(db, backend=backend, server=server, log=log) - if ver < SCHEMA_VERSION: - raise Exception("This file requires an older version of Anki.") - elif ver > SCHEMA_VERSION: - raise Exception("This file requires a newer version of Anki.") - elif create: + col = _Collection(db, backend=backend, server=server) + if create: # add in reverse order so basic is default addClozeModel(col) addBasicTypingModel(col) @@ -70,112 +58,15 @@ def Collection( addForwardReverse(col) addBasicModel(col) col.save() - if lock: - try: - col.lock() - except: - col.db.close() - raise return col -def _upgradeSchema(db: DBProxy) -> Any: - return db.scalar("select ver from col") - - # Creating a new collection ###################################################################### -def _createDB(db: DBProxy) -> int: - db.execute("pragma page_size = 4096") - db.execute("pragma legacy_file_format = 0") - db.execute("vacuum") - _addSchema(db) - _updateIndices(db) - db.execute("analyze") - return SCHEMA_VERSION - - -def _addSchema(db: DBProxy, setColConf: bool = True) -> None: - db.executescript( - """ -create table if not exists col ( - id integer primary key, - crt integer not null, - mod integer not null, - scm integer not null, - ver integer not null, - dty integer not null, - usn integer not null, - ls integer not null, - conf text not null, - models text not null, - decks text not null, - dconf text not null, - tags text not null -); - -create table if not exists notes ( - id integer primary key, /* 0 */ - guid text not null, /* 1 */ - mid integer not null, /* 2 */ - mod integer not null, /* 3 */ - usn integer not null, /* 4 */ - tags text not null, /* 5 */ - flds text not null, /* 6 */ - sfld integer not null, /* 7 */ - csum integer not null, /* 8 */ - flags integer not null, /* 9 */ - data text not null /* 10 */ -); - -create table if not exists cards ( - id integer primary key, /* 0 */ - nid integer not null, /* 1 */ - did integer not null, /* 2 */ - ord integer not null, /* 3 */ - mod integer not null, /* 4 */ - usn integer not null, /* 5 */ - type integer not null, /* 6 */ - queue integer not null, /* 7 */ - due integer not null, /* 8 */ - ivl integer not null, /* 9 */ - factor integer not null, /* 10 */ - reps integer not null, /* 11 */ - lapses integer not null, /* 12 */ - left integer not null, /* 13 */ - odue integer not null, /* 14 */ - odid integer not null, /* 15 */ - flags integer not null, /* 16 */ - data text not null /* 17 */ -); - -create table if not exists revlog ( - id integer primary key, - cid integer not null, - usn integer not null, - ease integer not null, - ivl integer not null, - lastIvl integer not null, - factor integer not null, - time integer not null, - type integer not null -); - -create table if not exists graves ( - usn integer not null, - oid integer not null, - type integer not null -); - -insert or ignore into col -values(1,0,0,%(s)s,%(v)s,0,0,0,'','{}','','','{}'); -""" - % ({"v": SCHEMA_VERSION, "s": intTime(1000)}) - ) - if setColConf: - _addColVars(db, *_getColVars(db)) +def initial_db_setup(db: DBProxy) -> None: + _addColVars(db, *_getColVars(db)) def _getColVars(db: DBProxy) -> Tuple[Any, Any, Dict[str, Any]]: @@ -202,23 +93,3 @@ update col set conf = ?, decks = ?, dconf = ?""", json.dumps({"1": g}), json.dumps({"1": gc}), ) - - -def _updateIndices(db: DBProxy) -> None: - "Add indices to the DB." - db.executescript( - """ --- syncing -create index if not exists ix_notes_usn on notes (usn); -create index if not exists ix_cards_usn on cards (usn); -create index if not exists ix_revlog_usn on revlog (usn); --- card spacing, etc -create index if not exists ix_cards_nid on cards (nid); --- scheduling and deck limiting -create index if not exists ix_cards_sched on cards (did, queue, due); --- revlog by card -create index if not exists ix_revlog_cid on revlog (cid); --- field uniqueness -create index if not exists ix_notes_csum on notes (csum); -""" - ) diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index f86be1b0b..0cec30aca 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -1,23 +1,26 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::backend_proto as pb; use crate::err::Result; use crate::storage::SqliteStorage; use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; use serde_derive::{Deserialize, Serialize}; #[derive(Deserialize)] -pub(super) struct DBRequest { - sql: String, - args: Vec, +#[serde(tag = "kind", rename_all = "lowercase")] +pub(super) enum DBRequest { + Query { sql: String, args: Vec }, + Begin, + Commit, + Rollback, } -// #[derive(Serialize)] -// pub(super) struct DBResult { -// rows: Vec>, -// } -type DBResult = Vec>; +#[derive(Serialize)] +#[serde(untagged)] +pub(super) enum DBResult { + Rows(Vec>), + None, +} #[derive(Serialize, Deserialize, Debug)] #[serde(untagged)] @@ -55,30 +58,41 @@ impl FromSql for SqlValue { } } -pub(super) fn db_query_json_str(db: &SqliteStorage, input: &[u8]) -> Result { +pub(super) fn db_command_bytes(db: &SqliteStorage, input: &[u8]) -> Result { let req: DBRequest = serde_json::from_slice(input)?; - let resp = db_query_json(db, req)?; + let resp = match req { + DBRequest::Query { sql, args } => db_query(db, &sql, &args)?, + DBRequest::Begin => { + db.begin()?; + DBResult::None + } + DBRequest::Commit => { + db.commit()?; + DBResult::None + } + DBRequest::Rollback => { + db.rollback()?; + DBResult::None + } + }; Ok(serde_json::to_string(&resp)?) } -pub(super) fn db_query_json(db: &SqliteStorage, input: DBRequest) -> Result { - let mut stmt = db.db.prepare_cached(&input.sql)?; +pub(super) fn db_query(db: &SqliteStorage, sql: &str, args: &[SqlValue]) -> Result { + let mut stmt = db.db.prepare_cached(sql)?; let columns = stmt.column_count(); - let mut rows = stmt.query(&input.args)?; + let res: std::result::Result>, rusqlite::Error> = stmt + .query_map(args, |row| { + let mut orow = Vec::with_capacity(columns); + for i in 0..columns { + let v: SqlValue = row.get(i)?; + orow.push(v); + } + Ok(orow) + })? + .collect(); - let mut output_rows = vec![]; - - while let Some(row) = rows.next()? { - let mut orow = Vec::with_capacity(columns); - for i in 0..columns { - let v: SqlValue = row.get(i)?; - orow.push(v); - } - - output_rows.push(orow); - } - - Ok(output_rows) + Ok(DBResult::Rows(res?)) } diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index bbf8b5266..7f479d7d8 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::backend::dbproxy::db_query_json_str; +use crate::backend::dbproxy::db_command_bytes; use crate::backend_proto::backend_input::Value; use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; @@ -37,7 +37,7 @@ pub struct Backend { media_folder: PathBuf, media_db: String, progress_callback: Option, - i18n: I18n, + pub i18n: I18n, log: Logger, } @@ -493,8 +493,8 @@ impl Backend { checker.restore_trash() } - pub fn db_query(&self, input: pb::DbQueryIn) -> Result { - db_query_proto(&self.col, input) + pub fn db_command(&self, input: &[u8]) -> Result { + db_command_bytes(&self.col, input) } } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 4f6a13bd1..e0d68c640 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -4,28 +4,12 @@ use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; use crate::time::i64_unix_timestamp; -use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ValueRef}; -use rusqlite::{params, Connection, OptionalExtension, NO_PARAMS}; -use serde::de::DeserializeOwned; -use serde_derive::{Deserialize, Serialize}; -use serde_json::Value; -use std::borrow::Cow; -use std::convert::TryFrom; -use std::fmt; +use rusqlite::{params, Connection, NO_PARAMS}; use std::path::{Path, PathBuf}; const SCHEMA_MIN_VERSION: u8 = 11; const SCHEMA_MAX_VERSION: u8 = 11; -macro_rules! cached_sql { - ( $label:expr, $db:expr, $sql:expr ) => {{ - if $label.is_none() { - $label = Some($db.prepare_cached($sql)?); - } - $label.as_mut().unwrap() - }}; -} - // currently public for dbproxy #[derive(Debug)] pub struct SqliteStorage { @@ -42,6 +26,8 @@ fn open_or_create_collection_db(path: &Path) -> Result { db.trace(Some(trace)); } + db.busy_timeout(std::time::Duration::from_secs(0))?; + db.pragma_update(None, "locking_mode", &"exclusive")?; db.pragma_update(None, "page_size", &4096)?; db.pragma_update(None, "cache_size", &(-40 * 1024))?; @@ -78,7 +64,6 @@ impl SqliteStorage { let (create, ver) = schema_version(&db)?; if create { - unimplemented!(); // todo db.prepare_cached("begin exclusive")?.execute(NO_PARAMS)?; db.execute_batch(include_str!("schema11.sql"))?; db.execute( @@ -118,12 +103,16 @@ impl SqliteStorage { } pub(crate) fn commit(&self) -> Result<()> { - self.db.prepare_cached("commit")?.execute(NO_PARAMS)?; + if !self.db.is_autocommit() { + self.db.prepare_cached("commit")?.execute(NO_PARAMS)?; + } Ok(()) } pub(crate) fn rollback(&self) -> Result<()> { - self.db.execute("rollback", NO_PARAMS)?; + if !self.db.is_autocommit() { + self.db.execute("rollback", NO_PARAMS)?; + } Ok(()) } } diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index d921aa799..7baad5569 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -4,9 +4,10 @@ use anki::backend::{ init_backend, init_i18n_backend, Backend as RustBackend, I18nBackend as RustI18nBackend, }; +use pyo3::exceptions::Exception; use pyo3::prelude::*; use pyo3::types::PyBytes; -use pyo3::{exceptions, wrap_pyfunction}; +use pyo3::{create_exception, exceptions, wrap_pyfunction}; // Regular backend ////////////////////////////////// @@ -16,6 +17,8 @@ struct Backend { backend: RustBackend, } +create_exception!(ankirspy, DBError, Exception); + #[pyfunction] fn buildhash() -> &'static str { include_str!("../../meta/buildhash").trim() @@ -71,11 +74,15 @@ impl Backend { } } - fn db_query(&mut self, py: Python, input: &PyBytes) -> PyObject { + fn db_command(&mut self, py: Python, input: &PyBytes) -> PyResult { let in_bytes = input.as_bytes(); - let out_string = self.backend.db_query(in_bytes).unwrap(); + let out_string = self + .backend + .db_command(in_bytes) + .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n)))?; + let out_obj = PyBytes::new(py, out_string.as_bytes()); - out_obj.into() + Ok(out_obj.into()) } } From ae06b9e44696d86c794a44abd27a36de278de40d Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 5 Mar 2020 14:35:48 +1000 Subject: [PATCH 029/150] add Collection struct, and get media check working again - media check no longer needs collection to be closed - use savepoints for operations initiated by Rust, so they are atomic without forcing a commit --- pylib/anki/collection.py | 8 +-- pylib/anki/dbproxy.py | 1 + pylib/anki/media.py | 7 ++- pylib/anki/storage.py | 2 +- pylib/tests/test_media.py | 2 - qt/aqt/mediacheck.py | 2 - rslib/src/backend/dbproxy.rs | 16 +++--- rslib/src/backend/mod.rs | 80 ++++++++++++++++-------------- rslib/src/collection.rs | 83 +++++++++++++++++++++++++++++++ rslib/src/lib.rs | 1 + rslib/src/media/check.rs | 96 +++++++++++++++++++----------------- rslib/src/media/col.rs | 14 ------ rslib/src/storage/mod.rs | 2 +- rslib/src/storage/sqlite.rs | 95 ++++++++++++++++++++++++++++++----- rspy/src/lib.rs | 2 +- 15 files changed, 281 insertions(+), 130 deletions(-) create mode 100644 rslib/src/collection.rs diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 989d18925..60ac9c0fe 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -264,10 +264,10 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", def reopen(self) -> None: "Reconnect to DB (after changing threads, etc)." raise Exception("fixme") - if not self.db: - #self.db = DBProxy(self.path) - self.media.connect() - self._openLog() + # if not self.db: + # # self.db = DBProxy(self.path) + # self.media.connect() + # self._openLog() def rollback(self) -> None: self.db.rollback() diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index ae5a8f779..71eff93f3 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Optional, Sequence, Union import anki +# fixme: threads # fixme: col.reopen() # fixme: setAutocommit() # fixme: transaction/lock handling diff --git a/pylib/anki/media.py b/pylib/anki/media.py index 5d100a6a8..dbfddd8fb 100644 --- a/pylib/anki/media.py +++ b/pylib/anki/media.py @@ -171,8 +171,11 @@ class MediaManager: ########################################################################## def check(self) -> MediaCheckOutput: - "This should be called while the collection is closed." - return self.col.backend.check_media() + output = self.col.backend.check_media() + # files may have been renamed on disk, so an undo at this point could + # break file references + self.col.save() + return output def render_all_latex( self, progress_cb: Optional[Callable[[int], bool]] = None diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index 0b626d1bb..e873a38f8 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -19,7 +19,7 @@ from anki.stdmodels import ( addForwardOptionalReverse, addForwardReverse, ) -from anki.utils import intTime, isWin +from anki.utils import intTime class ServerData: diff --git a/pylib/tests/test_media.py b/pylib/tests/test_media.py index ecd723e17..22ffba4cb 100644 --- a/pylib/tests/test_media.py +++ b/pylib/tests/test_media.py @@ -73,8 +73,6 @@ def test_deckIntegration(): with open(os.path.join(d.media.dir(), "foo.jpg"), "w") as f: f.write("test") # check media - d.close() ret = d.media.check() - d.reopen() assert ret.missing == ["fake2.png"] assert ret.unused == ["foo.jpg"] diff --git a/qt/aqt/mediacheck.py b/qt/aqt/mediacheck.py index 79275996e..b3c4c547e 100644 --- a/qt/aqt/mediacheck.py +++ b/qt/aqt/mediacheck.py @@ -40,7 +40,6 @@ class MediaChecker: def check(self) -> None: self.progress_dialog = self.mw.progress.start() hooks.bg_thread_progress_callback.append(self._on_progress) - self.mw.col.close() self.mw.taskman.run_in_background(self._check, self._on_finished) def _on_progress(self, proceed: bool, progress: Progress) -> bool: @@ -61,7 +60,6 @@ class MediaChecker: hooks.bg_thread_progress_callback.remove(self._on_progress) self.mw.progress.finish() self.progress_dialog = None - self.mw.col.reopen() exc = future.exception() if isinstance(exc, Interrupted): diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index 0cec30aca..f0c2512a2 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -2,7 +2,7 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::err::Result; -use crate::storage::SqliteStorage; +use crate::storage::StorageContext; use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; use serde_derive::{Deserialize, Serialize}; @@ -58,28 +58,28 @@ impl FromSql for SqlValue { } } -pub(super) fn db_command_bytes(db: &SqliteStorage, input: &[u8]) -> Result { +pub(super) fn db_command_bytes(ctx: &StorageContext, input: &[u8]) -> Result { let req: DBRequest = serde_json::from_slice(input)?; let resp = match req { - DBRequest::Query { sql, args } => db_query(db, &sql, &args)?, + DBRequest::Query { sql, args } => db_query(ctx, &sql, &args)?, DBRequest::Begin => { - db.begin()?; + ctx.begin_trx()?; DBResult::None } DBRequest::Commit => { - db.commit()?; + ctx.commit_trx()?; DBResult::None } DBRequest::Rollback => { - db.rollback()?; + ctx.rollback_trx()?; DBResult::None } }; Ok(serde_json::to_string(&resp)?) } -pub(super) fn db_query(db: &SqliteStorage, sql: &str, args: &[SqlValue]) -> Result { - let mut stmt = db.db.prepare_cached(sql)?; +pub(super) fn db_query(ctx: &StorageContext, sql: &str, args: &[SqlValue]) -> Result { + let mut stmt = ctx.db.prepare_cached(sql)?; let columns = stmt.column_count(); diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 7f479d7d8..43b18bb0e 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -4,16 +4,16 @@ use crate::backend::dbproxy::db_command_bytes; use crate::backend_proto::backend_input::Value; use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; +use crate::collection::{open_collection, Collection}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; use crate::i18n::{tr_args, FString, I18n}; use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}; -use crate::log::{default_logger, Logger}; +use crate::log::default_logger; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; -use crate::storage::SqliteStorage; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, @@ -31,14 +31,12 @@ mod dbproxy; pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { - col: SqliteStorage, + col: Collection, #[allow(dead_code)] col_path: PathBuf, media_folder: PathBuf, media_db: String, progress_callback: Option, - pub i18n: I18n, - log: Logger, } enum Progress<'a> { @@ -124,7 +122,7 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { log::terminal(), ); - let col = SqliteStorage::open_or_create(Path::new(&input.collection_path), input.server) + let col = open_collection(&input.collection_path, input.server, i18n, logger) .map_err(|e| format!("Unable to open collection: {:?}", e))?; match Backend::new( @@ -132,8 +130,6 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { &input.collection_path, &input.media_folder_path, &input.media_db_path, - i18n, - logger, ) { Ok(backend) => Ok(backend), Err(e) => Err(format!("{:?}", e)), @@ -142,12 +138,10 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { impl Backend { pub fn new( - col: SqliteStorage, + col: Collection, col_path: &str, media_folder: &str, media_db: &str, - i18n: I18n, - log: Logger, ) -> Result { Ok(Backend { col, @@ -155,11 +149,13 @@ impl Backend { media_folder: media_folder.into(), media_db: media_db.into(), progress_callback: None, - i18n, - log, }) } + pub fn i18n(&self) -> &I18n { + &self.col.i18n + } + /// Decode a request, process it, and return the encoded result. pub fn run_command_bytes(&mut self, req: &[u8]) -> Vec { let mut buf = vec![]; @@ -169,7 +165,7 @@ impl Backend { Err(_e) => { // unable to decode let err = AnkiError::invalid_input("couldn't decode backend request"); - let oerr = anki_error_to_proto_error(err, &self.i18n); + let oerr = anki_error_to_proto_error(err, &self.col.i18n); let output = pb::BackendOutput { value: Some(oerr.into()), }; @@ -187,12 +183,12 @@ impl Backend { let oval = if let Some(ival) = input.value { match self.run_command_inner(ival) { Ok(output) => output, - Err(err) => anki_error_to_proto_error(err, &self.i18n).into(), + Err(err) => anki_error_to_proto_error(err, &self.col.i18n).into(), } } else { anki_error_to_proto_error( AnkiError::invalid_input("unrecognized backend input value"), - &self.i18n, + &self.col.i18n, ) .into() }; @@ -237,12 +233,12 @@ impl Backend { Value::StudiedToday(input) => OValue::StudiedToday(studied_today( input.cards as usize, input.seconds as f32, - &self.i18n, + &self.col.i18n, )), Value::CongratsLearnMsg(input) => OValue::CongratsLearnMsg(learning_congrats( input.remaining as usize, input.next_due, - &self.i18n, + &self.col.i18n, )), Value::EmptyTrash(_) => { self.empty_trash()?; @@ -257,7 +253,7 @@ impl Backend { fn fire_progress_callback(&self, progress: Progress) -> bool { if let Some(cb) = &self.progress_callback { - let bytes = progress_to_proto_bytes(progress, &self.i18n); + let bytes = progress_to_proto_bytes(progress, &self.col.i18n); cb(bytes) } else { true @@ -337,7 +333,7 @@ impl Backend { &input.answer_template, &fields, input.card_ordinal as u16, - &self.i18n, + &self.col.i18n, )?; // return @@ -415,7 +411,7 @@ impl Backend { }; let mut rt = Runtime::new().unwrap(); - rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.log.clone())) + rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.col.log.clone())) } fn check_media(&self) -> Result { @@ -423,16 +419,18 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - let mut checker = MediaChecker::new(&mgr, &self.col_path, callback, &self.i18n, &self.log); - let mut output = checker.check()?; + self.col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); + let mut output = checker.check()?; - let report = checker.summarize_output(&mut output); + let report = checker.summarize_output(&mut output); - Ok(pb::MediaCheckOut { - unused: output.unused, - missing: output.missing, - report, - have_trash: output.trash_count > 0, + Ok(pb::MediaCheckOut { + unused: output.unused, + missing: output.missing, + report, + have_trash: output.trash_count > 0, + }) }) } @@ -454,7 +452,7 @@ impl Backend { .map(|(k, v)| (k.as_str(), translate_arg_to_fluent_val(&v))) .collect(); - self.i18n.trn(key, map) + self.col.i18n.trn(key, map) } fn format_time_span(&self, input: pb::FormatTimeSpanIn) -> String { @@ -463,12 +461,14 @@ impl Backend { None => return "".to_string(), }; match context { - pb::format_time_span_in::Context::Precise => time_span(input.seconds, &self.i18n, true), + pb::format_time_span_in::Context::Precise => { + time_span(input.seconds, &self.col.i18n, true) + } pb::format_time_span_in::Context::Intervals => { - time_span(input.seconds, &self.i18n, false) + time_span(input.seconds, &self.col.i18n, false) } pb::format_time_span_in::Context::AnswerButtons => { - answer_button_time(input.seconds, &self.i18n) + answer_button_time(input.seconds, &self.col.i18n) } } } @@ -478,9 +478,11 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - let mut checker = MediaChecker::new(&mgr, &self.col_path, callback, &self.i18n, &self.log); + self.col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); - checker.empty_trash() + checker.empty_trash() + }) } fn restore_trash(&self) -> Result<()> { @@ -488,13 +490,15 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - let mut checker = MediaChecker::new(&mgr, &self.col_path, callback, &self.i18n, &self.log); + self.col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); - checker.restore_trash() + checker.restore_trash() + }) } pub fn db_command(&self, input: &[u8]) -> Result { - db_command_bytes(&self.col, input) + db_command_bytes(&self.col.storage.context(self.col.server), input) } } diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs new file mode 100644 index 000000000..ef9837e8b --- /dev/null +++ b/rslib/src/collection.rs @@ -0,0 +1,83 @@ +use crate::err::Result; +use crate::i18n::I18n; +use crate::log::Logger; +use crate::storage::{SqliteStorage, StorageContext}; +use std::path::Path; + +pub fn open_collection>( + path: P, + server: bool, + i18n: I18n, + log: Logger, +) -> Result { + let storage = SqliteStorage::open_or_create(path.as_ref())?; + + let col = Collection { + storage, + server, + i18n, + log, + }; + + Ok(col) +} + +pub struct Collection { + pub(crate) storage: SqliteStorage, + pub(crate) server: bool, + pub(crate) i18n: I18n, + pub(crate) log: Logger, +} + +pub(crate) enum CollectionOp {} + +pub(crate) struct RequestContext<'a> { + pub storage: StorageContext<'a>, + pub i18n: &'a I18n, + pub log: &'a Logger, +} + +impl Collection { + /// Call the provided closure with a RequestContext that exists for + /// the duration of the call. The request will cache prepared sql + /// statements, so should be passed down the call tree. + /// + /// This function should be used for read-only requests. To mutate + /// the database, use transact() instead. + pub(crate) fn with_ctx(&self, func: F) -> Result + where + F: FnOnce(&mut RequestContext) -> Result, + { + let mut ctx = RequestContext { + storage: self.storage.context(self.server), + i18n: &self.i18n, + log: &self.log, + }; + func(&mut ctx) + } + + /// Execute the provided closure in a transaction, rolling back if + /// an error is returned. + pub(crate) fn transact(&self, op: Option, func: F) -> Result + where + F: FnOnce(&mut RequestContext) -> Result, + { + self.with_ctx(|ctx| { + ctx.storage.begin_rust_trx()?; + + let mut res = func(ctx); + + if res.is_ok() { + if let Err(e) = ctx.storage.commit_rust_op(op) { + res = Err(e); + } + } + + if res.is_err() { + ctx.storage.rollback_rust_trx()?; + } + + res + }) + } +} diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 82ce394fb..16849b2d2 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -11,6 +11,7 @@ pub fn version() -> &'static str { pub mod backend; pub mod cloze; +pub mod collection; pub mod err; pub mod i18n; pub mod latex; diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index a6f1fa76a..ee8b71115 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -1,14 +1,12 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::collection::RequestContext; use crate::err::{AnkiError, DBErrorKind, Result}; -use crate::i18n::{tr_args, tr_strs, FString, I18n}; +use crate::i18n::{tr_args, tr_strs, FString}; use crate::latex::extract_latex_expanding_clozes; -use crate::log::{debug, Logger}; -use crate::media::col::{ - for_every_note, get_note_types, mark_collection_modified, open_or_create_collection_db, - set_note, Note, -}; +use crate::log::debug; +use crate::media::col::{for_every_note, get_note_types, mark_collection_modified, set_note, Note}; use crate::media::database::MediaDatabaseContext; use crate::media::files::{ data_for_file, filename_if_normalized, trash_folder, MEDIA_SYNC_FILESIZE_LIMIT, @@ -19,14 +17,13 @@ use coarsetime::Instant; use lazy_static::lazy_static; use regex::Regex; use std::collections::{HashMap, HashSet}; -use std::path::Path; use std::{borrow::Cow, fs, io}; lazy_static! { static ref REMOTE_FILENAME: Regex = Regex::new("(?i)^https?://").unwrap(); } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct MediaCheckOutput { pub unused: Vec, pub missing: Vec, @@ -49,34 +46,28 @@ pub struct MediaChecker<'a, P> where P: FnMut(usize) -> bool, { + ctx: &'a RequestContext<'a>, mgr: &'a MediaManager, - col_path: &'a Path, progress_cb: P, checked: usize, progress_updated: Instant, - i18n: &'a I18n, - log: &'a Logger, } impl

MediaChecker<'_, P> where P: FnMut(usize) -> bool, { - pub fn new<'a>( + pub(crate) fn new<'a>( + ctx: &'a RequestContext<'a>, mgr: &'a MediaManager, - col_path: &'a Path, progress_cb: P, - i18n: &'a I18n, - log: &'a Logger, ) -> MediaChecker<'a, P> { MediaChecker { + ctx, mgr, - col_path, progress_cb, checked: 0, progress_updated: Instant::now(), - i18n, - log, } } @@ -100,7 +91,7 @@ where pub fn summarize_output(&self, output: &mut MediaCheckOutput) -> String { let mut buf = String::new(); - let i = &self.i18n; + let i = &self.ctx.i18n; // top summary area if output.trash_count > 0 { @@ -279,7 +270,7 @@ where } })?; let fname = self.mgr.add_file(ctx, disk_fname, &data)?; - debug!(self.log, "renamed"; "from"=>disk_fname, "to"=>&fname.as_ref()); + debug!(self.ctx.log, "renamed"; "from"=>disk_fname, "to"=>&fname.as_ref()); assert_ne!(fname.as_ref(), disk_fname); // remove the original file @@ -373,7 +364,7 @@ where self.mgr .add_file(&mut self.mgr.dbctx(), fname.as_ref(), &data)?; } else { - debug!(self.log, "file disappeared while restoring trash"; "fname"=>fname.as_ref()); + debug!(self.ctx.log, "file disappeared while restoring trash"; "fname"=>fname.as_ref()); } fs::remove_file(dentry.path())?; } @@ -387,14 +378,11 @@ where &mut self, renamed: &HashMap, ) -> Result> { - let mut db = open_or_create_collection_db(self.col_path)?; - let trx = db.transaction()?; - let mut referenced_files = HashSet::new(); - let note_types = get_note_types(&trx)?; + let note_types = get_note_types(&self.ctx.storage.db)?; let mut collection_modified = false; - for_every_note(&trx, |note| { + for_every_note(&self.ctx.storage.db, |note| { self.checked += 1; if self.checked % 10 == 0 { self.maybe_fire_progress_cb()?; @@ -407,7 +395,7 @@ where })?; if fix_and_extract_media_refs(note, &mut referenced_files, renamed)? { // note was modified, needs saving - set_note(&trx, note, nt)?; + set_note(&self.ctx.storage.db, note, nt)?; collection_modified = true; } @@ -417,8 +405,7 @@ where })?; if collection_modified { - mark_collection_modified(&trx)?; - trx.commit()?; + mark_collection_modified(&self.ctx.storage.db)?; } Ok(referenced_files) @@ -512,18 +499,18 @@ fn extract_latex_refs(note: &Note, seen_files: &mut HashSet, svg: bool) #[cfg(test)] mod test { + use crate::collection::{open_collection, Collection}; use crate::err::Result; use crate::i18n::I18n; use crate::log; - use crate::log::Logger; use crate::media::check::{MediaCheckOutput, MediaChecker}; use crate::media::files::trash_folder; use crate::media::MediaManager; - use std::path::{Path, PathBuf}; + use std::path::Path; use std::{fs, io}; use tempfile::{tempdir, TempDir}; - fn common_setup() -> Result<(TempDir, MediaManager, PathBuf, Logger, I18n)> { + fn common_setup() -> Result<(TempDir, MediaManager, Collection)> { let dir = tempdir()?; let media_dir = dir.path().join("media"); fs::create_dir(&media_dir)?; @@ -537,15 +524,16 @@ mod test { let mgr = MediaManager::new(&media_dir, media_db)?; let log = log::terminal(); - let i18n = I18n::new(&["zz"], "dummy", log.clone()); - Ok((dir, mgr, col_path, log, i18n)) + let col = open_collection(col_path, false, i18n, log)?; + + Ok((dir, mgr, col)) } #[test] fn media_check() -> Result<()> { - let (_dir, mgr, col_path, log, i18n) = common_setup()?; + let (_dir, mgr, col) = common_setup()?; // add some test files fs::write(&mgr.media_folder.join("zerobytes"), "")?; @@ -556,8 +544,13 @@ mod test { fs::write(&mgr.media_folder.join("unused.jpg"), "foo")?; let progress = |_n| true; - let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log); - let mut output = checker.check()?; + + let (output, report) = col.transact(None, |ctx| { + let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let output = checker.check()?; + let summary = checker.summarize_output(&mut output.clone()); + Ok((output, summary)) + })?; assert_eq!( output, @@ -577,7 +570,6 @@ mod test { assert!(fs::metadata(&mgr.media_folder.join("foo[.jpg")).is_err()); assert!(fs::metadata(&mgr.media_folder.join("foo.jpg")).is_ok()); - let report = checker.summarize_output(&mut output); assert_eq!( report, "Missing files: 1 @@ -617,14 +609,16 @@ Unused: unused.jpg #[test] fn trash_handling() -> Result<()> { - let (_dir, mgr, col_path, log, i18n) = common_setup()?; + let (_dir, mgr, col) = common_setup()?; let trash_folder = trash_folder(&mgr.media_folder)?; fs::write(trash_folder.join("test.jpg"), "test")?; let progress = |_n| true; - let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log); - checker.restore_trash()?; + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(&ctx, &mgr, progress); + checker.restore_trash() + })?; // file should have been moved to media folder assert_eq!(files_in_dir(&trash_folder), Vec::::new()); @@ -635,7 +629,10 @@ Unused: unused.jpg // if we repeat the process, restoring should do the same thing if the contents are equal fs::write(trash_folder.join("test.jpg"), "test")?; - checker.restore_trash()?; + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(&ctx, &mgr, progress); + checker.restore_trash() + })?; assert_eq!(files_in_dir(&trash_folder), Vec::::new()); assert_eq!( files_in_dir(&mgr.media_folder), @@ -644,7 +641,10 @@ Unused: unused.jpg // but rename if required fs::write(trash_folder.join("test.jpg"), "test2")?; - checker.restore_trash()?; + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(&ctx, &mgr, progress); + checker.restore_trash() + })?; assert_eq!(files_in_dir(&trash_folder), Vec::::new()); assert_eq!( files_in_dir(&mgr.media_folder), @@ -659,13 +659,17 @@ Unused: unused.jpg #[test] fn unicode_normalization() -> Result<()> { - let (_dir, mgr, col_path, log, i18n) = common_setup()?; + let (_dir, mgr, col) = common_setup()?; fs::write(&mgr.media_folder.join("ぱぱ.jpg"), "nfd encoding")?; let progress = |_n| true; - let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log); - let mut output = checker.check()?; + + let mut output = col.transact(None, |ctx| { + let mut checker = MediaChecker::new(&ctx, &mgr, progress); + checker.check() + })?; + output.missing.sort(); if cfg!(target_vendor = "apple") { diff --git a/rslib/src/media/col.rs b/rslib/src/media/col.rs index 460d64b5f..a87567a4c 100644 --- a/rslib/src/media/col.rs +++ b/rslib/src/media/col.rs @@ -11,7 +11,6 @@ use serde_aux::field_attributes::deserialize_number_from_string; use serde_derive::Deserialize; use std::collections::HashMap; use std::convert::TryInto; -use std::path::Path; #[derive(Debug)] pub(super) struct Note { @@ -45,19 +44,6 @@ fn field_checksum(text: &str) -> u32 { u32::from_be_bytes(digest[..4].try_into().unwrap()) } -pub(super) fn open_or_create_collection_db(path: &Path) -> Result { - let db = Connection::open(path)?; - - db.pragma_update(None, "locking_mode", &"exclusive")?; - db.pragma_update(None, "page_size", &4096)?; - db.pragma_update(None, "cache_size", &(-40 * 1024))?; - db.pragma_update(None, "legacy_file_format", &false)?; - db.pragma_update(None, "journal", &"wal")?; - db.set_prepared_statement_cache_capacity(5); - - Ok(db) -} - #[derive(Deserialize, Debug)] pub(super) struct NoteType { #[serde(deserialize_with = "deserialize_number_from_string")] diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs index 2b474f145..2ed04892c 100644 --- a/rslib/src/storage/mod.rs +++ b/rslib/src/storage/mod.rs @@ -1,3 +1,3 @@ mod sqlite; -pub(crate) use sqlite::SqliteStorage; +pub(crate) use sqlite::{SqliteStorage, StorageContext}; diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index e0d68c640..2e4bd37d1 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -1,9 +1,11 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::collection::CollectionOp; use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; -use crate::time::i64_unix_timestamp; +use crate::time::i64_unix_secs; +use crate::types::Usn; use rusqlite::{params, Connection, NO_PARAMS}; use std::path::{Path, PathBuf}; @@ -15,8 +17,9 @@ const SCHEMA_MAX_VERSION: u8 = 11; pub struct SqliteStorage { // currently crate-visible for dbproxy pub(crate) db: Connection, + + // fixme: stored in wrong location? path: PathBuf, - server: bool, } fn open_or_create_collection_db(path: &Path) -> Result { @@ -59,17 +62,14 @@ fn trace(s: &str) { } impl SqliteStorage { - pub(crate) fn open_or_create(path: &Path, server: bool) -> Result { + pub(crate) fn open_or_create(path: &Path) -> Result { let db = open_or_create_collection_db(path)?; let (create, ver) = schema_version(&db)?; if create { db.prepare_cached("begin exclusive")?.execute(NO_PARAMS)?; db.execute_batch(include_str!("schema11.sql"))?; - db.execute( - "update col set crt=?, ver=?", - params![i64_unix_timestamp(), ver], - )?; + db.execute("update col set crt=?, ver=?", params![i64_unix_secs(), ver])?; db.prepare_cached("commit")?.execute(NO_PARAMS)?; } else { if ver > SCHEMA_MAX_VERSION { @@ -89,30 +89,103 @@ impl SqliteStorage { let storage = Self { db, path: path.to_owned(), - server, }; Ok(storage) } - pub(crate) fn begin(&self) -> Result<()> { + pub(crate) fn context(&self, server: bool) -> StorageContext { + StorageContext::new(&self.db, server) + } +} + +pub(crate) struct StorageContext<'a> { + pub(crate) db: &'a Connection, + #[allow(dead_code)] + server: bool, + #[allow(dead_code)] + usn: Option, +} + +impl StorageContext<'_> { + fn new(db: &Connection, server: bool) -> StorageContext { + StorageContext { + db, + server, + usn: None, + } + } + + // Standard transaction start/stop + ////////////////////////////////////// + + pub(crate) fn begin_trx(&self) -> Result<()> { self.db .prepare_cached("begin exclusive")? .execute(NO_PARAMS)?; Ok(()) } - pub(crate) fn commit(&self) -> Result<()> { + pub(crate) fn commit_trx(&self) -> Result<()> { if !self.db.is_autocommit() { self.db.prepare_cached("commit")?.execute(NO_PARAMS)?; } Ok(()) } - pub(crate) fn rollback(&self) -> Result<()> { + pub(crate) fn rollback_trx(&self) -> Result<()> { if !self.db.is_autocommit() { self.db.execute("rollback", NO_PARAMS)?; } Ok(()) } + + // Savepoints + ////////////////////////////////////////// + // + // This is necessary at the moment because Anki's current architecture uses + // long-running transactions as an undo mechanism. Once a proper undo + // mechanism has been added to all existing functionality, we could + // transition these to standard commits. + + pub(crate) fn begin_rust_trx(&self) -> Result<()> { + self.db + .prepare_cached("savepoint rust")? + .execute(NO_PARAMS)?; + Ok(()) + } + + pub(crate) fn commit_rust_trx(&self) -> Result<()> { + self.db.prepare_cached("release rust")?.execute(NO_PARAMS)?; + Ok(()) + } + + pub(crate) fn commit_rust_op(&self, _op: Option) -> Result<()> { + self.commit_rust_trx() + } + + pub(crate) fn rollback_rust_trx(&self) -> Result<()> { + self.db + .prepare_cached("rollback to rust")? + .execute(NO_PARAMS)?; + Ok(()) + } + + ////////////////////////////////////////// + + #[allow(dead_code)] + pub(crate) fn usn(&mut self) -> Result { + if self.server { + if self.usn.is_none() { + self.usn = Some( + self.db + .prepare_cached("select usn from col")? + .query_row(NO_PARAMS, |row| row.get(0))?, + ); + } + Ok(*self.usn.as_ref().unwrap()) + } else { + Ok(-1) + } + } } diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index 7baad5569..78b222c88 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -79,7 +79,7 @@ impl Backend { let out_string = self .backend .db_command(in_bytes) - .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n)))?; + .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n())))?; let out_obj = PyBytes::new(py, out_string.as_bytes()); Ok(out_obj.into()) From 47c142a74c7153617dc61c5f3d83a65f2a3835ef Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 5 Mar 2020 16:29:04 +1000 Subject: [PATCH 030/150] move note code into notes.rs, add ability to rollback when unchanged --- rslib/src/collection.rs | 13 ++++++++++--- rslib/src/lib.rs | 1 + rslib/src/media/check.rs | 28 ++++++++++++++-------------- rslib/src/media/mod.rs | 1 - rslib/src/{media/col.rs => notes.rs} | 10 +++------- rslib/src/storage/sqlite.rs | 9 ++++++++- 6 files changed, 36 insertions(+), 26 deletions(-) rename rslib/src/{media/col.rs => notes.rs} (93%) diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs index ef9837e8b..c728edca6 100644 --- a/rslib/src/collection.rs +++ b/rslib/src/collection.rs @@ -1,3 +1,6 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + use crate::err::Result; use crate::i18n::I18n; use crate::log::Logger; @@ -35,6 +38,7 @@ pub(crate) struct RequestContext<'a> { pub storage: StorageContext<'a>, pub i18n: &'a I18n, pub log: &'a Logger, + pub should_commit: bool, } impl Collection { @@ -52,6 +56,7 @@ impl Collection { storage: self.storage.context(self.server), i18n: &self.i18n, log: &self.log, + should_commit: true, }; func(&mut ctx) } @@ -67,13 +72,15 @@ impl Collection { let mut res = func(ctx); - if res.is_ok() { - if let Err(e) = ctx.storage.commit_rust_op(op) { + if res.is_ok() && ctx.should_commit { + if let Err(e) = ctx.storage.mark_modified() { + res = Err(e); + } else if let Err(e) = ctx.storage.commit_rust_op(op) { res = Err(e); } } - if res.is_err() { + if res.is_err() || !ctx.should_commit { ctx.storage.rollback_rust_trx()?; } diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 16849b2d2..8f815dce8 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -17,6 +17,7 @@ pub mod i18n; pub mod latex; pub mod log; pub mod media; +pub mod notes; pub mod sched; pub mod storage; pub mod template; diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index ee8b71115..b5338d631 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -6,11 +6,11 @@ use crate::err::{AnkiError, DBErrorKind, Result}; use crate::i18n::{tr_args, tr_strs, FString}; use crate::latex::extract_latex_expanding_clozes; use crate::log::debug; -use crate::media::col::{for_every_note, get_note_types, mark_collection_modified, set_note, Note}; use crate::media::database::MediaDatabaseContext; use crate::media::files::{ data_for_file, filename_if_normalized, trash_folder, MEDIA_SYNC_FILESIZE_LIMIT, }; +use crate::notes::{for_every_note, get_note_types, set_note, Note}; use crate::text::{normalize_to_nfc, MediaRef}; use crate::{media::MediaManager, text::extract_media_refs}; use coarsetime::Instant; @@ -42,26 +42,26 @@ struct MediaFolderCheck { oversize: Vec, } -pub struct MediaChecker<'a, P> +pub struct MediaChecker<'a, 'b, P> where P: FnMut(usize) -> bool, { - ctx: &'a RequestContext<'a>, + ctx: &'a mut RequestContext<'b>, mgr: &'a MediaManager, progress_cb: P, checked: usize, progress_updated: Instant, } -impl

MediaChecker<'_, P> +impl

MediaChecker<'_, '_, P> where P: FnMut(usize) -> bool, { - pub(crate) fn new<'a>( - ctx: &'a RequestContext<'a>, + pub(crate) fn new<'a, 'b>( + ctx: &'a mut RequestContext<'b>, mgr: &'a MediaManager, progress_cb: P, - ) -> MediaChecker<'a, P> { + ) -> MediaChecker<'a, 'b, P> { MediaChecker { ctx, mgr, @@ -404,8 +404,8 @@ where Ok(()) })?; - if collection_modified { - mark_collection_modified(&self.ctx.storage.db)?; + if !collection_modified { + self.ctx.should_commit = false; } Ok(referenced_files) @@ -546,7 +546,7 @@ mod test { let progress = |_n| true; let (output, report) = col.transact(None, |ctx| { - let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let mut checker = MediaChecker::new(ctx, &mgr, progress); let output = checker.check()?; let summary = checker.summarize_output(&mut output.clone()); Ok((output, summary)) @@ -616,7 +616,7 @@ Unused: unused.jpg let progress = |_n| true; col.transact(None, |ctx| { - let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let mut checker = MediaChecker::new(ctx, &mgr, progress); checker.restore_trash() })?; @@ -630,7 +630,7 @@ Unused: unused.jpg // if we repeat the process, restoring should do the same thing if the contents are equal fs::write(trash_folder.join("test.jpg"), "test")?; col.transact(None, |ctx| { - let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let mut checker = MediaChecker::new(ctx, &mgr, progress); checker.restore_trash() })?; assert_eq!(files_in_dir(&trash_folder), Vec::::new()); @@ -642,7 +642,7 @@ Unused: unused.jpg // but rename if required fs::write(trash_folder.join("test.jpg"), "test2")?; col.transact(None, |ctx| { - let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let mut checker = MediaChecker::new(ctx, &mgr, progress); checker.restore_trash() })?; assert_eq!(files_in_dir(&trash_folder), Vec::::new()); @@ -666,7 +666,7 @@ Unused: unused.jpg let progress = |_n| true; let mut output = col.transact(None, |ctx| { - let mut checker = MediaChecker::new(&ctx, &mgr, progress); + let mut checker = MediaChecker::new(ctx, &mgr, progress); checker.check() })?; diff --git a/rslib/src/media/mod.rs b/rslib/src/media/mod.rs index c0dcfcca8..a3351b95c 100644 --- a/rslib/src/media/mod.rs +++ b/rslib/src/media/mod.rs @@ -12,7 +12,6 @@ use std::path::{Path, PathBuf}; pub mod changetracker; pub mod check; -pub mod col; pub mod database; pub mod files; pub mod sync; diff --git a/rslib/src/media/col.rs b/rslib/src/notes.rs similarity index 93% rename from rslib/src/media/col.rs rename to rslib/src/notes.rs index a87567a4c..c4289e9b1 100644 --- a/rslib/src/media/col.rs +++ b/rslib/src/notes.rs @@ -1,10 +1,11 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -/// Basic note reading/updating functionality for the media DB check. +/// At the moment, this is just basic note reading/updating functionality for +/// the media DB check. use crate::err::{AnkiError, DBErrorKind, Result}; use crate::text::strip_html_preserving_image_filenames; -use crate::time::{i64_unix_millis, i64_unix_secs}; +use crate::time::i64_unix_secs; use crate::types::{ObjID, Timestamp, Usn}; use rusqlite::{params, Connection, Row, NO_PARAMS}; use serde_aux::field_attributes::deserialize_number_from_string; @@ -140,8 +141,3 @@ pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) - Ok(()) } - -pub(super) fn mark_collection_modified(db: &Connection) -> Result<()> { - db.execute("update col set mod=?", params![i64_unix_millis()])?; - Ok(()) -} diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 2e4bd37d1..886a8f801 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -4,7 +4,7 @@ use crate::collection::CollectionOp; use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; -use crate::time::i64_unix_secs; +use crate::time::{i64_unix_millis, i64_unix_secs}; use crate::types::Usn; use rusqlite::{params, Connection, NO_PARAMS}; use std::path::{Path, PathBuf}; @@ -173,6 +173,13 @@ impl StorageContext<'_> { ////////////////////////////////////////// + pub(crate) fn mark_modified(&self) -> Result<()> { + self.db + .prepare_cached("update col set mod=?")? + .execute(params![i64_unix_millis()])?; + Ok(()) + } + #[allow(dead_code)] pub(crate) fn usn(&mut self) -> Result { if self.server { From 14546c8a8b417347bd448a1b869158c3ceef856b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 09:22:46 +1000 Subject: [PATCH 031/150] wrap the collection in a mutex so DB access is thread safe --- rslib/src/backend/mod.rs | 63 +++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 43b18bb0e..ca51c560f 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -8,7 +8,7 @@ use crate::collection::{open_collection, Collection}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; use crate::i18n::{tr_args, FString, I18n}; use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}; -use crate::log::default_logger; +use crate::log::{default_logger, Logger}; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; @@ -24,6 +24,7 @@ use fluent::FluentValue; use prost::Message; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; +use std::sync::{Arc, Mutex}; use tokio::runtime::Runtime; mod dbproxy; @@ -31,12 +32,14 @@ mod dbproxy; pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { - col: Collection, + col: Arc>, #[allow(dead_code)] col_path: PathBuf, media_folder: PathBuf, media_db: String, progress_callback: Option, + i18n: I18n, + log: Logger, } enum Progress<'a> { @@ -122,14 +125,21 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { log::terminal(), ); - let col = open_collection(&input.collection_path, input.server, i18n, logger) - .map_err(|e| format!("Unable to open collection: {:?}", e))?; + let col = open_collection( + &input.collection_path, + input.server, + i18n.clone(), + logger.clone(), + ) + .map_err(|e| format!("Unable to open collection: {:?}", e))?; match Backend::new( col, &input.collection_path, &input.media_folder_path, &input.media_db_path, + i18n, + logger, ) { Ok(backend) => Ok(backend), Err(e) => Err(format!("{:?}", e)), @@ -142,18 +152,22 @@ impl Backend { col_path: &str, media_folder: &str, media_db: &str, + i18n: I18n, + log: Logger, ) -> Result { Ok(Backend { - col, + col: Arc::new(Mutex::new(col)), col_path: col_path.into(), media_folder: media_folder.into(), media_db: media_db.into(), progress_callback: None, + i18n, + log, }) } pub fn i18n(&self) -> &I18n { - &self.col.i18n + &self.i18n } /// Decode a request, process it, and return the encoded result. @@ -165,7 +179,7 @@ impl Backend { Err(_e) => { // unable to decode let err = AnkiError::invalid_input("couldn't decode backend request"); - let oerr = anki_error_to_proto_error(err, &self.col.i18n); + let oerr = anki_error_to_proto_error(err, &self.i18n); let output = pb::BackendOutput { value: Some(oerr.into()), }; @@ -183,12 +197,12 @@ impl Backend { let oval = if let Some(ival) = input.value { match self.run_command_inner(ival) { Ok(output) => output, - Err(err) => anki_error_to_proto_error(err, &self.col.i18n).into(), + Err(err) => anki_error_to_proto_error(err, &self.i18n).into(), } } else { anki_error_to_proto_error( AnkiError::invalid_input("unrecognized backend input value"), - &self.col.i18n, + &self.i18n, ) .into() }; @@ -233,12 +247,12 @@ impl Backend { Value::StudiedToday(input) => OValue::StudiedToday(studied_today( input.cards as usize, input.seconds as f32, - &self.col.i18n, + &self.i18n, )), Value::CongratsLearnMsg(input) => OValue::CongratsLearnMsg(learning_congrats( input.remaining as usize, input.next_due, - &self.col.i18n, + &self.i18n, )), Value::EmptyTrash(_) => { self.empty_trash()?; @@ -253,7 +267,7 @@ impl Backend { fn fire_progress_callback(&self, progress: Progress) -> bool { if let Some(cb) = &self.progress_callback { - let bytes = progress_to_proto_bytes(progress, &self.col.i18n); + let bytes = progress_to_proto_bytes(progress, &self.i18n); cb(bytes) } else { true @@ -333,7 +347,7 @@ impl Backend { &input.answer_template, &fields, input.card_ordinal as u16, - &self.col.i18n, + &self.i18n, )?; // return @@ -411,7 +425,7 @@ impl Backend { }; let mut rt = Runtime::new().unwrap(); - rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.col.log.clone())) + rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.log.clone())) } fn check_media(&self) -> Result { @@ -419,7 +433,7 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.transact(None, |ctx| { + self.col.lock().unwrap().transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); let mut output = checker.check()?; @@ -452,7 +466,7 @@ impl Backend { .map(|(k, v)| (k.as_str(), translate_arg_to_fluent_val(&v))) .collect(); - self.col.i18n.trn(key, map) + self.i18n.trn(key, map) } fn format_time_span(&self, input: pb::FormatTimeSpanIn) -> String { @@ -461,14 +475,12 @@ impl Backend { None => return "".to_string(), }; match context { - pb::format_time_span_in::Context::Precise => { - time_span(input.seconds, &self.col.i18n, true) - } + pb::format_time_span_in::Context::Precise => time_span(input.seconds, &self.i18n, true), pb::format_time_span_in::Context::Intervals => { - time_span(input.seconds, &self.col.i18n, false) + time_span(input.seconds, &self.i18n, false) } pb::format_time_span_in::Context::AnswerButtons => { - answer_button_time(input.seconds, &self.col.i18n) + answer_button_time(input.seconds, &self.i18n) } } } @@ -478,7 +490,7 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.transact(None, |ctx| { + self.col.lock().unwrap().transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); checker.empty_trash() @@ -490,7 +502,7 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.transact(None, |ctx| { + self.col.lock().unwrap().transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); checker.restore_trash() @@ -498,7 +510,10 @@ impl Backend { } pub fn db_command(&self, input: &[u8]) -> Result { - db_command_bytes(&self.col.storage.context(self.col.server), input) + self.col + .lock() + .unwrap() + .with_ctx(|ctx| db_command_bytes(&ctx.storage, input)) } } From e14c5e4745a8ef5f4a556816151fd03f609c458b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 10:05:58 +1000 Subject: [PATCH 032/150] proper implementation of executemany(); drop executescript() --- pylib/anki/dbproxy.py | 15 +++++++-------- pylib/anki/rsbackend.py | 5 ++++- rslib/src/backend/dbproxy.rs | 25 +++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 71eff93f3..80cb3795d 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -100,12 +100,11 @@ class DBProxy: # Updates ################ - def executemany(self, sql: str, args: Iterable[Iterable[ValueForDB]]) -> None: + def executemany(self, sql: str, args: Iterable[Sequence[ValueForDB]]) -> None: self.mod = True - # fixme - for row in args: - self.execute(sql, *row) - - def executescript(self, sql: str) -> None: - self.mod = True - raise Exception("fixme") + assert ":" not in sql + if isinstance(args, list): + list_args = args + else: + list_args = list(args) + self._backend.db_execute_many(sql, list_args) diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index ab2534a54..db427b4a2 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -386,9 +386,12 @@ class RustBackend: def restore_trash(self): self._run_command(pb.BackendInput(restore_trash=pb.Empty())) - def db_query(self, sql: str, args: Iterable[ValueForDB]) -> List[DBRow]: + def db_query(self, sql: str, args: List[ValueForDB]) -> List[DBRow]: return self._db_command(dict(kind="query", sql=sql, args=args)) + def db_execute_many(self, sql: str, args: List[List[ValueForDB]]) -> List[DBRow]: + return self._db_command(dict(kind="executemany", sql=sql, args=args)) + def db_begin(self) -> None: return self._db_command(dict(kind="begin")) diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index f0c2512a2..f4ebaf8b6 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -9,10 +9,17 @@ use serde_derive::{Deserialize, Serialize}; #[derive(Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub(super) enum DBRequest { - Query { sql: String, args: Vec }, + Query { + sql: String, + args: Vec, + }, Begin, Commit, Rollback, + ExecuteMany { + sql: String, + args: Vec>, + }, } #[derive(Serialize)] @@ -74,13 +81,13 @@ pub(super) fn db_command_bytes(ctx: &StorageContext, input: &[u8]) -> Result db_execute_many(ctx, &sql, &args)?, }; Ok(serde_json::to_string(&resp)?) } pub(super) fn db_query(ctx: &StorageContext, sql: &str, args: &[SqlValue]) -> Result { let mut stmt = ctx.db.prepare_cached(sql)?; - let columns = stmt.column_count(); let res: std::result::Result>, rusqlite::Error> = stmt @@ -96,3 +103,17 @@ pub(super) fn db_query(ctx: &StorageContext, sql: &str, args: &[SqlValue]) -> Re Ok(DBResult::Rows(res?)) } + +pub(super) fn db_execute_many( + ctx: &StorageContext, + sql: &str, + args: &[Vec], +) -> Result { + let mut stmt = ctx.db.prepare_cached(sql)?; + + for params in args { + stmt.execute(params)?; + } + + Ok(DBResult::None) +} From db1508e27c813dc24e9c846e7d730e4cf0d46db8 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 10:11:05 +1000 Subject: [PATCH 033/150] support first_row_only --- pylib/anki/dbproxy.py | 3 +-- pylib/anki/rsbackend.py | 8 ++++++-- rslib/src/backend/dbproxy.rs | 38 +++++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 80cb3795d..197e5ef21 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -67,8 +67,7 @@ class DBProxy: self.mod = True assert ":" not in sql # fetch rows - # fixme: first_row_only - return self._backend.db_query(sql, args) + return self._backend.db_query(sql, args, first_row_only) # Query shortcuts ################### diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index db427b4a2..b098aaa10 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -386,8 +386,12 @@ class RustBackend: def restore_trash(self): self._run_command(pb.BackendInput(restore_trash=pb.Empty())) - def db_query(self, sql: str, args: List[ValueForDB]) -> List[DBRow]: - return self._db_command(dict(kind="query", sql=sql, args=args)) + def db_query( + self, sql: str, args: List[ValueForDB], first_row_only: bool + ) -> List[DBRow]: + return self._db_command( + dict(kind="query", sql=sql, args=args, first_row_only=first_row_only) + ) def db_execute_many(self, sql: str, args: List[List[ValueForDB]]) -> List[DBRow]: return self._db_command(dict(kind="executemany", sql=sql, args=args)) diff --git a/rslib/src/backend/dbproxy.rs b/rslib/src/backend/dbproxy.rs index f4ebaf8b6..aa4cb173c 100644 --- a/rslib/src/backend/dbproxy.rs +++ b/rslib/src/backend/dbproxy.rs @@ -4,6 +4,7 @@ use crate::err::Result; use crate::storage::StorageContext; use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput, ValueRef}; +use rusqlite::OptionalExtension; use serde_derive::{Deserialize, Serialize}; #[derive(Deserialize)] @@ -12,6 +13,7 @@ pub(super) enum DBRequest { Query { sql: String, args: Vec, + first_row_only: bool, }, Begin, Commit, @@ -68,7 +70,17 @@ impl FromSql for SqlValue { pub(super) fn db_command_bytes(ctx: &StorageContext, input: &[u8]) -> Result { let req: DBRequest = serde_json::from_slice(input)?; let resp = match req { - DBRequest::Query { sql, args } => db_query(ctx, &sql, &args)?, + DBRequest::Query { + sql, + args, + first_row_only, + } => { + if first_row_only { + db_query_row(ctx, &sql, &args)? + } else { + db_query(ctx, &sql, &args)? + } + } DBRequest::Begin => { ctx.begin_trx()?; DBResult::None @@ -86,6 +98,30 @@ pub(super) fn db_command_bytes(ctx: &StorageContext, input: &[u8]) -> Result Result { + let mut stmt = ctx.db.prepare_cached(sql)?; + let columns = stmt.column_count(); + + let row = stmt + .query_row(args, |row| { + let mut orow = Vec::with_capacity(columns); + for i in 0..columns { + let v: SqlValue = row.get(i)?; + orow.push(v); + } + Ok(orow) + }) + .optional()?; + + let rows = if let Some(row) = row { + vec![row] + } else { + vec![] + }; + + Ok(DBResult::Rows(rows)) +} + pub(super) fn db_query(ctx: &StorageContext, sql: &str, args: &[SqlValue]) -> Result { let mut stmt = ctx.db.prepare_cached(sql)?; let columns = stmt.column_count(); From 63e335706812633fc0f226d29c6363f9f1232f8c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 10:16:39 +1000 Subject: [PATCH 034/150] pass weakref in from storage to fix type checking/code completion --- pylib/anki/dbproxy.py | 3 +-- pylib/anki/rsbackend.py | 3 ++- pylib/anki/storage.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 197e5ef21..e2392fbef 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -3,7 +3,6 @@ from __future__ import annotations -import weakref from typing import Any, Iterable, List, Optional, Sequence, Union import anki @@ -28,7 +27,7 @@ class DBProxy: ############### def __init__(self, backend: anki.rsbackend.RustBackend, path: str) -> None: - self._backend = weakref.proxy(backend) + self._backend = backend self._path = path self.mod = False diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index b098aaa10..a15444a53 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -14,6 +14,7 @@ from typing import ( NewType, NoReturn, Optional, + Sequence, Tuple, Union, ) @@ -387,7 +388,7 @@ class RustBackend: self._run_command(pb.BackendInput(restore_trash=pb.Empty())) def db_query( - self, sql: str, args: List[ValueForDB], first_row_only: bool + self, sql: str, args: Sequence[ValueForDB], first_row_only: bool ) -> List[DBRow]: return self._db_command( dict(kind="query", sql=sql, args=args, first_row_only=first_row_only) diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index e873a38f8..cdb5fd830 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -4,6 +4,7 @@ import copy import json import os +import weakref from typing import Any, Dict, Optional, Tuple from anki.collection import _Collection @@ -38,7 +39,7 @@ def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: backend = RustBackend( path, media_dir, media_db, log_path, server=server is not None ) - db = DBProxy(backend, path) + db = DBProxy(weakref.proxy(backend), path) db.begin() db.setAutocommit(True) From 90de4a267d8fc6dd21b644c12e3f195310f31a77 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 10:56:34 +1000 Subject: [PATCH 035/150] drop lock() and setAutocommit() We no longer need to worry about pysqlite implicitly beginning transactions, and can be more explicit about beginning/ending transactions save() now also has a trx argument controlling whether a transaction should be started / left open --- pylib/anki/collection.py | 34 +++++++++++++++++----------------- pylib/anki/db.py | 8 ++++++++ pylib/anki/dbproxy.py | 10 ++-------- pylib/anki/importing/anki2.py | 5 +---- pylib/anki/storage.py | 6 +++--- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 60ac9c0fe..95fef5eaa 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -219,8 +219,10 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", json.dumps(self.conf), ) - def save(self, name: Optional[str] = None, mod: Optional[int] = None) -> None: - "Flush, commit DB, and take out another write lock." + def save( + self, name: Optional[str] = None, mod: Optional[int] = None, trx: bool = True + ) -> None: + "Flush, commit DB, and take out another write lock if trx=True." # let the managers conditionally flush self.models.flush() self.decks.flush() @@ -229,8 +231,14 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", if self.db.mod: self.flush(mod=mod) self.db.commit() - self.lock() self.db.mod = False + if trx: + self.db.begin() + elif not trx: + # if no changes were pending but calling code expects to be + # outside of a transaction, we need to roll back + self.db.rollback() + self._markOp(name) self._lastSave = time.time() @@ -241,20 +249,13 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", return True return None - def lock(self) -> None: - self.db.begin() - def close(self, save: bool = True) -> None: "Disconnect from DB." if self.db: if save: - self.save() - else: - self.db.rollback() + self.save(trx=False) if not self.server: - self.db.setAutocommit(True) self.db.execute("pragma journal_mode = delete") - self.db.setAutocommit(False) self.db.close() self.db = None self.backend = None @@ -271,8 +272,8 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", def rollback(self) -> None: self.db.rollback() + self.db.begin() self.load() - self.lock() def modSchema(self, check: bool) -> None: "Mark schema modified. Call this first so user can abort if necessary." @@ -303,10 +304,10 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.modSchema(check=False) self.ls = self.scm # ensure db is compacted before upload - self.db.setAutocommit(True) + self.save(trx=False) self.db.execute("vacuum") self.db.execute("analyze") - self.close() + self.close(save=False) # Object creation helpers ########################################################################## @@ -1012,11 +1013,10 @@ and type=0""", return len(to_fix) def optimize(self) -> None: - self.db.setAutocommit(True) + self.save(trx=False) self.db.execute("vacuum") self.db.execute("analyze") - self.db.setAutocommit(False) - self.lock() + self.db.begin() # Logging ########################################################################## diff --git a/pylib/anki/db.py b/pylib/anki/db.py index 786ec7b77..1800ca14b 100644 --- a/pylib/anki/db.py +++ b/pylib/anki/db.py @@ -1,6 +1,14 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +""" +A convenience wrapper over pysqlite. + +Anki's Collection class now uses dbproxy.py instead of this class, +but this class is still used by aqt's profile manager, and a number +of add-ons rely on it. +""" + import os import time from sqlite3 import Cursor diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index e2392fbef..ae6136ace 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -7,8 +7,8 @@ from typing import Any, Iterable, List, Optional, Sequence, Union import anki -# fixme: threads -# fixme: col.reopen() +# fixme: col.close()/col.reopen() & journal_mode=delete + # fixme: setAutocommit() # fixme: transaction/lock handling # fixme: progress @@ -47,12 +47,6 @@ class DBProxy: def rollback(self) -> None: self._backend.db_rollback() - def setAutocommit(self, autocommit: bool) -> None: - if autocommit: - self.commit() - else: - self.begin() - # Querying ################ diff --git a/pylib/anki/importing/anki2.py b/pylib/anki/importing/anki2.py index 50ba9cfab..342877206 100644 --- a/pylib/anki/importing/anki2.py +++ b/pylib/anki/importing/anki2.py @@ -65,10 +65,7 @@ class Anki2Importer(Importer): self._importCards() self._importStaticMedia() self._postImport() - self.dst.db.setAutocommit(True) - self.dst.db.execute("vacuum") - self.dst.db.execute("analyze") - self.dst.db.setAutocommit(False) + self.dst.optimize() # Notes ###################################################################### diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index cdb5fd830..10dca3cd1 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -40,15 +40,12 @@ def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: path, media_dir, media_db, log_path, server=server is not None ) db = DBProxy(weakref.proxy(backend), path) - db.begin() - db.setAutocommit(True) # initial setup required? create = db.scalar("select models = '{}' from col") if create: initial_db_setup(db) - db.setAutocommit(False) # add db to col and do any remaining upgrades col = _Collection(db, backend=backend, server=server) if create: @@ -59,6 +56,8 @@ def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: addForwardReverse(col) addBasicModel(col) col.save() + else: + db.begin() return col @@ -67,6 +66,7 @@ def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: def initial_db_setup(db: DBProxy) -> None: + db.begin() _addColVars(db, *_getColVars(db)) From daaf8bdc7007ad8d567d01e23e05989142295fe1 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 12:30:19 +1000 Subject: [PATCH 036/150] release the GIL during a DB request --- rspy/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index 78b222c88..8d74ed3c9 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -76,11 +76,12 @@ impl Backend { fn db_command(&mut self, py: Python, input: &PyBytes) -> PyResult { let in_bytes = input.as_bytes(); - let out_string = self - .backend - .db_command(in_bytes) - .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n())))?; - + let out_res = py.allow_threads(move || { + self.backend + .db_command(in_bytes) + .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n()))) + }); + let out_string = out_res?; let out_obj = PyBytes::new(py, out_string.as_bytes()); Ok(out_obj.into()) } From 32555b2857bd2dfc6456b18e3436abc0f243ef2b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 12:35:02 +1000 Subject: [PATCH 037/150] run Check DB in a background thread Since the DB is now stored behind a mutex, we're no longer limited to accessing the database on the main thread. --- qt/aqt/main.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 7129f9c7d..ecf841f2b 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -12,6 +12,7 @@ import signal import time import zipfile from argparse import Namespace +from concurrent.futures import Future from threading import Thread from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple @@ -1260,25 +1261,29 @@ will be lost. Continue?""" def onCheckDB(self): "True if no problems" - self.progress.start(immediate=True) - ret, ok = self.col.fixIntegrity() - self.progress.finish() - if not ok: - showText(ret) - else: - tooltip(ret) + self.progress.start() - # if an error has directed the user to check the database, - # silently clean up any broken reset hooks which distract from - # the underlying issue - while True: - try: - self.reset() - break - except Exception as e: - print("swallowed exception in reset hook:", e) - continue - return ret + def onDone(future: Future): + self.progress.finish() + ret, ok = future.result() + + if not ok: + showText(ret) + else: + tooltip(ret) + + # if an error has directed the user to check the database, + # silently clean up any broken reset hooks which distract from + # the underlying issue + while True: + try: + self.reset() + break + except Exception as e: + print("swallowed exception in reset hook:", e) + continue + + self.taskman.run_in_background(self.col.fixIntegrity, onDone) def on_check_media_db(self) -> None: check_media_db(self) From 90d4d62c48b0a333c58e62ba74902a110a6e2005 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 13:14:37 +1000 Subject: [PATCH 038/150] use a timer to automatically show progress window We were previously relying on the DB progress hook to cause the progress window to display. Qt's progress dialogs do have built in support for automatically showing, but it's easier to add a timer than change the existing code to use it. --- qt/aqt/progress.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index e072bcb81..de8b9aef5 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -25,6 +25,7 @@ class ProgressManager: self.app = QApplication.instance() self.inDB = False self.blockUpdates = False + self._show_timer: Optional[QTimer] = None self._win = None self._levels = 0 @@ -114,6 +115,10 @@ class ProgressManager: self._firstTime = time.time() self._lastUpdate = time.time() self._updating = False + self._show_timer = QTimer(self.mw) + self._show_timer.setSingleShot(True) + self._show_timer.start(600) + self._show_timer.timeout.connect(self._on_show_timer) return self._win def update(self, label=None, value=None, process=True, maybeShow=True): @@ -143,6 +148,9 @@ class ProgressManager: if self._win: self._closeWin() self._unsetBusy() + if self._show_timer: + self._show_timer.stop() + self._show_timer = None def clear(self): "Restore the interface after an error." @@ -189,6 +197,10 @@ class ProgressManager: "True if processing." return self._levels + def _on_show_timer(self): + self._show_timer = None + self._showWin() + class ProgressDialog(QDialog): def __init__(self, parent): From 0f38514ad705aa42cbe02276cd5929e0be6cf139 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 13:21:24 +1000 Subject: [PATCH 039/150] drop the DB progress handler code This code was an awful hack to provide some semblance of UI responsiveness while executing DB statements on the main thread. Instead, we can just run DB statements in a background thread now, keeping the UI responsive. --- pylib/anki/dbproxy.py | 4 ---- qt/aqt/browser.py | 3 --- qt/aqt/main.py | 2 -- qt/aqt/progress.py | 43 ++++++------------------------------------- 4 files changed, 6 insertions(+), 46 deletions(-) diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index ae6136ace..5b6fa0bbf 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -9,10 +9,6 @@ import anki # fixme: col.close()/col.reopen() & journal_mode=delete -# fixme: setAutocommit() -# fixme: transaction/lock handling -# fixme: progress - # DBValue is actually Union[str, int, float, None], but if defined # that way, every call site needs to do a type check prior to using # the return values. diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index eb74e3f73..2760bec79 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -391,15 +391,12 @@ class StatusDelegate(QItemDelegate): self.model = model def paint(self, painter, option, index): - self.browser.mw.progress.blockUpdates = True try: c = self.model.getCard(index) except: # in the the middle of a reset; return nothing so this row is not # rendered until we have a chance to reset the model return - finally: - self.browser.mw.progress.blockUpdates = True if self.model.isRTL(index): option.direction = Qt.RightToLeft diff --git a/qt/aqt/main.py b/qt/aqt/main.py index ecf841f2b..dd3ab2566 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -463,7 +463,6 @@ close the profile or restart Anki.""" self.col = Collection(cpath) self.setEnabled(True) - self.progress.setupDB(self.col.db) self.maybeEnableUndo() self.moveToState("deckBrowser") return True @@ -1531,7 +1530,6 @@ Please ensure a profile is open and Anki is not busy, then try again.""" gc.disable() def doGC(self) -> None: - assert not self.progress.inDB gc.collect() # Crash log diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index de8b9aef5..cc8852299 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -11,10 +11,6 @@ import aqt.forms from anki.lang import _ from aqt.qt import * -# fixme: if mw->subwindow opens a progress dialog with mw as the parent, mw -# gets raised on finish on compiz. perhaps we should be using the progress -# dialog as the parent? - # Progress info ########################################################################## @@ -29,44 +25,14 @@ class ProgressManager: self._win = None self._levels = 0 - # SQLite progress handler - ########################################################################## - - def setupDB(self, db): - "Install a handler in the current DB." - self.lastDbProgress = 0 - self.inDB = False - # db.set_progress_handler(self._dbProgress, 10000) - - def _dbProgress(self): - "Called from SQLite." - # do nothing if we don't have a progress window - if not self._win: - return - # make sure we're not executing too frequently - if (time.time() - self.lastDbProgress) < 0.01: - return - self.lastDbProgress = time.time() - # and we're in the main thread - if not self.mw.inMainThread(): - return - # ensure timers don't fire - self.inDB = True - # handle GUI events - if not self.blockUpdates: - self._maybeShow() - self.app.processEvents(QEventLoop.ExcludeUserInputEvents) - self.inDB = False - # Safer timers ########################################################################## - # QTimer may fire in processEvents(). We provide a custom timer which - # automatically defers until the DB is not busy, and avoids running - # while a progress window is visible. + # A custom timer which avoids firing while a progress dialog is active + # (likely due to some long-running DB operation) def timer(self, ms, func, repeat, requiresCollection=True): def handler(): - if self.inDB or self._levels: + if self._levels: # retry in 100ms self.timer(100, func, False, requiresCollection) elif not self.mw.col and requiresCollection: @@ -123,6 +89,9 @@ class ProgressManager: def update(self, label=None, value=None, process=True, maybeShow=True): # print self._min, self._counter, self._max, label, time.time() - self._lastTime + if not self.mw.inMainThread(): + print("progress.update() called on wrong thread") + return if self._updating: return if maybeShow: From fe59d11047097ebd00f4dcf8c589352a782776db Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 13:52:16 +1000 Subject: [PATCH 040/150] fix mypy warning --- qt/aqt/progress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index cc8852299..acb14e362 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -84,7 +84,7 @@ class ProgressManager: self._show_timer = QTimer(self.mw) self._show_timer.setSingleShot(True) self._show_timer.start(600) - self._show_timer.timeout.connect(self._on_show_timer) + self._show_timer.timeout.connect(self._on_show_timer) # type: ignore return self._win def update(self, label=None, value=None, process=True, maybeShow=True): From fa12213e987a86e538a1082a6e0e06f335e830dd Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:03:23 +1000 Subject: [PATCH 041/150] move .reopen() to mw; fix exporting --- pylib/anki/collection.py | 8 -------- pylib/anki/exporting.py | 6 +++--- qt/aqt/exporting.py | 3 ++- qt/aqt/main.py | 5 ++++- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 95fef5eaa..06442fe95 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -262,14 +262,6 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.media.close() self._closeLog() - def reopen(self) -> None: - "Reconnect to DB (after changing threads, etc)." - raise Exception("fixme") - # if not self.db: - # # self.db = DBProxy(self.path) - # self.media.connect() - # self._openLog() - def rollback(self) -> None: self.db.rollback() self.db.begin() diff --git a/pylib/anki/exporting.py b/pylib/anki/exporting.py index 67fd5faa6..92a77d2a9 100644 --- a/pylib/anki/exporting.py +++ b/pylib/anki/exporting.py @@ -397,20 +397,20 @@ class AnkiCollectionPackageExporter(AnkiPackageExporter): AnkiPackageExporter.__init__(self, col) def doExport(self, z, path): - # close our deck & write it into the zip file, and reopen + "Export collection. Caller must re-open afterwards." + # close our deck & write it into the zip file self.count = self.col.cardCount() v2 = self.col.schedVer() != 1 + mdir = self.col.media.dir() self.col.close() if not v2: z.write(self.col.path, "collection.anki2") else: self._addDummyCollection(z) z.write(self.col.path, "collection.anki21") - self.col.reopen() # copy all media if not self.includeMedia: return {} - mdir = self.col.media.dir() return self._exportMedia(z, os.listdir(mdir), mdir) diff --git a/qt/aqt/exporting.py b/qt/aqt/exporting.py index 8b6657f8d..2e63e65e7 100644 --- a/qt/aqt/exporting.py +++ b/qt/aqt/exporting.py @@ -25,7 +25,7 @@ class ExportDialog(QDialog): ): QDialog.__init__(self, mw, Qt.Window) self.mw = mw - self.col = mw.col + self.col = mw.col.weakref() self.frm = aqt.forms.exporting.Ui_ExportDialog() self.frm.setupUi(self) self.exporter = None @@ -151,6 +151,7 @@ class ExportDialog(QDialog): period = 3000 if self.isVerbatim: msg = _("Collection exported.") + self.mw.reopen() else: if self.isTextNote: msg = ( diff --git a/qt/aqt/main.py b/qt/aqt/main.py index dd3ab2566..661608dcc 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -459,7 +459,6 @@ close the profile or restart Anki.""" def _loadCollection(self) -> bool: cpath = self.pm.collectionPath() - self.col = Collection(cpath) self.setEnabled(True) @@ -467,6 +466,10 @@ close the profile or restart Anki.""" self.moveToState("deckBrowser") return True + def reopen(self): + cpath = self.pm.collectionPath() + self.col = Collection(cpath) + def unloadCollection(self, onsuccess: Callable) -> None: def callback(): self.setEnabled(False) From 7986a795308b5796b39c3acf1449f40608d86607 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:06:33 +1000 Subject: [PATCH 042/150] remove db.close() --- pylib/anki/collection.py | 1 - pylib/anki/dbproxy.py | 6 ------ pylib/tests/shared.py | 2 +- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 06442fe95..680b98434 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -256,7 +256,6 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.save(trx=False) if not self.server: self.db.execute("pragma journal_mode = delete") - self.db.close() self.db = None self.backend = None self.media.close() diff --git a/pylib/anki/dbproxy.py b/pylib/anki/dbproxy.py index 5b6fa0bbf..a6ea0fec0 100644 --- a/pylib/anki/dbproxy.py +++ b/pylib/anki/dbproxy.py @@ -7,8 +7,6 @@ from typing import Any, Iterable, List, Optional, Sequence, Union import anki -# fixme: col.close()/col.reopen() & journal_mode=delete - # DBValue is actually Union[str, int, float, None], but if defined # that way, every call site needs to do a type check prior to using # the return values. @@ -27,10 +25,6 @@ class DBProxy: self._path = path self.mod = False - def close(self) -> None: - # fixme - pass - # Transactions ############### diff --git a/pylib/tests/shared.py b/pylib/tests/shared.py index 5294a84f0..df83eac73 100644 --- a/pylib/tests/shared.py +++ b/pylib/tests/shared.py @@ -22,7 +22,7 @@ def getEmptyCol(): os.close(fd) os.unlink(nam) col = aopen(nam) - col.db.close() + col.close() getEmptyCol.master = nam (fd, nam) = tempfile.mkstemp(suffix=".anki2") shutil.copy(getEmptyCol.master, nam) From 53952ba1319a1c88a97e23b8fcd69abc0b7a6590 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:19:25 +1000 Subject: [PATCH 043/150] export in a background thread --- qt/aqt/exporting.py | 79 ++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/qt/aqt/exporting.py b/qt/aqt/exporting.py index 2e63e65e7..0b2bfdf21 100644 --- a/qt/aqt/exporting.py +++ b/qt/aqt/exporting.py @@ -6,6 +6,7 @@ from __future__ import annotations import os import re import time +from concurrent.futures import Future from typing import List, Optional import aqt @@ -131,7 +132,7 @@ class ExportDialog(QDialog): break self.hide() if file: - self.mw.progress.start(immediate=True) + # check we can write to file try: f = open(file, "wb") f.close() @@ -139,39 +140,51 @@ class ExportDialog(QDialog): showWarning(_("Couldn't save file: %s") % str(e)) else: os.unlink(file) - exportedMedia = lambda cnt: self.mw.progress.update( - label=ngettext( - "Exported %d media file", "Exported %d media files", cnt + + # progress handler + def exported_media(cnt): + self.mw.taskman.run_on_main( + lambda: self.mw.progress.update( + label=ngettext( + "Exported %d media file", "Exported %d media files", cnt + ) + % cnt ) - % cnt ) - hooks.media_files_did_export.append(exportedMedia) + + def do_export(): self.exporter.exportInto(file) - hooks.media_files_did_export.remove(exportedMedia) - period = 3000 - if self.isVerbatim: - msg = _("Collection exported.") - self.mw.reopen() - else: - if self.isTextNote: - msg = ( - ngettext( - "%d note exported.", - "%d notes exported.", - self.exporter.count, - ) - % self.exporter.count - ) - else: - msg = ( - ngettext( - "%d card exported.", - "%d cards exported.", - self.exporter.count, - ) - % self.exporter.count - ) - tooltip(msg, period=period) - finally: + + def on_done(future: Future): self.mw.progress.finish() - QDialog.accept(self) + hooks.media_files_did_export.remove(exported_media) + # raises if exporter failed + future.result() + self.on_export_finished() + + self.mw.progress.start(immediate=True) + hooks.media_files_did_export.append(exported_media) + + self.mw.taskman.run_in_background(do_export, on_done) + + def on_export_finished(self): + if self.isVerbatim: + msg = _("Collection exported.") + self.mw.reopen() + else: + if self.isTextNote: + msg = ( + ngettext( + "%d note exported.", "%d notes exported.", self.exporter.count, + ) + % self.exporter.count + ) + else: + msg = ( + ngettext( + "%d card exported.", "%d cards exported.", self.exporter.count, + ) + % self.exporter.count + ) + tooltip(msg, period=3000) + QDialog.reject(self) From 231fa30a86b99073e5bfa9fbc49612c6deced01f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:36:05 +1000 Subject: [PATCH 044/150] import mapped files like csv in a background thread --- qt/aqt/importing.py | 59 +++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/qt/aqt/importing.py b/qt/aqt/importing.py index d03a80c4f..af685a323 100644 --- a/qt/aqt/importing.py +++ b/qt/aqt/importing.py @@ -9,6 +9,7 @@ import shutil import traceback import unicodedata import zipfile +from concurrent.futures import Future import anki.importing as importing import aqt.deckchooser @@ -74,6 +75,7 @@ class ChangeMap(QDialog): self.accept() +# called by importFile() when importing a mappable file like .csv class ImportDialog(QDialog): def __init__(self, mw: AnkiQt, importer) -> None: QDialog.__init__(self, mw, Qt.Window) @@ -192,30 +194,35 @@ you can enter it here. Use \\t to represent tab.""" self.mw.col.decks.select(did) self.mw.progress.start(immediate=True) self.mw.checkpoint(_("Import")) - try: - self.importer.run() - except UnicodeDecodeError: - showUnicodeWarning() - return - except Exception as e: - msg = tr(TR.IMPORTING_FAILED_DEBUG_INFO) + "\n" - err = repr(str(e)) - if "1-character string" in err: - msg += err - elif "invalidTempFolder" in err: - msg += self.mw.errorHandler.tempFolderMsg() - else: - msg += traceback.format_exc() - showText(msg) - return - finally: + + def on_done(future: Future): self.mw.progress.finish() - txt = _("Importing complete.") + "\n" - if self.importer.log: - txt += "\n".join(self.importer.log) - self.close() - showText(txt) - self.mw.reset() + + try: + future.result() + except UnicodeDecodeError: + showUnicodeWarning() + return + except Exception as e: + msg = tr(TR.IMPORTING_FAILED_DEBUG_INFO) + "\n" + err = repr(str(e)) + if "1-character string" in err: + msg += err + elif "invalidTempFolder" in err: + msg += self.mw.errorHandler.tempFolderMsg() + else: + msg += traceback.format_exc() + showText(msg) + return + else: + txt = _("Importing complete.") + "\n" + if self.importer.log: + txt += "\n".join(self.importer.log) + self.close() + showText(txt) + self.mw.reset() + + self.mw.taskman.run_in_background(self.importer.run, on_done) def setupMappingFrame(self): # qt seems to have a bug with adding/removing from a grid, so we add @@ -380,9 +387,13 @@ def importFile(mw, file): except: showWarning(invalidZipMsg()) return - # we need to ask whether to import/replace + # we need to ask whether to import/replace; if it's + # a colpkg file then the rest of the import process + # will happen in setupApkgImport() if not setupApkgImport(mw, importer): return + + # importing non-colpkg files mw.progress.start(immediate=True) try: try: From ad9dad874827f449852bbe2641f5ee3e5a2de443 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:38:35 +1000 Subject: [PATCH 045/150] import .apkg files in a background thread --- qt/aqt/importing.py | 65 ++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/qt/aqt/importing.py b/qt/aqt/importing.py index af685a323..501ff8c52 100644 --- a/qt/aqt/importing.py +++ b/qt/aqt/importing.py @@ -395,41 +395,44 @@ def importFile(mw, file): # importing non-colpkg files mw.progress.start(immediate=True) - try: + + def on_done(future: Future): + mw.progress.finish() try: - importer.run() - finally: - mw.progress.finish() - except zipfile.BadZipfile: - showWarning(invalidZipMsg()) - except Exception as e: - err = repr(str(e)) - if "invalidFile" in err: - msg = _( - """\ -Invalid file. Please restore from backup.""" - ) - showWarning(msg) - elif "invalidTempFolder" in err: - showWarning(mw.errorHandler.tempFolderMsg()) - elif "readonly" in err: - showWarning( - _( + future.result() + except zipfile.BadZipfile: + showWarning(invalidZipMsg()) + except Exception as e: + err = repr(str(e)) + if "invalidFile" in err: + msg = _( """\ -Unable to import from a read-only file.""" + Invalid file. Please restore from backup.""" ) - ) + showWarning(msg) + elif "invalidTempFolder" in err: + showWarning(mw.errorHandler.tempFolderMsg()) + elif "readonly" in err: + showWarning( + _( + """\ + Unable to import from a read-only file.""" + ) + ) + else: + msg = tr(TR.IMPORTING_FAILED_DEBUG_INFO) + "\n" + msg += str(traceback.format_exc()) + showText(msg) else: - msg = tr(TR.IMPORTING_FAILED_DEBUG_INFO) + "\n" - msg += str(traceback.format_exc()) - showText(msg) - else: - log = "\n".join(importer.log) - if "\n" not in log: - tooltip(log) - else: - showText(log) - mw.reset() + log = "\n".join(importer.log) + if "\n" not in log: + tooltip(log) + else: + showText(log) + + mw.reset() + + mw.taskman.run_in_background(importer.run, on_done) def invalidZipMsg(): From 8d429cd192f89bc017eee88d17cc55bb06915fec Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 6 Mar 2020 14:55:15 +1000 Subject: [PATCH 046/150] import .colpkg in a background thread --- qt/aqt/importing.py | 85 +++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/qt/aqt/importing.py b/qt/aqt/importing.py index 501ff8c52..92ffd64cf 100644 --- a/qt/aqt/importing.py +++ b/qt/aqt/importing.py @@ -473,48 +473,57 @@ def replaceWithApkg(mw, file, backup): mw.unloadCollection(lambda: _replaceWithApkg(mw, file, backup)) -def _replaceWithApkg(mw, file, backup): +def _replaceWithApkg(mw, filename, backup): mw.progress.start(immediate=True) - z = zipfile.ZipFile(file) + def do_import(): + z = zipfile.ZipFile(filename) - # v2 scheduler? - colname = "collection.anki21" - try: - z.getinfo(colname) - except KeyError: - colname = "collection.anki2" + # v2 scheduler? + colname = "collection.anki21" + try: + z.getinfo(colname) + except KeyError: + colname = "collection.anki2" - try: with z.open(colname) as source, open(mw.pm.collectionPath(), "wb") as target: shutil.copyfileobj(source, target) - except: + + d = os.path.join(mw.pm.profileFolder(), "collection.media") + for n, (cStr, file) in enumerate( + json.loads(z.read("media").decode("utf8")).items() + ): + mw.taskman.run_on_main( + lambda n=n: mw.progress.update( + ngettext("Processed %d media file", "Processed %d media files", n) + % n + ) + ) + size = z.getinfo(cStr).file_size + dest = os.path.join(d, unicodedata.normalize("NFC", file)) + # if we have a matching file size + if os.path.exists(dest) and size == os.stat(dest).st_size: + continue + data = z.read(cStr) + open(dest, "wb").write(data) + + z.close() + + def on_done(future: Future): mw.progress.finish() - showWarning(_("The provided file is not a valid .apkg file.")) - return - # because users don't have a backup of media, it's safer to import new - # data and rely on them running a media db check to get rid of any - # unwanted media. in the future we might also want to deduplicate this - # step - d = os.path.join(mw.pm.profileFolder(), "collection.media") - for n, (cStr, file) in enumerate( - json.loads(z.read("media").decode("utf8")).items() - ): - mw.progress.update( - ngettext("Processed %d media file", "Processed %d media files", n) % n - ) - size = z.getinfo(cStr).file_size - dest = os.path.join(d, unicodedata.normalize("NFC", file)) - # if we have a matching file size - if os.path.exists(dest) and size == os.stat(dest).st_size: - continue - data = z.read(cStr) - open(dest, "wb").write(data) - z.close() - # reload - if not mw.loadCollection(): - mw.progress.finish() - return - if backup: - mw.col.modSchema(check=False) - mw.progress.finish() + + try: + future.result() + except Exception as e: + print(e) + showWarning(_("The provided file is not a valid .apkg file.")) + return + + if not mw.loadCollection(): + return + if backup: + mw.col.modSchema(check=False) + + tooltip(_("Importing complete.")) + + mw.taskman.run_in_background(do_import, on_done) From d0d6aa1433cea27b6ac9d85a94592497f0b35a45 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 8 Mar 2020 18:55:10 +1000 Subject: [PATCH 047/150] drop usage of flushSched() --- pylib/anki/cards.py | 26 ++------------------------ pylib/anki/sched.py | 2 +- pylib/anki/schedv2.py | 2 +- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/pylib/anki/cards.py b/pylib/anki/cards.py index 9c35b0c4e..f8ac57314 100644 --- a/pylib/anki/cards.py +++ b/pylib/anki/cards.py @@ -124,30 +124,6 @@ insert or replace into cards values ) self.col.log(self) - def flushSched(self) -> None: - self._preFlush() - # bug checks - self.col.db.execute( - """update cards set -mod=?, usn=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?, -lapses=?, left=?, odue=?, odid=?, did=? where id = ?""", - self.mod, - self.usn, - self.type, - self.queue, - self.due, - self.ivl, - self.factor, - self.reps, - self.lapses, - self.left, - self.odue, - self.odid, - self.did, - self.id, - ) - self.col.log(self) - def question(self, reload: bool = False, browser: bool = False) -> str: return self.css() + self.render_output(reload, browser).question_text @@ -181,6 +157,8 @@ lapses=?, left=?, odue=?, odid=?, did=? where id = ?""", def note_type(self) -> NoteType: return self.col.models.get(self.note().mid) + # legacy aliases + flushSched = flush q = question a = answer model = note_type diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index 982da8b59..af426d744 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -80,7 +80,7 @@ class Scheduler(V2): self._updateStats(card, "time", card.timeTaken()) card.mod = intTime() card.usn = self.col.usn() - card.flushSched() + card.flush() def counts(self, card: Optional[Card] = None) -> Tuple[int, int, int]: counts = [self.newCount, self.lrnCount, self.revCount] diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 67aa6c4db..c6ecc6efc 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -82,7 +82,7 @@ class Scheduler: self._updateStats(card, "time", card.timeTaken()) card.mod = intTime() card.usn = self.col.usn() - card.flushSched() + card.flush() def _answerCard(self, card: Card, ease: int) -> None: if self._previewingCard(card): From 1322d8c617342625d751d66e5e3c4cfe5849b69b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 13 Mar 2020 20:49:01 +1000 Subject: [PATCH 048/150] make the collection optional --- proto/backend.proto | 1 + rslib/src/backend/mod.rs | 63 ++++++++++++++++++++++++++-------------- rslib/src/err.rs | 3 ++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 230158eb2..16eadd120 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -91,6 +91,7 @@ message BackendError { SyncError sync_error = 7; // user interrupted operation Empty interrupted = 8; + Empty collection_not_open = 9; } } diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index ca51c560f..b32f63fd7 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -32,7 +32,7 @@ mod dbproxy; pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { - col: Arc>, + col: Arc>>, #[allow(dead_code)] col_path: PathBuf, media_folder: PathBuf, @@ -61,6 +61,7 @@ fn anki_error_to_proto_error(err: AnkiError, i18n: &I18n) -> pb::BackendError { } AnkiError::SyncError { kind, .. } => V::SyncError(pb::SyncError { kind: kind.into() }), AnkiError::Interrupted => V::Interrupted(Empty {}), + AnkiError::CollectionNotOpen => V::CollectionNotOpen(pb::Empty {}), }; pb::BackendError { @@ -156,7 +157,7 @@ impl Backend { log: Logger, ) -> Result { Ok(Backend { - col: Arc::new(Mutex::new(col)), + col: Arc::new(Mutex::new(Some(col))), col_path: col_path.into(), media_folder: media_folder.into(), media_db: media_db.into(), @@ -193,6 +194,22 @@ impl Backend { buf } + /// If collection is open, run the provided closure while holding + /// the mutex. + /// If collection is not open, return an error. + fn with_col(&self, func: F) -> Result + where + F: FnOnce(&Collection) -> Result, + { + func( + self.col + .lock() + .unwrap() + .as_ref() + .ok_or(AnkiError::CollectionNotOpen)?, + ) + } + fn run_command(&mut self, input: pb::BackendInput) -> pb::BackendOutput { let oval = if let Some(ival) = input.value { match self.run_command_inner(ival) { @@ -433,17 +450,20 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.lock().unwrap().transact(None, |ctx| { - let mut checker = MediaChecker::new(ctx, &mgr, callback); - let mut output = checker.check()?; - let report = checker.summarize_output(&mut output); + self.with_col(|col| { + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); + let mut output = checker.check()?; - Ok(pb::MediaCheckOut { - unused: output.unused, - missing: output.missing, - report, - have_trash: output.trash_count > 0, + let report = checker.summarize_output(&mut output); + + Ok(pb::MediaCheckOut { + unused: output.unused, + missing: output.missing, + report, + have_trash: output.trash_count > 0, + }) }) }) } @@ -490,10 +510,12 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.lock().unwrap().transact(None, |ctx| { - let mut checker = MediaChecker::new(ctx, &mgr, callback); + self.with_col(|col| { + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); - checker.empty_trash() + checker.empty_trash() + }) }) } @@ -502,18 +524,17 @@ impl Backend { |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.col.lock().unwrap().transact(None, |ctx| { - let mut checker = MediaChecker::new(ctx, &mgr, callback); + self.with_col(|col| { + col.transact(None, |ctx| { + let mut checker = MediaChecker::new(ctx, &mgr, callback); - checker.restore_trash() + checker.restore_trash() + }) }) } pub fn db_command(&self, input: &[u8]) -> Result { - self.col - .lock() - .unwrap() - .with_ctx(|ctx| db_command_bytes(&ctx.storage, input)) + self.with_col(|col| col.with_ctx(|ctx| db_command_bytes(&ctx.storage, input))) } } diff --git a/rslib/src/err.rs b/rslib/src/err.rs index 66e4bc66c..1ec149bc4 100644 --- a/rslib/src/err.rs +++ b/rslib/src/err.rs @@ -33,6 +33,9 @@ pub enum AnkiError { #[fail(display = "The user interrupted the operation.")] Interrupted, + + #[fail(display = "Operation requires an open collection.")] + CollectionNotOpen, } // error helpers From 649b40371bbf976df5c83184da896f17b2c48bfb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 08:12:37 +1000 Subject: [PATCH 049/150] drop unused col_path --- rslib/src/backend/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index b32f63fd7..87b7c6dd7 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -33,8 +33,6 @@ pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { col: Arc>>, - #[allow(dead_code)] - col_path: PathBuf, media_folder: PathBuf, media_db: String, progress_callback: Option, @@ -136,7 +134,6 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { match Backend::new( col, - &input.collection_path, &input.media_folder_path, &input.media_db_path, i18n, @@ -150,7 +147,6 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { impl Backend { pub fn new( col: Collection, - col_path: &str, media_folder: &str, media_db: &str, i18n: I18n, @@ -158,7 +154,6 @@ impl Backend { ) -> Result { Ok(Backend { col: Arc::new(Mutex::new(Some(col))), - col_path: col_path.into(), media_folder: media_folder.into(), media_db: media_db.into(), progress_callback: None, From 94e4c40ebfec57ce5f7194a45e2891b491253996 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 08:26:42 +1000 Subject: [PATCH 050/150] move media folder/db paths into collection this breaks background media syncing for now --- rslib/src/backend/mod.rs | 64 ++++++++++++++++++---------------------- rslib/src/collection.rs | 16 ++++++++-- rslib/src/media/check.rs | 4 +-- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 87b7c6dd7..2b0957865 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -23,7 +23,6 @@ use crate::{backend_proto as pb, log}; use fluent::FluentValue; use prost::Message; use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::runtime::Runtime; @@ -33,8 +32,6 @@ pub type ProtoProgressCallback = Box) -> bool + Send>; pub struct Backend { col: Arc>>, - media_folder: PathBuf, - media_db: String, progress_callback: Option, i18n: I18n, log: Logger, @@ -125,37 +122,25 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { ); let col = open_collection( - &input.collection_path, + input.collection_path, + input.media_folder_path, + input.media_db_path, input.server, i18n.clone(), logger.clone(), ) .map_err(|e| format!("Unable to open collection: {:?}", e))?; - match Backend::new( - col, - &input.media_folder_path, - &input.media_db_path, - i18n, - logger, - ) { + match Backend::new(col, i18n, logger) { Ok(backend) => Ok(backend), Err(e) => Err(format!("{:?}", e)), } } impl Backend { - pub fn new( - col: Collection, - media_folder: &str, - media_db: &str, - i18n: I18n, - log: Logger, - ) -> Result { + pub fn new(col: Collection, i18n: I18n, log: Logger) -> Result { Ok(Backend { col: Arc::new(Mutex::new(Some(col))), - media_folder: media_folder.into(), - media_db: media_db.into(), progress_callback: None, i18n, log, @@ -422,31 +407,35 @@ impl Backend { } fn add_media_file(&mut self, input: pb::AddMediaFileIn) -> Result { - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - let mut ctx = mgr.dbctx(); - Ok(mgr - .add_file(&mut ctx, &input.desired_name, &input.data)? - .into()) + self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mut ctx = mgr.dbctx(); + Ok(mgr + .add_file(&mut ctx, &input.desired_name, &input.data)? + .into()) + }) } - fn sync_media(&self, input: SyncMediaIn) -> Result<()> { - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; + // fixme: will block other db access + fn sync_media(&self, input: SyncMediaIn) -> Result<()> { let callback = |progress: &MediaSyncProgress| { self.fire_progress_callback(Progress::MediaSync(progress)) }; - let mut rt = Runtime::new().unwrap(); - rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.log.clone())) + self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mut rt = Runtime::new().unwrap(); + rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.log.clone())) + }) } fn check_media(&self) -> Result { let callback = |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; col.transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); let mut output = checker.check()?; @@ -464,9 +453,11 @@ impl Backend { } fn remove_media_files(&self, fnames: &[String]) -> Result<()> { - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; - let mut ctx = mgr.dbctx(); - mgr.remove_files(&mut ctx, fnames) + self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mut ctx = mgr.dbctx(); + mgr.remove_files(&mut ctx, fnames) + }) } fn translate_string(&self, input: pb::TranslateStringIn) -> String { @@ -504,8 +495,8 @@ impl Backend { let callback = |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; col.transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); @@ -518,8 +509,9 @@ impl Backend { let callback = |progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32)); - let mgr = MediaManager::new(&self.media_folder, &self.media_db)?; self.with_col(|col| { + let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + col.transact(None, |ctx| { let mut checker = MediaChecker::new(ctx, &mgr, callback); diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs index c728edca6..92e96044c 100644 --- a/rslib/src/collection.rs +++ b/rslib/src/collection.rs @@ -5,18 +5,24 @@ use crate::err::Result; use crate::i18n::I18n; use crate::log::Logger; use crate::storage::{SqliteStorage, StorageContext}; -use std::path::Path; +use std::path::PathBuf; -pub fn open_collection>( +pub fn open_collection>( path: P, + media_folder: P, + media_db: P, server: bool, i18n: I18n, log: Logger, ) -> Result { - let storage = SqliteStorage::open_or_create(path.as_ref())?; + let col_path = path.into(); + let storage = SqliteStorage::open_or_create(&col_path)?; let col = Collection { storage, + col_path, + media_folder: media_folder.into(), + media_db: media_db.into(), server, i18n, log, @@ -27,6 +33,10 @@ pub fn open_collection>( pub struct Collection { pub(crate) storage: SqliteStorage, + #[allow(dead_code)] + pub(crate) col_path: PathBuf, + pub(crate) media_folder: PathBuf, + pub(crate) media_db: PathBuf, pub(crate) server: bool, pub(crate) i18n: I18n, pub(crate) log: Logger, diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index b5338d631..2273a4f97 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -521,12 +521,12 @@ mod test { &include_bytes!("../../tests/support/mediacheck.anki2")[..], )?; - let mgr = MediaManager::new(&media_dir, media_db)?; + let mgr = MediaManager::new(&media_dir, media_db.clone())?; let log = log::terminal(); let i18n = I18n::new(&["zz"], "dummy", log.clone()); - let col = open_collection(col_path, false, i18n, log)?; + let col = open_collection(col_path, media_dir, media_db, false, i18n, log)?; Ok((dir, mgr, col)) } From 874bc085fe35515c05200a1dd231b172af816846 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 09:08:02 +1000 Subject: [PATCH 051/150] support opening and closing the DB while keeping backend alive This is safer than just dropping the backend, as .close() will block if something else is holding the mutex. Also means we can drop the extra I18nBackend code. Media syncing still needs fixing. --- proto/backend.proto | 22 ++++++---- pylib/anki/collection.py | 3 +- pylib/anki/rsbackend.py | 30 ++++++++------ pylib/anki/storage.py | 14 +++++-- rslib/src/backend/mod.rs | 86 ++++++++++++++++++++++++++-------------- rslib/src/err.rs | 3 ++ 6 files changed, 103 insertions(+), 55 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 16eadd120..18b6c174c 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -7,13 +7,9 @@ package backend_proto; message Empty {} message BackendInit { - string collection_path = 1; - string media_folder_path = 2; - string media_db_path = 3; - repeated string preferred_langs = 4; - string locale_folder_path = 5; - string log_path = 6; - bool server = 7; + repeated string preferred_langs = 1; + string locale_folder_path = 2; + bool server = 3; } message I18nBackendInit { @@ -45,6 +41,8 @@ message BackendInput { CongratsLearnMsgIn congrats_learn_msg = 33; Empty empty_trash = 34; Empty restore_trash = 35; + OpenCollectionIn open_collection = 36; + Empty close_collection = 37; } } @@ -73,6 +71,8 @@ message BackendOutput { Empty trash_media_files = 29; Empty empty_trash = 34; Empty restore_trash = 35; + Empty open_collection = 36; + Empty close_collection = 37; BackendError error = 2047; } @@ -91,7 +91,6 @@ message BackendError { SyncError sync_error = 7; // user interrupted operation Empty interrupted = 8; - Empty collection_not_open = 9; } } @@ -326,3 +325,10 @@ message CongratsLearnMsgIn { float next_due = 1; uint32 remaining = 2; } + +message OpenCollectionIn { + string collection_path = 1; + string media_folder_path = 2; + string media_db_path = 3; + string log_path = 4; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 680b98434..3408f2e2a 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -256,8 +256,7 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.save(trx=False) if not self.server: self.db.execute("pragma journal_mode = delete") - self.db = None - self.backend = None + self.backend.close_collection() self.media.close() self._closeLog() diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index a15444a53..5b57224cc 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -200,22 +200,11 @@ def _on_progress(progress_bytes: bytes) -> bool: class RustBackend: - def __init__( - self, - col_path: str, - media_folder_path: str, - media_db_path: str, - log_path: str, - server: bool, - ) -> None: + def __init__(self, server: bool = False) -> None: ftl_folder = os.path.join(anki.lang.locale_folder, "fluent") init_msg = pb.BackendInit( - collection_path=col_path, - media_folder_path=media_folder_path, - media_db_path=media_db_path, locale_folder_path=ftl_folder, preferred_langs=[anki.lang.currentLang], - log_path=log_path, server=server, ) self._backend = ankirspy.open_backend(init_msg.SerializeToString()) @@ -234,6 +223,23 @@ class RustBackend: else: return output + def open_collection( + self, col_path: str, media_folder_path: str, media_db_path: str, log_path: str + ): + self._run_command( + pb.BackendInput( + open_collection=pb.OpenCollectionIn( + collection_path=col_path, + media_folder_path=media_folder_path, + media_db_path=media_db_path, + log_path=log_path, + ) + ) + ) + + def close_collection(self): + self._run_command(pb.BackendInput(close_collection=pb.Empty())) + def template_requirements( self, template_fronts: List[str], field_map: Dict[str, int] ) -> AllTemplateReqs: diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index 10dca3cd1..f19c4d327 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -27,18 +27,24 @@ class ServerData: minutes_west: Optional[int] = None -def Collection(path: str, server: Optional[ServerData] = None) -> _Collection: +def Collection( + path: str, + backend: Optional[RustBackend] = None, + server: Optional[ServerData] = None, +) -> _Collection: "Open a new or existing collection. Path must be unicode." assert path.endswith(".anki2") + if backend is None: + backend = RustBackend(server=server is not None) + (media_dir, media_db) = media_paths_from_col_path(path) log_path = "" if not server: log_path = path.replace(".anki2", "2.log") path = os.path.abspath(path) + # connect - backend = RustBackend( - path, media_dir, media_db, log_path, server=server is not None - ) + backend.open_collection(path, media_dir, media_db, log_path) db = DBProxy(weakref.proxy(backend), path) # initial setup required? diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 2b0957865..0025e6f97 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -35,6 +35,7 @@ pub struct Backend { progress_callback: Option, i18n: I18n, log: Logger, + server: bool, } enum Progress<'a> { @@ -56,7 +57,8 @@ fn anki_error_to_proto_error(err: AnkiError, i18n: &I18n) -> pb::BackendError { } AnkiError::SyncError { kind, .. } => V::SyncError(pb::SyncError { kind: kind.into() }), AnkiError::Interrupted => V::Interrupted(Empty {}), - AnkiError::CollectionNotOpen => V::CollectionNotOpen(pb::Empty {}), + AnkiError::CollectionNotOpen => V::InvalidInput(pb::Empty {}), + AnkiError::CollectionAlreadyOpen => V::InvalidInput(pb::Empty {}), }; pb::BackendError { @@ -105,46 +107,24 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { Err(_) => return Err("couldn't decode init request".into()), }; - let mut path = input.collection_path.clone(); - path.push_str(".log"); - - let log_path = match input.log_path.as_str() { - "" => None, - path => Some(path), - }; - let logger = - default_logger(log_path).map_err(|e| format!("Unable to open log file: {:?}", e))?; - let i18n = I18n::new( &input.preferred_langs, input.locale_folder_path, log::terminal(), ); - let col = open_collection( - input.collection_path, - input.media_folder_path, - input.media_db_path, - input.server, - i18n.clone(), - logger.clone(), - ) - .map_err(|e| format!("Unable to open collection: {:?}", e))?; - - match Backend::new(col, i18n, logger) { - Ok(backend) => Ok(backend), - Err(e) => Err(format!("{:?}", e)), - } + Ok(Backend::new(i18n, log::terminal(), input.server)) } impl Backend { - pub fn new(col: Collection, i18n: I18n, log: Logger) -> Result { - Ok(Backend { - col: Arc::new(Mutex::new(Some(col))), + pub fn new(i18n: I18n, log: Logger, server: bool) -> Backend { + Backend { + col: Arc::new(Mutex::new(None)), progress_callback: None, i18n, log, - }) + server, + } } pub fn i18n(&self) -> &I18n { @@ -259,9 +239,57 @@ impl Backend { self.restore_trash()?; OValue::RestoreTrash(Empty {}) } + Value::OpenCollection(input) => { + self.open_collection(input)?; + OValue::OpenCollection(Empty {}) + } + Value::CloseCollection(_) => { + self.close_collection()?; + OValue::CloseCollection(Empty {}) + } }) } + fn open_collection(&self, input: pb::OpenCollectionIn) -> Result<()> { + let mut mutex = self.col.lock().unwrap(); + if mutex.is_some() { + return Err(AnkiError::CollectionAlreadyOpen); + } + + let mut path = input.collection_path.clone(); + path.push_str(".log"); + + let log_path = match input.log_path.as_str() { + "" => None, + path => Some(path), + }; + let logger = default_logger(log_path)?; + + let col = open_collection( + input.collection_path, + input.media_folder_path, + input.media_db_path, + self.server, + self.i18n.clone(), + logger, + )?; + + *mutex = Some(col); + + Ok(()) + } + + fn close_collection(&self) -> Result<()> { + let mut mutex = self.col.lock().unwrap(); + if mutex.is_none() { + return Err(AnkiError::CollectionNotOpen); + } + + *mutex = None; + + Ok(()) + } + fn fire_progress_callback(&self, progress: Progress) -> bool { if let Some(cb) = &self.progress_callback { let bytes = progress_to_proto_bytes(progress, &self.i18n); diff --git a/rslib/src/err.rs b/rslib/src/err.rs index 1ec149bc4..8208370d5 100644 --- a/rslib/src/err.rs +++ b/rslib/src/err.rs @@ -36,6 +36,9 @@ pub enum AnkiError { #[fail(display = "Operation requires an open collection.")] CollectionNotOpen, + + #[fail(display = "Close the existing collection first.")] + CollectionAlreadyOpen, } // error helpers From ba17567617ba3a6a4ed7ff6d130dab9f3a25fe11 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 09:45:00 +1000 Subject: [PATCH 052/150] drop the separate i18n backend --- pylib/anki/lang.py | 8 +++---- pylib/anki/rsbackend.py | 31 +++++++++++-------------- qt/aqt/__init__.py | 17 ++++++++------ qt/aqt/main.py | 3 +++ rslib/src/backend/mod.rs | 49 ---------------------------------------- rspy/src/lib.rs | 29 +----------------------- 6 files changed, 31 insertions(+), 106 deletions(-) diff --git a/pylib/anki/lang.py b/pylib/anki/lang.py index 629029017..e679b5d85 100644 --- a/pylib/anki/lang.py +++ b/pylib/anki/lang.py @@ -145,7 +145,7 @@ current_catalog: Optional[ ] = None # the current Fluent translation instance -current_i18n: Optional[anki.rsbackend.I18nBackend] +current_i18n: Optional[anki.rsbackend.RustBackend] # path to locale folder locale_folder = "" @@ -175,9 +175,9 @@ def set_lang(lang: str, locale_dir: str) -> None: current_catalog = gettext.translation( "anki", gettext_dir, languages=[lang], fallback=True ) - current_i18n = anki.rsbackend.I18nBackend( - preferred_langs=[lang], ftl_folder=ftl_dir - ) + + current_i18n = anki.rsbackend.RustBackend(ftl_folder=ftl_dir, langs=[lang]) + locale_folder = locale_dir diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 5b57224cc..a77e0f367 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -200,12 +200,20 @@ def _on_progress(progress_bytes: bytes) -> bool: class RustBackend: - def __init__(self, server: bool = False) -> None: - ftl_folder = os.path.join(anki.lang.locale_folder, "fluent") + def __init__( + self, + ftl_folder: Optional[str] = None, + langs: Optional[List[str]] = None, + server: bool = False, + ) -> None: + # pick up global defaults if not provided + if ftl_folder is None: + ftl_folder = os.path.join(anki.lang.locale_folder, "fluent") + if langs is None: + langs = [anki.lang.currentLang] + init_msg = pb.BackendInit( - locale_folder_path=ftl_folder, - preferred_langs=[anki.lang.currentLang], - server=server, + locale_folder_path=ftl_folder, preferred_langs=langs, server=server, ) self._backend = ankirspy.open_backend(init_msg.SerializeToString()) self._backend.set_progress_callback(_on_progress) @@ -428,19 +436,6 @@ def translate_string_in( return pb.TranslateStringIn(key=key, args=args) -class I18nBackend: - def __init__(self, preferred_langs: List[str], ftl_folder: str) -> None: - init_msg = pb.I18nBackendInit( - locale_folder_path=ftl_folder, preferred_langs=preferred_langs - ) - self._backend = ankirspy.open_i18n(init_msg.SerializeToString()) - - def translate(self, key: TR, **kwargs: Union[str, int, float]) -> str: - return self._backend.translate( - translate_string_in(key, **kwargs).SerializeToString() - ) - - # temporarily force logging of media handling if "RUST_LOG" not in os.environ: os.environ["RUST_LOG"] = "warn,anki::media=debug" diff --git a/qt/aqt/__init__.py b/qt/aqt/__init__.py index 925250405..b7183af15 100644 --- a/qt/aqt/__init__.py +++ b/qt/aqt/__init__.py @@ -17,6 +17,7 @@ import anki.lang import aqt.buildinfo from anki import version as _version from anki.consts import HELP_SITE +from anki.rsbackend import RustBackend from anki.utils import checksum, isLin, isMac from aqt.qt import * from aqt.utils import locale_dir @@ -162,15 +163,15 @@ dialogs = DialogManager() # Qt requires its translator to be installed before any GUI widgets are # loaded, and we need the Qt language to match the gettext language or # translated shortcuts will not work. -# -# The Qt translator needs to be retained to work. +# A reference to the Qt translator needs to be held to prevent it from +# being immediately deallocated. _qtrans: Optional[QTranslator] = None -def setupLang( +def setupLangAndBackend( pm: ProfileManager, app: QApplication, force: Optional[str] = None -) -> None: +) -> RustBackend: global _qtrans try: locale.setlocale(locale.LC_ALL, "") @@ -218,6 +219,8 @@ def setupLang( if _qtrans.load("qtbase_" + qt_lang, qt_dir): app.installTranslator(_qtrans) + return anki.lang.current_i18n + # App initialisation ########################################################################## @@ -465,8 +468,8 @@ environment points to a valid, writable folder.""", if opts.profile: pm.openProfile(opts.profile) - # i18n - setupLang(pm, app, opts.lang) + # i18n & backend + backend = setupLangAndBackend(pm, app, opts.lang) if isLin and pm.glMode() == "auto": from aqt.utils import gfxDriverIsBroken @@ -483,7 +486,7 @@ environment points to a valid, writable folder.""", # load the main window import aqt.main - mw = aqt.main.AnkiQt(app, pm, opts, args) + mw = aqt.main.AnkiQt(app, pm, backend, opts, args) if exec: app.exec() else: diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 661608dcc..ed56974c3 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -29,6 +29,7 @@ from anki import hooks from anki.collection import _Collection from anki.hooks import runHook from anki.lang import _, ngettext +from anki.rsbackend import RustBackend from anki.sound import AVTag, SoundOrVideoTag from anki.storage import Collection from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields @@ -78,10 +79,12 @@ class AnkiQt(QMainWindow): self, app: QApplication, profileManager: ProfileManagerType, + backend: RustBackend, opts: Namespace, args: List[Any], ) -> None: QMainWindow.__init__(self) + self.backend = backend self.state = "startup" self.opts = opts self.col: Optional[_Collection] = None diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 0025e6f97..59de1e1fa 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -621,52 +621,3 @@ fn media_sync_progress(p: &MediaSyncProgress, i18n: &I18n) -> pb::MediaSyncProgr ), } } - -/// Standalone I18n backend -/// This is a hack to allow translating strings in the GUI -/// when a collection is not open, and in the future it should -/// either be shared with or merged into the backend object. -/////////////////////////////////////////////////////// - -pub struct I18nBackend { - i18n: I18n, -} - -pub fn init_i18n_backend(init_msg: &[u8]) -> Result { - let input: pb::I18nBackendInit = match pb::I18nBackendInit::decode(init_msg) { - Ok(req) => req, - Err(_) => return Err(AnkiError::invalid_input("couldn't decode init msg")), - }; - - let log = log::terminal(); - - let i18n = I18n::new(&input.preferred_langs, input.locale_folder_path, log); - - Ok(I18nBackend { i18n }) -} - -impl I18nBackend { - pub fn translate(&self, req: &[u8]) -> String { - let req = match pb::TranslateStringIn::decode(req) { - Ok(req) => req, - Err(_e) => return "decoding error".into(), - }; - - self.translate_string(req) - } - - fn translate_string(&self, input: pb::TranslateStringIn) -> String { - let key = match pb::FluentString::from_i32(input.key) { - Some(key) => key, - None => return "invalid key".to_string(), - }; - - let map = input - .args - .iter() - .map(|(k, v)| (k.as_str(), translate_arg_to_fluent_val(&v))) - .collect(); - - self.i18n.trn(key, map) - } -} diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs index 8d74ed3c9..5ec2375f4 100644 --- a/rspy/src/lib.rs +++ b/rspy/src/lib.rs @@ -1,9 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use anki::backend::{ - init_backend, init_i18n_backend, Backend as RustBackend, I18nBackend as RustI18nBackend, -}; +use anki::backend::{init_backend, Backend as RustBackend}; use pyo3::exceptions::Exception; use pyo3::prelude::*; use pyo3::types::PyBytes; @@ -87,30 +85,6 @@ impl Backend { } } -// I18n backend -////////////////////////////////// - -#[pyclass] -struct I18nBackend { - backend: RustI18nBackend, -} - -#[pyfunction] -fn open_i18n(init_msg: &PyBytes) -> PyResult { - match init_i18n_backend(init_msg.as_bytes()) { - Ok(backend) => Ok(I18nBackend { backend }), - Err(e) => Err(exceptions::Exception::py_err(format!("{:?}", e))), - } -} - -#[pymethods] -impl I18nBackend { - fn translate(&self, input: &PyBytes) -> String { - let in_bytes = input.as_bytes(); - self.backend.translate(in_bytes) - } -} - // Module definition ////////////////////////////////// @@ -119,7 +93,6 @@ fn ankirspy(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_wrapped(wrap_pyfunction!(buildhash)).unwrap(); m.add_wrapped(wrap_pyfunction!(open_backend)).unwrap(); - m.add_wrapped(wrap_pyfunction!(open_i18n)).unwrap(); Ok(()) } From d03e13a1bdcc1cf55932085c491d5117a79e75b6 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 09:50:06 +1000 Subject: [PATCH 053/150] reuse the existing backend instead of creating a new one --- qt/aqt/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index ed56974c3..2fa5c4f53 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -462,7 +462,7 @@ close the profile or restart Anki.""" def _loadCollection(self) -> bool: cpath = self.pm.collectionPath() - self.col = Collection(cpath) + self.col = Collection(cpath, backend=self.backend) self.setEnabled(True) self.maybeEnableUndo() @@ -471,7 +471,7 @@ close the profile or restart Anki.""" def reopen(self): cpath = self.pm.collectionPath() - self.col = Collection(cpath) + self.col = Collection(cpath, backend=self.backend) def unloadCollection(self, onsuccess: Callable) -> None: def callback(): From d7daa63dbdaae4e5b4b876523f4c58387a1262ff Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 10:05:55 +1000 Subject: [PATCH 054/150] make sure we set db to None so we can tell when we've closed the DB --- pylib/anki/collection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 3408f2e2a..471f2c5a7 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -257,6 +257,7 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", if not self.server: self.db.execute("pragma journal_mode = delete") self.backend.close_collection() + self.db = None self.media.close() self._closeLog() From 5f19048c93caacbf0c4bca7d02f5a9fcf22e91a2 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 10:06:22 +1000 Subject: [PATCH 055/150] fix media sync being logged to console --- rslib/src/backend/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 59de1e1fa..02f743ae0 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -8,7 +8,7 @@ use crate::collection::{open_collection, Collection}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; use crate::i18n::{tr_args, FString, I18n}; use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}; -use crate::log::{default_logger, Logger}; +use crate::log::default_logger; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; @@ -34,7 +34,6 @@ pub struct Backend { col: Arc>>, progress_callback: Option, i18n: I18n, - log: Logger, server: bool, } @@ -113,16 +112,15 @@ pub fn init_backend(init_msg: &[u8]) -> std::result::Result { log::terminal(), ); - Ok(Backend::new(i18n, log::terminal(), input.server)) + Ok(Backend::new(i18n, input.server)) } impl Backend { - pub fn new(i18n: I18n, log: Logger, server: bool) -> Backend { + pub fn new(i18n: I18n, server: bool) -> Backend { Backend { col: Arc::new(Mutex::new(None)), progress_callback: None, i18n, - log, server, } } @@ -454,7 +452,7 @@ impl Backend { self.with_col(|col| { let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; let mut rt = Runtime::new().unwrap(); - rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, self.log.clone())) + rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, col.log.clone())) }) } From 72bcef917e65b9c43b3719b8597edecfac4663fe Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 11:52:39 +1000 Subject: [PATCH 056/150] release mutex before beginning media sync And check media sync is not running on close --- rslib/src/backend/mod.rs | 58 +++++++++++++++++++++++++++++----------- rslib/src/collection.rs | 33 ++++++++++++++++++++++- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 02f743ae0..197004aca 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -8,7 +8,7 @@ use crate::collection::{open_collection, Collection}; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; use crate::i18n::{tr_args, FString, I18n}; use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}; -use crate::log::default_logger; +use crate::log::{default_logger, Logger}; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; @@ -23,6 +23,7 @@ use crate::{backend_proto as pb, log}; use fluent::FluentValue; use prost::Message; use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::runtime::Runtime; @@ -157,13 +158,13 @@ impl Backend { /// If collection is not open, return an error. fn with_col(&self, func: F) -> Result where - F: FnOnce(&Collection) -> Result, + F: FnOnce(&mut Collection) -> Result, { func( self.col .lock() .unwrap() - .as_ref() + .as_mut() .ok_or(AnkiError::CollectionNotOpen)?, ) } @@ -249,8 +250,8 @@ impl Backend { } fn open_collection(&self, input: pb::OpenCollectionIn) -> Result<()> { - let mut mutex = self.col.lock().unwrap(); - if mutex.is_some() { + let mut col = self.col.lock().unwrap(); + if col.is_some() { return Err(AnkiError::CollectionAlreadyOpen); } @@ -263,7 +264,7 @@ impl Backend { }; let logger = default_logger(log_path)?; - let col = open_collection( + let new_col = open_collection( input.collection_path, input.media_folder_path, input.media_db_path, @@ -272,18 +273,22 @@ impl Backend { logger, )?; - *mutex = Some(col); + *col = Some(new_col); Ok(()) } fn close_collection(&self) -> Result<()> { - let mut mutex = self.col.lock().unwrap(); - if mutex.is_none() { + let mut col = self.col.lock().unwrap(); + if col.is_none() { return Err(AnkiError::CollectionNotOpen); } - *mutex = None; + if !col.as_ref().unwrap().can_close() { + return Err(AnkiError::invalid_input("can't close yet")); + } + + *col = None; Ok(()) } @@ -445,15 +450,38 @@ impl Backend { // fixme: will block other db access fn sync_media(&self, input: SyncMediaIn) -> Result<()> { + let mut guard = self.col.lock().unwrap(); + + let col = guard.as_mut().unwrap(); + col.set_media_sync_running()?; + + let folder = col.media_folder.clone(); + let db = col.media_db.clone(); + let log = col.log.clone(); + + drop(guard); + + let res = self.sync_media_inner(input, folder, db, log); + + self.with_col(|col| col.set_media_sync_finished())?; + + res + } + + fn sync_media_inner( + &self, + input: pb::SyncMediaIn, + folder: PathBuf, + db: PathBuf, + log: Logger, + ) -> Result<()> { let callback = |progress: &MediaSyncProgress| { self.fire_progress_callback(Progress::MediaSync(progress)) }; - self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; - let mut rt = Runtime::new().unwrap(); - rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, col.log.clone())) - }) + let mgr = MediaManager::new(&folder, &db)?; + let mut rt = Runtime::new().unwrap(); + rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey, log)) } fn check_media(&self) -> Result { diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs index 92e96044c..5db81874e 100644 --- a/rslib/src/collection.rs +++ b/rslib/src/collection.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::err::Result; +use crate::err::{AnkiError, Result}; use crate::i18n::I18n; use crate::log::Logger; use crate::storage::{SqliteStorage, StorageContext}; @@ -26,11 +26,19 @@ pub fn open_collection>( server, i18n, log, + state: CollectionState::Normal, }; Ok(col) } +#[derive(Debug, PartialEq)] +pub enum CollectionState { + Normal, + // in this state, the DB must not be closed + MediaSyncRunning, +} + pub struct Collection { pub(crate) storage: SqliteStorage, #[allow(dead_code)] @@ -40,6 +48,7 @@ pub struct Collection { pub(crate) server: bool, pub(crate) i18n: I18n, pub(crate) log: Logger, + state: CollectionState, } pub(crate) enum CollectionOp {} @@ -97,4 +106,26 @@ impl Collection { res }) } + + pub(crate) fn set_media_sync_running(&mut self) -> Result<()> { + if self.state == CollectionState::Normal { + self.state = CollectionState::MediaSyncRunning; + Ok(()) + } else { + Err(AnkiError::invalid_input("media sync already running")) + } + } + + pub(crate) fn set_media_sync_finished(&mut self) -> Result<()> { + if self.state == CollectionState::MediaSyncRunning { + self.state = CollectionState::Normal; + Ok(()) + } else { + Err(AnkiError::invalid_input("media sync not running")) + } + } + + pub(crate) fn can_close(&self) -> bool { + self.state == CollectionState::Normal + } } From 55c9f5dbebf2fa441e01d2b53a8d972c007feb7f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 12:09:11 +1000 Subject: [PATCH 057/150] wait for media sync to complete before unloading collection --- qt/aqt/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 2fa5c4f53..efb046f66 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -397,7 +397,7 @@ close the profile or restart Anki.""" # at this point there should be no windows left self._checkForUnclosedWidgets() - self.maybeAutoSync(True) + self.maybeAutoSync() def _checkForUnclosedWidgets(self) -> None: for w in self.app.topLevelWidgets(): @@ -476,6 +476,7 @@ close the profile or restart Anki.""" def unloadCollection(self, onsuccess: Callable) -> None: def callback(): self.setEnabled(False) + self.media_syncer.show_diag_until_finished() self._unloadCollection() onsuccess() @@ -845,7 +846,7 @@ title="%s" %s>%s""" % ( self.media_syncer.start() # expects a current profile, but no collection loaded - def maybeAutoSync(self, closing=False) -> None: + def maybeAutoSync(self) -> None: if ( not self.pm.profile["syncKey"] or not self.pm.profile["autoSync"] @@ -857,10 +858,6 @@ title="%s" %s>%s""" % ( # ok to sync self._sync() - # if media still syncing at this point, pop up progress diag - if closing: - self.media_syncer.show_diag_until_finished() - def maybe_auto_sync_media(self) -> None: if not self.pm.profile["autoSync"] or self.safeMode or self.restoringBackup: return From 794c8a984bae77149aa417db3425f3d46716b429 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 14 Mar 2020 14:05:54 +1000 Subject: [PATCH 058/150] add string about waiting for completion --- rslib/ftl/sync.ftl | 1 + 1 file changed, 1 insertion(+) diff --git a/rslib/ftl/sync.ftl b/rslib/ftl/sync.ftl index 97dec7e7e..b76a15505 100644 --- a/rslib/ftl/sync.ftl +++ b/rslib/ftl/sync.ftl @@ -31,3 +31,4 @@ sync-client-too-old = sync-wrong-pass = AnkiWeb ID or password was incorrect; please try again. sync-resync-required = Please sync again. If this message keeps appearing, please post on the support site. +sync-must-wait-for-end = Anki is currently syncing. Please wait for the sync to complete, then try again. From f623f19b3d4bfffed9da76850eb644a79e103cc7 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 15 Mar 2020 19:11:45 +1000 Subject: [PATCH 059/150] basic search parsing --- rslib/src/lib.rs | 1 + rslib/src/search/mod.rs | 1 + rslib/src/search/parser.rs | 132 +++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 rslib/src/search/mod.rs create mode 100644 rslib/src/search/parser.rs diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 8f815dce8..f66dd1bd6 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -19,6 +19,7 @@ pub mod log; pub mod media; pub mod notes; pub mod sched; +pub mod search; pub mod storage; pub mod template; pub mod template_filters; diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs new file mode 100644 index 000000000..b93e263bb --- /dev/null +++ b/rslib/src/search/mod.rs @@ -0,0 +1 @@ +mod parser; diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs new file mode 100644 index 000000000..3517fa817 --- /dev/null +++ b/rslib/src/search/parser.rs @@ -0,0 +1,132 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use nom::branch::alt; +use nom::bytes::complete::{escaped, is_not, take_while1}; +use nom::character::complete::{char, one_of}; +use nom::combinator::{all_consuming, map}; +use nom::sequence::{delimited, preceded}; +use nom::{multi::many0, IResult}; + +#[derive(Debug, PartialEq)] +pub(super) enum Node<'a> { + And, + Or, + Not(Box>), + Group(Vec>), + Text(&'a str), +} + +/// Parse the input string into a list of nodes. +#[allow(dead_code)] +pub(super) fn parse(input: &str) -> std::result::Result, String> { + let (_, nodes) = all_consuming(group_inner)(input).map_err(|e| format!("{:?}", e))?; + Ok(nodes) +} + +/// One or more nodes surrounded by brackets, eg (one OR two) +fn group(s: &str) -> IResult<&str, Node> { + map(delimited(char('('), group_inner, char(')')), |nodes| { + Node::Group(nodes) + })(s) +} + +/// One or more nodes inside brackets, er 'one OR two -three' +fn group_inner(input: &str) -> IResult<&str, Vec> { + let mut remaining = input; + let mut nodes = vec![]; + + loop { + match node(remaining) { + Ok((rem, node)) => { + remaining = rem; + + if nodes.len() % 2 == 0 { + // before adding the node, if the length is even then the node + // must not be a boolean + if matches!(node, Node::And | Node::Or) { + return Err(nom::Err::Failure(("", nom::error::ErrorKind::NoneOf))); + } + } else { + // if the length is odd, the next item must be a boolean. if it's + // not, add an implicit and + if !matches!(node, Node::And | Node::Or) { + nodes.push(Node::And); + } + } + nodes.push(node); + } + Err(e) => match e { + nom::Err::Error(_) => break, + _ => return Err(e), + }, + }; + } + + Ok((remaining, nodes)) +} + +/// Optional leading space, then a (negated) group or text +fn node(s: &str) -> IResult<&str, Node> { + let whitespace0 = many0(one_of(" \u{3000}")); + preceded(whitespace0, alt((negated_node, group, text)))(s) +} + +fn negated_node(s: &str) -> IResult<&str, Node> { + map(preceded(char('-'), alt((group, text))), |node| { + Node::Not(Box::new(node)) + })(s) +} + +/// Either quoted or unquoted text +fn text(s: &str) -> IResult<&str, Node> { + alt((quoted_term, unquoted_term))(s) +} + +/// Unquoted text, terminated by a space or ) +fn unquoted_term(s: &str) -> IResult<&str, Node> { + map(take_while1(|c| c != ' ' && c != ')'), |text: &str| { + if text.len() == 2 && text.to_ascii_lowercase() == "or" { + Node::Or + } else if text.len() == 3 && text.to_ascii_lowercase() == "and" { + Node::And + } else { + Node::Text(text) + } + })(s) +} + +// Quoted text, including the outer double quotes. +fn quoted_term(s: &str) -> IResult<&str, Node> { + delimited(char('"'), quoted_term_inner, char('"'))(s) +} + +/// Quoted text, terminated by a non-escaped double quote +/// Can escape :, " and \ +fn quoted_term_inner(s: &str) -> IResult<&str, Node> { + map(escaped(is_not(r#""\"#), '\\', one_of(r#"":\"#)), |o| { + Node::Text(o) + })(s) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parsing() -> Result<(), String> { + use Node::*; + assert_eq!( + parse(r#"hello -(world and "foo bar") OR test"#)?, + vec![ + Text("hello"), + And, + Not(Box::new(Group(vec![Text("world"), And, Text("foo bar")]))), + Or, + Text("test") + ] + ); + + Ok(()) + } +} From 289318d92c0b601d2812a93854f6b6b24f7292b8 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 15 Mar 2020 20:41:49 +1000 Subject: [PATCH 060/150] split up searches with a qualifier --- rslib/src/search/parser.rs | 57 ++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 3517fa817..fc2e4ea98 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -7,6 +7,7 @@ use nom::character::complete::{char, one_of}; use nom::combinator::{all_consuming, map}; use nom::sequence::{delimited, preceded}; use nom::{multi::many0, IResult}; +use std::borrow::Cow; #[derive(Debug, PartialEq)] pub(super) enum Node<'a> { @@ -14,7 +15,11 @@ pub(super) enum Node<'a> { Or, Not(Box>), Group(Vec>), - Text(&'a str), + UnqualifiedText(Cow<'a, str>), + QualifiedText { + key: Cow<'a, str>, + val: Cow<'a, str>, + }, } /// Parse the input string into a list of nodes. @@ -83,6 +88,32 @@ fn text(s: &str) -> IResult<&str, Node> { alt((quoted_term, unquoted_term))(s) } +/// Determine if text is a qualified search, and handle escaped chars. +fn node_for_text(s: &str) -> Node { + let mut it = s.splitn(2, ':'); + let (head, tail) = ( + without_escapes(it.next().unwrap()), + it.next().map(without_escapes), + ); + + if let Some(tail) = tail { + Node::QualifiedText { + key: head, + val: tail, + } + } else { + Node::UnqualifiedText(head) + } +} + +fn without_escapes(s: &str) -> Cow { + if s.find('\\').is_some() { + s.replace('\\', "").into() + } else { + s.into() + } +} + /// Unquoted text, terminated by a space or ) fn unquoted_term(s: &str) -> IResult<&str, Node> { map(take_while1(|c| c != ' ' && c != ')'), |text: &str| { @@ -91,7 +122,7 @@ fn unquoted_term(s: &str) -> IResult<&str, Node> { } else if text.len() == 3 && text.to_ascii_lowercase() == "and" { Node::And } else { - Node::Text(text) + node_for_text(text) } })(s) } @@ -102,10 +133,10 @@ fn quoted_term(s: &str) -> IResult<&str, Node> { } /// Quoted text, terminated by a non-escaped double quote -/// Can escape :, " and \ +/// Can escape " and \ fn quoted_term_inner(s: &str) -> IResult<&str, Node> { - map(escaped(is_not(r#""\"#), '\\', one_of(r#"":\"#)), |o| { - Node::Text(o) + map(escaped(is_not(r#""\"#), '\\', one_of(r#""\"#)), |o| { + node_for_text(o) })(s) } @@ -116,14 +147,22 @@ mod test { #[test] fn parsing() -> Result<(), String> { use Node::*; + assert_eq!( - parse(r#"hello -(world and "foo bar") OR test"#)?, + parse(r#"hello -(world and "foo:bar baz") OR test"#)?, vec![ - Text("hello"), + UnqualifiedText("hello".into()), And, - Not(Box::new(Group(vec![Text("world"), And, Text("foo bar")]))), + Not(Box::new(Group(vec![ + UnqualifiedText("world".into()), + And, + QualifiedText { + key: "foo".into(), + val: "bar baz".into() + } + ]))), Or, - Text("test") + UnqualifiedText("test".into()) ] ); From 08d205d37770fa0cacf409841cf14916be5b3ccc Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 16 Mar 2020 09:48:22 +1000 Subject: [PATCH 061/150] decode search terms in parser --- rslib/src/search/parser.rs | 301 +++++++++++++++++++++++++++++++++---- 1 file changed, 269 insertions(+), 32 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index fc2e4ea98..b3768aaae 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -1,13 +1,39 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::types::ObjID; use nom::branch::alt; -use nom::bytes::complete::{escaped, is_not, take_while1}; +use nom::bytes::complete::{escaped, is_not, tag, take_while1}; use nom::character::complete::{char, one_of}; -use nom::combinator::{all_consuming, map}; +use nom::character::is_digit; +use nom::combinator::{all_consuming, map, map_res}; use nom::sequence::{delimited, preceded}; use nom::{multi::many0, IResult}; -use std::borrow::Cow; +use std::{borrow::Cow, num}; + +// fixme: need to preserve \ when used twice in string + +struct ParseError {} + +impl From for ParseError { + fn from(_: num::ParseIntError) -> Self { + ParseError {} + } +} + +impl From for ParseError { + fn from(_: num::ParseFloatError) -> Self { + ParseError {} + } +} + +impl From> for ParseError { + fn from(_: nom::Err<(I, nom::error::ErrorKind)>) -> Self { + ParseError {} + } +} + +type ParseResult = Result; #[derive(Debug, PartialEq)] pub(super) enum Node<'a> { @@ -15,11 +41,59 @@ pub(super) enum Node<'a> { Or, Not(Box>), Group(Vec>), + Search(SearchNode<'a>), +} + +#[derive(Debug, PartialEq)] +pub(super) enum SearchNode<'a> { + // text without a colon UnqualifiedText(Cow<'a, str>), - QualifiedText { - key: Cow<'a, str>, - val: Cow<'a, str>, + // foo:bar, where foo doesn't match a term below + SingleField { + field: Cow<'a, str>, + text: Cow<'a, str>, }, + AddedInDays(u32), + CardTemplate(Cow<'a, str>), + Deck(Cow<'a, str>), + NoteTypeID(ObjID), + NoteType(Cow<'a, str>), + Rated { + days: u32, + ease: Option, + }, + Tag(Cow<'a, str>), + Duplicates { + note_type_id: ObjID, + text: String, + }, + State(StateKind), + Flag(u8), + NoteIDs(Cow<'a, str>), + CardIDs(Cow<'a, str>), + Property { + operator: String, + kind: PropertyKind, + }, +} + +#[derive(Debug, PartialEq)] +pub(super) enum PropertyKind { + Due(i32), + Interval(u32), + Reps(u32), + Lapses(u32), + Ease(f32), +} + +#[derive(Debug, PartialEq)] +pub(super) enum StateKind { + New, + Review, + Learning, + Due, + Buried, + Suspended, } /// Parse the input string into a list of nodes. @@ -89,7 +163,7 @@ fn text(s: &str) -> IResult<&str, Node> { } /// Determine if text is a qualified search, and handle escaped chars. -fn node_for_text(s: &str) -> Node { +fn search_node_for_text(s: &str) -> ParseResult { let mut it = s.splitn(2, ':'); let (head, tail) = ( without_escapes(it.next().unwrap()), @@ -97,15 +171,13 @@ fn node_for_text(s: &str) -> Node { ); if let Some(tail) = tail { - Node::QualifiedText { - key: head, - val: tail, - } + search_node_for_text_with_argument(head, tail) } else { - Node::UnqualifiedText(head) + Ok(SearchNode::UnqualifiedText(head)) } } +/// Strip the \ escaping character fn without_escapes(s: &str) -> Cow { if s.find('\\').is_some() { s.replace('\\', "").into() @@ -116,18 +188,21 @@ fn without_escapes(s: &str) -> Cow { /// Unquoted text, terminated by a space or ) fn unquoted_term(s: &str) -> IResult<&str, Node> { - map(take_while1(|c| c != ' ' && c != ')'), |text: &str| { - if text.len() == 2 && text.to_ascii_lowercase() == "or" { - Node::Or - } else if text.len() == 3 && text.to_ascii_lowercase() == "and" { - Node::And - } else { - node_for_text(text) - } - })(s) + map_res( + take_while1(|c| c != ' ' && c != ')'), + |text: &str| -> ParseResult { + Ok(if text.len() == 2 && text.to_ascii_lowercase() == "or" { + Node::Or + } else if text.len() == 3 && text.to_ascii_lowercase() == "and" { + Node::And + } else { + Node::Search(search_node_for_text(text)?) + }) + }, + )(s) } -// Quoted text, including the outer double quotes. +/// Quoted text, including the outer double quotes. fn quoted_term(s: &str) -> IResult<&str, Node> { delimited(char('"'), quoted_term_inner, char('"'))(s) } @@ -135,9 +210,136 @@ fn quoted_term(s: &str) -> IResult<&str, Node> { /// Quoted text, terminated by a non-escaped double quote /// Can escape " and \ fn quoted_term_inner(s: &str) -> IResult<&str, Node> { - map(escaped(is_not(r#""\"#), '\\', one_of(r#""\"#)), |o| { - node_for_text(o) - })(s) + map_res( + escaped(is_not(r#""\"#), '\\', one_of(r#""\"#)), + |o| -> ParseResult { Ok(Node::Search(search_node_for_text(o)?)) }, + )(s) +} + +/// Convert a colon-separated key/val pair into the relevant search type. +fn search_node_for_text_with_argument<'a>( + key: Cow<'a, str>, + val: Cow<'a, str>, +) -> ParseResult> { + Ok(match key.to_ascii_lowercase().as_str() { + "added" => SearchNode::AddedInDays(val.parse()?), + "card" => SearchNode::CardTemplate(val), + "deck" => SearchNode::Deck(val), + "note" => SearchNode::NoteType(val), + "tag" => SearchNode::Tag(val), + "mid" => SearchNode::NoteTypeID(val.parse()?), + "nid" => SearchNode::NoteIDs(check_id_list(val)?), + "cid" => SearchNode::CardIDs(check_id_list(val)?), + "is" => parse_state(val.as_ref())?, + "flag" => parse_flag(val.as_ref())?, + "rated" => parse_rated(val.as_ref())?, + "dupes" => parse_dupes(val.as_ref())?, + "prop" => parse_prop(val.as_ref())?, + + // anything else is a field search + _ => SearchNode::SingleField { + field: key, + text: val, + }, + }) +} + +/// ensure a list of ids contains only numbers and commas, returning unchanged if true +/// used by nid: and cid: +fn check_id_list(s: Cow) -> ParseResult> { + if s.as_bytes().iter().any(|&c| !is_digit(c) && c != b',') { + Err(ParseError {}) + } else { + Ok(s) + } +} + +/// eg is:due +fn parse_state(s: &str) -> ParseResult> { + use StateKind::*; + Ok(SearchNode::State(match s { + "new" => New, + "review" => Review, + "learn" => Learning, + "due" => Due, + "buried" => Buried, + "suspended" => Suspended, + _ => return Err(ParseError {}), + })) +} + +/// flag:0-4 +fn parse_flag(s: &str) -> ParseResult> { + let n: u8 = s.parse()?; + if n > 4 { + Err(ParseError {}) + } else { + Ok(SearchNode::Flag(n)) + } +} + +/// eg rated:3 or rated:10:2 +fn parse_rated(val: &str) -> ParseResult> { + let mut it = val.splitn(2, ':'); + let days = it.next().unwrap().parse()?; + let ease = match it.next() { + Some(v) => Some(v.parse()?), + None => None, + }; + + Ok(SearchNode::Rated { days, ease }) +} + +/// eg dupes:1231,hello +fn parse_dupes(val: &str) -> ParseResult> { + let mut it = val.splitn(2, ","); + let mid: ObjID = it.next().unwrap().parse()?; + let text = it.next().ok_or(ParseError {})?; + Ok(SearchNode::Duplicates { + note_type_id: mid, + text: text.into(), + }) +} + +/// eg prop:ivl>3, prop:ease!=2.5 +fn parse_prop(val: &str) -> ParseResult> { + let (val, key) = alt(( + tag("ivl"), + tag("due"), + tag("reps"), + tag("lapses"), + tag("ease"), + ))(val)?; + + let (val, operator) = alt(( + tag("<="), + tag(">="), + tag("!="), + tag("="), + tag("<"), + tag(">"), + ))(val)?; + + let kind = if key == "ease" { + let num: f32 = val.parse()?; + PropertyKind::Ease(num) + } else if key == "due" { + let num: i32 = val.parse()?; + PropertyKind::Due(num) + } else { + let num: u32 = val.parse()?; + match key { + "ivl" => PropertyKind::Interval(num), + "reps" => PropertyKind::Reps(num), + "lapses" => PropertyKind::Lapses(num), + _ => unreachable!(), + } + }; + + Ok(SearchNode::Property { + operator: operator.to_string(), + kind, + }) } #[cfg(test)] @@ -147,25 +349,60 @@ mod test { #[test] fn parsing() -> Result<(), String> { use Node::*; + use SearchNode::*; assert_eq!( parse(r#"hello -(world and "foo:bar baz") OR test"#)?, vec![ - UnqualifiedText("hello".into()), + Search(UnqualifiedText("hello".into())), And, Not(Box::new(Group(vec![ - UnqualifiedText("world".into()), + Search(UnqualifiedText("world".into())), And, - QualifiedText { - key: "foo".into(), - val: "bar baz".into() - } + Search(SingleField { + field: "foo".into(), + text: "bar baz".into() + }) ]))), Or, - UnqualifiedText("test".into()) + Search(UnqualifiedText("test".into())) ] ); + assert_eq!(parse("added:3")?, vec![Search(AddedInDays(3))]); + assert_eq!( + parse("card:front")?, + vec![Search(CardTemplate("front".into()))] + ); + assert_eq!(parse("deck:default")?, vec![Search(Deck("default".into()))]); + assert_eq!(parse("note:basic")?, vec![Search(NoteType("basic".into()))]); + assert_eq!(parse("tag:hard")?, vec![Search(Tag("hard".into()))]); + assert_eq!( + parse("nid:1237123712,2,3")?, + vec![Search(NoteIDs("1237123712,2,3".into()))] + ); + assert!(parse("nid:1237123712_2,3").is_err()); + assert_eq!(parse("is:due")?, vec![Search(State(StateKind::Due))]); + assert_eq!(parse("flag:3")?, vec![Search(Flag(3))]); + assert!(parse("flag:-1").is_err()); + assert!(parse("flag:5").is_err()); + + assert_eq!( + parse("prop:ivl>3")?, + vec![Search(Property { + operator: ">".into(), + kind: PropertyKind::Interval(3) + })] + ); + assert!(parse("prop:ivl>3.3").is_err()); + assert_eq!( + parse("prop:ease<=3.3")?, + vec![Search(Property { + operator: "<=".into(), + kind: PropertyKind::Ease(3.3) + })] + ); + Ok(()) } } From e790367b1ed54eeea98937e67c59d80432a177d2 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 16 Mar 2020 13:16:43 +1000 Subject: [PATCH 062/150] ensure id list not empty --- rslib/src/search/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index b3768aaae..e102070a2 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -247,7 +247,7 @@ fn search_node_for_text_with_argument<'a>( /// ensure a list of ids contains only numbers and commas, returning unchanged if true /// used by nid: and cid: fn check_id_list(s: Cow) -> ParseResult> { - if s.as_bytes().iter().any(|&c| !is_digit(c) && c != b',') { + if s.is_empty() || s.as_bytes().iter().any(|&c| !is_digit(c) && c != b',') { Err(ParseError {}) } else { Ok(s) From 00300bb24d8b0c1b6178ac5a45d405107a84fe5d Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 16 Mar 2020 13:16:59 +1000 Subject: [PATCH 063/150] ensure rated ease in range --- rslib/src/search/parser.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index e102070a2..74101ba56 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -279,11 +279,19 @@ fn parse_flag(s: &str) -> ParseResult> { } /// eg rated:3 or rated:10:2 +/// second arg must be between 1-4 fn parse_rated(val: &str) -> ParseResult> { let mut it = val.splitn(2, ':'); let days = it.next().unwrap().parse()?; let ease = match it.next() { - Some(v) => Some(v.parse()?), + Some(v) => { + let n: u8 = v.parse()?; + if n < 5 && n > 0 { + Some(n) + } else { + return Err(ParseError {}); + } + } None => None, }; From 4f93ae4b6dfa7117ec16fcf898d83cbadfca434f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 16 Mar 2020 13:19:05 +1000 Subject: [PATCH 064/150] start of searching sql --- rslib/src/search/mod.rs | 1 + rslib/src/search/searcher.rs | 146 +++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 rslib/src/search/searcher.rs diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs index b93e263bb..2732ec547 100644 --- a/rslib/src/search/mod.rs +++ b/rslib/src/search/mod.rs @@ -1 +1,2 @@ mod parser; +mod searcher; diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs new file mode 100644 index 000000000..7e0020af7 --- /dev/null +++ b/rslib/src/search/searcher.rs @@ -0,0 +1,146 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::parser::{Node, PropertyKind, SearchNode, StateKind}; +use crate::{collection::RequestContext, types::ObjID}; +use rusqlite::types::ToSqlOutput; +use std::fmt::Write; + +struct SearchContext<'a> { + ctx: &'a mut RequestContext<'a>, + sql: String, + args: Vec>, +} + +#[allow(dead_code)] +fn node_to_sql<'a>( + ctx: &'a mut RequestContext<'a>, + node: &'a Node, +) -> (String, Vec>) { + let sql = String::new(); + let args = vec![]; + let mut sctx = SearchContext { ctx, sql, args }; + write_node_to_sql(&mut sctx, node); + (sctx.sql, sctx.args) +} + +fn write_node_to_sql(ctx: &mut SearchContext, node: &Node) { + match node { + Node::And => write!(ctx.sql, " and ").unwrap(), + Node::Or => write!(ctx.sql, " or ").unwrap(), + Node::Not(node) => { + write!(ctx.sql, "not ").unwrap(); + write_node_to_sql(ctx, node); + } + Node::Group(nodes) => { + write!(ctx.sql, "(").unwrap(); + for node in nodes { + write_node_to_sql(ctx, node); + } + write!(ctx.sql, ")").unwrap(); + } + Node::Search(search) => write_search_node_to_sql(ctx, search), + } +} + +fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { + match node { + SearchNode::UnqualifiedText(text) => write_unqualified(ctx, text), + SearchNode::SingleField { field, text } => { + write_single_field(ctx, field.as_ref(), text.as_ref()) + } + SearchNode::AddedInDays(days) => { + write!(ctx.sql, "c.id > {}", days).unwrap(); + } + SearchNode::CardTemplate(template) => write_template(ctx, template.as_ref()), + SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref()), + SearchNode::NoteTypeID(ntid) => { + write!(ctx.sql, "n.mid = {}", ntid).unwrap(); + } + SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref()), + SearchNode::Rated { days, ease } => write_rated(ctx, days, ease), + SearchNode::Tag(tag) => write_tag(ctx, tag), + SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, note_type_id, text), + SearchNode::State(state) => write_state(ctx, state), + SearchNode::Flag(flag) => { + write!(ctx.sql, "(c.flags & 7) == {}", flag).unwrap(); + } + SearchNode::NoteIDs(nids) => { + write!(ctx.sql, "n.id in ({})", nids).unwrap(); + } + SearchNode::CardIDs(cids) => { + write!(ctx.sql, "c.id in ({})", cids).unwrap(); + } + SearchNode::Property { operator, kind } => write_prop(ctx, operator, kind), + } +} + +fn write_unqualified(ctx: &mut SearchContext, text: &str) { + // implicitly wrap in % + let text = format!("%{}%", text); + write!( + ctx.sql, + "(n.sfld like ? escape '\\' or n.flds like ? escape '\\')" + ) + .unwrap(); + ctx.args.push(text.clone().into()); + ctx.args.push(text.into()); +} + +fn write_tag(ctx: &mut SearchContext, text: &str) { + if text == "none" { + write!(ctx.sql, "n.tags = ''").unwrap(); + return; + } + + let tag = format!(" %{}% ", text.replace('*', "%")); + write!(ctx.sql, "n.tags like ?").unwrap(); + ctx.args.push(tag.into()); +} + +// fixme: need day cutoff +fn write_rated(ctx: &mut SearchContext, days: &u32, ease: &Option) {} + +// fixme: need current day +fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) {} + +// fixme: need db +fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) {} + +// fixme: need cutoff & current day +fn write_state(ctx: &mut SearchContext, state: &StateKind) {} + +// fixme: need deck manager +fn write_deck(ctx: &mut SearchContext, deck: &str) {} + +// fixme: need note type manager +fn write_template(ctx: &mut SearchContext, template: &str) {} + +// fixme: need note type manager +fn write_note_type(ctx: &mut SearchContext, notetype: &str) {} + +// fixme: need note type manager +fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) {} + +#[cfg(test)] +mod test { + use super::super::parser::parse; + use super::*; + + // parse + fn p(search: &str) -> Node { + Node::Group(parse(search).unwrap()) + } + + // get sql + // fn s<'a>(n: &'a Node) -> (String, Vec>) { + // node_to_sql(n) + // } + + #[test] + fn tosql() -> Result<(), String> { + // assert_eq!(s(&p("added:1")), ("(c.id > 1)".into(), vec![])); + + Ok(()) + } +} From 68657c7166b4d0849394e414bfa4aa59f514773c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 12:31:54 +1000 Subject: [PATCH 065/150] field_checksum needs to strip HTML --- rslib/src/notes.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rslib/src/notes.rs b/rslib/src/notes.rs index c4289e9b1..212e51c1e 100644 --- a/rslib/src/notes.rs +++ b/rslib/src/notes.rs @@ -40,8 +40,9 @@ impl Note { } } -fn field_checksum(text: &str) -> u32 { - let digest = sha1::Sha1::from(text).digest().bytes(); +pub(crate) fn field_checksum(text: &str) -> u32 { + let text = strip_html_preserving_image_filenames(text); + let digest = sha1::Sha1::from(text.as_ref()).digest().bytes(); u32::from_be_bytes(digest[..4].try_into().unwrap()) } From dcb2b46d1b81bc78970805da9d2e08f23da6bc23 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 12:34:39 +1000 Subject: [PATCH 066/150] use .eq_ignore_ascii_case() to avoid allocating --- rslib/src/search/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 74101ba56..117906ab4 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -191,9 +191,9 @@ fn unquoted_term(s: &str) -> IResult<&str, Node> { map_res( take_while1(|c| c != ' ' && c != ')'), |text: &str| -> ParseResult { - Ok(if text.len() == 2 && text.to_ascii_lowercase() == "or" { + Ok(if text.eq_ignore_ascii_case("or") { Node::Or - } else if text.len() == 3 && text.to_ascii_lowercase() == "and" { + } else if text.eq_ignore_ascii_case("and") { Node::And } else { Node::Search(search_node_for_text(text)?) From 91d7b02325be2b0aafec835c213333dd1a9bf35a Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 12:35:02 +1000 Subject: [PATCH 067/150] separate out template ordinal and name search --- rslib/src/search/parser.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 117906ab4..400346c2a 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -54,7 +54,7 @@ pub(super) enum SearchNode<'a> { text: Cow<'a, str>, }, AddedInDays(u32), - CardTemplate(Cow<'a, str>), + CardTemplate(TemplateKind), Deck(Cow<'a, str>), NoteTypeID(ObjID), NoteType(Cow<'a, str>), @@ -96,6 +96,12 @@ pub(super) enum StateKind { Suspended, } +#[derive(Debug, PartialEq)] +pub(super) enum TemplateKind { + Ordinal(u16), + Name(String), +} + /// Parse the input string into a list of nodes. #[allow(dead_code)] pub(super) fn parse(input: &str) -> std::result::Result, String> { @@ -223,13 +229,13 @@ fn search_node_for_text_with_argument<'a>( ) -> ParseResult> { Ok(match key.to_ascii_lowercase().as_str() { "added" => SearchNode::AddedInDays(val.parse()?), - "card" => SearchNode::CardTemplate(val), "deck" => SearchNode::Deck(val), "note" => SearchNode::NoteType(val), "tag" => SearchNode::Tag(val), "mid" => SearchNode::NoteTypeID(val.parse()?), "nid" => SearchNode::NoteIDs(check_id_list(val)?), "cid" => SearchNode::CardIDs(check_id_list(val)?), + "card" => parse_template(val.as_ref()), "is" => parse_state(val.as_ref())?, "flag" => parse_flag(val.as_ref())?, "rated" => parse_rated(val.as_ref())?, @@ -350,6 +356,13 @@ fn parse_prop(val: &str) -> ParseResult> { }) } +fn parse_template(val: &str) -> SearchNode<'static> { + SearchNode::CardTemplate(match val.parse::() { + Ok(n) => TemplateKind::Ordinal(n), + Err(_) => TemplateKind::Name(val.into()), + }) +} + #[cfg(test)] mod test { use super::*; @@ -380,7 +393,11 @@ mod test { assert_eq!(parse("added:3")?, vec![Search(AddedInDays(3))]); assert_eq!( parse("card:front")?, - vec![Search(CardTemplate("front".into()))] + vec![Search(CardTemplate(TemplateKind::Name("front".into())))] + ); + assert_eq!( + parse("card:3")?, + vec![Search(CardTemplate(TemplateKind::Ordinal(3)))] ); assert_eq!(parse("deck:default")?, vec![Search(Deck("default".into()))]); assert_eq!(parse("note:basic")?, vec![Search(NoteType("basic".into()))]); From 761d1d181232e08fdedf500eed7cac0038911f70 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 12:33:29 +1000 Subject: [PATCH 068/150] add card queue/type enums --- rslib/Cargo.toml | 2 ++ rslib/src/card.rs | 33 +++++++++++++++++++++++++++++++++ rslib/src/lib.rs | 1 + 3 files changed, 36 insertions(+) create mode 100644 rslib/src/card.rs diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 4d4f51e86..7250b185c 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -36,6 +36,8 @@ slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_de slog-term = "2.5.0" slog-async = "2.4.0" slog-envlogger = "2.2.0" +serde_repr = "0.1.5" +num_enum = "0.4.2" [target.'cfg(target_vendor="apple")'.dependencies] rusqlite = { version = "0.21.0", features = ["trace"] } diff --git a/rslib/src/card.rs b/rslib/src/card.rs new file mode 100644 index 000000000..1ee1386b0 --- /dev/null +++ b/rslib/src/card.rs @@ -0,0 +1,33 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use num_enum::TryFromPrimitive; +use serde_repr::{Deserialize_repr, Serialize_repr}; + +#[derive(Serialize_repr, Deserialize_repr, Debug, PartialEq, TryFromPrimitive, Clone, Copy)] +#[repr(u8)] +pub enum CardType { + New = 0, + Learn = 1, + Review = 2, + Relearn = 3, +} + +#[derive(Serialize_repr, Deserialize_repr, Debug, PartialEq, TryFromPrimitive, Clone, Copy)] +#[repr(i8)] +pub enum CardQueue { + /// due is the order cards are shown in + New = 0, + /// due is a unix timestamp + Learn = 1, + /// due is days since creation date + Review = 2, + DayLearn = 3, + /// due is a unix timestamp. + /// preview cards only placed here when failed. + PreviewRepeat = 4, + /// cards are not due in these states + Suspended = -1, + UserBuried = -2, + SchedBuried = -3, +} diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index f66dd1bd6..b2e410d8c 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -10,6 +10,7 @@ pub fn version() -> &'static str { } pub mod backend; +pub mod card; pub mod cloze; pub mod collection; pub mod err; From cffa52ff8284fc32a9a080ce707496950f70608b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 12:39:13 +1000 Subject: [PATCH 069/150] more searching work --- rslib/src/search/searcher.rs | 164 +++++++++++++++++++++++++++++++---- 1 file changed, 147 insertions(+), 17 deletions(-) diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 7e0020af7..188aff410 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -1,12 +1,15 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use super::parser::{Node, PropertyKind, SearchNode, StateKind}; +use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind}; +use crate::card::CardQueue; +use crate::notes::field_checksum; use crate::{collection::RequestContext, types::ObjID}; use rusqlite::types::ToSqlOutput; use std::fmt::Write; struct SearchContext<'a> { + #[allow(dead_code)] ctx: &'a mut RequestContext<'a>, sql: String, args: Vec>, @@ -52,7 +55,7 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { SearchNode::AddedInDays(days) => { write!(ctx.sql, "c.id > {}", days).unwrap(); } - SearchNode::CardTemplate(template) => write_template(ctx, template.as_ref()), + SearchNode::CardTemplate(template) => write_template(ctx, template), SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref()), SearchNode::NoteTypeID(ntid) => { write!(ctx.sql, "n.mid = {}", ntid).unwrap(); @@ -99,38 +102,165 @@ fn write_tag(ctx: &mut SearchContext, text: &str) { } // fixme: need day cutoff -fn write_rated(ctx: &mut SearchContext, days: &u32, ease: &Option) {} +fn write_rated(ctx: &mut SearchContext, days: &u32, ease: &Option) { + let today_cutoff = 0; // fixme + let days = *days.min(&31); + let target_cutoff = today_cutoff - 86_400 * days; + write!( + ctx.sql, + "c.id in (select cid from revlog where id>{}", + target_cutoff + ) + .unwrap(); + if let Some(ease) = ease { + write!(ctx.sql, "and ease={})", ease).unwrap(); + } else { + write!(ctx.sql, ")").unwrap(); + } +} // fixme: need current day -fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) {} - -// fixme: need db -fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) {} +fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) { + match kind { + PropertyKind::Due(days) => { + let day = days; // fixme: + sched_today + write!( + ctx.sql, + "(c.queue in ({rev},{daylrn}) and due {op} {day})", + rev = CardQueue::Review as u8, + daylrn = CardQueue::DayLearn as u8, + op = op, + day = day + ) + } + PropertyKind::Interval(ivl) => write!(ctx.sql, "ivl {} {}", op, ivl), + PropertyKind::Reps(reps) => write!(ctx.sql, "reps {} {}", op, reps), + PropertyKind::Lapses(days) => write!(ctx.sql, "lapses {} {}", op, days), + PropertyKind::Ease(ease) => write!(ctx.sql, "ease {} {}", op, (ease * 1000.0) as u32), + } + .unwrap(); +} // fixme: need cutoff & current day -fn write_state(ctx: &mut SearchContext, state: &StateKind) {} +fn write_state(ctx: &mut SearchContext, state: &StateKind) { + match state { + StateKind::New => write!(ctx.sql, "c.queue = {}", CardQueue::New as u8), + StateKind::Review => write!(ctx.sql, "c.queue = {}", CardQueue::Review as u8), + StateKind::Learning => write!( + ctx.sql, + "c.queue in ({},{})", + CardQueue::Learn as u8, + CardQueue::DayLearn as u8 + ), + StateKind::Buried => write!( + ctx.sql, + "c.queue in ({},{})", + CardQueue::SchedBuried as u8, + CardQueue::UserBuried as u8 + ), + StateKind::Suspended => write!(ctx.sql, "c.queue = {}", CardQueue::Suspended as u8), + StateKind::Due => { + let today = 0; // fixme: today + let day_cutoff = 0; // fixme: day_cutoff + write!( + ctx.sql, + " +(c.queue in ({rev},{daylrn}) and c.due <= {today}) or +(c.queue = {lrn} and c.due <= {daycutoff})", + rev = CardQueue::Review as u8, + daylrn = CardQueue::DayLearn as u8, + today = today, + lrn = CardQueue::Learn as u8, + daycutoff = day_cutoff, + ) + } + } + .unwrap() +} // fixme: need deck manager -fn write_deck(ctx: &mut SearchContext, deck: &str) {} +fn write_deck(ctx: &mut SearchContext, deck: &str) { + match deck { + "*" => write!(ctx.sql, "true").unwrap(), + "filtered" => write!(ctx.sql, "c.odid > 0").unwrap(), + "current" => { + todo!() // fixme: need current deck and child decks + } + _deck => { + // fixme: narrow to dids matching possible wildcard; include children + todo!() + } + } +} // fixme: need note type manager -fn write_template(ctx: &mut SearchContext, template: &str) {} +fn write_template(ctx: &mut SearchContext, template: &TemplateKind) { + match template { + TemplateKind::Ordinal(n) => { + write!(ctx.sql, "c.ord = {}", n).unwrap(); + } + TemplateKind::Name(_name) => { + // fixme: search through note types loooking for template name + } + } +} // fixme: need note type manager -fn write_note_type(ctx: &mut SearchContext, notetype: &str) {} +fn write_note_type(ctx: &mut SearchContext, _notetype: &str) { + let ntid: Option = None; // fixme: get id via name search + if let Some(ntid) = ntid { + write!(ctx.sql, "n.mid = {}", ntid).unwrap(); + } else { + write!(ctx.sql, "false").unwrap(); + } +} // fixme: need note type manager -fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) {} +// fixme: need field_at_index() +fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { + let _ = field; + let fields = vec![(0, 0)]; // fixme: get list of (ntid, ordinal) + + if fields.is_empty() { + write!(ctx.sql, "false").unwrap(); + return; + } + + write!(ctx.sql, "(").unwrap(); + ctx.args.push(val.to_string().into()); + let arg_idx = ctx.args.len(); + for (ntid, ord) in fields { + write!( + ctx.sql, + "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", + ntid, ord, arg_idx + ) + .unwrap(); + } + write!(ctx.sql, ")").unwrap(); +} + +// fixme: need field_at_index() +fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) { + let csum = field_checksum(text); + write!( + ctx.sql, + "(n.mid = {} and n.csum = {} and field_at_index(n.flds, 0) = ?", + ntid, csum + ) + .unwrap(); + ctx.args.push(text.to_string().into()) +} #[cfg(test)] mod test { - use super::super::parser::parse; - use super::*; + // use super::super::parser::parse; + // use super::*; // parse - fn p(search: &str) -> Node { - Node::Group(parse(search).unwrap()) - } + // fn p(search: &str) -> Node { + // Node::Group(parse(search).unwrap()) + // } // get sql // fn s<'a>(n: &'a Node) -> (String, Vec>) { From 7eab5041269cc2aef5974a06d4e63836833955aa Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 13:41:56 +1000 Subject: [PATCH 070/150] add field_at_index() sql func --- rslib/Cargo.toml | 4 ++-- rslib/src/search/searcher.rs | 2 -- rslib/src/storage/sqlite.rs | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 7250b185c..58e5d5c90 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -40,10 +40,10 @@ serde_repr = "0.1.5" num_enum = "0.4.2" [target.'cfg(target_vendor="apple")'.dependencies] -rusqlite = { version = "0.21.0", features = ["trace"] } +rusqlite = { version = "0.21.0", features = ["trace", "functions"] } [target.'cfg(not(target_vendor="apple"))'.dependencies] -rusqlite = { version = "0.21.0", features = ["trace", "bundled"] } +rusqlite = { version = "0.21.0", features = ["trace", "functions", "bundled"] } [target.'cfg(linux)'.dependencies] reqwest = { version = "0.10.1", features = ["json", "native-tls-vendored"] } diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 188aff410..9a2c67e96 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -216,7 +216,6 @@ fn write_note_type(ctx: &mut SearchContext, _notetype: &str) { } // fixme: need note type manager -// fixme: need field_at_index() fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { let _ = field; let fields = vec![(0, 0)]; // fixme: get list of (ntid, ordinal) @@ -240,7 +239,6 @@ fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { write!(ctx.sql, ")").unwrap(); } -// fixme: need field_at_index() fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) { let csum = field_checksum(text); write!( diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 886a8f801..967e79727 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -36,11 +36,26 @@ fn open_or_create_collection_db(path: &Path) -> Result { db.pragma_update(None, "cache_size", &(-40 * 1024))?; db.pragma_update(None, "legacy_file_format", &false)?; db.pragma_update(None, "journal", &"wal")?; + db.set_prepared_statement_cache_capacity(50); + add_field_index_function(&db)?; + Ok(db) } +/// Adds sql function field_at_index(flds, index) +/// to split provided fields and return field at zero-based index. +/// If out of range, returns empty string. +fn add_field_index_function(db: &Connection) -> Result<()> { + db.create_scalar_function("field_at_index", 2, true, |ctx| { + let mut fields = ctx.get_raw(0).as_str()?.split('\x1f'); + let idx: u16 = ctx.get(1)?; + Ok(fields.nth(idx as usize).unwrap_or("").to_string()) + }) + .map_err(Into::into) +} + /// Fetch schema version from database. /// Return (must_create, version) fn schema_version(db: &Connection) -> Result<(bool, u8)> { From cc54e9275617fa9be1ce5c4e82f39cb73c9df300 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 14:52:55 +1000 Subject: [PATCH 071/150] move html stripping out of field_checksum into caller --- rslib/src/notes.rs | 28 +++++++++++++++++----------- rslib/src/search/searcher.rs | 7 +++++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/rslib/src/notes.rs b/rslib/src/notes.rs index 212e51c1e..99efb9410 100644 --- a/rslib/src/notes.rs +++ b/rslib/src/notes.rs @@ -40,9 +40,10 @@ impl Note { } } +/// Text must be passed to strip_html_preserving_image_filenames() by +/// caller prior to passing in here. pub(crate) fn field_checksum(text: &str) -> u32 { - let text = strip_html_preserving_image_filenames(text); - let digest = sha1::Sha1::from(text.as_ref()).digest().bytes(); + let digest = sha1::Sha1::from(text).digest().bytes(); u32::from_be_bytes(digest[..4].try_into().unwrap()) } @@ -119,15 +120,20 @@ pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) - note.mtime_secs = i64_unix_secs(); // hard-coded for now note.usn = -1; - let csum = field_checksum(¬e.fields()[0]); - let sort_field = strip_html_preserving_image_filenames( - note.fields() - .get(note_type.sort_field_idx as usize) - .ok_or_else(|| AnkiError::DBError { - info: "sort field out of range".to_string(), - kind: DBErrorKind::MissingEntity, - })?, - ); + let field1_nohtml = strip_html_preserving_image_filenames(¬e.fields()[0]); + let csum = field_checksum(field1_nohtml.as_ref()); + let sort_field = if note_type.sort_field_idx == 0 { + field1_nohtml + } else { + strip_html_preserving_image_filenames( + note.fields() + .get(note_type.sort_field_idx as usize) + .ok_or_else(|| AnkiError::DBError { + info: "sort field out of range".to_string(), + kind: DBErrorKind::MissingEntity, + })?, + ) + }; let mut stmt = db.prepare_cached("update notes set mod=?,usn=?,flds=?,sfld=?,csum=? where id=?")?; diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 9a2c67e96..4a3978724 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -4,7 +4,9 @@ use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind}; use crate::card::CardQueue; use crate::notes::field_checksum; -use crate::{collection::RequestContext, types::ObjID}; +use crate::{ + collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID, +}; use rusqlite::types::ToSqlOutput; use std::fmt::Write; @@ -240,7 +242,8 @@ fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { } fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) { - let csum = field_checksum(text); + let text_nohtml = strip_html_preserving_image_filenames(text); + let csum = field_checksum(text_nohtml.as_ref()); write!( ctx.sql, "(n.mid = {} and n.csum = {} and field_at_index(n.flds, 0) = ?", From f559ae3ef8ad68615037f12d80c762dc34106d85 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 15:17:38 +1000 Subject: [PATCH 072/150] address some clippy lints --- rslib/src/search/parser.rs | 2 +- rslib/src/search/searcher.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 400346c2a..62824fe9f 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -306,7 +306,7 @@ fn parse_rated(val: &str) -> ParseResult> { /// eg dupes:1231,hello fn parse_dupes(val: &str) -> ParseResult> { - let mut it = val.splitn(2, ","); + let mut it = val.splitn(2, ','); let mid: ObjID = it.next().unwrap().parse()?; let text = it.next().ok_or(ParseError {})?; Ok(SearchNode::Duplicates { diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 4a3978724..65086e5d6 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -63,9 +63,9 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { write!(ctx.sql, "n.mid = {}", ntid).unwrap(); } SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref()), - SearchNode::Rated { days, ease } => write_rated(ctx, days, ease), + SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease), SearchNode::Tag(tag) => write_tag(ctx, tag), - SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, note_type_id, text), + SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, *note_type_id, text), SearchNode::State(state) => write_state(ctx, state), SearchNode::Flag(flag) => { write!(ctx.sql, "(c.flags & 7) == {}", flag).unwrap(); @@ -104,9 +104,9 @@ fn write_tag(ctx: &mut SearchContext, text: &str) { } // fixme: need day cutoff -fn write_rated(ctx: &mut SearchContext, days: &u32, ease: &Option) { +fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) { let today_cutoff = 0; // fixme - let days = *days.min(&31); + let days = days.min(31); let target_cutoff = today_cutoff - 86_400 * days; write!( ctx.sql, @@ -241,7 +241,7 @@ fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { write!(ctx.sql, ")").unwrap(); } -fn write_dupes(ctx: &mut SearchContext, ntid: &ObjID, text: &str) { +fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { let text_nohtml = strip_html_preserving_image_filenames(text); let csum = field_checksum(text_nohtml.as_ref()); write!( From 9f3cc0982d23a936903d37f76e6c1d11349e677f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 17 Mar 2020 17:02:58 +1000 Subject: [PATCH 073/150] deck searching A bit more complicated than it needs to be, as we don't have the full deck manager infrastructure yet. --- rslib/src/config.rs | 11 ++++++ rslib/src/decks.rs | 29 +++++++++++++++ rslib/src/lib.rs | 2 ++ rslib/src/search/searcher.rs | 70 +++++++++++++++++++++++++----------- rslib/src/storage/sqlite.rs | 17 ++++++++- rslib/src/text.rs | 21 +++++++++++ 6 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 rslib/src/config.rs create mode 100644 rslib/src/decks.rs diff --git a/rslib/src/config.rs b/rslib/src/config.rs new file mode 100644 index 000000000..2394819c9 --- /dev/null +++ b/rslib/src/config.rs @@ -0,0 +1,11 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::types::ObjID; +use serde_derive::Deserialize; + +#[derive(Deserialize)] +pub struct Config { + #[serde(rename = "curDeck")] + pub(crate) current_deck_id: ObjID, +} diff --git a/rslib/src/decks.rs b/rslib/src/decks.rs new file mode 100644 index 000000000..9bf8e7e54 --- /dev/null +++ b/rslib/src/decks.rs @@ -0,0 +1,29 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::types::ObjID; +use serde_derive::Deserialize; + +#[derive(Deserialize)] +pub struct Deck { + pub(crate) id: ObjID, + pub(crate) name: String, +} + +pub(crate) fn child_ids<'a>(decks: &'a [Deck], name: &str) -> impl Iterator + 'a { + let prefix = format!("{}::", name); + decks + .iter() + .filter(move |d| d.name.starts_with(&prefix)) + .map(|d| d.id) +} + +pub(crate) fn get_deck(decks: &[Deck], id: ObjID) -> Option<&Deck> { + for d in decks { + if d.id == id { + return Some(d); + } + } + + None +} diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index b2e410d8c..37a8db7f6 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -13,6 +13,8 @@ pub mod backend; pub mod card; pub mod cloze; pub mod collection; +pub mod config; +pub mod decks; pub mod err; pub mod i18n; pub mod latex; diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 65086e5d6..dc9f375a9 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -3,7 +3,11 @@ use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind}; use crate::card::CardQueue; +use crate::decks::child_ids; +use crate::decks::get_deck; +use crate::err::{AnkiError, Result}; use crate::notes::field_checksum; +use crate::text::matches_wildcard; use crate::{ collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID, }; @@ -21,34 +25,35 @@ struct SearchContext<'a> { fn node_to_sql<'a>( ctx: &'a mut RequestContext<'a>, node: &'a Node, -) -> (String, Vec>) { +) -> Result<(String, Vec>)> { let sql = String::new(); let args = vec![]; let mut sctx = SearchContext { ctx, sql, args }; - write_node_to_sql(&mut sctx, node); - (sctx.sql, sctx.args) + write_node_to_sql(&mut sctx, node)?; + Ok((sctx.sql, sctx.args)) } -fn write_node_to_sql(ctx: &mut SearchContext, node: &Node) { +fn write_node_to_sql(ctx: &mut SearchContext, node: &Node) -> Result<()> { match node { Node::And => write!(ctx.sql, " and ").unwrap(), Node::Or => write!(ctx.sql, " or ").unwrap(), Node::Not(node) => { write!(ctx.sql, "not ").unwrap(); - write_node_to_sql(ctx, node); + write_node_to_sql(ctx, node)?; } Node::Group(nodes) => { write!(ctx.sql, "(").unwrap(); for node in nodes { - write_node_to_sql(ctx, node); + write_node_to_sql(ctx, node)?; } write!(ctx.sql, ")").unwrap(); } - Node::Search(search) => write_search_node_to_sql(ctx, search), - } + Node::Search(search) => write_search_node_to_sql(ctx, search)?, + }; + Ok(()) } -fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { +fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Result<()> { match node { SearchNode::UnqualifiedText(text) => write_unqualified(ctx, text), SearchNode::SingleField { field, text } => { @@ -58,7 +63,7 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { write!(ctx.sql, "c.id > {}", days).unwrap(); } SearchNode::CardTemplate(template) => write_template(ctx, template), - SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref()), + SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref())?, SearchNode::NoteTypeID(ntid) => { write!(ctx.sql, "n.mid = {}", ntid).unwrap(); } @@ -77,7 +82,8 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) { write!(ctx.sql, "c.id in ({})", cids).unwrap(); } SearchNode::Property { operator, kind } => write_prop(ctx, operator, kind), - } + }; + Ok(()) } fn write_unqualified(ctx: &mut SearchContext, text: &str) { @@ -180,19 +186,43 @@ fn write_state(ctx: &mut SearchContext, state: &StateKind) { .unwrap() } -// fixme: need deck manager -fn write_deck(ctx: &mut SearchContext, deck: &str) { +fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { match deck { "*" => write!(ctx.sql, "true").unwrap(), "filtered" => write!(ctx.sql, "c.odid > 0").unwrap(), - "current" => { - todo!() // fixme: need current deck and child decks + deck => { + let all_decks = ctx.ctx.storage.all_decks()?; + let dids_with_children = if deck == "current" { + let config = ctx.ctx.storage.all_config()?; + let mut dids_with_children = vec![config.current_deck_id]; + let current = get_deck(&all_decks, config.current_deck_id) + .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?; + for child_did in child_ids(&all_decks, ¤t.name) { + dids_with_children.push(child_did); + } + dids_with_children + } else { + let mut dids_with_children = vec![]; + for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) { + dids_with_children.push(deck.id); + for child_id in child_ids(&all_decks, &deck.name) { + dids_with_children.push(child_id); + } + } + dids_with_children + }; + + if dids_with_children.is_empty() { + write!(ctx.sql, "false") + } else { + let did_strings: Vec = + dids_with_children.iter().map(ToString::to_string).collect(); + write!(ctx.sql, "c.did in ({})", did_strings.join(",")) + } + .unwrap(); } - _deck => { - // fixme: narrow to dids matching possible wildcard; include children - todo!() - } - } + }; + Ok(()) } // fixme: need note type manager diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 967e79727..a689b34c5 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -2,10 +2,11 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::collection::CollectionOp; +use crate::config::Config; use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; use crate::time::{i64_unix_millis, i64_unix_secs}; -use crate::types::Usn; +use crate::{decks::Deck, types::Usn}; use rusqlite::{params, Connection, NO_PARAMS}; use std::path::{Path, PathBuf}; @@ -210,4 +211,18 @@ impl StorageContext<'_> { Ok(-1) } } + + pub(crate) fn all_decks(&self) -> Result> { + self.db + .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> { + Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) + }) + } + + pub(crate) fn all_config(&self) -> Result { + self.db + .query_row_and_then("select conf from col", NO_PARAMS, |row| -> Result<_> { + Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) + }) + } } diff --git a/rslib/src/text.rs b/rslib/src/text.rs index 4d5ff286d..f7a770cc7 100644 --- a/rslib/src/text.rs +++ b/rslib/src/text.rs @@ -219,8 +219,20 @@ pub(crate) fn normalize_to_nfc(s: &str) -> Cow { } } +/// True if search is equal to text, folding ascii case. +/// Supports '*' to match 0 or more characters. +pub(crate) fn matches_wildcard(text: &str, search: &str) -> bool { + if search.contains('*') { + let search = format!("^(?i){}$", regex::escape(search).replace(r"\*", ".*")); + Regex::new(&search).unwrap().is_match(text) + } else { + text.eq_ignore_ascii_case(search) + } +} + #[cfg(test)] mod test { + use super::matches_wildcard; use crate::text::{ extract_av_tags, strip_av_tags, strip_html, strip_html_preserving_image_filenames, AVTag, }; @@ -265,4 +277,13 @@ mod test { ] ); } + + #[test] + fn wildcard() { + assert_eq!(matches_wildcard("foo", "bar"), false); + assert_eq!(matches_wildcard("foo", "Foo"), true); + assert_eq!(matches_wildcard("foo", "F*"), true); + assert_eq!(matches_wildcard("foo", "F*oo"), true); + assert_eq!(matches_wildcard("foo", "b*"), false); + } } From 2beccd377b4733fd2bbe8f70017df6868de46a48 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Mar 2020 11:53:02 +1000 Subject: [PATCH 074/150] add v1 and v2 legacy timing code --- rslib/src/backend/mod.rs | 4 +- rslib/src/config.rs | 4 ++ rslib/src/sched/cutoff.rs | 119 ++++++++++++++++++++++++++++++++++-- rslib/src/storage/sqlite.rs | 29 ++++++++- 4 files changed, 147 insertions(+), 9 deletions(-) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 197004aca..84d40129e 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -12,7 +12,7 @@ use crate::log::{default_logger, Logger}; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; -use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today}; +use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today_v2_new}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, @@ -348,7 +348,7 @@ impl Backend { } fn sched_timing_today(&self, input: pb::SchedTimingTodayIn) -> pb::SchedTimingTodayOut { - let today = sched_timing_today( + let today = sched_timing_today_v2_new( input.created_secs as i64, input.created_mins_west, input.now_secs as i64, diff --git a/rslib/src/config.rs b/rslib/src/config.rs index 2394819c9..42ae62067 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -5,7 +5,11 @@ use crate::types::ObjID; use serde_derive::Deserialize; #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] pub struct Config { #[serde(rename = "curDeck")] pub(crate) current_deck_id: ObjID, + pub(crate) rollover: Option, + pub(crate) creation_offset: Option, + pub(crate) local_offset: Option, } diff --git a/rslib/src/sched/cutoff.rs b/rslib/src/sched/cutoff.rs index 5c99b57a5..04bdb60fa 100644 --- a/rslib/src/sched/cutoff.rs +++ b/rslib/src/sched/cutoff.rs @@ -1,8 +1,10 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::time::i64_unix_secs; use chrono::{Date, Duration, FixedOffset, Local, TimeZone}; +#[derive(Debug, PartialEq, Clone, Copy)] pub struct SchedTimingToday { /// The number of days that have passed since the collection was created. pub days_elapsed: u32, @@ -17,7 +19,7 @@ pub struct SchedTimingToday { /// - now_secs is a timestamp of the current time /// - now_mins_west is the current offset west of UTC /// - rollover_hour is the hour of the day the rollover happens (eg 4 for 4am) -pub fn sched_timing_today( +pub fn sched_timing_today_v2_new( created_secs: i64, created_mins_west: i32, now_secs: i64, @@ -90,11 +92,86 @@ pub fn local_minutes_west_for_stamp(stamp: i64) -> i32 { Local.timestamp(stamp, 0).offset().utc_minus_local() / 60 } +// Legacy code +// ---------------------------------- + +fn sched_timing_today_v1(crt: i64, now: i64) -> SchedTimingToday { + let days_elapsed = (now - crt) / 86_400; + let next_day_at = crt + (days_elapsed + 1) * 86_400; + SchedTimingToday { + days_elapsed: days_elapsed as u32, + next_day_at, + } +} + +fn sched_timing_today_v2_legacy( + crt: i64, + rollover: i8, + now: i64, + mins_west: i32, +) -> SchedTimingToday { + let normalized_rollover = normalized_rollover_hour(rollover); + let offset = fixed_offset_from_minutes(mins_west); + + let crt_at_rollover = offset + .timestamp(crt, 0) + .date() + .and_hms(normalized_rollover as u32, 0, 0) + .timestamp(); + let days_elapsed = (now - crt_at_rollover) / 86_400; + + let mut next_day_at = offset + .timestamp(now, 0) + .date() + .and_hms(normalized_rollover as u32, 0, 0) + .timestamp(); + if next_day_at < now { + next_day_at += 86_400; + } + + SchedTimingToday { + days_elapsed: days_elapsed as u32, + next_day_at, + } +} + +// ---------------------------------- + +/// Based on provided input, get timing info from the relevant function. +pub(crate) fn sched_timing_today( + created_secs: i64, + created_mins_west: Option, + now_mins_west: Option, + rollover_hour: Option, +) -> SchedTimingToday { + let now = i64_unix_secs(); + + match (rollover_hour, created_mins_west) { + (None, _) => { + // if rollover unset, v1 scheduler + sched_timing_today_v1(created_secs, now) + } + (Some(roll), None) => { + // if creation offset unset, v2 legacy cutoff using local timezone + let offset = local_minutes_west_for_stamp(now); + sched_timing_today_v2_legacy(created_secs, roll, now, offset) + } + (Some(roll), Some(crt_west)) => { + // new cutoff code, using provided current timezone, falling back on local timezone + let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now)); + sched_timing_today_v2_new(created_secs, crt_west, now, now_west, roll) + } + } +} + #[cfg(test)] mod test { + use super::SchedTimingToday; + use crate::sched::cutoff::sched_timing_today_v1; + use crate::sched::cutoff::sched_timing_today_v2_legacy; use crate::sched::cutoff::{ fixed_offset_from_minutes, local_minutes_west_for_stamp, normalized_rollover_hour, - sched_timing_today, + sched_timing_today_v2_new, }; use chrono::{FixedOffset, Local, TimeZone, Utc}; @@ -117,7 +194,7 @@ mod test { // helper fn elap(start: i64, end: i64, start_west: i32, end_west: i32, rollhour: i8) -> u32 { - let today = sched_timing_today(start, start_west, end, end_west, rollhour); + let today = sched_timing_today_v2_new(start, start_west, end, end_west, rollhour); today.days_elapsed } @@ -228,7 +305,7 @@ mod test { // before the rollover, the next day should be later on the same day let now = Local.ymd(2019, 1, 3).and_hms(2, 0, 0); let next_day_at = Local.ymd(2019, 1, 3).and_hms(rollhour, 0, 0); - let today = sched_timing_today( + let today = sched_timing_today_v2_new( crt.timestamp(), crt.offset().utc_minus_local() / 60, now.timestamp(), @@ -240,7 +317,7 @@ mod test { // after the rollover, the next day should be the next day let now = Local.ymd(2019, 1, 3).and_hms(rollhour, 0, 0); let next_day_at = Local.ymd(2019, 1, 4).and_hms(rollhour, 0, 0); - let today = sched_timing_today( + let today = sched_timing_today_v2_new( crt.timestamp(), crt.offset().utc_minus_local() / 60, now.timestamp(), @@ -252,7 +329,7 @@ mod test { // after the rollover, the next day should be the next day let now = Local.ymd(2019, 1, 3).and_hms(rollhour + 3, 0, 0); let next_day_at = Local.ymd(2019, 1, 4).and_hms(rollhour, 0, 0); - let today = sched_timing_today( + let today = sched_timing_today_v2_new( crt.timestamp(), crt.offset().utc_minus_local() / 60, now.timestamp(), @@ -261,4 +338,34 @@ mod test { ); assert_eq!(today.next_day_at, next_day_at.timestamp()); } + + #[test] + fn legacy_timing() { + let now = 1584491078; + let mins_west = -600; + + assert_eq!( + sched_timing_today_v1(1575226800, now), + SchedTimingToday { + days_elapsed: 107, + next_day_at: 1584558000 + } + ); + + assert_eq!( + sched_timing_today_v2_legacy(1533564000, 0, now, mins_west), + SchedTimingToday { + days_elapsed: 589, + next_day_at: 1584540000 + } + ); + + assert_eq!( + sched_timing_today_v2_legacy(1524038400, 4, now, mins_west), + SchedTimingToday { + days_elapsed: 700, + next_day_at: 1584554400 + } + ); + } } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index a689b34c5..bcf702110 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -6,7 +6,11 @@ use crate::config::Config; use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; use crate::time::{i64_unix_millis, i64_unix_secs}; -use crate::{decks::Deck, types::Usn}; +use crate::{ + decks::Deck, + sched::cutoff::{sched_timing_today, SchedTimingToday}, + types::Usn, +}; use rusqlite::{params, Connection, NO_PARAMS}; use std::path::{Path, PathBuf}; @@ -121,6 +125,8 @@ pub(crate) struct StorageContext<'a> { server: bool, #[allow(dead_code)] usn: Option, + + timing_today: Option, } impl StorageContext<'_> { @@ -129,6 +135,7 @@ impl StorageContext<'_> { db, server, usn: None, + timing_today: None, } } @@ -225,4 +232,24 @@ impl StorageContext<'_> { Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) }) } + + #[allow(dead_code)] + pub(crate) fn timing_today(&mut self) -> Result { + if self.timing_today.is_none() { + let crt: i64 = self + .db + .prepare_cached("select crt from col")? + .query_row(NO_PARAMS, |row| row.get(0))?; + let conf = self.all_config()?; + let now_offset = if self.server { conf.local_offset } else { None }; + + self.timing_today = Some(sched_timing_today( + crt, + conf.creation_offset, + now_offset, + conf.rollover, + )); + } + Ok(*self.timing_today.as_ref().unwrap()) + } } From dc12c23ce9fcc9b13f580cc36c34188e231ced1b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Mar 2020 12:00:23 +1000 Subject: [PATCH 075/150] add timing to search --- rslib/src/search/searcher.rs | 51 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index dc9f375a9..78ba58297 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -68,10 +68,10 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Resul write!(ctx.sql, "n.mid = {}", ntid).unwrap(); } SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref()), - SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease), + SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease)?, SearchNode::Tag(tag) => write_tag(ctx, tag), SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, *note_type_id, text), - SearchNode::State(state) => write_state(ctx, state), + SearchNode::State(state) => write_state(ctx, state)?, SearchNode::Flag(flag) => { write!(ctx.sql, "(c.flags & 7) == {}", flag).unwrap(); } @@ -81,7 +81,7 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Resul SearchNode::CardIDs(cids) => { write!(ctx.sql, "c.id in ({})", cids).unwrap(); } - SearchNode::Property { operator, kind } => write_prop(ctx, operator, kind), + SearchNode::Property { operator, kind } => write_prop(ctx, operator, kind)?, }; Ok(()) } @@ -109,10 +109,9 @@ fn write_tag(ctx: &mut SearchContext, text: &str) { ctx.args.push(tag.into()); } -// fixme: need day cutoff -fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) { - let today_cutoff = 0; // fixme - let days = days.min(31); +fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) -> Result<()> { + let today_cutoff = ctx.ctx.storage.timing_today()?.next_day_at; + let days = days.min(31) as i64; let target_cutoff = today_cutoff - 86_400 * days; write!( ctx.sql, @@ -125,13 +124,15 @@ fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) { } else { write!(ctx.sql, ")").unwrap(); } + + Ok(()) } -// fixme: need current day -fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) { +fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) -> Result<()> { + let timing = ctx.ctx.storage.timing_today()?; match kind { PropertyKind::Due(days) => { - let day = days; // fixme: + sched_today + let day = days + (timing.days_elapsed as i32); write!( ctx.sql, "(c.queue in ({rev},{daylrn}) and due {op} {day})", @@ -147,10 +148,11 @@ fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) { PropertyKind::Ease(ease) => write!(ctx.sql, "ease {} {}", op, (ease * 1000.0) as u32), } .unwrap(); + Ok(()) } -// fixme: need cutoff & current day -fn write_state(ctx: &mut SearchContext, state: &StateKind) { +fn write_state(ctx: &mut SearchContext, state: &StateKind) -> Result<()> { + let timing = ctx.ctx.storage.timing_today()?; match state { StateKind::New => write!(ctx.sql, "c.queue = {}", CardQueue::New as u8), StateKind::Review => write!(ctx.sql, "c.queue = {}", CardQueue::Review as u8), @@ -167,23 +169,20 @@ fn write_state(ctx: &mut SearchContext, state: &StateKind) { CardQueue::UserBuried as u8 ), StateKind::Suspended => write!(ctx.sql, "c.queue = {}", CardQueue::Suspended as u8), - StateKind::Due => { - let today = 0; // fixme: today - let day_cutoff = 0; // fixme: day_cutoff - write!( - ctx.sql, - " + StateKind::Due => write!( + ctx.sql, + " (c.queue in ({rev},{daylrn}) and c.due <= {today}) or (c.queue = {lrn} and c.due <= {daycutoff})", - rev = CardQueue::Review as u8, - daylrn = CardQueue::DayLearn as u8, - today = today, - lrn = CardQueue::Learn as u8, - daycutoff = day_cutoff, - ) - } + rev = CardQueue::Review as u8, + daylrn = CardQueue::DayLearn as u8, + today = timing.days_elapsed, + lrn = CardQueue::Learn as u8, + daycutoff = timing.next_day_at, + ), } - .unwrap() + .unwrap(); + Ok(()) } fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { From 85af35509dba03c0aebd4138caa38241c09ad07c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Mar 2020 12:01:41 +1000 Subject: [PATCH 076/150] ctx->req --- rslib/src/search/searcher.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 78ba58297..5f7f419b8 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -16,19 +16,19 @@ use std::fmt::Write; struct SearchContext<'a> { #[allow(dead_code)] - ctx: &'a mut RequestContext<'a>, + req: &'a mut RequestContext<'a>, sql: String, args: Vec>, } #[allow(dead_code)] fn node_to_sql<'a>( - ctx: &'a mut RequestContext<'a>, + req: &'a mut RequestContext<'a>, node: &'a Node, ) -> Result<(String, Vec>)> { let sql = String::new(); let args = vec![]; - let mut sctx = SearchContext { ctx, sql, args }; + let mut sctx = SearchContext { req, sql, args }; write_node_to_sql(&mut sctx, node)?; Ok((sctx.sql, sctx.args)) } @@ -110,7 +110,7 @@ fn write_tag(ctx: &mut SearchContext, text: &str) { } fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) -> Result<()> { - let today_cutoff = ctx.ctx.storage.timing_today()?.next_day_at; + let today_cutoff = ctx.req.storage.timing_today()?.next_day_at; let days = days.min(31) as i64; let target_cutoff = today_cutoff - 86_400 * days; write!( @@ -129,7 +129,7 @@ fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) -> Result<( } fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) -> Result<()> { - let timing = ctx.ctx.storage.timing_today()?; + let timing = ctx.req.storage.timing_today()?; match kind { PropertyKind::Due(days) => { let day = days + (timing.days_elapsed as i32); @@ -152,7 +152,7 @@ fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) -> Result< } fn write_state(ctx: &mut SearchContext, state: &StateKind) -> Result<()> { - let timing = ctx.ctx.storage.timing_today()?; + let timing = ctx.req.storage.timing_today()?; match state { StateKind::New => write!(ctx.sql, "c.queue = {}", CardQueue::New as u8), StateKind::Review => write!(ctx.sql, "c.queue = {}", CardQueue::Review as u8), @@ -190,9 +190,9 @@ fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { "*" => write!(ctx.sql, "true").unwrap(), "filtered" => write!(ctx.sql, "c.odid > 0").unwrap(), deck => { - let all_decks = ctx.ctx.storage.all_decks()?; + let all_decks = ctx.req.storage.all_decks()?; let dids_with_children = if deck == "current" { - let config = ctx.ctx.storage.all_config()?; + let config = ctx.req.storage.all_config()?; let mut dids_with_children = vec![config.current_deck_id]; let current = get_deck(&all_decks, config.current_deck_id) .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?; From 9752de5aaadd59e1cca77453eacd9fa7735aa60e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Mar 2020 13:51:19 +1000 Subject: [PATCH 077/150] finish the remaining searches Searches that require multiple deck or note type lookups won't perform very well at the moment - it either needs caching or to be split up at the DB level. Nothing tested yet. --- rslib/src/lib.rs | 1 + rslib/src/media/check.rs | 4 +- rslib/src/notes.rs | 40 ++---------- rslib/src/notetypes.rs | 39 +++++++++++ rslib/src/search/searcher.rs | 123 +++++++++++++++++++++++++---------- rslib/src/storage/sqlite.rs | 23 ++++++- 6 files changed, 156 insertions(+), 74 deletions(-) create mode 100644 rslib/src/notetypes.rs diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 37a8db7f6..da4d91107 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -21,6 +21,7 @@ pub mod latex; pub mod log; pub mod media; pub mod notes; +pub mod notetypes; pub mod sched; pub mod search; pub mod storage; diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 2273a4f97..87726d458 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -10,7 +10,7 @@ use crate::media::database::MediaDatabaseContext; use crate::media::files::{ data_for_file, filename_if_normalized, trash_folder, MEDIA_SYNC_FILESIZE_LIMIT, }; -use crate::notes::{for_every_note, get_note_types, set_note, Note}; +use crate::notes::{for_every_note, set_note, Note}; use crate::text::{normalize_to_nfc, MediaRef}; use crate::{media::MediaManager, text::extract_media_refs}; use coarsetime::Instant; @@ -379,7 +379,7 @@ where renamed: &HashMap, ) -> Result> { let mut referenced_files = HashSet::new(); - let note_types = get_note_types(&self.ctx.storage.db)?; + let note_types = self.ctx.storage.all_note_types()?; let mut collection_modified = false; for_every_note(&self.ctx.storage.db, |note| { diff --git a/rslib/src/notes.rs b/rslib/src/notes.rs index 99efb9410..cad1f614c 100644 --- a/rslib/src/notes.rs +++ b/rslib/src/notes.rs @@ -6,11 +6,11 @@ use crate::err::{AnkiError, DBErrorKind, Result}; use crate::text::strip_html_preserving_image_filenames; use crate::time::i64_unix_secs; -use crate::types::{ObjID, Timestamp, Usn}; +use crate::{ + notetypes::NoteType, + types::{ObjID, Timestamp, Usn}, +}; use rusqlite::{params, Connection, Row, NO_PARAMS}; -use serde_aux::field_attributes::deserialize_number_from_string; -use serde_derive::Deserialize; -use std::collections::HashMap; use std::convert::TryInto; #[derive(Debug)] @@ -47,38 +47,6 @@ pub(crate) fn field_checksum(text: &str) -> u32 { u32::from_be_bytes(digest[..4].try_into().unwrap()) } -#[derive(Deserialize, Debug)] -pub(super) struct NoteType { - #[serde(deserialize_with = "deserialize_number_from_string")] - id: ObjID, - #[serde(rename = "sortf")] - sort_field_idx: u16, - - #[serde(rename = "latexsvg", default)] - latex_svg: bool, -} - -impl NoteType { - pub fn latex_uses_svg(&self) -> bool { - self.latex_svg - } -} - -pub(super) fn get_note_types(db: &Connection) -> Result> { - let mut stmt = db.prepare("select models from col")?; - let note_types = stmt - .query_and_then(NO_PARAMS, |row| -> Result> { - let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?; - Ok(v) - })? - .next() - .ok_or_else(|| AnkiError::DBError { - info: "col table empty".to_string(), - kind: DBErrorKind::MissingEntity, - })??; - Ok(note_types) -} - #[allow(dead_code)] fn get_note(db: &Connection, nid: ObjID) -> Result> { let mut stmt = db.prepare_cached("select id, mid, mod, usn, flds from notes where id=?")?; diff --git a/rslib/src/notetypes.rs b/rslib/src/notetypes.rs new file mode 100644 index 000000000..4b9295939 --- /dev/null +++ b/rslib/src/notetypes.rs @@ -0,0 +1,39 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::types::ObjID; +use serde_aux::field_attributes::deserialize_number_from_string; +use serde_derive::Deserialize; + +#[derive(Deserialize, Debug)] +pub(crate) struct NoteType { + #[serde(deserialize_with = "deserialize_number_from_string")] + pub id: ObjID, + pub name: String, + #[serde(rename = "sortf")] + pub sort_field_idx: u16, + #[serde(rename = "latexsvg", default)] + pub latex_svg: bool, + #[serde(rename = "tmpls")] + pub templates: Vec, + #[serde(rename = "flds")] + pub fields: Vec, +} + +#[derive(Deserialize, Debug)] +pub(crate) struct CardTemplate { + pub name: String, + pub ord: u16, +} + +#[derive(Deserialize, Debug)] +pub(crate) struct NoteField { + pub name: String, + pub ord: u16, +} + +impl NoteType { + pub fn latex_uses_svg(&self) -> bool { + self.latex_svg + } +} diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 5f7f419b8..35229a21d 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -57,17 +57,17 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Resul match node { SearchNode::UnqualifiedText(text) => write_unqualified(ctx, text), SearchNode::SingleField { field, text } => { - write_single_field(ctx, field.as_ref(), text.as_ref()) + write_single_field(ctx, field.as_ref(), text.as_ref())? } SearchNode::AddedInDays(days) => { write!(ctx.sql, "c.id > {}", days).unwrap(); } - SearchNode::CardTemplate(template) => write_template(ctx, template), + SearchNode::CardTemplate(template) => write_template(ctx, template)?, SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref())?, SearchNode::NoteTypeID(ntid) => { write!(ctx.sql, "n.mid = {}", ntid).unwrap(); } - SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref()), + SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref())?, SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease)?, SearchNode::Tag(tag) => write_tag(ctx, tag), SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, *note_type_id, text), @@ -211,55 +211,74 @@ fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { dids_with_children }; - if dids_with_children.is_empty() { - write!(ctx.sql, "false") - } else { - let did_strings: Vec = - dids_with_children.iter().map(ToString::to_string).collect(); - write!(ctx.sql, "c.did in ({})", did_strings.join(",")) - } - .unwrap(); + ctx.sql.push_str("c.did in "); + ids_to_string(&mut ctx.sql, &dids_with_children); } }; Ok(()) } -// fixme: need note type manager -fn write_template(ctx: &mut SearchContext, template: &TemplateKind) { +fn write_template(ctx: &mut SearchContext, template: &TemplateKind) -> Result<()> { match template { TemplateKind::Ordinal(n) => { write!(ctx.sql, "c.ord = {}", n).unwrap(); } - TemplateKind::Name(_name) => { - // fixme: search through note types loooking for template name + TemplateKind::Name(name) => { + let note_types = ctx.req.storage.all_note_types()?; + let mut id_ords = vec![]; + for nt in note_types.values() { + for tmpl in &nt.templates { + if matches_wildcard(&tmpl.name, name) { + id_ords.push(format!("(n.mid = {} and c.ord = {})", nt.id, tmpl.ord)); + } + } + } + + if id_ords.is_empty() { + ctx.sql.push_str("false"); + } else { + write!(ctx.sql, "({})", id_ords.join(",")).unwrap(); + } + } + }; + Ok(()) +} + +fn write_note_type(ctx: &mut SearchContext, nt_name: &str) -> Result<()> { + let ntids: Vec<_> = ctx + .req + .storage + .all_note_types()? + .values() + .filter(|nt| matches_wildcard(&nt.name, nt_name)) + .map(|nt| nt.id) + .collect(); + ctx.sql.push_str("n.mid in "); + ids_to_string(&mut ctx.sql, &ntids); + Ok(()) +} + +fn write_single_field(ctx: &mut SearchContext, field_name: &str, val: &str) -> Result<()> { + let note_types = ctx.req.storage.all_note_types()?; + + let mut field_map = vec![]; + for nt in note_types.values() { + for field in &nt.fields { + if field.name.eq_ignore_ascii_case(field_name) { + field_map.push((nt.id, field.ord)); + } } } -} -// fixme: need note type manager -fn write_note_type(ctx: &mut SearchContext, _notetype: &str) { - let ntid: Option = None; // fixme: get id via name search - if let Some(ntid) = ntid { - write!(ctx.sql, "n.mid = {}", ntid).unwrap(); - } else { + if field_map.is_empty() { write!(ctx.sql, "false").unwrap(); - } -} - -// fixme: need note type manager -fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { - let _ = field; - let fields = vec![(0, 0)]; // fixme: get list of (ntid, ordinal) - - if fields.is_empty() { - write!(ctx.sql, "false").unwrap(); - return; + return Ok(()); } write!(ctx.sql, "(").unwrap(); ctx.args.push(val.to_string().into()); let arg_idx = ctx.args.len(); - for (ntid, ord) in fields { + for (ntid, ord) in field_map { write!( ctx.sql, "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", @@ -268,6 +287,8 @@ fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { .unwrap(); } write!(ctx.sql, ")").unwrap(); + + Ok(()) } fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { @@ -282,8 +303,42 @@ fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { ctx.args.push(text.to_string().into()) } +// Write a list of IDs as '(x,y,...)' into the provided string. +fn ids_to_string(buf: &mut String, ids: &[T]) +where + T: std::fmt::Display, +{ + buf.push('('); + if !ids.is_empty() { + for id in ids.iter().skip(1) { + write!(buf, "{},", id).unwrap(); + } + write!(buf, "{}", ids[0]).unwrap(); + } + buf.push(')'); +} + #[cfg(test)] mod test { + use super::ids_to_string; + + #[test] + fn ids_string() { + let mut s = String::new(); + ids_to_string::(&mut s, &[]); + assert_eq!(s, "()"); + s.clear(); + ids_to_string(&mut s, &[7]); + assert_eq!(s, "(7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6]); + assert_eq!(s, "(6,7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6, 5]); + assert_eq!(s, "(6,5,7)"); + s.clear(); + } + // use super::super::parser::parse; // use super::*; diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index bcf702110..a36497fa1 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -8,11 +8,15 @@ use crate::err::{AnkiError, DBErrorKind}; use crate::time::{i64_unix_millis, i64_unix_secs}; use crate::{ decks::Deck, + notetypes::NoteType, sched::cutoff::{sched_timing_today, SchedTimingToday}, - types::Usn, + types::{ObjID, Usn}, }; use rusqlite::{params, Connection, NO_PARAMS}; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; const SCHEMA_MIN_VERSION: u8 = 11; const SCHEMA_MAX_VERSION: u8 = 11; @@ -233,6 +237,21 @@ impl StorageContext<'_> { }) } + pub(crate) fn all_note_types(&self) -> Result> { + let mut stmt = self.db.prepare("select models from col")?; + let note_types = stmt + .query_and_then(NO_PARAMS, |row| -> Result> { + let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?; + Ok(v) + })? + .next() + .ok_or_else(|| AnkiError::DBError { + info: "col table empty".to_string(), + kind: DBErrorKind::MissingEntity, + })??; + Ok(note_types) + } + #[allow(dead_code)] pub(crate) fn timing_today(&mut self) -> Result { if self.timing_today.is_none() { From bca5f2ddffb022b34d92680837e29c8a2007fc67 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 08:20:47 +1000 Subject: [PATCH 078/150] prototype of integration no ordering yet, and no tests --- proto/backend.proto | 11 ++ pylib/anki/rsbackend.py | 5 + rslib/src/backend/mod.rs | 11 ++ rslib/src/search/cards.rs | 28 +++ rslib/src/search/mod.rs | 5 +- rslib/src/search/parser.rs | 10 +- rslib/src/search/searcher.rs | 361 ---------------------------------- rslib/src/search/sqlwriter.rs | 361 ++++++++++++++++++++++++++++++++++ 8 files changed, 426 insertions(+), 366 deletions(-) create mode 100644 rslib/src/search/cards.rs delete mode 100644 rslib/src/search/searcher.rs create mode 100644 rslib/src/search/sqlwriter.rs diff --git a/proto/backend.proto b/proto/backend.proto index 18b6c174c..638d9bc57 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -43,6 +43,7 @@ message BackendInput { Empty restore_trash = 35; OpenCollectionIn open_collection = 36; Empty close_collection = 37; + SearchCardsIn search_cards = 38; } } @@ -73,6 +74,7 @@ message BackendOutput { Empty restore_trash = 35; Empty open_collection = 36; Empty close_collection = 37; + SearchCardsOut search_cards = 38; BackendError error = 2047; } @@ -332,3 +334,12 @@ message OpenCollectionIn { string media_db_path = 3; string log_path = 4; } + +message SearchCardsIn { + string search = 1; +} + +message SearchCardsOut { + repeated int64 card_ids = 1; + +} diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index a77e0f367..eb3f69d63 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -423,6 +423,11 @@ class RustBackend: def _db_command(self, input: Dict[str, Any]) -> Any: return orjson.loads(self._backend.db_command(orjson.dumps(input))) + def search_cards(self, search: str) -> Sequence[int]: + return self._run_command( + pb.BackendInput(search_cards=pb.SearchCardsIn(search=search)) + ).search_cards.card_ids + def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 84d40129e..8640be245 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -14,6 +14,7 @@ use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today_v2_new}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; +use crate::search::search_cards; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, @@ -246,6 +247,7 @@ impl Backend { self.close_collection()?; OValue::CloseCollection(Empty {}) } + Value::SearchCards(input) => OValue::SearchCards(self.search_cards(input)?), }) } @@ -577,6 +579,15 @@ impl Backend { pub fn db_command(&self, input: &[u8]) -> Result { self.with_col(|col| col.with_ctx(|ctx| db_command_bytes(&ctx.storage, input))) } + + fn search_cards(&self, input: pb::SearchCardsIn) -> Result { + self.with_col(|col| { + col.with_ctx(|ctx| { + let cids = search_cards(ctx, &input.search)?; + Ok(pb::SearchCardsOut { card_ids: cids }) + }) + }) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs new file mode 100644 index 000000000..00a3f21e9 --- /dev/null +++ b/rslib/src/search/cards.rs @@ -0,0 +1,28 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::{parser::Node, sqlwriter::node_to_sql}; +use crate::collection::RequestContext; +use crate::err::Result; +use crate::search::parser::parse; +use crate::types::ObjID; + +pub(crate) fn search_cards<'a, 'b>( + req: &'a mut RequestContext<'b>, + search: &'a str, +) -> Result> { + let top_node = Node::Group(parse(search)?); + let (sql, args) = node_to_sql(req, &top_node)?; + + let sql = format!( + "select c.id from cards c, notes n where c.nid=n.id and {} order by c.id", + sql + ); + let mut stmt = req.storage.db.prepare(&sql)?; + let ids: Vec = stmt + .query_map(&args, |row| row.get(0))? + .collect::>()?; + + println!("sql {}\nargs {:?} count {}", sql, args, ids.len()); + Ok(ids) +} diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs index 2732ec547..417241051 100644 --- a/rslib/src/search/mod.rs +++ b/rslib/src/search/mod.rs @@ -1,2 +1,5 @@ +mod cards; mod parser; -mod searcher; +mod sqlwriter; + +pub(crate) use cards::search_cards; diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 62824fe9f..d60112a2d 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::err::{AnkiError, Result}; use crate::types::ObjID; use nom::branch::alt; use nom::bytes::complete::{escaped, is_not, tag, take_while1}; @@ -33,7 +34,7 @@ impl From> for ParseError { } } -type ParseResult = Result; +type ParseResult = std::result::Result; #[derive(Debug, PartialEq)] pub(super) enum Node<'a> { @@ -104,8 +105,9 @@ pub(super) enum TemplateKind { /// Parse the input string into a list of nodes. #[allow(dead_code)] -pub(super) fn parse(input: &str) -> std::result::Result, String> { - let (_, nodes) = all_consuming(group_inner)(input).map_err(|e| format!("{:?}", e))?; +pub(super) fn parse(input: &str) -> Result> { + let (_, nodes) = all_consuming(group_inner)(input) + .map_err(|_e| AnkiError::invalid_input("unable to parse search"))?; Ok(nodes) } @@ -368,7 +370,7 @@ mod test { use super::*; #[test] - fn parsing() -> Result<(), String> { + fn parsing() -> Result<()> { use Node::*; use SearchNode::*; diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs deleted file mode 100644 index 35229a21d..000000000 --- a/rslib/src/search/searcher.rs +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind}; -use crate::card::CardQueue; -use crate::decks::child_ids; -use crate::decks::get_deck; -use crate::err::{AnkiError, Result}; -use crate::notes::field_checksum; -use crate::text::matches_wildcard; -use crate::{ - collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID, -}; -use rusqlite::types::ToSqlOutput; -use std::fmt::Write; - -struct SearchContext<'a> { - #[allow(dead_code)] - req: &'a mut RequestContext<'a>, - sql: String, - args: Vec>, -} - -#[allow(dead_code)] -fn node_to_sql<'a>( - req: &'a mut RequestContext<'a>, - node: &'a Node, -) -> Result<(String, Vec>)> { - let sql = String::new(); - let args = vec![]; - let mut sctx = SearchContext { req, sql, args }; - write_node_to_sql(&mut sctx, node)?; - Ok((sctx.sql, sctx.args)) -} - -fn write_node_to_sql(ctx: &mut SearchContext, node: &Node) -> Result<()> { - match node { - Node::And => write!(ctx.sql, " and ").unwrap(), - Node::Or => write!(ctx.sql, " or ").unwrap(), - Node::Not(node) => { - write!(ctx.sql, "not ").unwrap(); - write_node_to_sql(ctx, node)?; - } - Node::Group(nodes) => { - write!(ctx.sql, "(").unwrap(); - for node in nodes { - write_node_to_sql(ctx, node)?; - } - write!(ctx.sql, ")").unwrap(); - } - Node::Search(search) => write_search_node_to_sql(ctx, search)?, - }; - Ok(()) -} - -fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Result<()> { - match node { - SearchNode::UnqualifiedText(text) => write_unqualified(ctx, text), - SearchNode::SingleField { field, text } => { - write_single_field(ctx, field.as_ref(), text.as_ref())? - } - SearchNode::AddedInDays(days) => { - write!(ctx.sql, "c.id > {}", days).unwrap(); - } - SearchNode::CardTemplate(template) => write_template(ctx, template)?, - SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref())?, - SearchNode::NoteTypeID(ntid) => { - write!(ctx.sql, "n.mid = {}", ntid).unwrap(); - } - SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref())?, - SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease)?, - SearchNode::Tag(tag) => write_tag(ctx, tag), - SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, *note_type_id, text), - SearchNode::State(state) => write_state(ctx, state)?, - SearchNode::Flag(flag) => { - write!(ctx.sql, "(c.flags & 7) == {}", flag).unwrap(); - } - SearchNode::NoteIDs(nids) => { - write!(ctx.sql, "n.id in ({})", nids).unwrap(); - } - SearchNode::CardIDs(cids) => { - write!(ctx.sql, "c.id in ({})", cids).unwrap(); - } - SearchNode::Property { operator, kind } => write_prop(ctx, operator, kind)?, - }; - Ok(()) -} - -fn write_unqualified(ctx: &mut SearchContext, text: &str) { - // implicitly wrap in % - let text = format!("%{}%", text); - write!( - ctx.sql, - "(n.sfld like ? escape '\\' or n.flds like ? escape '\\')" - ) - .unwrap(); - ctx.args.push(text.clone().into()); - ctx.args.push(text.into()); -} - -fn write_tag(ctx: &mut SearchContext, text: &str) { - if text == "none" { - write!(ctx.sql, "n.tags = ''").unwrap(); - return; - } - - let tag = format!(" %{}% ", text.replace('*', "%")); - write!(ctx.sql, "n.tags like ?").unwrap(); - ctx.args.push(tag.into()); -} - -fn write_rated(ctx: &mut SearchContext, days: u32, ease: Option) -> Result<()> { - let today_cutoff = ctx.req.storage.timing_today()?.next_day_at; - let days = days.min(31) as i64; - let target_cutoff = today_cutoff - 86_400 * days; - write!( - ctx.sql, - "c.id in (select cid from revlog where id>{}", - target_cutoff - ) - .unwrap(); - if let Some(ease) = ease { - write!(ctx.sql, "and ease={})", ease).unwrap(); - } else { - write!(ctx.sql, ")").unwrap(); - } - - Ok(()) -} - -fn write_prop(ctx: &mut SearchContext, op: &str, kind: &PropertyKind) -> Result<()> { - let timing = ctx.req.storage.timing_today()?; - match kind { - PropertyKind::Due(days) => { - let day = days + (timing.days_elapsed as i32); - write!( - ctx.sql, - "(c.queue in ({rev},{daylrn}) and due {op} {day})", - rev = CardQueue::Review as u8, - daylrn = CardQueue::DayLearn as u8, - op = op, - day = day - ) - } - PropertyKind::Interval(ivl) => write!(ctx.sql, "ivl {} {}", op, ivl), - PropertyKind::Reps(reps) => write!(ctx.sql, "reps {} {}", op, reps), - PropertyKind::Lapses(days) => write!(ctx.sql, "lapses {} {}", op, days), - PropertyKind::Ease(ease) => write!(ctx.sql, "ease {} {}", op, (ease * 1000.0) as u32), - } - .unwrap(); - Ok(()) -} - -fn write_state(ctx: &mut SearchContext, state: &StateKind) -> Result<()> { - let timing = ctx.req.storage.timing_today()?; - match state { - StateKind::New => write!(ctx.sql, "c.queue = {}", CardQueue::New as u8), - StateKind::Review => write!(ctx.sql, "c.queue = {}", CardQueue::Review as u8), - StateKind::Learning => write!( - ctx.sql, - "c.queue in ({},{})", - CardQueue::Learn as u8, - CardQueue::DayLearn as u8 - ), - StateKind::Buried => write!( - ctx.sql, - "c.queue in ({},{})", - CardQueue::SchedBuried as u8, - CardQueue::UserBuried as u8 - ), - StateKind::Suspended => write!(ctx.sql, "c.queue = {}", CardQueue::Suspended as u8), - StateKind::Due => write!( - ctx.sql, - " -(c.queue in ({rev},{daylrn}) and c.due <= {today}) or -(c.queue = {lrn} and c.due <= {daycutoff})", - rev = CardQueue::Review as u8, - daylrn = CardQueue::DayLearn as u8, - today = timing.days_elapsed, - lrn = CardQueue::Learn as u8, - daycutoff = timing.next_day_at, - ), - } - .unwrap(); - Ok(()) -} - -fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { - match deck { - "*" => write!(ctx.sql, "true").unwrap(), - "filtered" => write!(ctx.sql, "c.odid > 0").unwrap(), - deck => { - let all_decks = ctx.req.storage.all_decks()?; - let dids_with_children = if deck == "current" { - let config = ctx.req.storage.all_config()?; - let mut dids_with_children = vec![config.current_deck_id]; - let current = get_deck(&all_decks, config.current_deck_id) - .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?; - for child_did in child_ids(&all_decks, ¤t.name) { - dids_with_children.push(child_did); - } - dids_with_children - } else { - let mut dids_with_children = vec![]; - for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) { - dids_with_children.push(deck.id); - for child_id in child_ids(&all_decks, &deck.name) { - dids_with_children.push(child_id); - } - } - dids_with_children - }; - - ctx.sql.push_str("c.did in "); - ids_to_string(&mut ctx.sql, &dids_with_children); - } - }; - Ok(()) -} - -fn write_template(ctx: &mut SearchContext, template: &TemplateKind) -> Result<()> { - match template { - TemplateKind::Ordinal(n) => { - write!(ctx.sql, "c.ord = {}", n).unwrap(); - } - TemplateKind::Name(name) => { - let note_types = ctx.req.storage.all_note_types()?; - let mut id_ords = vec![]; - for nt in note_types.values() { - for tmpl in &nt.templates { - if matches_wildcard(&tmpl.name, name) { - id_ords.push(format!("(n.mid = {} and c.ord = {})", nt.id, tmpl.ord)); - } - } - } - - if id_ords.is_empty() { - ctx.sql.push_str("false"); - } else { - write!(ctx.sql, "({})", id_ords.join(",")).unwrap(); - } - } - }; - Ok(()) -} - -fn write_note_type(ctx: &mut SearchContext, nt_name: &str) -> Result<()> { - let ntids: Vec<_> = ctx - .req - .storage - .all_note_types()? - .values() - .filter(|nt| matches_wildcard(&nt.name, nt_name)) - .map(|nt| nt.id) - .collect(); - ctx.sql.push_str("n.mid in "); - ids_to_string(&mut ctx.sql, &ntids); - Ok(()) -} - -fn write_single_field(ctx: &mut SearchContext, field_name: &str, val: &str) -> Result<()> { - let note_types = ctx.req.storage.all_note_types()?; - - let mut field_map = vec![]; - for nt in note_types.values() { - for field in &nt.fields { - if field.name.eq_ignore_ascii_case(field_name) { - field_map.push((nt.id, field.ord)); - } - } - } - - if field_map.is_empty() { - write!(ctx.sql, "false").unwrap(); - return Ok(()); - } - - write!(ctx.sql, "(").unwrap(); - ctx.args.push(val.to_string().into()); - let arg_idx = ctx.args.len(); - for (ntid, ord) in field_map { - write!( - ctx.sql, - "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", - ntid, ord, arg_idx - ) - .unwrap(); - } - write!(ctx.sql, ")").unwrap(); - - Ok(()) -} - -fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { - let text_nohtml = strip_html_preserving_image_filenames(text); - let csum = field_checksum(text_nohtml.as_ref()); - write!( - ctx.sql, - "(n.mid = {} and n.csum = {} and field_at_index(n.flds, 0) = ?", - ntid, csum - ) - .unwrap(); - ctx.args.push(text.to_string().into()) -} - -// Write a list of IDs as '(x,y,...)' into the provided string. -fn ids_to_string(buf: &mut String, ids: &[T]) -where - T: std::fmt::Display, -{ - buf.push('('); - if !ids.is_empty() { - for id in ids.iter().skip(1) { - write!(buf, "{},", id).unwrap(); - } - write!(buf, "{}", ids[0]).unwrap(); - } - buf.push(')'); -} - -#[cfg(test)] -mod test { - use super::ids_to_string; - - #[test] - fn ids_string() { - let mut s = String::new(); - ids_to_string::(&mut s, &[]); - assert_eq!(s, "()"); - s.clear(); - ids_to_string(&mut s, &[7]); - assert_eq!(s, "(7)"); - s.clear(); - ids_to_string(&mut s, &[7, 6]); - assert_eq!(s, "(6,7)"); - s.clear(); - ids_to_string(&mut s, &[7, 6, 5]); - assert_eq!(s, "(6,5,7)"); - s.clear(); - } - - // use super::super::parser::parse; - // use super::*; - - // parse - // fn p(search: &str) -> Node { - // Node::Group(parse(search).unwrap()) - // } - - // get sql - // fn s<'a>(n: &'a Node) -> (String, Vec>) { - // node_to_sql(n) - // } - - #[test] - fn tosql() -> Result<(), String> { - // assert_eq!(s(&p("added:1")), ("(c.id > 1)".into(), vec![])); - - Ok(()) - } -} diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs new file mode 100644 index 000000000..c7a0ea671 --- /dev/null +++ b/rslib/src/search/sqlwriter.rs @@ -0,0 +1,361 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind}; +use crate::card::CardQueue; +use crate::decks::child_ids; +use crate::decks::get_deck; +use crate::err::{AnkiError, Result}; +use crate::notes::field_checksum; +use crate::text::matches_wildcard; +use crate::{ + collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID, +}; +use std::fmt::Write; + +struct SqlWriter<'a, 'b> { + req: &'a mut RequestContext<'b>, + sql: String, + args: Vec, +} + +pub(super) fn node_to_sql(req: &mut RequestContext, node: &Node) -> Result<(String, Vec)> { + let mut sctx = SqlWriter::new(req); + sctx.write_node_to_sql(&node)?; + Ok((sctx.sql, sctx.args)) +} + +impl SqlWriter<'_, '_> { + fn new<'a, 'b>(req: &'a mut RequestContext<'b>) -> SqlWriter<'a, 'b> { + let sql = String::new(); + let args = vec![]; + SqlWriter { req, sql, args } + } + + fn write_node_to_sql(&mut self, node: &Node) -> Result<()> { + match node { + Node::And => write!(self.sql, " and ").unwrap(), + Node::Or => write!(self.sql, " or ").unwrap(), + Node::Not(node) => { + write!(self.sql, "not ").unwrap(); + self.write_node_to_sql(node)?; + } + Node::Group(nodes) => { + write!(self.sql, "(").unwrap(); + for node in nodes { + self.write_node_to_sql(node)?; + } + write!(self.sql, ")").unwrap(); + } + Node::Search(search) => self.write_search_node_to_sql(search)?, + }; + Ok(()) + } + + fn write_search_node_to_sql(&mut self, node: &SearchNode) -> Result<()> { + match node { + SearchNode::UnqualifiedText(text) => self.write_unqualified(text), + SearchNode::SingleField { field, text } => { + self.write_single_field(field.as_ref(), text.as_ref())? + } + SearchNode::AddedInDays(days) => { + write!(self.sql, "c.id > {}", days).unwrap(); + } + SearchNode::CardTemplate(template) => self.write_template(template)?, + SearchNode::Deck(deck) => self.write_deck(deck.as_ref())?, + SearchNode::NoteTypeID(ntid) => { + write!(self.sql, "n.mid = {}", ntid).unwrap(); + } + SearchNode::NoteType(notetype) => self.write_note_type(notetype.as_ref())?, + SearchNode::Rated { days, ease } => self.write_rated(*days, *ease)?, + SearchNode::Tag(tag) => self.write_tag(tag), + SearchNode::Duplicates { note_type_id, text } => self.write_dupes(*note_type_id, text), + SearchNode::State(state) => self.write_state(state)?, + SearchNode::Flag(flag) => { + write!(self.sql, "(c.flags & 7) == {}", flag).unwrap(); + } + SearchNode::NoteIDs(nids) => { + write!(self.sql, "n.id in ({})", nids).unwrap(); + } + SearchNode::CardIDs(cids) => { + write!(self.sql, "c.id in ({})", cids).unwrap(); + } + SearchNode::Property { operator, kind } => self.write_prop(operator, kind)?, + }; + Ok(()) + } + + fn write_unqualified(&mut self, text: &str) { + // implicitly wrap in % + let text = format!("%{}%", text); + write!( + self.sql, + "(n.sfld like ? escape '\\' or n.flds like ? escape '\\')" + ) + .unwrap(); + self.args.push(text.clone().into()); + self.args.push(text.into()); + } + + fn write_tag(&mut self, text: &str) { + if text == "none" { + write!(self.sql, "n.tags = ''").unwrap(); + return; + } + + let tag = format!(" %{}% ", text.replace('*', "%")); + write!(self.sql, "n.tags like ?").unwrap(); + self.args.push(tag.into()); + } + + fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { + let today_cutoff = self.req.storage.timing_today()?.next_day_at; + let days = days.min(31) as i64; + let target_cutoff = today_cutoff - 86_400 * days; + write!( + self.sql, + "c.id in (select cid from revlog where id>{}", + target_cutoff + ) + .unwrap(); + if let Some(ease) = ease { + write!(self.sql, "and ease={})", ease).unwrap(); + } else { + write!(self.sql, ")").unwrap(); + } + + Ok(()) + } + + fn write_prop(&mut self, op: &str, kind: &PropertyKind) -> Result<()> { + let timing = self.req.storage.timing_today()?; + match kind { + PropertyKind::Due(days) => { + let day = days + (timing.days_elapsed as i32); + write!( + self.sql, + "(c.queue in ({rev},{daylrn}) and due {op} {day})", + rev = CardQueue::Review as u8, + daylrn = CardQueue::DayLearn as u8, + op = op, + day = day + ) + } + PropertyKind::Interval(ivl) => write!(self.sql, "ivl {} {}", op, ivl), + PropertyKind::Reps(reps) => write!(self.sql, "reps {} {}", op, reps), + PropertyKind::Lapses(days) => write!(self.sql, "lapses {} {}", op, days), + PropertyKind::Ease(ease) => write!(self.sql, "ease {} {}", op, (ease * 1000.0) as u32), + } + .unwrap(); + Ok(()) + } + + fn write_state(&mut self, state: &StateKind) -> Result<()> { + let timing = self.req.storage.timing_today()?; + match state { + StateKind::New => write!(self.sql, "c.queue = {}", CardQueue::New as u8), + StateKind::Review => write!(self.sql, "c.queue = {}", CardQueue::Review as u8), + StateKind::Learning => write!( + self.sql, + "c.queue in ({},{})", + CardQueue::Learn as u8, + CardQueue::DayLearn as u8 + ), + StateKind::Buried => write!( + self.sql, + "c.queue in ({},{})", + CardQueue::SchedBuried as u8, + CardQueue::UserBuried as u8 + ), + StateKind::Suspended => write!(self.sql, "c.queue = {}", CardQueue::Suspended as u8), + StateKind::Due => write!( + self.sql, + " + (c.queue in ({rev},{daylrn}) and c.due <= {today}) or + (c.queue = {lrn} and c.due <= {daycutoff})", + rev = CardQueue::Review as u8, + daylrn = CardQueue::DayLearn as u8, + today = timing.days_elapsed, + lrn = CardQueue::Learn as u8, + daycutoff = timing.next_day_at, + ), + } + .unwrap(); + Ok(()) + } + + fn write_deck(&mut self, deck: &str) -> Result<()> { + match deck { + "*" => write!(self.sql, "true").unwrap(), + "filtered" => write!(self.sql, "c.odid > 0").unwrap(), + deck => { + let all_decks = self.req.storage.all_decks()?; + let dids_with_children = if deck == "current" { + let config = self.req.storage.all_config()?; + let mut dids_with_children = vec![config.current_deck_id]; + let current = get_deck(&all_decks, config.current_deck_id) + .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?; + for child_did in child_ids(&all_decks, ¤t.name) { + dids_with_children.push(child_did); + } + dids_with_children + } else { + let mut dids_with_children = vec![]; + for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) { + dids_with_children.push(deck.id); + for child_id in child_ids(&all_decks, &deck.name) { + dids_with_children.push(child_id); + } + } + dids_with_children + }; + + self.sql.push_str("c.did in "); + ids_to_string(&mut self.sql, &dids_with_children); + } + }; + Ok(()) + } + + fn write_template(&mut self, template: &TemplateKind) -> Result<()> { + match template { + TemplateKind::Ordinal(n) => { + write!(self.sql, "c.ord = {}", n).unwrap(); + } + TemplateKind::Name(name) => { + let note_types = self.req.storage.all_note_types()?; + let mut id_ords = vec![]; + for nt in note_types.values() { + for tmpl in &nt.templates { + if matches_wildcard(&tmpl.name, name) { + id_ords.push(format!("(n.mid = {} and c.ord = {})", nt.id, tmpl.ord)); + } + } + } + + if id_ords.is_empty() { + self.sql.push_str("false"); + } else { + write!(self.sql, "({})", id_ords.join(",")).unwrap(); + } + } + }; + Ok(()) + } + + fn write_note_type(&mut self, nt_name: &str) -> Result<()> { + let ntids: Vec<_> = self + .req + .storage + .all_note_types()? + .values() + .filter(|nt| matches_wildcard(&nt.name, nt_name)) + .map(|nt| nt.id) + .collect(); + self.sql.push_str("n.mid in "); + ids_to_string(&mut self.sql, &ntids); + Ok(()) + } + + fn write_single_field(&mut self, field_name: &str, val: &str) -> Result<()> { + let note_types = self.req.storage.all_note_types()?; + + let mut field_map = vec![]; + for nt in note_types.values() { + for field in &nt.fields { + if field.name.eq_ignore_ascii_case(field_name) { + field_map.push((nt.id, field.ord)); + } + } + } + + if field_map.is_empty() { + write!(self.sql, "false").unwrap(); + return Ok(()); + } + + write!(self.sql, "(").unwrap(); + self.args.push(val.to_string().into()); + let arg_idx = self.args.len(); + for (ntid, ord) in field_map { + write!( + self.sql, + "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", + ntid, ord, arg_idx + ) + .unwrap(); + } + write!(self.sql, ")").unwrap(); + + Ok(()) + } + + fn write_dupes(&mut self, ntid: ObjID, text: &str) { + let text_nohtml = strip_html_preserving_image_filenames(text); + let csum = field_checksum(text_nohtml.as_ref()); + write!( + self.sql, + "(n.mid = {} and n.csum = {} and field_at_index(n.flds, 0) = ?", + ntid, csum + ) + .unwrap(); + self.args.push(text.to_string().into()) + } +} + +// Write a list of IDs as '(x,y,...)' into the provided string. +fn ids_to_string(buf: &mut String, ids: &[T]) +where + T: std::fmt::Display, +{ + buf.push('('); + if !ids.is_empty() { + for id in ids.iter().skip(1) { + write!(buf, "{},", id).unwrap(); + } + write!(buf, "{}", ids[0]).unwrap(); + } + buf.push(')'); +} + +#[cfg(test)] +mod test { + use super::ids_to_string; + + #[test] + fn ids_string() { + let mut s = String::new(); + ids_to_string::(&mut s, &[]); + assert_eq!(s, "()"); + s.clear(); + ids_to_string(&mut s, &[7]); + assert_eq!(s, "(7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6]); + assert_eq!(s, "(6,7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6, 5]); + assert_eq!(s, "(6,5,7)"); + s.clear(); + } + + // use super::super::parser::parse; + // use super::*; + + // parse + // fn p(search: &str) -> Node { + // Node::Group(parse(search).unwrap()) + // } + + // get sql + // fn s<'a>(n: &'a Node) -> (String, Vec>) { + // node_to_sql(n) + // } + + #[test] + fn tosql() -> Result<(), String> { + // assert_eq!(s(&p("added:1")), ("(c.id > 1)".into(), vec![])); + + Ok(()) + } +} From 1f9e8e388a4cd1931cc8e1a43938a3a7d2e735c1 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 09:26:31 +1000 Subject: [PATCH 079/150] start on search tests --- rslib/src/media/check.rs | 10 +++--- rslib/src/search/sqlwriter.rs | 58 +++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 87726d458..843d14493 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -498,7 +498,10 @@ fn extract_latex_refs(note: &Note, seen_files: &mut HashSet, svg: bool) } #[cfg(test)] -mod test { +pub(crate) mod test { + pub(crate) const MEDIACHECK_ANKI2: &'static [u8] = + include_bytes!("../../tests/support/mediacheck.anki2"); + use crate::collection::{open_collection, Collection}; use crate::err::Result; use crate::i18n::I18n; @@ -516,10 +519,7 @@ mod test { fs::create_dir(&media_dir)?; let media_db = dir.path().join("media.db"); let col_path = dir.path().join("col.anki2"); - fs::write( - &col_path, - &include_bytes!("../../tests/support/mediacheck.anki2")[..], - )?; + fs::write(&col_path, MEDIACHECK_ANKI2)?; let mgr = MediaManager::new(&media_dir, media_db.clone())?; diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index c7a0ea671..ecd4bfd99 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -321,6 +321,9 @@ where #[cfg(test)] mod test { use super::ids_to_string; + use crate::{collection::open_collection, i18n::I18n, log}; + use std::{fs, path::PathBuf}; + use tempfile::tempdir; #[test] fn ids_string() { @@ -339,22 +342,51 @@ mod test { s.clear(); } - // use super::super::parser::parse; - // use super::*; + use super::super::parser::parse; + use super::*; - // parse - // fn p(search: &str) -> Node { - // Node::Group(parse(search).unwrap()) - // } - - // get sql - // fn s<'a>(n: &'a Node) -> (String, Vec>) { - // node_to_sql(n) - // } + // shortcut + fn s(req: &mut RequestContext, search: &str) -> (String, Vec) { + let node = Node::Group(parse(search).unwrap()); + node_to_sql(req, &node).unwrap() + } #[test] - fn tosql() -> Result<(), String> { - // assert_eq!(s(&p("added:1")), ("(c.id > 1)".into(), vec![])); + fn sql() -> Result<()> { + // re-use the mediacheck .anki2 file for now + use crate::media::check::test::MEDIACHECK_ANKI2; + let dir = tempdir().unwrap(); + let col_path = dir.path().join("col.anki2"); + fs::write(&col_path, MEDIACHECK_ANKI2).unwrap(); + + let i18n = I18n::new(&[""], "", log::terminal()); + let col = open_collection( + &col_path, + &PathBuf::new(), + &PathBuf::new(), + false, + i18n, + log::terminal(), + ) + .unwrap(); + + col.with_ctx(|ctx| { + // unqualified search + assert_eq!( + s(ctx, "test"), + ( + "((n.sfld like ?1 escape '\\' or n.flds like ?1 escape '\\'))".into(), + vec!["%test%".into()] + ) + ); + assert_eq!(s(ctx, "te%st").1, vec!["%te%st%".to_string()]); + // user should be able to escape sql wildcards + assert_eq!(s(ctx, r#"te\%s\_t"#).1, vec!["%te\\%s\\_t%".to_string()]); + + + Ok(()) + }) + .unwrap(); Ok(()) } From b70668d31ccc4026dc04d69fda2d2f7e62e73d70 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 09:26:49 +1000 Subject: [PATCH 080/150] avoid extra sql binding in unqualified search --- rslib/src/search/sqlwriter.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index ecd4bfd99..066a4b2be 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -88,13 +88,13 @@ impl SqlWriter<'_, '_> { fn write_unqualified(&mut self, text: &str) { // implicitly wrap in % let text = format!("%{}%", text); + self.args.push(text.into()); write!( self.sql, - "(n.sfld like ? escape '\\' or n.flds like ? escape '\\')" + "(n.sfld like ?{n} escape '\\' or n.flds like ?{n} escape '\\')", + n = self.args.len(), ) .unwrap(); - self.args.push(text.clone().into()); - self.args.push(text.into()); } fn write_tag(&mut self, text: &str) { From c723adea17b4a6caf5318d1d62e94aa9cacc3e36 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 09:27:19 +1000 Subject: [PATCH 081/150] fix escape handling, and handle sql wildcards --- rslib/src/search/parser.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index d60112a2d..03ad4692b 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -174,8 +174,8 @@ fn text(s: &str) -> IResult<&str, Node> { fn search_node_for_text(s: &str) -> ParseResult { let mut it = s.splitn(2, ':'); let (head, tail) = ( - without_escapes(it.next().unwrap()), - it.next().map(without_escapes), + unescape_quotes(it.next().unwrap()), + it.next().map(unescape_quotes), ); if let Some(tail) = tail { @@ -185,10 +185,10 @@ fn search_node_for_text(s: &str) -> ParseResult { } } -/// Strip the \ escaping character -fn without_escapes(s: &str) -> Cow { - if s.find('\\').is_some() { - s.replace('\\', "").into() +/// \" -> " +fn unescape_quotes(s: &str) -> Cow { + if s.find(r#"\""#).is_some() { + s.replace(r#"\""#, "\"").into() } else { s.into() } @@ -216,10 +216,10 @@ fn quoted_term(s: &str) -> IResult<&str, Node> { } /// Quoted text, terminated by a non-escaped double quote -/// Can escape " and \ +/// Can escape %, _, " and \ fn quoted_term_inner(s: &str) -> IResult<&str, Node> { map_res( - escaped(is_not(r#""\"#), '\\', one_of(r#""\"#)), + escaped(is_not(r#""\"#), '\\', one_of(r#""\%_"#)), |o| -> ParseResult { Ok(Node::Search(search_node_for_text(o)?)) }, )(s) } From 5df04b161cc3cc25d29abbbbe2965235ac219a8e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 09:48:19 +1000 Subject: [PATCH 082/150] fix qualified search --- rslib/src/search/sqlwriter.rs | 39 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 066a4b2be..d974a9c67 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -269,23 +269,28 @@ impl SqlWriter<'_, '_> { } } + // for now, sort the map for the benefit of unit tests + field_map.sort(); + if field_map.is_empty() { write!(self.sql, "false").unwrap(); return Ok(()); } - write!(self.sql, "(").unwrap(); self.args.push(val.to_string().into()); let arg_idx = self.args.len(); - for (ntid, ord) in field_map { - write!( - self.sql, - "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", - ntid, ord, arg_idx - ) - .unwrap(); - } - write!(self.sql, ")").unwrap(); + let searches: Vec<_> = field_map + .iter() + .map(|(ntid, ord)| { + format!( + "(n.mid = {mid} and field_at_index(n.flds, {ord}) like ?{n})", + mid = ntid, + ord = ord, + n = arg_idx + ) + }) + .collect(); + write!(self.sql, "({})", searches.join(" or ")).unwrap(); Ok(()) } @@ -383,6 +388,20 @@ mod test { // user should be able to escape sql wildcards assert_eq!(s(ctx, r#"te\%s\_t"#).1, vec!["%te\\%s\\_t%".to_string()]); + // qualified search + assert_eq!( + s(ctx, "front:test"), + ( + concat!( + "(((n.mid = 1581236385344 and field_at_index(n.flds, 0) like ?1) or ", + "(n.mid = 1581236385345 and field_at_index(n.flds, 0) like ?1) or ", + "(n.mid = 1581236385346 and field_at_index(n.flds, 0) like ?1) or ", + "(n.mid = 1581236385347 and field_at_index(n.flds, 0) like ?1)))" + ) + .into(), + vec!["test".into()] + ) + ); Ok(()) }) From 2693e142aa3e87319c78b54320b4e8fce05e1d68 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 09:57:12 +1000 Subject: [PATCH 083/150] fix added --- rslib/src/search/sqlwriter.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index d974a9c67..1dbe8cfe6 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -58,9 +58,7 @@ impl SqlWriter<'_, '_> { SearchNode::SingleField { field, text } => { self.write_single_field(field.as_ref(), text.as_ref())? } - SearchNode::AddedInDays(days) => { - write!(self.sql, "c.id > {}", days).unwrap(); - } + SearchNode::AddedInDays(days) => self.write_added(*days)?, SearchNode::CardTemplate(template) => self.write_template(template)?, SearchNode::Deck(deck) => self.write_deck(deck.as_ref())?, SearchNode::NoteTypeID(ntid) => { @@ -306,6 +304,13 @@ impl SqlWriter<'_, '_> { .unwrap(); self.args.push(text.to_string().into()) } + + fn write_added(&mut self, days: u32) -> Result<()> { + let timing = self.req.storage.timing_today()?; + let cutoff = timing.next_day_at - (86_400 * (days as i64)); + write!(self.sql, "c.id > {}", cutoff).unwrap(); + Ok(()) + } } // Write a list of IDs as '(x,y,...)' into the provided string. @@ -403,6 +408,13 @@ mod test { ) ); + // added + let t = ctx.storage.timing_today().unwrap(); + assert_eq!( + s(ctx, "added:3").0, + format!("(c.id > {})", t.next_day_at - (86_400 * 3)) + ); + Ok(()) }) .unwrap(); From 425a9d04caa0acda039ef82db1dbd9d833efe065 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 10:03:58 +1000 Subject: [PATCH 084/150] fix decks --- rslib/src/search/sqlwriter.rs | 13 ++++++++++++- rslib/src/storage/sqlite.rs | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 1dbe8cfe6..f1b015470 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -187,7 +187,13 @@ impl SqlWriter<'_, '_> { "*" => write!(self.sql, "true").unwrap(), "filtered" => write!(self.sql, "c.odid > 0").unwrap(), deck => { - let all_decks = self.req.storage.all_decks()?; + let all_decks: Vec<_> = self + .req + .storage + .all_decks()? + .into_iter() + .map(|(_, v)| v) + .collect(); let dids_with_children = if deck == "current" { let config = self.req.storage.all_config()?; let mut dids_with_children = vec![config.current_deck_id]; @@ -415,6 +421,11 @@ mod test { format!("(c.id > {})", t.next_day_at - (86_400 * 3)) ); + // deck + assert_eq!(s(ctx, "deck:default"), ("(c.did in (1))".into(), vec![],)); + assert_eq!(s(ctx, "deck:current"), ("(c.did in (1))".into(), vec![],)); + assert_eq!(s(ctx, "deck:filtered"), ("(c.odid > 0)".into(), vec![],)); + Ok(()) }) .unwrap(); diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index a36497fa1..8b61a6730 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -223,7 +223,7 @@ impl StorageContext<'_> { } } - pub(crate) fn all_decks(&self) -> Result> { + pub(crate) fn all_decks(&self) -> Result> { self.db .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> { Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) From fa654a0e2235c494bdddc9dfc7b08d85e0a80925 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 10:16:08 +1000 Subject: [PATCH 085/150] fix cards --- rslib/src/search/sqlwriter.rs | 42 +++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index f1b015470..eefd1cda6 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -232,15 +232,22 @@ impl SqlWriter<'_, '_> { for nt in note_types.values() { for tmpl in &nt.templates { if matches_wildcard(&tmpl.name, name) { - id_ords.push(format!("(n.mid = {} and c.ord = {})", nt.id, tmpl.ord)); + id_ords.push((nt.id, tmpl.ord)); } } } + // sort for the benefit of unit tests + id_ords.sort(); + if id_ords.is_empty() { self.sql.push_str("false"); } else { - write!(self.sql, "({})", id_ords.join(",")).unwrap(); + let v: Vec<_> = id_ords + .iter() + .map(|(ntid, ord)| format!("(n.mid = {} and c.ord = {})", ntid, ord)) + .collect(); + write!(self.sql, "({})", v.join(" or ")).unwrap(); } } }; @@ -424,8 +431,39 @@ mod test { // deck assert_eq!(s(ctx, "deck:default"), ("(c.did in (1))".into(), vec![],)); assert_eq!(s(ctx, "deck:current"), ("(c.did in (1))".into(), vec![],)); + assert_eq!(s(ctx, "deck:missing"), ("(c.did in ())".into(), vec![],)); + assert_eq!(s(ctx, "deck:d*"), ("(c.did in (1))".into(), vec![],)); assert_eq!(s(ctx, "deck:filtered"), ("(c.odid > 0)".into(), vec![],)); + // card + assert_eq!(s(ctx, "card:front"), ("(false)".into(), vec![],)); + assert_eq!( + s(ctx, r#""card:card 1""#), + ( + concat!( + "(((n.mid = 1581236385344 and c.ord = 0) or ", + "(n.mid = 1581236385345 and c.ord = 0) or ", + "(n.mid = 1581236385346 and c.ord = 0) or ", + "(n.mid = 1581236385347 and c.ord = 0)))" + ) + .into(), + vec![], + ) + ); + + // todo: + // card + // mid + // nid + // note + // rated + // tag + // is + // dupe + // flag + // cid + // prop + Ok(()) }) .unwrap(); From 37ad664afc18cc858a6125a6f5255ded244b4bd7 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 10:30:26 +1000 Subject: [PATCH 086/150] fix tags, more tests --- rslib/src/search/sqlwriter.rs | 40 ++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index eefd1cda6..52a39df4a 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -101,7 +101,7 @@ impl SqlWriter<'_, '_> { return; } - let tag = format!(" %{}% ", text.replace('*', "%")); + let tag = format!("% {} %", text.replace('*', "%")); write!(self.sql, "n.tags like ?").unwrap(); self.args.push(tag.into()); } @@ -451,17 +451,41 @@ mod test { ) ); + // IDs + assert_eq!(s(ctx, "mid:3"), ("(n.mid = 3)".into(), vec![])); + assert_eq!(s(ctx, "nid:3"), ("(n.id in (3))".into(), vec![])); + assert_eq!(s(ctx, "nid:3,4"), ("(n.id in (3,4))".into(), vec![])); + assert_eq!(s(ctx, "cid:3,4"), ("(c.id in (3,4))".into(), vec![])); + + // flags + assert_eq!(s(ctx, "flag:2"), ("((c.flags & 7) == 2)".into(), vec![])); + assert_eq!(s(ctx, "flag:0"), ("((c.flags & 7) == 0)".into(), vec![])); + + // dupes + assert_eq!( + s(ctx, "dupes:123,test"), + ( + "((n.mid = 123 and n.csum = 2840236005 and field_at_index(n.flds, 0) = ?)" + .into(), + vec!["test".into()] + ) + ); + + // tags + assert_eq!( + s(ctx, "tag:one"), + ("(n.tags like ?)".into(), vec!["% one %".into()]) + ); + assert_eq!( + s(ctx, "tag:o*e"), + ("(n.tags like ?)".into(), vec!["% o%e %".into()]) + ); + assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![])); + // todo: - // card - // mid - // nid // note // rated - // tag // is - // dupe - // flag - // cid // prop Ok(()) From 8c158a389721399058c21c1ff2ab179e62f97e34 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 10:47:32 +1000 Subject: [PATCH 087/150] fix rated and state searches --- rslib/src/search/sqlwriter.rs | 54 ++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 52a39df4a..1f2b16447 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -109,15 +109,15 @@ impl SqlWriter<'_, '_> { fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { let today_cutoff = self.req.storage.timing_today()?.next_day_at; let days = days.min(31) as i64; - let target_cutoff = today_cutoff - 86_400 * days; + let target_cutoff_ms = (today_cutoff - 86_400 * days) * 1_000; write!( self.sql, "c.id in (select cid from revlog where id>{}", - target_cutoff + target_cutoff_ms ) .unwrap(); if let Some(ease) = ease { - write!(self.sql, "and ease={})", ease).unwrap(); + write!(self.sql, " and ease={})", ease).unwrap(); } else { write!(self.sql, ")").unwrap(); } @@ -151,30 +151,30 @@ impl SqlWriter<'_, '_> { fn write_state(&mut self, state: &StateKind) -> Result<()> { let timing = self.req.storage.timing_today()?; match state { - StateKind::New => write!(self.sql, "c.queue = {}", CardQueue::New as u8), - StateKind::Review => write!(self.sql, "c.queue = {}", CardQueue::Review as u8), + StateKind::New => write!(self.sql, "c.queue = {}", CardQueue::New as i8), + StateKind::Review => write!(self.sql, "c.queue = {}", CardQueue::Review as i8), StateKind::Learning => write!( self.sql, "c.queue in ({},{})", - CardQueue::Learn as u8, - CardQueue::DayLearn as u8 + CardQueue::Learn as i8, + CardQueue::DayLearn as i8 ), StateKind::Buried => write!( self.sql, "c.queue in ({},{})", - CardQueue::SchedBuried as u8, - CardQueue::UserBuried as u8 + CardQueue::SchedBuried as i8, + CardQueue::UserBuried as i8 ), - StateKind::Suspended => write!(self.sql, "c.queue = {}", CardQueue::Suspended as u8), + StateKind::Suspended => write!(self.sql, "c.queue = {}", CardQueue::Suspended as i8), StateKind::Due => write!( self.sql, " (c.queue in ({rev},{daylrn}) and c.due <= {today}) or (c.queue = {lrn} and c.due <= {daycutoff})", - rev = CardQueue::Review as u8, - daylrn = CardQueue::DayLearn as u8, + rev = CardQueue::Review as i8, + daylrn = CardQueue::DayLearn as i8, today = timing.days_elapsed, - lrn = CardQueue::Learn as u8, + lrn = CardQueue::Learn as i8, daycutoff = timing.next_day_at, ), } @@ -422,10 +422,10 @@ mod test { ); // added - let t = ctx.storage.timing_today().unwrap(); + let timing = ctx.storage.timing_today().unwrap(); assert_eq!( s(ctx, "added:3").0, - format!("(c.id > {})", t.next_day_at - (86_400 * 3)) + format!("(c.id > {})", timing.next_day_at - (86_400 * 3)) ); // deck @@ -482,10 +482,30 @@ mod test { ); assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![])); + // state + assert_eq!( + s(ctx, "is:suspended").0, + format!("(c.queue = {})", CardQueue::Suspended as i8) + ); + + // rated + assert_eq!( + s(ctx, "rated:2").0, + format!( + "(c.id in (select cid from revlog where id>{}))", + (timing.next_day_at - (86_400 * 2)) * 1_000 + ) + ); + assert_eq!( + s(ctx, "rated:40:1").0, + format!( + "(c.id in (select cid from revlog where id>{} and ease=1))", + (timing.next_day_at - (86_400 * 31)) * 1_000 + ) + ); + // todo: // note - // rated - // is // prop Ok(()) From 67cb27badaba734f1d564c9d503c8f6dfaafffdc Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 11:02:51 +1000 Subject: [PATCH 088/150] add remaining tests and fix some clippy lints --- rslib/src/search/sqlwriter.rs | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 1f2b16447..268df9e2a 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -86,7 +86,7 @@ impl SqlWriter<'_, '_> { fn write_unqualified(&mut self, text: &str) { // implicitly wrap in % let text = format!("%{}%", text); - self.args.push(text.into()); + self.args.push(text); write!( self.sql, "(n.sfld like ?{n} escape '\\' or n.flds like ?{n} escape '\\')", @@ -103,7 +103,7 @@ impl SqlWriter<'_, '_> { let tag = format!("% {} %", text.replace('*', "%")); write!(self.sql, "n.tags like ?").unwrap(); - self.args.push(tag.into()); + self.args.push(tag); } fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { @@ -255,7 +255,7 @@ impl SqlWriter<'_, '_> { } fn write_note_type(&mut self, nt_name: &str) -> Result<()> { - let ntids: Vec<_> = self + let mut ntids: Vec<_> = self .req .storage .all_note_types()? @@ -264,6 +264,8 @@ impl SqlWriter<'_, '_> { .map(|nt| nt.id) .collect(); self.sql.push_str("n.mid in "); + // sort for the benefit of unit tests + ntids.sort(); ids_to_string(&mut self.sql, &ntids); Ok(()) } @@ -288,7 +290,7 @@ impl SqlWriter<'_, '_> { return Ok(()); } - self.args.push(val.to_string().into()); + self.args.push(val.to_string()); let arg_idx = self.args.len(); let searches: Vec<_> = field_map .iter() @@ -315,7 +317,7 @@ impl SqlWriter<'_, '_> { ntid, csum ) .unwrap(); - self.args.push(text.to_string().into()) + self.args.push(text.to_string()); } fn write_added(&mut self, days: u32) -> Result<()> { @@ -504,9 +506,23 @@ mod test { ) ); - // todo: - // note - // prop + // props + assert_eq!(s(ctx, "prop:lapses=3").0, "(lapses = 3)".to_string()); + assert_eq!(s(ctx, "prop:ease>=2.5").0, "(ease >= 2500)".to_string()); + assert_eq!( + s(ctx, "prop:due!=-1").0, + format!( + "((c.queue in (2,3) and due != {}))", + timing.days_elapsed - 1 + ) + ); + + // note types by name + assert_eq!(&s(ctx, "note:basic").0, "(n.mid in (1581236385347))"); + assert_eq!( + &s(ctx, "note:basic*").0, + "(n.mid in (1581236385345,1581236385346,1581236385347,1581236385344))" + ); Ok(()) }) From 224bad25667e407325db9b248c23da1ed5e458d3 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 11:54:39 +1000 Subject: [PATCH 089/150] handle empty searches and leading/trailing whitespace --- rslib/src/search/parser.rs | 30 ++++++++++++++++++++++++++++-- rslib/src/search/sqlwriter.rs | 1 + 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 03ad4692b..98252e90a 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -76,6 +76,7 @@ pub(super) enum SearchNode<'a> { operator: String, kind: PropertyKind, }, + WholeCollection, } #[derive(Debug, PartialEq)] @@ -106,6 +107,11 @@ pub(super) enum TemplateKind { /// Parse the input string into a list of nodes. #[allow(dead_code)] pub(super) fn parse(input: &str) -> Result> { + let input = input.trim(); + if input.is_empty() { + return Ok(vec![Node::Search(SearchNode::WholeCollection)]); + } + let (_, nodes) = all_consuming(group_inner)(input) .map_err(|_e| AnkiError::invalid_input("unable to parse search"))?; Ok(nodes) @@ -150,12 +156,19 @@ fn group_inner(input: &str) -> IResult<&str, Vec> { }; } - Ok((remaining, nodes)) + if nodes.is_empty() { + Err(nom::Err::Error((remaining, nom::error::ErrorKind::Many1))) + } else { + Ok((remaining, nodes)) + } +} + +fn whitespace0(s: &str) -> IResult<&str, Vec> { + many0(one_of(" \u{3000}"))(s) } /// Optional leading space, then a (negated) group or text fn node(s: &str) -> IResult<&str, Node> { - let whitespace0 = many0(one_of(" \u{3000}")); preceded(whitespace0, alt((negated_node, group, text)))(s) } @@ -374,6 +387,19 @@ mod test { use Node::*; use SearchNode::*; + assert_eq!(parse("")?, vec![Search(SearchNode::WholeCollection)]); + assert_eq!(parse(" ")?, vec![Search(SearchNode::WholeCollection)]); + + // leading/trailing/interspersed whitespace + assert_eq!( + parse(" t t2 ")?, + vec![ + Search(UnqualifiedText("t".into())), + And, + Search(UnqualifiedText("t2".into())) + ] + ); + assert_eq!( parse(r#"hello -(world and "foo:bar baz") OR test"#)?, vec![ diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 268df9e2a..2501f806c 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -79,6 +79,7 @@ impl SqlWriter<'_, '_> { write!(self.sql, "c.id in ({})", cids).unwrap(); } SearchNode::Property { operator, kind } => self.write_prop(operator, kind)?, + SearchNode::WholeCollection => write!(self.sql, "true").unwrap(), }; Ok(()) } From c90670ec3a93fec9d95b87baee8d13b854c15bb2 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 12:08:22 +1000 Subject: [PATCH 090/150] tolerate some string IDs --- rslib/src/config.rs | 6 +++++- rslib/src/decks.rs | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rslib/src/config.rs b/rslib/src/config.rs index 42ae62067..9514d2cab 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -2,12 +2,16 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::types::ObjID; +use serde_aux::field_attributes::deserialize_number_from_string; use serde_derive::Deserialize; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct Config { - #[serde(rename = "curDeck")] + #[serde( + rename = "curDeck", + deserialize_with = "deserialize_number_from_string" + )] pub(crate) current_deck_id: ObjID, pub(crate) rollover: Option, pub(crate) creation_offset: Option, diff --git a/rslib/src/decks.rs b/rslib/src/decks.rs index 9bf8e7e54..d00c6d4f2 100644 --- a/rslib/src/decks.rs +++ b/rslib/src/decks.rs @@ -2,10 +2,12 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::types::ObjID; +use serde_aux::field_attributes::deserialize_number_from_string; use serde_derive::Deserialize; #[derive(Deserialize)] pub struct Deck { + #[serde(deserialize_with = "deserialize_number_from_string")] pub(crate) id: ObjID, pub(crate) name: String, } From 79697746a4c591def796d498d1013b95ed96b9e0 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 14:04:28 +1000 Subject: [PATCH 091/150] added needs to use milliseconds --- rslib/src/search/sqlwriter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 2501f806c..b4077fbb9 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -323,7 +323,7 @@ impl SqlWriter<'_, '_> { fn write_added(&mut self, days: u32) -> Result<()> { let timing = self.req.storage.timing_today()?; - let cutoff = timing.next_day_at - (86_400 * (days as i64)); + let cutoff = (timing.next_day_at - (86_400 * (days as i64))) * 1_000; write!(self.sql, "c.id > {}", cutoff).unwrap(); Ok(()) } @@ -428,7 +428,7 @@ mod test { let timing = ctx.storage.timing_today().unwrap(); assert_eq!( s(ctx, "added:3").0, - format!("(c.id > {})", timing.next_day_at - (86_400 * 3)) + format!("(c.id > {})", (timing.next_day_at - (86_400 * 3)) * 1_000) ); // deck From d94effcdc76f6b5d7bc2af0349da9e6ee6d177da Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 14:07:36 +1000 Subject: [PATCH 092/150] fix is:new/is:review --- rslib/src/search/sqlwriter.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index b4077fbb9..d6685d9f0 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -152,8 +152,8 @@ impl SqlWriter<'_, '_> { fn write_state(&mut self, state: &StateKind) -> Result<()> { let timing = self.req.storage.timing_today()?; match state { - StateKind::New => write!(self.sql, "c.queue = {}", CardQueue::New as i8), - StateKind::Review => write!(self.sql, "c.queue = {}", CardQueue::Review as i8), + StateKind::New => write!(self.sql, "c.type = {}", CardQueue::New as i8), + StateKind::Review => write!(self.sql, "c.type = {}", CardQueue::Review as i8), StateKind::Learning => write!( self.sql, "c.queue in ({},{})", @@ -490,6 +490,10 @@ mod test { s(ctx, "is:suspended").0, format!("(c.queue = {})", CardQueue::Suspended as i8) ); + assert_eq!( + s(ctx, "is:new").0, + format!("(c.type = {})", CardQueue::New as i8) + ); // rated assert_eq!( From 2c362d699106da4fbe81701664c438107f74f4fb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 15:35:57 +1000 Subject: [PATCH 093/150] search order --- rslib/src/config.rs | 27 +++++++++++++++++++++++++++ rslib/src/search/cards.rs | 32 +++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/rslib/src/config.rs b/rslib/src/config.rs index 9514d2cab..2224b68f3 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -16,4 +16,31 @@ pub struct Config { pub(crate) rollover: Option, pub(crate) creation_offset: Option, pub(crate) local_offset: Option, + #[serde(rename = "sortType")] + pub(crate) browser_sort_kind: SortKind, + #[serde(rename = "sortBackwards", default)] + pub(crate) browser_sort_reverse: bool, +} + +#[derive(Deserialize, PartialEq, Debug)] +#[serde(rename_all = "camelCase")] +pub enum SortKind { + #[serde(rename = "noteCrt")] + NoteCreation, + NoteMod, + #[serde(rename = "noteFld")] + NoteField, + CardMod, + CardReps, + CardDue, + CardEase, + CardLapses, + #[serde(rename = "cardIvl")] + CardInterval, +} + +impl Default for SortKind { + fn default() -> Self { + Self::NoteCreation + } } diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 00a3f21e9..a0ab40562 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -2,7 +2,9 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use super::{parser::Node, sqlwriter::node_to_sql}; +use crate::card::CardType; use crate::collection::RequestContext; +use crate::config::SortKind; use crate::err::Result; use crate::search::parser::parse; use crate::types::ObjID; @@ -13,11 +15,12 @@ pub(crate) fn search_cards<'a, 'b>( ) -> Result> { let top_node = Node::Group(parse(search)?); let (sql, args) = node_to_sql(req, &top_node)?; - - let sql = format!( - "select c.id from cards c, notes n where c.nid=n.id and {} order by c.id", + let mut sql = format!( + "select c.id from cards c, notes n where c.nid=n.id and {} order by ", sql ); + write_order(req, &mut sql)?; + let mut stmt = req.storage.db.prepare(&sql)?; let ids: Vec = stmt .query_map(&args, |row| row.get(0))? @@ -26,3 +29,26 @@ pub(crate) fn search_cards<'a, 'b>( println!("sql {}\nargs {:?} count {}", sql, args, ids.len()); Ok(ids) } + +fn write_order(req: &mut RequestContext, sql: &mut String) -> Result<()> { + let conf = req.storage.all_config()?; + let tmp_str; + sql.push_str(match conf.browser_sort_kind { + SortKind::NoteCreation => "n.id, c.ord", + SortKind::NoteMod => "n.mod, c.ord", + SortKind::NoteField => "n.sfld collate nocase, c.ord", + SortKind::CardMod => "c.mod", + SortKind::CardReps => "c.reps", + SortKind::CardDue => "c.type, c.due", + SortKind::CardEase => { + tmp_str = format!("c.type = {}, c.factor", CardType::New as i8); + &tmp_str + } + SortKind::CardLapses => "c.lapses", + SortKind::CardInterval => "c.ivl", + }); + if conf.browser_sort_reverse { + sql.push_str(" desc"); + } + Ok(()) +} From 131811846169e6bb1e4e94a2fb9287315387daa5 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 15:44:35 +1000 Subject: [PATCH 094/150] flush config on sort order change --- qt/aqt/browser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 2760bec79..f7b5f7804 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -947,10 +947,14 @@ by clicking on one on the left.""" if type == "noteFld": ord = not ord self.col.conf["sortBackwards"] = ord + self.col.setMod() + self.col.save() self.search() else: if self.col.conf["sortBackwards"] != ord: self.col.conf["sortBackwards"] = ord + self.col.setMod() + self.col.save() self.model.reverse() self.setSortIndicator() From 00d0447ecbaf0d742efc1cc1427a269d2d13c57c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 18:09:15 +1000 Subject: [PATCH 095/150] support sorting on note type, card template and decks --- rslib/src/config.rs | 18 ++++++- rslib/src/search/cards.rs | 86 +++++++++++++++++++++++++++++--- rslib/src/search/sort_order.sql | 2 + rslib/src/search/sort_order2.sql | 2 + rslib/src/storage/sqlite.rs | 1 + 5 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 rslib/src/search/sort_order.sql create mode 100644 rslib/src/search/sort_order2.sql diff --git a/rslib/src/config.rs b/rslib/src/config.rs index 2224b68f3..ce1f68bc8 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -2,9 +2,19 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::types::ObjID; +use serde::Deserialize as DeTrait; use serde_aux::field_attributes::deserialize_number_from_string; use serde_derive::Deserialize; +use serde_json::Value; +pub(crate) fn default_on_invalid<'de, T, D>(deserializer: D) -> Result +where + T: Default + DeTrait<'de>, + D: serde::de::Deserializer<'de>, +{ + let v: Value = DeTrait::deserialize(deserializer)?; + Ok(T::deserialize(v).unwrap_or_default()) +} #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct Config { @@ -16,7 +26,7 @@ pub struct Config { pub(crate) rollover: Option, pub(crate) creation_offset: Option, pub(crate) local_offset: Option, - #[serde(rename = "sortType")] + #[serde(rename = "sortType", deserialize_with = "default_on_invalid")] pub(crate) browser_sort_kind: SortKind, #[serde(rename = "sortBackwards", default)] pub(crate) browser_sort_reverse: bool, @@ -30,6 +40,8 @@ pub enum SortKind { NoteMod, #[serde(rename = "noteFld")] NoteField, + #[serde(rename = "note")] + NoteType, CardMod, CardReps, CardDue, @@ -37,6 +49,10 @@ pub enum SortKind { CardLapses, #[serde(rename = "cardIvl")] CardInterval, + #[serde(rename = "deck")] + CardDeck, + #[serde(rename = "template")] + CardTemplate, } impl Default for SortKind { diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index a0ab40562..774cff59a 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -8,6 +8,7 @@ use crate::config::SortKind; use crate::err::Result; use crate::search::parser::parse; use crate::types::ObjID; +use rusqlite::params; pub(crate) fn search_cards<'a, 'b>( req: &'a mut RequestContext<'b>, @@ -15,11 +16,15 @@ pub(crate) fn search_cards<'a, 'b>( ) -> Result> { let top_node = Node::Group(parse(search)?); let (sql, args) = node_to_sql(req, &top_node)?; + + let conf = req.storage.all_config()?; + prepare_sort(req, &conf.browser_sort_kind)?; + let mut sql = format!( - "select c.id from cards c, notes n where c.nid=n.id and {} order by ", + "select c.id from cards c, notes n where c.nid=n.id and {}", sql ); - write_order(req, &mut sql)?; + write_order(&mut sql, &conf.browser_sort_kind, conf.browser_sort_reverse)?; let mut stmt = req.storage.db.prepare(&sql)?; let ids: Vec = stmt @@ -30,10 +35,10 @@ pub(crate) fn search_cards<'a, 'b>( Ok(ids) } -fn write_order(req: &mut RequestContext, sql: &mut String) -> Result<()> { - let conf = req.storage.all_config()?; +/// Add the order clause to the sql. +fn write_order(sql: &mut String, kind: &SortKind, reverse: bool) -> Result<()> { let tmp_str; - sql.push_str(match conf.browser_sort_kind { + let order = match kind { SortKind::NoteCreation => "n.id, c.ord", SortKind::NoteMod => "n.mod, c.ord", SortKind::NoteField => "n.sfld collate nocase, c.ord", @@ -46,9 +51,76 @@ fn write_order(req: &mut RequestContext, sql: &mut String) -> Result<()> { } SortKind::CardLapses => "c.lapses", SortKind::CardInterval => "c.ivl", - }); - if conf.browser_sort_reverse { + SortKind::CardDeck => "(select v from sort_order where k = c.did)", + SortKind::NoteType => "(select v from sort_order where k = n.mid)", + SortKind::CardTemplate => "(select v from sort_order where k1 = n.mid and k2 = c.ord)", + }; + if order.is_empty() { + return Ok(()); + } + sql.push_str(" order by "); + sql.push_str(order); + if reverse { sql.push_str(" desc"); } Ok(()) } + +// In the future these items should be moved from JSON into separate SQL tables, +// - for now we use a temporary deck to sort them. +fn prepare_sort(req: &mut RequestContext, kind: &SortKind) -> Result<()> { + use SortKind::*; + match kind { + CardDeck | NoteType => { + prepare_sort_order_table(req)?; + let mut stmt = req + .storage + .db + .prepare("insert into sort_order (k,v) values (?,?)")?; + + match kind { + CardDeck => { + for (k, v) in req.storage.all_decks()? { + stmt.execute(params![k, v.name])?; + } + } + NoteType => { + for (k, v) in req.storage.all_note_types()? { + stmt.execute(params![k, v.name])?; + } + } + _ => unreachable!(), + } + } + CardTemplate => { + prepare_sort_order_table2(req)?; + let mut stmt = req + .storage + .db + .prepare("insert into sort_order (k1,k2,v) values (?,?,?)")?; + + for (ntid, nt) in req.storage.all_note_types()? { + for tmpl in nt.templates { + stmt.execute(params![ntid, tmpl.ord, tmpl.name])?; + } + } + } + _ => (), + } + + Ok(()) +} + +fn prepare_sort_order_table(req: &mut RequestContext) -> Result<()> { + req.storage + .db + .execute_batch(include_str!("sort_order.sql"))?; + Ok(()) +} + +fn prepare_sort_order_table2(req: &mut RequestContext) -> Result<()> { + req.storage + .db + .execute_batch(include_str!("sort_order2.sql"))?; + Ok(()) +} diff --git a/rslib/src/search/sort_order.sql b/rslib/src/search/sort_order.sql new file mode 100644 index 000000000..3c77a5a7d --- /dev/null +++ b/rslib/src/search/sort_order.sql @@ -0,0 +1,2 @@ +drop table if exists sort_order; +create temporary table sort_order (k int primary key, v text); diff --git a/rslib/src/search/sort_order2.sql b/rslib/src/search/sort_order2.sql new file mode 100644 index 000000000..62db35752 --- /dev/null +++ b/rslib/src/search/sort_order2.sql @@ -0,0 +1,2 @@ +drop table if exists sort_order; +create temporary table sort_order (k1 int, k2 int, v text, primary key (k1, k2)) without rowid; \ No newline at end of file diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 8b61a6730..42d559ab8 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -45,6 +45,7 @@ fn open_or_create_collection_db(path: &Path) -> Result { db.pragma_update(None, "cache_size", &(-40 * 1024))?; db.pragma_update(None, "legacy_file_format", &false)?; db.pragma_update(None, "journal", &"wal")?; + db.pragma_update(None, "temp_store", &"memory")?; db.set_prepared_statement_cache_capacity(50); From 09a76967e70a50c14db301731c96587a3e687455 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 18:20:16 +1000 Subject: [PATCH 096/150] support sorting on tags I don't personally understand it, but some users seem to want it. --- rslib/src/config.rs | 1 + rslib/src/search/cards.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/rslib/src/config.rs b/rslib/src/config.rs index ce1f68bc8..3ee0114d0 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -42,6 +42,7 @@ pub enum SortKind { NoteField, #[serde(rename = "note")] NoteType, + NoteTags, CardMod, CardReps, CardDue, diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 774cff59a..c20a8f93e 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -51,6 +51,7 @@ fn write_order(sql: &mut String, kind: &SortKind, reverse: bool) -> Result<()> { } SortKind::CardLapses => "c.lapses", SortKind::CardInterval => "c.ivl", + SortKind::NoteTags => "n.tags", SortKind::CardDeck => "(select v from sort_order where k = c.did)", SortKind::NoteType => "(select v from sort_order where k = n.mid)", SortKind::CardTemplate => "(select v from sort_order where k1 = n.mid and k2 = c.ord)", From 13f3719650413877ce2248a80cae2ba7125a9772 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 18:33:35 +1000 Subject: [PATCH 097/150] ensure endReset() is called even if an exception is raised --- qt/aqt/browser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index f7b5f7804..11ac3e25e 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -184,8 +184,9 @@ class DataModel(QAbstractTableModel): invalid = True else: raise - # print "fetch cards in %dms" % ((time.time() - t)*1000) - self.endReset() + finally: + # print "fetch cards in %dms" % ((time.time() - t)*1000) + self.endReset() if invalid: showWarning(_("Invalid search - please check for typing mistakes.")) From 9ee82d55b11168b24f8c6b78efed735e2523bc66 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 20 Mar 2020 18:34:03 +1000 Subject: [PATCH 098/150] disable word wrap in browser rows --- qt/aqt/browser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 11ac3e25e..257ea0e10 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -834,6 +834,7 @@ class Browser(QMainWindow): self.form.tableView.selectionModel() self.form.tableView.setItemDelegate(StatusDelegate(self, self.model)) self.form.tableView.selectionModel().selectionChanged.connect(self.onRowChanged) + self.form.tableView.setWordWrap(False) if not theme_manager.night_mode: self.form.tableView.setStyleSheet( "QTableView{ selection-background-color: rgba(150, 150, 150, 50); " From 1f8a1126a4f0edaffe7a84f74b3468e56a51ac47 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:49:44 +1000 Subject: [PATCH 099/150] don't require trailing whitespace in .sql files --- .github/scripts/trailing-newlines.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/trailing-newlines.sh b/.github/scripts/trailing-newlines.sh index e2009df8c..332e65280 100755 --- a/.github/scripts/trailing-newlines.sh +++ b/.github/scripts/trailing-newlines.sh @@ -2,7 +2,7 @@ set -e -files=$(rg -l '[^\n]\z' -g '!*.{svg,scss,json}' || true) +files=$(rg -l '[^\n]\z' -g '!*.{svg,scss,json,sql}' || true) if [ "$files" != "" ]; then echo "the following files are missing a newline on the last line:" echo $files From 5debd3e0f836a350be6c59456c08966332eab1f9 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:54:43 +1000 Subject: [PATCH 100/150] add the ability to provide a custom sort order; use backend for find --- proto/backend.proto | 36 +++++++++++++----------------------- pylib/anki/collection.py | 6 +++--- pylib/anki/pybackend.py | 26 +++++++++++++------------- pylib/anki/rsbackend.py | 10 ++++++++-- pylib/anki/sched.py | 8 ++++---- pylib/anki/schedv2.py | 4 ++-- rslib/src/backend/mod.rs | 17 +++++++++++++---- rslib/src/search/cards.rs | 26 +++++++++++++++++++++----- rslib/src/search/mod.rs | 2 +- 9 files changed, 78 insertions(+), 57 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 638d9bc57..1cb0c1a43 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -24,8 +24,8 @@ message BackendInput { TemplateRequirementsIn template_requirements = 16; SchedTimingTodayIn sched_timing_today = 17; Empty deck_tree = 18; - FindCardsIn find_cards = 19; - BrowserRowsIn browser_rows = 20; + SearchCardsIn search_cards = 19; +// BrowserRowsIn browser_rows = 20; RenderCardIn render_card = 21; int64 local_minutes_west = 22; string strip_av_tags = 23; @@ -43,7 +43,6 @@ message BackendInput { Empty restore_trash = 35; OpenCollectionIn open_collection = 36; Empty close_collection = 37; - SearchCardsIn search_cards = 38; } } @@ -63,8 +62,8 @@ message BackendOutput { // fallible commands TemplateRequirementsOut template_requirements = 16; DeckTreeOut deck_tree = 18; - FindCardsOut find_cards = 19; - BrowserRowsOut browser_rows = 20; + SearchCardsOut search_cards = 19; +// BrowserRowsOut browser_rows = 20; RenderCardOut render_card = 21; string add_media_file = 26; Empty sync_media = 27; @@ -74,7 +73,6 @@ message BackendOutput { Empty restore_trash = 35; Empty open_collection = 36; Empty close_collection = 37; - SearchCardsOut search_cards = 38; BackendError error = 2047; } @@ -191,23 +189,6 @@ message DeckTreeNode { bool collapsed = 7; } -message FindCardsIn { - string search = 1; -} - -message FindCardsOut { - repeated int64 card_ids = 1; -} - -message BrowserRowsIn { - repeated int64 card_ids = 1; -} - -message BrowserRowsOut { - // just sort fields for proof of concept - repeated string sort_fields = 1; -} - message RenderCardIn { string question_template = 1; string answer_template = 2; @@ -337,9 +318,18 @@ message OpenCollectionIn { message SearchCardsIn { string search = 1; + SortOrder order = 2; } message SearchCardsOut { repeated int64 card_ids = 1; } + +message SortOrder { + oneof value { + Empty from_config = 1; + Empty none = 2; + string custom = 3; + } +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 471f2c5a7..fcc99b7c3 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -15,7 +15,7 @@ import time import traceback import unicodedata import weakref -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union import anki.find import anki.latex # sets up hook @@ -616,8 +616,8 @@ where c.nid = n.id and c.id in %s group by nid""" # Finding cards ########################################################################## - def findCards(self, query: str, order: Union[bool, str] = False) -> Any: - return anki.find.Finder(self).findCards(query, order) + def findCards(self, query: str, order: Union[bool, str] = False) -> Sequence[int]: + return self.backend.search_cards(query, order) def findNotes(self, query: str) -> Any: return anki.find.Finder(self).findNotes(query) diff --git a/pylib/anki/pybackend.py b/pylib/anki/pybackend.py index 5e27e7447..008ecc9ef 100644 --- a/pylib/anki/pybackend.py +++ b/pylib/anki/pybackend.py @@ -48,19 +48,19 @@ class PythonBackend: native = self.col.sched.deckDueTree() return native_deck_tree_to_proto(native) - def find_cards(self, input: pb.FindCardsIn) -> pb.FindCardsOut: - cids = self.col.findCards(input.search) - return pb.FindCardsOut(card_ids=cids) - - def browser_rows(self, input: pb.BrowserRowsIn) -> pb.BrowserRowsOut: - sort_fields = [] - for cid in input.card_ids: - sort_fields.append( - self.col.db.scalar( - "select sfld from notes n,cards c where n.id=c.nid and c.id=?", cid - ) - ) - return pb.BrowserRowsOut(sort_fields=sort_fields) + # def find_cards(self, input: pb.FindCardsIn) -> pb.FindCardsOut: + # cids = self.col.findCards(input.search) + # return pb.FindCardsOut(card_ids=cids) + # + # def browser_rows(self, input: pb.BrowserRowsIn) -> pb.BrowserRowsOut: + # sort_fields = [] + # for cid in input.card_ids: + # sort_fields.append( + # self.col.db.scalar( + # "select sfld from notes n,cards c where n.id=c.nid and c.id=?", cid + # ) + # ) + # return pb.BrowserRowsOut(sort_fields=sort_fields) def native_deck_tree_to_proto(native): diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index eb3f69d63..56507a79c 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -423,9 +423,15 @@ class RustBackend: def _db_command(self, input: Dict[str, Any]) -> Any: return orjson.loads(self._backend.db_command(orjson.dumps(input))) - def search_cards(self, search: str) -> Sequence[int]: + def search_cards(self, search: str, order: Union[bool, str]) -> Sequence[int]: + if isinstance(order, str): + mode = pb.SortOrder(custom=order) + elif not order: + mode = pb.SortOrder(none=pb.Empty()) + else: + mode = pb.SortOrder(from_config=pb.Empty()) return self._run_command( - pb.BackendInput(search_cards=pb.SearchCardsIn(search=search)) + pb.BackendInput(search_cards=pb.SearchCardsIn(search=search, order=mode)) ).search_cards.card_ids diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index af426d744..0f696934b 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -8,7 +8,7 @@ import random import time from heapq import * from operator import itemgetter -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import anki from anki import hooks @@ -707,7 +707,7 @@ did = ? and queue = {QUEUE_TYPE_REV} and due <= ? limit ?""", # Dynamic deck handling ########################################################################## - def rebuildDyn(self, did: Optional[int] = None) -> Optional[List[int]]: # type: ignore[override] + def rebuildDyn(self, did: Optional[int] = None) -> Optional[Sequence[int]]: # type: ignore[override] "Rebuild a dynamic deck." did = did or self.col.decks.selected() deck = self.col.decks.get(did) @@ -721,7 +721,7 @@ did = ? and queue = {QUEUE_TYPE_REV} and due <= ? limit ?""", self.col.decks.select(did) return ids - def _fillDyn(self, deck: Dict[str, Any]) -> List[int]: # type: ignore[override] + def _fillDyn(self, deck: Dict[str, Any]) -> Sequence[int]: # type: ignore[override] search, limit, order = deck["terms"][0] orderlimit = self._dynOrder(order, limit) if search.strip(): @@ -751,7 +751,7 @@ due = odue, odue = 0, odid = 0, usn = ? where %s""" self.col.usn(), ) - def _moveToDyn(self, did: int, ids: List[int]) -> None: # type: ignore[override] + def _moveToDyn(self, did: int, ids: Sequence[int]) -> None: # type: ignore[override] deck = self.col.decks.get(did) data = [] t = intTime() diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index c6ecc6efc..ad72381aa 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -11,7 +11,7 @@ from heapq import * from operator import itemgetter # from anki.collection import _Collection -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union import anki # pylint: disable=unused-import from anki import hooks @@ -1215,7 +1215,7 @@ due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? whe t = "c.due, c.ord" return t + " limit %d" % l - def _moveToDyn(self, did: int, ids: List[int], start: int = -100000) -> None: + def _moveToDyn(self, did: int, ids: Sequence[int], start: int = -100000) -> None: deck = self.col.decks.get(did) data = [] u = self.col.usn() diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 8640be245..06f0c30ac 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -14,7 +14,7 @@ use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today_v2_new}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; -use crate::search::search_cards; +use crate::search::{search_cards, SortMode}; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, @@ -200,8 +200,6 @@ impl Backend { OValue::SchedTimingToday(self.sched_timing_today(input)) } Value::DeckTree(_) => todo!(), - Value::FindCards(_) => todo!(), - Value::BrowserRows(_) => todo!(), Value::RenderCard(input) => OValue::RenderCard(self.render_template(input)?), Value::LocalMinutesWest(stamp) => { OValue::LocalMinutesWest(local_minutes_west_for_stamp(stamp)) @@ -583,7 +581,18 @@ impl Backend { fn search_cards(&self, input: pb::SearchCardsIn) -> Result { self.with_col(|col| { col.with_ctx(|ctx| { - let cids = search_cards(ctx, &input.search)?; + let order = if let Some(order) = input.order { + use pb::sort_order::Value as V; + match order.value { + Some(V::None(_)) => SortMode::NoOrder, + Some(V::Custom(s)) => SortMode::Custom(s), + Some(V::FromConfig(_)) => SortMode::FromConfig, + None => SortMode::FromConfig, + } + } else { + SortMode::FromConfig + }; + let cids = search_cards(ctx, &input.search, order)?; Ok(pb::SearchCardsOut { card_ids: cids }) }) }) diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index c20a8f93e..fd1953131 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -10,21 +10,38 @@ use crate::search::parser::parse; use crate::types::ObjID; use rusqlite::params; +pub(crate) enum SortMode { + NoOrder, + FromConfig, + Custom(String), +} + pub(crate) fn search_cards<'a, 'b>( req: &'a mut RequestContext<'b>, search: &'a str, + order: SortMode, ) -> Result> { let top_node = Node::Group(parse(search)?); let (sql, args) = node_to_sql(req, &top_node)?; - let conf = req.storage.all_config()?; - prepare_sort(req, &conf.browser_sort_kind)?; - let mut sql = format!( "select c.id from cards c, notes n where c.nid=n.id and {}", sql ); - write_order(&mut sql, &conf.browser_sort_kind, conf.browser_sort_reverse)?; + + match order { + SortMode::NoOrder => (), + SortMode::FromConfig => { + let conf = req.storage.all_config()?; + prepare_sort(req, &conf.browser_sort_kind)?; + sql.push_str(" order by "); + write_order(&mut sql, &conf.browser_sort_kind, conf.browser_sort_reverse)?; + } + SortMode::Custom(order_clause) => { + sql.push_str(" order by "); + sql.push_str(&order_clause); + } + } let mut stmt = req.storage.db.prepare(&sql)?; let ids: Vec = stmt @@ -59,7 +76,6 @@ fn write_order(sql: &mut String, kind: &SortKind, reverse: bool) -> Result<()> { if order.is_empty() { return Ok(()); } - sql.push_str(" order by "); sql.push_str(order); if reverse { sql.push_str(" desc"); diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs index 417241051..2313c1d39 100644 --- a/rslib/src/search/mod.rs +++ b/rslib/src/search/mod.rs @@ -2,4 +2,4 @@ mod cards; mod parser; mod sqlwriter; -pub(crate) use cards::search_cards; +pub(crate) use cards::{search_cards, SortMode}; From 63ce44aaa72f4d4d8dbdfc29cc4e5f72382b8b17 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:55:53 +1000 Subject: [PATCH 101/150] enable sorting on the extra browser columns --- qt/aqt/browser.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 257ea0e10..efcea412d 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -917,31 +917,11 @@ QTableView {{ gridline-color: {grid} }} def _onSortChanged(self, idx, ord): type = self.model.activeCols[idx] - noSort = ("question", "answer", "template", "deck", "note", "noteTags") + noSort = ("question", "answer") if type in noSort: - if type == "template": - showInfo( - _( - """\ -This column can't be sorted on, but you can search for individual card types, \ -such as 'card:1'.""" - ) - ) - elif type == "deck": - showInfo( - _( - """\ -This column can't be sorted on, but you can search for specific decks \ -by clicking on one on the left.""" - ) - ) - else: - showInfo( - _( - "Sorting on this column is not supported. Please " - "choose another." - ) - ) + showInfo( + _("Sorting on this column is not supported. Please " "choose another.") + ) type = self.col.conf["sortType"] if self.col.conf["sortType"] != type: self.col.conf["sortType"] = type From 949252d4386f8513120d135928fd5119cfb53443 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:56:09 +1000 Subject: [PATCH 102/150] fix ease search --- rslib/src/search/sqlwriter.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index d6685d9f0..b73dcb227 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -143,7 +143,9 @@ impl SqlWriter<'_, '_> { PropertyKind::Interval(ivl) => write!(self.sql, "ivl {} {}", op, ivl), PropertyKind::Reps(reps) => write!(self.sql, "reps {} {}", op, reps), PropertyKind::Lapses(days) => write!(self.sql, "lapses {} {}", op, days), - PropertyKind::Ease(ease) => write!(self.sql, "ease {} {}", op, (ease * 1000.0) as u32), + PropertyKind::Ease(ease) => { + write!(self.sql, "factor {} {}", op, (ease * 1000.0) as u32) + } } .unwrap(); Ok(()) @@ -513,7 +515,7 @@ mod test { // props assert_eq!(s(ctx, "prop:lapses=3").0, "(lapses = 3)".to_string()); - assert_eq!(s(ctx, "prop:ease>=2.5").0, "(ease >= 2500)".to_string()); + assert_eq!(s(ctx, "prop:ease>=2.5").0, "(factor >= 2500)".to_string()); assert_eq!( s(ctx, "prop:due!=-1").0, format!( From 124357bd829604e660779ba420e0accd79879b5f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:56:32 +1000 Subject: [PATCH 103/150] handle * in single-field search --- rslib/src/search/sqlwriter.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index b73dcb227..d4cf396ce 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -293,7 +293,7 @@ impl SqlWriter<'_, '_> { return Ok(()); } - self.args.push(val.to_string()); + self.args.push(val.replace('*', "%")); let arg_idx = self.args.len(); let searches: Vec<_> = field_map .iter() @@ -413,7 +413,7 @@ mod test { // qualified search assert_eq!( - s(ctx, "front:test"), + s(ctx, "front:te*st"), ( concat!( "(((n.mid = 1581236385344 and field_at_index(n.flds, 0) like ?1) or ", @@ -422,7 +422,7 @@ mod test { "(n.mid = 1581236385347 and field_at_index(n.flds, 0) like ?1)))" ) .into(), - vec!["test".into()] + vec!["te%st".into()] ) ); From c3314d3689f5892f8eac217b03963d419cde5e3f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:56:55 +1000 Subject: [PATCH 104/150] don't crash when card:0 passed in --- rslib/src/search/parser.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 98252e90a..a50aeb46f 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -373,7 +373,7 @@ fn parse_prop(val: &str) -> ParseResult> { fn parse_template(val: &str) -> SearchNode<'static> { SearchNode::CardTemplate(match val.parse::() { - Ok(n) => TemplateKind::Ordinal(n), + Ok(n) => TemplateKind::Ordinal(n.max(1) - 1), Err(_) => TemplateKind::Name(val.into()), }) } @@ -425,7 +425,12 @@ mod test { ); assert_eq!( parse("card:3")?, - vec![Search(CardTemplate(TemplateKind::Ordinal(3)))] + vec![Search(CardTemplate(TemplateKind::Ordinal(2)))] + ); + // 0 must not cause a crash due to underflow + assert_eq!( + parse("card:0")?, + vec![Search(CardTemplate(TemplateKind::Ordinal(0)))] ); assert_eq!(parse("deck:default")?, vec![Search(Deck("default".into()))]); assert_eq!(parse("note:basic")?, vec![Search(NoteType("basic".into()))]); From ad09c89c3c0991fd1237af99dc58e0c945235f62 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 07:57:07 +1000 Subject: [PATCH 105/150] check for child decks case-insensitively --- rslib/src/decks.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rslib/src/decks.rs b/rslib/src/decks.rs index d00c6d4f2..a833e3a1d 100644 --- a/rslib/src/decks.rs +++ b/rslib/src/decks.rs @@ -13,10 +13,12 @@ pub struct Deck { } pub(crate) fn child_ids<'a>(decks: &'a [Deck], name: &str) -> impl Iterator + 'a { - let prefix = format!("{}::", name); + let prefix = format!("{}::", name.to_ascii_lowercase()); decks .iter() - .filter(move |d| d.name.starts_with(&prefix)) + .filter(move |d| { + d.name.to_ascii_lowercase().starts_with(&prefix) + }) .map(|d| d.id) } From 3a4146560cbbecd077a67448989e906acfde96ee Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 08:09:28 +1000 Subject: [PATCH 106/150] handle escaped tag searches and tag:* special case --- rslib/src/search/sqlwriter.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index d4cf396ce..e6f8163ef 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -97,14 +97,19 @@ impl SqlWriter<'_, '_> { } fn write_tag(&mut self, text: &str) { - if text == "none" { - write!(self.sql, "n.tags = ''").unwrap(); - return; + match text { + "none" => { + write!(self.sql, "n.tags = ''").unwrap(); + } + "*" | "%" => { + write!(self.sql, "true").unwrap(); + } + text => { + let tag = format!("% {} %", text.replace('*', "%")); + write!(self.sql, "n.tags like ? escape '\\'").unwrap(); + self.args.push(tag); + } } - - let tag = format!("% {} %", text.replace('*', "%")); - write!(self.sql, "n.tags like ?").unwrap(); - self.args.push(tag); } fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { @@ -486,6 +491,7 @@ mod test { ("(n.tags like ?)".into(), vec!["% o%e %".into()]) ); assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![])); + assert_eq!(s(ctx, "tag:*"), ("(true)".into(), vec![])); // state assert_eq!( From 868c463fb1484812d108eb891b201d46a40a96e7 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 08:10:52 +1000 Subject: [PATCH 107/150] tests need to flush before searching --- pylib/tests/test_find.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index 3ad041e6b..3d73c33ce 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -68,6 +68,7 @@ def test_findCards(): f["Front"] = "test" f["Back"] = "foo bar" deck.addNote(f) + deck.save() latestCardIds = [c.id for c in f.cards()] # tag searches assert len(deck.findCards("tag:*")) == 5 @@ -133,15 +134,19 @@ def test_findCards(): assert len(deck.findCards("front:*")) == 5 # ordering deck.conf["sortType"] = "noteCrt" + deck.flush() assert deck.findCards("front:*", order=True)[-1] in latestCardIds assert deck.findCards("", order=True)[-1] in latestCardIds deck.conf["sortType"] = "noteFld" + deck.flush() assert deck.findCards("", order=True)[0] == catCard.id assert deck.findCards("", order=True)[-1] in latestCardIds deck.conf["sortType"] = "cardMod" + deck.flush() assert deck.findCards("", order=True)[-1] in latestCardIds assert deck.findCards("", order=True)[0] == firstCardId deck.conf["sortBackwards"] = True + deck.flush() assert deck.findCards("", order=True)[0] in latestCardIds # model assert len(deck.findCards("note:basic")) == 5 @@ -177,6 +182,7 @@ def test_findCards(): deck.db.execute( "update cards set did = ? where id = ?", deck.decks.id("Default::Child"), id ) + deck.save() assert len(deck.findCards("deck:default")) == 7 assert len(deck.findCards("deck:default::child")) == 1 assert len(deck.findCards("deck:default -deck:default::*")) == 6 From daa848bb4d6fc67e405bf547fd50d018b8c39e69 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 08:12:01 +1000 Subject: [PATCH 108/150] update tests to reflect what now constitutes an error --- pylib/tests/test_find.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index 3d73c33ce..e1cfc76b7 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -118,9 +118,8 @@ def test_findCards(): assert len(deck.findCards("nid:%d" % f.id)) == 2 assert len(deck.findCards("nid:%d,%d" % (f1id, f2id))) == 2 # templates - with pytest.raises(Exception): - deck.findCards("card:foo") - assert len(deck.findCards("'card:card 1'")) == 4 + assert len(deck.findCards("card:foo")) == 0 + assert len(deck.findCards('"card:card 1"')) == 4 assert len(deck.findCards("card:reverse")) == 1 assert len(deck.findCards("card:1")) == 4 assert len(deck.findCards("card:2")) == 1 @@ -158,8 +157,7 @@ def test_findCards(): assert len(deck.findCards("-deck:foo")) == 5 assert len(deck.findCards("deck:def*")) == 5 assert len(deck.findCards("deck:*EFAULT")) == 5 - with pytest.raises(Exception): - deck.findCards("deck:*cefault") + assert len(deck.findCards("deck:*cefault")) == 0 # full search f = deck.newNote() f["Front"] = "helloworld" @@ -241,17 +239,12 @@ def test_findCards(): assert len(deck.findCards("-(tag:monkey OR tag:sheep)")) == 6 assert len(deck.findCards("tag:monkey or (tag:sheep sheep)")) == 2 assert len(deck.findCards("tag:monkey or (tag:sheep octopus)")) == 1 - # invalid grouping shouldn't error - assert len(deck.findCards(")")) == 0 - assert len(deck.findCards("(()")) == 0 # added assert len(deck.findCards("added:0")) == 0 deck.db.execute("update cards set id = id - 86400*1000 where id = ?", id) assert len(deck.findCards("added:1")) == deck.cardCount() - 1 assert len(deck.findCards("added:2")) == deck.cardCount() # flag - with pytest.raises(Exception): - deck.findCards("flag:01") with pytest.raises(Exception): deck.findCards("flag:12") From 307aadfd8a3088c14dee18f0028784e16157ded5 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 08:12:32 +1000 Subject: [PATCH 109/150] don't set deck.sched.today in test --- pylib/tests/test_find.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index e1cfc76b7..2591211e2 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -199,14 +199,8 @@ def test_findCards(): assert len(deck.findCards("prop:ivl!=10")) > 1 assert len(deck.findCards("prop:due>0")) == 1 # due dates should work - deck.sched.today = 15 - assert len(deck.findCards("prop:due=14")) == 0 - assert len(deck.findCards("prop:due=15")) == 1 - assert len(deck.findCards("prop:due=16")) == 0 - # including negatives - deck.sched.today = 32 - assert len(deck.findCards("prop:due=-1")) == 0 - assert len(deck.findCards("prop:due=-2")) == 1 + assert len(deck.findCards("prop:due=29")) == 0 + assert len(deck.findCards("prop:due=30")) == 1 # ease factors assert len(deck.findCards("prop:ease=2.3")) == 0 assert len(deck.findCards("prop:ease=2.2")) == 1 From aee64016acda5a4d092a6ad982be1d49a5f28cfc Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 08:17:24 +1000 Subject: [PATCH 110/150] fix formatting and unit test --- rslib/src/decks.rs | 4 +--- rslib/src/search/sqlwriter.rs | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/rslib/src/decks.rs b/rslib/src/decks.rs index a833e3a1d..16b80b261 100644 --- a/rslib/src/decks.rs +++ b/rslib/src/decks.rs @@ -16,9 +16,7 @@ pub(crate) fn child_ids<'a>(decks: &'a [Deck], name: &str) -> impl Iterator Date: Sat, 21 Mar 2020 09:00:05 +1000 Subject: [PATCH 111/150] add note searching --- proto/backend.proto | 12 ++++++++++-- pylib/anki/collection.py | 9 ++++++--- pylib/anki/find.py | 2 +- pylib/anki/rsbackend.py | 5 +++++ rslib/src/backend/mod.rs | 12 +++++++++++- rslib/src/search/mod.rs | 2 ++ rslib/src/search/notes.rs | 28 ++++++++++++++++++++++++++++ 7 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 rslib/src/search/notes.rs diff --git a/proto/backend.proto b/proto/backend.proto index 1cb0c1a43..638786a5a 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -25,7 +25,7 @@ message BackendInput { SchedTimingTodayIn sched_timing_today = 17; Empty deck_tree = 18; SearchCardsIn search_cards = 19; -// BrowserRowsIn browser_rows = 20; + SearchNotesIn search_notes = 20; RenderCardIn render_card = 21; int64 local_minutes_west = 22; string strip_av_tags = 23; @@ -63,7 +63,7 @@ message BackendOutput { TemplateRequirementsOut template_requirements = 16; DeckTreeOut deck_tree = 18; SearchCardsOut search_cards = 19; -// BrowserRowsOut browser_rows = 20; + SearchNotesOut search_notes = 20; RenderCardOut render_card = 21; string add_media_file = 26; Empty sync_media = 27; @@ -333,3 +333,11 @@ message SortOrder { string custom = 3; } } + +message SearchNotesIn { + string search = 1; +} + +message SearchNotesOut { + repeated int64 note_ids = 2; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index fcc99b7c3..fb03431a3 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -616,11 +616,11 @@ where c.nid = n.id and c.id in %s group by nid""" # Finding cards ########################################################################## - def findCards(self, query: str, order: Union[bool, str] = False) -> Sequence[int]: + def find_cards(self, query: str, order: Union[bool, str] = False) -> Sequence[int]: return self.backend.search_cards(query, order) - def findNotes(self, query: str) -> Any: - return anki.find.Finder(self).findNotes(query) + def find_notes(self, query: str) -> Sequence[int]: + return self.backend.search_notes(query) def findReplace( self, @@ -636,6 +636,9 @@ where c.nid = n.id and c.id in %s group by nid""" def findDupes(self, fieldName: str, search: str = "") -> List[Tuple[Any, list]]: return anki.find.findDupes(self, fieldName, search) + findCards = find_cards + findNotes = find_notes + # Stats ########################################################################## diff --git a/pylib/anki/find.py b/pylib/anki/find.py index a2c2b19c6..5beb1d044 100644 --- a/pylib/anki/find.py +++ b/pylib/anki/find.py @@ -595,7 +595,7 @@ def findDupes( # limit search to notes with applicable field name if search: search = "(" + search + ") " - search += "'%s:*'" % fieldName + search += '"%s:*"' % fieldName.replace('"', '"') # go through notes vals: Dict[str, List[int]] = {} dupes = [] diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 56507a79c..6f7c98628 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -434,6 +434,11 @@ class RustBackend: pb.BackendInput(search_cards=pb.SearchCardsIn(search=search, order=mode)) ).search_cards.card_ids + def search_notes(self, search: str) -> Sequence[int]: + return self._run_command( + pb.BackendInput(search_notes=pb.SearchNotesIn(search=search)) + ).search_notes.note_ids + def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 06f0c30ac..cc9815778 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -14,7 +14,7 @@ use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today_v2_new}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; -use crate::search::{search_cards, SortMode}; +use crate::search::{search_cards, search_notes, SortMode}; use crate::template::{ render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate, RenderedNode, @@ -246,6 +246,7 @@ impl Backend { OValue::CloseCollection(Empty {}) } Value::SearchCards(input) => OValue::SearchCards(self.search_cards(input)?), + Value::SearchNotes(input) => OValue::SearchNotes(self.search_notes(input)?), }) } @@ -597,6 +598,15 @@ impl Backend { }) }) } + + fn search_notes(&self, input: pb::SearchNotesIn) -> Result { + self.with_col(|col| { + col.with_ctx(|ctx| { + let nids = search_notes(ctx, &input.search)?; + Ok(pb::SearchNotesOut { note_ids: nids }) + }) + }) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs index 2313c1d39..14cdb1181 100644 --- a/rslib/src/search/mod.rs +++ b/rslib/src/search/mod.rs @@ -1,5 +1,7 @@ mod cards; +mod notes; mod parser; mod sqlwriter; pub(crate) use cards::{search_cards, SortMode}; +pub(crate) use notes::search_notes; diff --git a/rslib/src/search/notes.rs b/rslib/src/search/notes.rs new file mode 100644 index 000000000..50021a92e --- /dev/null +++ b/rslib/src/search/notes.rs @@ -0,0 +1,28 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::{parser::Node, sqlwriter::node_to_sql}; +use crate::collection::RequestContext; +use crate::err::Result; +use crate::search::parser::parse; +use crate::types::ObjID; + +pub(crate) fn search_notes<'a, 'b>( + req: &'a mut RequestContext<'b>, + search: &'a str, +) -> Result> { + let top_node = Node::Group(parse(search)?); + let (sql, args) = node_to_sql(req, &top_node)?; + + let sql = format!( + "select n.id from cards c, notes n where c.nid=n.id and {}", + sql + ); + + let mut stmt = req.storage.db.prepare(&sql)?; + let ids: Vec = stmt + .query_map(&args, |row| row.get(0))? + .collect::>()?; + + Ok(ids) +} From 2aab44d9cef4fb5f8d2be7f7150f2b1219292c79 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 09:34:24 +1000 Subject: [PATCH 112/150] support deck:"foo bar" style searches --- rslib/src/search/parser.rs | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index a50aeb46f..4fc860232 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -8,7 +8,7 @@ use nom::bytes::complete::{escaped, is_not, tag, take_while1}; use nom::character::complete::{char, one_of}; use nom::character::is_digit; use nom::combinator::{all_consuming, map, map_res}; -use nom::sequence::{delimited, preceded}; +use nom::sequence::{delimited, preceded, tuple}; use nom::{multi::many0, IResult}; use std::{borrow::Cow, num}; @@ -180,7 +180,7 @@ fn negated_node(s: &str) -> IResult<&str, Node> { /// Either quoted or unquoted text fn text(s: &str) -> IResult<&str, Node> { - alt((quoted_term, unquoted_term))(s) + alt((quoted_term, partially_quoted_term, unquoted_term))(s) } /// Determine if text is a qualified search, and handle escaped chars. @@ -210,7 +210,7 @@ fn unescape_quotes(s: &str) -> Cow { /// Unquoted text, terminated by a space or ) fn unquoted_term(s: &str) -> IResult<&str, Node> { map_res( - take_while1(|c| c != ' ' && c != ')'), + take_while1(|c| c != ' ' && c != ')' && c != '"'), |text: &str| -> ParseResult { Ok(if text.eq_ignore_ascii_case("or") { Node::Or @@ -225,16 +225,30 @@ fn unquoted_term(s: &str) -> IResult<&str, Node> { /// Quoted text, including the outer double quotes. fn quoted_term(s: &str) -> IResult<&str, Node> { + map_res(quoted_term_str, |o| -> ParseResult { + Ok(Node::Search(search_node_for_text(o)?)) + })(s) +} + +fn quoted_term_str(s: &str) -> IResult<&str, &str> { delimited(char('"'), quoted_term_inner, char('"'))(s) } /// Quoted text, terminated by a non-escaped double quote /// Can escape %, _, " and \ -fn quoted_term_inner(s: &str) -> IResult<&str, Node> { - map_res( - escaped(is_not(r#""\"#), '\\', one_of(r#""\%_"#)), - |o| -> ParseResult { Ok(Node::Search(search_node_for_text(o)?)) }, - )(s) +fn quoted_term_inner(s: &str) -> IResult<&str, &str> { + escaped(is_not(r#""\"#), '\\', one_of(r#""\%_"#))(s) +} + +/// eg deck:"foo bar" - quotes must come after the : +fn partially_quoted_term(s: &str) -> IResult<&str, Node> { + let term = take_while1(|c| c != ' ' && c != ')' && c != ':'); + let (s, (term, _, quoted_val)) = tuple((term, char(':'), quoted_term_str))(s)?; + + match search_node_for_text_with_argument(term.into(), quoted_val.into()) { + Ok(search) => Ok((s, Node::Search(search))), + Err(_) => Err(nom::Err::Failure((s, nom::error::ErrorKind::NoneOf))), + } } /// Convert a colon-separated key/val pair into the relevant search type. @@ -433,6 +447,11 @@ mod test { vec![Search(CardTemplate(TemplateKind::Ordinal(0)))] ); assert_eq!(parse("deck:default")?, vec![Search(Deck("default".into()))]); + assert_eq!( + parse("deck:\"default one\"")?, + vec![Search(Deck("default one".into()))] + ); + assert_eq!(parse("note:basic")?, vec![Search(NoteType("basic".into()))]); assert_eq!(parse("tag:hard")?, vec![Search(Tag("hard".into()))]); assert_eq!( From 2dc1b5c982c73a8138cf5664a670f87eb9fe36d0 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 10:23:11 +1000 Subject: [PATCH 113/150] add regexp() to sqlite --- rslib/src/storage/sqlite.rs | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 42d559ab8..e5502f90c 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -12,6 +12,7 @@ use crate::{ sched::cutoff::{sched_timing_today, SchedTimingToday}, types::{ObjID, Usn}, }; +use regex::Regex; use rusqlite::{params, Connection, NO_PARAMS}; use std::{ collections::HashMap, @@ -50,6 +51,7 @@ fn open_or_create_collection_db(path: &Path) -> Result { db.set_prepared_statement_cache_capacity(50); add_field_index_function(&db)?; + add_regexp_function(&db)?; Ok(db) } @@ -57,13 +59,49 @@ fn open_or_create_collection_db(path: &Path) -> Result { /// Adds sql function field_at_index(flds, index) /// to split provided fields and return field at zero-based index. /// If out of range, returns empty string. -fn add_field_index_function(db: &Connection) -> Result<()> { +fn add_field_index_function(db: &Connection) -> rusqlite::Result<()> { db.create_scalar_function("field_at_index", 2, true, |ctx| { let mut fields = ctx.get_raw(0).as_str()?.split('\x1f'); let idx: u16 = ctx.get(1)?; Ok(fields.nth(idx as usize).unwrap_or("").to_string()) }) - .map_err(Into::into) +} + +/// Adds sql function regexp(regex, string) -> is_match +/// Taken from the rusqlite docs +fn add_regexp_function(db: &Connection) -> rusqlite::Result<()> { + db.create_scalar_function("regexp", 2, true, move |ctx| { + assert_eq!(ctx.len(), 2, "called with unexpected number of arguments"); + + let saved_re: Option<&Regex> = ctx.get_aux(0)?; + let new_re = match saved_re { + None => { + let s = ctx.get::(0)?; + match Regex::new(&s) { + Ok(r) => Some(r), + Err(err) => return Err(rusqlite::Error::UserFunctionError(Box::new(err))), + } + } + Some(_) => None, + }; + + let is_match = { + let re = saved_re.unwrap_or_else(|| new_re.as_ref().unwrap()); + + let text = ctx + .get_raw(1) + .as_str() + .map_err(|e| rusqlite::Error::UserFunctionError(e.into()))?; + + re.is_match(text) + }; + + if let Some(re) = new_re { + ctx.set_aux(0, re); + } + + Ok(is_match) + }) } /// Fetch schema version from database. From d1ebdbdcceae6be5f7c7e5b2472c051a8f3a8e45 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 12:00:48 +1000 Subject: [PATCH 114/150] support regex searches --- rslib/src/search/parser.rs | 14 ++++++++++---- rslib/src/search/sqlwriter.rs | 12 ++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 4fc860232..f59378be4 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -5,7 +5,7 @@ use crate::err::{AnkiError, Result}; use crate::types::ObjID; use nom::branch::alt; use nom::bytes::complete::{escaped, is_not, tag, take_while1}; -use nom::character::complete::{char, one_of}; +use nom::character::complete::{anychar, char, one_of}; use nom::character::is_digit; use nom::combinator::{all_consuming, map, map_res}; use nom::sequence::{delimited, preceded, tuple}; @@ -77,6 +77,7 @@ pub(super) enum SearchNode<'a> { kind: PropertyKind, }, WholeCollection, + Regex(Cow<'a, str>), } #[derive(Debug, PartialEq)] @@ -235,9 +236,8 @@ fn quoted_term_str(s: &str) -> IResult<&str, &str> { } /// Quoted text, terminated by a non-escaped double quote -/// Can escape %, _, " and \ fn quoted_term_inner(s: &str) -> IResult<&str, &str> { - escaped(is_not(r#""\"#), '\\', one_of(r#""\%_"#))(s) + escaped(is_not(r#""\"#), '\\', anychar)(s) } /// eg deck:"foo bar" - quotes must come after the : @@ -270,7 +270,7 @@ fn search_node_for_text_with_argument<'a>( "rated" => parse_rated(val.as_ref())?, "dupes" => parse_dupes(val.as_ref())?, "prop" => parse_prop(val.as_ref())?, - + "re" => SearchNode::Regex(val), // anything else is a field search _ => SearchNode::SingleField { field: key, @@ -432,6 +432,12 @@ mod test { ] ); + // any character should be escapable in quotes + assert_eq!( + parse(r#""re:\btest""#)?, + vec![Search(Regex(r"\btest".into()))] + ); + assert_eq!(parse("added:3")?, vec![Search(AddedInDays(3))]); assert_eq!( parse("card:front")?, diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index da3a3e6b8..58519631e 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -80,6 +80,7 @@ impl SqlWriter<'_, '_> { } SearchNode::Property { operator, kind } => self.write_prop(operator, kind)?, SearchNode::WholeCollection => write!(self.sql, "true").unwrap(), + SearchNode::Regex(re) => self.write_regex(re.as_ref()), }; Ok(()) } @@ -334,6 +335,11 @@ impl SqlWriter<'_, '_> { write!(self.sql, "c.id > {}", cutoff).unwrap(); Ok(()) } + + fn write_regex(&mut self, word: &str) { + self.sql.push_str("n.flds regexp ?"); + self.args.push(format!(r"(?i){}", word)); + } } // Write a list of IDs as '(x,y,...)' into the provided string. @@ -537,6 +543,12 @@ mod test { "(n.mid in (1581236385345,1581236385346,1581236385347,1581236385344))" ); + // regex + assert_eq!( + s(ctx, r"re:\bone"), + ("(n.flds regexp ?)".into(), vec![r"(?i)\bone".into()]) + ); + Ok(()) }) .unwrap(); From 4ff17d31b3d35ca39944a0f2829e457e6b5dcdda Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 12:40:20 +1000 Subject: [PATCH 115/150] add unicase collation sqlite's like is hard-coded to use ASCII comparisons, so we can't take advantage of this yet --- rslib/Cargo.toml | 5 +++-- rslib/src/storage/sqlite.rs | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 58e5d5c90..627d96efa 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -38,12 +38,13 @@ slog-async = "2.4.0" slog-envlogger = "2.2.0" serde_repr = "0.1.5" num_enum = "0.4.2" +unicase = "2.6.0" [target.'cfg(target_vendor="apple")'.dependencies] -rusqlite = { version = "0.21.0", features = ["trace", "functions"] } +rusqlite = { version = "0.21.0", features = ["trace", "functions", "collation"] } [target.'cfg(not(target_vendor="apple"))'.dependencies] -rusqlite = { version = "0.21.0", features = ["trace", "functions", "bundled"] } +rusqlite = { version = "0.21.0", features = ["trace", "functions", "collation", "bundled"] } [target.'cfg(linux)'.dependencies] reqwest = { version = "0.10.1", features = ["json", "native-tls-vendored"] } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index e5502f90c..0d8a3936d 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -14,14 +14,20 @@ use crate::{ }; use regex::Regex; use rusqlite::{params, Connection, NO_PARAMS}; +use std::cmp::Ordering; use std::{ collections::HashMap, path::{Path, PathBuf}, }; +use unicase::UniCase; const SCHEMA_MIN_VERSION: u8 = 11; const SCHEMA_MAX_VERSION: u8 = 11; +fn unicase_compare(s1: &str, s2: &str) -> Ordering { + UniCase::new(s1).cmp(&UniCase::new(s2)) +} + // currently public for dbproxy #[derive(Debug)] pub struct SqliteStorage { @@ -53,6 +59,8 @@ fn open_or_create_collection_db(path: &Path) -> Result { add_field_index_function(&db)?; add_regexp_function(&db)?; + db.create_collation("unicase", unicase_compare)?; + Ok(db) } From 08e64d246dd0d6092c295ea50cb740b630acd24b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 12:44:56 +1000 Subject: [PATCH 116/150] don't require wildcard for unicode case folding in search --- rslib/src/text.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rslib/src/text.rs b/rslib/src/text.rs index f7a770cc7..0a799c5df 100644 --- a/rslib/src/text.rs +++ b/rslib/src/text.rs @@ -5,6 +5,7 @@ use lazy_static::lazy_static; use regex::{Captures, Regex}; use std::borrow::Cow; use std::ptr; +use unicase::eq as uni_eq; use unicode_normalization::{is_nfc, UnicodeNormalization}; #[derive(Debug, PartialEq)] @@ -219,14 +220,14 @@ pub(crate) fn normalize_to_nfc(s: &str) -> Cow { } } -/// True if search is equal to text, folding ascii case. +/// True if search is equal to text, folding case. /// Supports '*' to match 0 or more characters. pub(crate) fn matches_wildcard(text: &str, search: &str) -> bool { if search.contains('*') { let search = format!("^(?i){}$", regex::escape(search).replace(r"\*", ".*")); Regex::new(&search).unwrap().is_match(text) } else { - text.eq_ignore_ascii_case(search) + uni_eq(text, search) } } From 97577dbc16aef2373d2ddb2b61f6ea680d2640fb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 12:45:25 +1000 Subject: [PATCH 117/150] support wildcard in field*:val search --- rslib/src/search/sqlwriter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 58519631e..d259d88a6 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -285,7 +285,7 @@ impl SqlWriter<'_, '_> { let mut field_map = vec![]; for nt in note_types.values() { for field in &nt.fields { - if field.name.eq_ignore_ascii_case(field_name) { + if matches_wildcard(&field.name, field_name) { field_map.push((nt.id, field.ord)); } } From f0ed34d79b2e1a8b2796583d05f63729e609776e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 13:06:46 +1000 Subject: [PATCH 118/150] support regexp search in single field --- rslib/src/search/parser.rs | 32 +++++++++++++++++++++++++++----- rslib/src/search/sqlwriter.rs | 19 ++++++++++++++----- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index f59378be4..46575350c 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -53,6 +53,7 @@ pub(super) enum SearchNode<'a> { SingleField { field: Cow<'a, str>, text: Cow<'a, str>, + is_re: bool, }, AddedInDays(u32), CardTemplate(TemplateKind), @@ -272,10 +273,7 @@ fn search_node_for_text_with_argument<'a>( "prop" => parse_prop(val.as_ref())?, "re" => SearchNode::Regex(val), // anything else is a field search - _ => SearchNode::SingleField { - field: key, - text: val, - }, + _ => parse_single_field(key.as_ref(), val.as_ref()), }) } @@ -392,6 +390,20 @@ fn parse_template(val: &str) -> SearchNode<'static> { }) } +fn parse_single_field(key: &str, mut val: &str) -> SearchNode<'static> { + let is_re = if val.starts_with("re:") { + val = val.trim_start_matches("re:"); + true + } else { + false + }; + SearchNode::SingleField { + field: key.to_string().into(), + text: val.to_string().into(), + is_re, + } +} + #[cfg(test)] mod test { use super::*; @@ -424,7 +436,8 @@ mod test { And, Search(SingleField { field: "foo".into(), - text: "bar baz".into() + text: "bar baz".into(), + is_re: false, }) ]))), Or, @@ -432,6 +445,15 @@ mod test { ] ); + assert_eq!( + parse("foo:re:bar")?, + vec![Search(SingleField { + field: "foo".into(), + text: "bar".into(), + is_re: true + })] + ); + // any character should be escapable in quotes assert_eq!( parse(r#""re:\btest""#)?, diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index d259d88a6..dee776f45 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -55,8 +55,8 @@ impl SqlWriter<'_, '_> { fn write_search_node_to_sql(&mut self, node: &SearchNode) -> Result<()> { match node { SearchNode::UnqualifiedText(text) => self.write_unqualified(text), - SearchNode::SingleField { field, text } => { - self.write_single_field(field.as_ref(), text.as_ref())? + SearchNode::SingleField { field, text, is_re } => { + self.write_single_field(field.as_ref(), text.as_ref(), *is_re)? } SearchNode::AddedInDays(days) => self.write_added(*days)?, SearchNode::CardTemplate(template) => self.write_template(template)?, @@ -279,7 +279,7 @@ impl SqlWriter<'_, '_> { Ok(()) } - fn write_single_field(&mut self, field_name: &str, val: &str) -> Result<()> { + fn write_single_field(&mut self, field_name: &str, val: &str, is_re: bool) -> Result<()> { let note_types = self.req.storage.all_note_types()?; let mut field_map = vec![]; @@ -299,15 +299,24 @@ impl SqlWriter<'_, '_> { return Ok(()); } - self.args.push(val.replace('*', "%")); + let cmp; + if is_re { + cmp = "regexp"; + self.args.push(format!("(?i){}", val)); + } else { + cmp = "like"; + self.args.push(val.replace('*', "%")); + } + let arg_idx = self.args.len(); let searches: Vec<_> = field_map .iter() .map(|(ntid, ord)| { format!( - "(n.mid = {mid} and field_at_index(n.flds, {ord}) like ?{n})", + "(n.mid = {mid} and field_at_index(n.flds, {ord}) {cmp} ?{n})", mid = ntid, ord = ord, + cmp = cmp, n = arg_idx ) }) From 51a379de23d8134b90e81935724bf6cdf2da00da Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 15:15:59 +1000 Subject: [PATCH 119/150] add search that ignores combining chars On a test of a ~40k card collection, the 'ignore accents' add-on takes about 1150ms, and this code takes about 70ms. --- rslib/src/search/parser.rs | 2 ++ rslib/src/search/sqlwriter.rs | 16 ++++++++++++++++ rslib/src/storage/sqlite.rs | 13 +++++++++++++ rslib/src/text.rs | 30 +++++++++++++++++++++++++++++- 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 46575350c..92a7a3744 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -79,6 +79,7 @@ pub(super) enum SearchNode<'a> { }, WholeCollection, Regex(Cow<'a, str>), + NoCombining(Cow<'a, str>), } #[derive(Debug, PartialEq)] @@ -272,6 +273,7 @@ fn search_node_for_text_with_argument<'a>( "dupes" => parse_dupes(val.as_ref())?, "prop" => parse_prop(val.as_ref())?, "re" => SearchNode::Regex(val), + "nc" => SearchNode::NoCombining(val), // anything else is a field search _ => parse_single_field(key.as_ref(), val.as_ref()), }) diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index dee776f45..9b4107245 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -8,6 +8,7 @@ use crate::decks::get_deck; use crate::err::{AnkiError, Result}; use crate::notes::field_checksum; use crate::text::matches_wildcard; +use crate::text::without_combining; use crate::{ collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID, }; @@ -81,6 +82,7 @@ impl SqlWriter<'_, '_> { SearchNode::Property { operator, kind } => self.write_prop(operator, kind)?, SearchNode::WholeCollection => write!(self.sql, "true").unwrap(), SearchNode::Regex(re) => self.write_regex(re.as_ref()), + SearchNode::NoCombining(text) => self.write_no_combining(text.as_ref()), }; Ok(()) } @@ -97,6 +99,20 @@ impl SqlWriter<'_, '_> { .unwrap(); } + fn write_no_combining(&mut self, text: &str) { + let text = format!("%{}%", without_combining(text)); + self.args.push(text); + write!( + self.sql, + concat!( + "(coalesce(without_combining(cast(n.sfld as text)), n.sfld) like ?{n} escape '\\' ", + "or coalesce(without_combining(n.flds), n.flds) like ?{n} escape '\\')" + ), + n = self.args.len(), + ) + .unwrap(); + } + fn write_tag(&mut self, text: &str) { match text { "none" => { diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 0d8a3936d..23d6eec1b 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -10,12 +10,14 @@ use crate::{ decks::Deck, notetypes::NoteType, sched::cutoff::{sched_timing_today, SchedTimingToday}, + text::without_combining, types::{ObjID, Usn}, }; use regex::Regex; use rusqlite::{params, Connection, NO_PARAMS}; use std::cmp::Ordering; use std::{ + borrow::Cow, collections::HashMap, path::{Path, PathBuf}, }; @@ -58,6 +60,7 @@ fn open_or_create_collection_db(path: &Path) -> Result { add_field_index_function(&db)?; add_regexp_function(&db)?; + add_without_combining_function(&db)?; db.create_collation("unicase", unicase_compare)?; @@ -75,6 +78,16 @@ fn add_field_index_function(db: &Connection) -> rusqlite::Result<()> { }) } +fn add_without_combining_function(db: &Connection) -> rusqlite::Result<()> { + db.create_scalar_function("without_combining", 1, true, |ctx| { + let text = ctx.get_raw(0).as_str()?; + Ok(match without_combining(text) { + Cow::Borrowed(_) => None, + Cow::Owned(o) => Some(o), + }) + }) +} + /// Adds sql function regexp(regex, string) -> is_match /// Taken from the rusqlite docs fn add_regexp_function(db: &Connection) -> rusqlite::Result<()> { diff --git a/rslib/src/text.rs b/rslib/src/text.rs index 0a799c5df..a88bd4a4f 100644 --- a/rslib/src/text.rs +++ b/rslib/src/text.rs @@ -6,7 +6,9 @@ use regex::{Captures, Regex}; use std::borrow::Cow; use std::ptr; use unicase::eq as uni_eq; -use unicode_normalization::{is_nfc, UnicodeNormalization}; +use unicode_normalization::{ + char::is_combining_mark, is_nfc, is_nfkd_quick, IsNormalized, UnicodeNormalization, +}; #[derive(Debug, PartialEq)] pub enum AVTag { @@ -231,12 +233,32 @@ pub(crate) fn matches_wildcard(text: &str, search: &str) -> bool { } } +/// Convert provided string to NFKD form and strip combining characters. +pub(crate) fn without_combining(s: &str) -> Cow { + // if the string is already normalized + if matches!(is_nfkd_quick(s.chars()), IsNormalized::Yes) { + // and no combining characters found, return unchanged + if !s.chars().any(is_combining_mark) { + return s.into(); + } + } + + // we need to create a new string without the combining marks + s.chars() + .nfkd() + .filter(|c| !is_combining_mark(*c)) + .collect::() + .into() +} + #[cfg(test)] mod test { use super::matches_wildcard; + use crate::text::without_combining; use crate::text::{ extract_av_tags, strip_av_tags, strip_html, strip_html_preserving_image_filenames, AVTag, }; + use std::borrow::Cow; #[test] fn stripping() { @@ -287,4 +309,10 @@ mod test { assert_eq!(matches_wildcard("foo", "F*oo"), true); assert_eq!(matches_wildcard("foo", "b*"), false); } + + #[test] + fn combining() { + assert!(matches!(without_combining("test"), Cow::Borrowed(_))); + assert!(matches!(without_combining("Über"), Cow::Owned(_))); + } } From 9696e959be4250145ec6428ee530b1ef6b401194 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 15:30:35 +1000 Subject: [PATCH 120/150] increase the rated search cap to 365, and allow searches for ease 0 An add-on appears to use ease 0 when rescheduling cards, and it may make sense for Anki to do the same in the future as well. --- rslib/src/search/parser.rs | 4 ++-- rslib/src/search/sqlwriter.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index 92a7a3744..cca8a1817 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -314,14 +314,14 @@ fn parse_flag(s: &str) -> ParseResult> { } /// eg rated:3 or rated:10:2 -/// second arg must be between 1-4 +/// second arg must be between 0-4 fn parse_rated(val: &str) -> ParseResult> { let mut it = val.splitn(2, ':'); let days = it.next().unwrap().parse()?; let ease = match it.next() { Some(v) => { let n: u8 = v.parse()?; - if n < 5 && n > 0 { + if n < 5 { Some(n) } else { return Err(ParseError {}); diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 9b4107245..1f5e7ef0c 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -131,7 +131,7 @@ impl SqlWriter<'_, '_> { fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> { let today_cutoff = self.req.storage.timing_today()?.next_day_at; - let days = days.min(31) as i64; + let days = days.min(365) as i64; let target_cutoff_ms = (today_cutoff - 86_400 * days) * 1_000; write!( self.sql, @@ -543,10 +543,10 @@ mod test { ) ); assert_eq!( - s(ctx, "rated:40:1").0, + s(ctx, "rated:400:1").0, format!( "(c.id in (select cid from revlog where id>{} and ease=1))", - (timing.next_day_at - (86_400 * 31)) * 1_000 + (timing.next_day_at - (86_400 * 365)) * 1_000 ) ); From 9afbcd4178a746a599a5c50648ddc85d3c559cd0 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 16:38:46 +1000 Subject: [PATCH 121/150] remove old finder code; add search hooks to browser & remove old one --- pylib/anki/find.py | 489 +-------------------------------------- pylib/anki/hooks.py | 26 --- pylib/tests/test_find.py | 27 --- pylib/tools/genhooks.py | 5 - qt/aqt/browser.py | 31 +-- qt/aqt/gui_hooks.py | 62 +++++ qt/tools/genhooks_gui.py | 20 ++ 7 files changed, 107 insertions(+), 553 deletions(-) diff --git a/pylib/anki/find.py b/pylib/anki/find.py index 5beb1d044..8ecc8211e 100644 --- a/pylib/anki/find.py +++ b/pylib/anki/find.py @@ -4,500 +4,25 @@ from __future__ import annotations import re -import sre_constants -import unicodedata -from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple, Union, cast +from typing import TYPE_CHECKING, Optional, Set -from anki import hooks -from anki.consts import * from anki.hooks import * -from anki.utils import ( - fieldChecksum, - ids2str, - intTime, - joinFields, - splitFields, - stripHTMLMedia, -) +from anki.utils import ids2str, intTime, joinFields, splitFields, stripHTMLMedia if TYPE_CHECKING: from anki.collection import _Collection -# Find -########################################################################## - class Finder: def __init__(self, col: Optional[_Collection]) -> None: self.col = col.weakref() - self.search = dict( - added=self._findAdded, - card=self._findTemplate, - deck=self._findDeck, - mid=self._findMid, - nid=self._findNids, - cid=self._findCids, - note=self._findModel, - prop=self._findProp, - rated=self._findRated, - tag=self._findTag, - dupe=self._findDupes, - flag=self._findFlag, - ) - self.search["is"] = self._findCardState - hooks.search_terms_prepared(self.search) + print("Finder() is deprecated, please use col.find_cards() or .find_notes()") - def findCards(self, query: str, order: Union[bool, str] = False) -> List[Any]: - "Return a list of card ids for QUERY." - tokens = self._tokenize(query) - preds, args = self._where(tokens) - if preds is None: - raise Exception("invalidSearch") - order, rev = self._order(order) - sql = self._query(preds, order) - try: - res = self.col.db.list(sql, *args) - except: - # invalid grouping - return [] - if rev: - res.reverse() - return res + def findCards(self, query, order): + return self.col.find_cards(query, order) - def findNotes(self, query: str) -> List[Any]: - tokens = self._tokenize(query) - preds, args = self._where(tokens) - if preds is None: - return [] - if preds: - preds = "(" + preds + ")" - else: - preds = "1" - sql = ( - """ -select distinct(n.id) from cards c, notes n where c.nid=n.id and """ - + preds - ) - try: - res = self.col.db.list(sql, *args) - except: - # invalid grouping - return [] - return res - - # Tokenizing - ###################################################################### - - def _tokenize(self, query: str) -> List[str]: - inQuote: Union[bool, str] = False - tokens = [] - token = "" - for c in query: - # quoted text - if c in ("'", '"'): - if inQuote: - if c == inQuote: - inQuote = False - else: - token += c - elif token: - # quotes are allowed to start directly after a : - if token[-1] == ":": - inQuote = c - else: - token += c - else: - inQuote = c - # separator (space and ideographic space) - elif c in (" ", "\u3000"): - if inQuote: - token += c - elif token: - # space marks token finished - tokens.append(token) - token = "" - # nesting - elif c in ("(", ")"): - if inQuote: - token += c - else: - if c == ")" and token: - tokens.append(token) - token = "" - tokens.append(c) - # negation - elif c == "-": - if token: - token += c - elif not tokens or tokens[-1] != "-": - tokens.append("-") - # normal character - else: - token += c - # if we finished in a token, add it - if token: - tokens.append(token) - return tokens - - # Query building - ###################################################################### - - def _where(self, tokens: List[str]) -> Tuple[str, Optional[List[str]]]: - # state and query - s: Dict[str, Any] = dict(isnot=False, isor=False, join=False, q="", bad=False) - args: List[Any] = [] - - def add(txt, wrap=True): - # failed command? - if not txt: - # if it was to be negated then we can just ignore it - if s["isnot"]: - s["isnot"] = False - return None, None - else: - s["bad"] = True - return None, None - elif txt == "skip": - return None, None - # do we need a conjunction? - if s["join"]: - if s["isor"]: - s["q"] += " or " - s["isor"] = False - else: - s["q"] += " and " - if s["isnot"]: - s["q"] += " not " - s["isnot"] = False - if wrap: - txt = "(" + txt + ")" - s["q"] += txt - s["join"] = True - - for token in tokens: - if s["bad"]: - return None, None - # special tokens - if token == "-": - s["isnot"] = True - elif token.lower() == "or": - s["isor"] = True - elif token == "(": - add(token, wrap=False) - s["join"] = False - elif token == ")": - s["q"] += ")" - # commands - elif ":" in token: - cmd, val = token.split(":", 1) - cmd = cmd.lower() - if cmd in self.search: - add(self.search[cmd]((val, args))) - else: - add(self._findField(cmd, val)) - # normal text search - else: - add(self._findText(token, args)) - if s["bad"]: - return None, None - return s["q"], args - - def _query(self, preds: str, order: str) -> str: - # can we skip the note table? - if "n." not in preds and "n." not in order: - sql = "select c.id from cards c where " - else: - sql = "select c.id from cards c, notes n where c.nid=n.id and " - # combine with preds - if preds: - sql += "(" + preds + ")" - else: - sql += "1" - # order - if order: - sql += " " + order - return sql - - # Ordering - ###################################################################### - - def _order(self, order: Union[bool, str]) -> Tuple[str, bool]: - if not order: - return "", False - elif order is not True: - # custom order string provided - return " order by " + cast(str, order), False - # use deck default - type = self.col.conf["sortType"] - sort = None - if type.startswith("note"): - if type == "noteCrt": - sort = "n.id, c.ord" - elif type == "noteMod": - sort = "n.mod, c.ord" - elif type == "noteFld": - sort = "n.sfld collate nocase, c.ord" - elif type.startswith("card"): - if type == "cardMod": - sort = "c.mod" - elif type == "cardReps": - sort = "c.reps" - elif type == "cardDue": - sort = "c.type, c.due" - elif type == "cardEase": - sort = f"c.type == {CARD_TYPE_NEW}, c.factor" - elif type == "cardLapses": - sort = "c.lapses" - elif type == "cardIvl": - sort = "c.ivl" - if not sort: - # deck has invalid sort order; revert to noteCrt - sort = "n.id, c.ord" - return " order by " + sort, self.col.conf["sortBackwards"] - - # Commands - ###################################################################### - - def _findTag(self, args: Tuple[str, List[Any]]) -> str: - (val, list_args) = args - if val == "none": - return 'n.tags = ""' - val = val.replace("*", "%") - if not val.startswith("%"): - val = "% " + val - if not val.endswith("%") or val.endswith("\\%"): - val += " %" - list_args.append(val) - return "n.tags like ? escape '\\'" - - def _findCardState(self, args: Tuple[str, List[Any]]) -> Optional[str]: - (val, __) = args - if val in ("review", "new", "learn"): - if val == "review": - n = 2 - elif val == "new": - n = CARD_TYPE_NEW - else: - return f"queue in ({QUEUE_TYPE_LRN}, {QUEUE_TYPE_DAY_LEARN_RELEARN})" - return "type = %d" % n - elif val == "suspended": - return "c.queue = -1" - elif val == "buried": - return f"c.queue in ({QUEUE_TYPE_SIBLING_BURIED}, {QUEUE_TYPE_MANUALLY_BURIED})" - elif val == "due": - return f""" -(c.queue in ({QUEUE_TYPE_REV},{QUEUE_TYPE_DAY_LEARN_RELEARN}) and c.due <= %d) or -(c.queue = {QUEUE_TYPE_LRN} and c.due <= %d)""" % ( - self.col.sched.today, - self.col.sched.dayCutoff, - ) - else: - # unknown - return None - - def _findFlag(self, args: Tuple[str, List[Any]]) -> Optional[str]: - (val, __) = args - if not val or len(val) != 1 or val not in "01234": - return None - mask = 2 ** 3 - 1 - return "(c.flags & %d) == %d" % (mask, int(val)) - - def _findRated(self, args: Tuple[str, List[Any]]) -> Optional[str]: - # days(:optional_ease) - (val, __) = args - r = val.split(":") - try: - days = int(r[0]) - except ValueError: - return None - days = min(days, 31) - # ease - ease = "" - if len(r) > 1: - if r[1] not in ("1", "2", "3", "4"): - return None - ease = "and ease=%s" % r[1] - cutoff = (self.col.sched.dayCutoff - 86400 * days) * 1000 - return "c.id in (select cid from revlog where id>%d %s)" % (cutoff, ease) - - def _findAdded(self, args: Tuple[str, List[Any]]) -> Optional[str]: - (val, __) = args - try: - days = int(val) - except ValueError: - return None - cutoff = (self.col.sched.dayCutoff - 86400 * days) * 1000 - return "c.id > %d" % cutoff - - def _findProp(self, args: Tuple[str, List[Any]]) -> Optional[str]: - # extract - (strval, __) = args - m = re.match("(^.+?)(<=|>=|!=|=|<|>)(.+?$)", strval) - if not m: - return None - prop, cmp, strval = m.groups() - prop = prop.lower() # pytype: disable=attribute-error - # is val valid? - try: - if prop == "ease": - val = float(strval) - else: - val = int(strval) - except ValueError: - return None - # is prop valid? - if prop not in ("due", "ivl", "reps", "lapses", "ease"): - return None - # query - q = [] - if prop == "due": - val += self.col.sched.today - # only valid for review/daily learning - q.append(f"(c.queue in ({QUEUE_TYPE_REV},{QUEUE_TYPE_DAY_LEARN_RELEARN}))") - elif prop == "ease": - prop = "factor" - val = int(val * 1000) - q.append("(%s %s %s)" % (prop, cmp, val)) - return " and ".join(q) - - def _findText(self, val: str, args: List[str]) -> str: - val = val.replace("*", "%") - args.append("%" + val + "%") - args.append("%" + val + "%") - return "(n.sfld like ? escape '\\' or n.flds like ? escape '\\')" - - def _findNids(self, args: Tuple[str, List[Any]]) -> Optional[str]: - (val, __) = args - if re.search("[^0-9,]", val): - return None - return "n.id in (%s)" % val - - def _findCids(self, args) -> Optional[str]: - (val, __) = args - if re.search("[^0-9,]", val): - return None - return "c.id in (%s)" % val - - def _findMid(self, args) -> Optional[str]: - (val, __) = args - if re.search("[^0-9]", val): - return None - return "n.mid = %s" % val - - def _findModel(self, args: Tuple[str, List[Any]]) -> str: - (val, __) = args - ids = [] - val = val.lower() - for m in self.col.models.all(): - if unicodedata.normalize("NFC", m["name"].lower()) == val: - ids.append(m["id"]) - return "n.mid in %s" % ids2str(ids) - - def _findDeck(self, args: Tuple[str, List[Any]]) -> Optional[str]: - # if searching for all decks, skip - (val, __) = args - if val == "*": - return "skip" - # deck types - elif val == "filtered": - return "c.odid" - - def dids(did): - if not did: - return None - return [did] + [a[1] for a in self.col.decks.children(did)] - - # current deck? - ids = None - if val.lower() == "current": - ids = dids(self.col.decks.current()["id"]) - elif "*" not in val: - # single deck - ids = dids(self.col.decks.id(val, create=False)) - else: - # wildcard - ids = set() - val = re.escape(val).replace(r"\*", ".*") - for d in self.col.decks.all(): - if re.match("(?i)" + val, unicodedata.normalize("NFC", d["name"])): - ids.update(dids(d["id"])) - if not ids: - return None - sids = ids2str(ids) - return "c.did in %s or c.odid in %s" % (sids, sids) - - def _findTemplate(self, args: Tuple[str, List[Any]]) -> str: - # were we given an ordinal number? - (val, __) = args - try: - num = int(val) - 1 - except: - num = None - if num is not None: - return "c.ord = %d" % num - # search for template names - lims = [] - for m in self.col.models.all(): - for t in m["tmpls"]: - if unicodedata.normalize("NFC", t["name"].lower()) == val.lower(): - if m["type"] == MODEL_CLOZE: - # if the user has asked for a cloze card, we want - # to give all ordinals, so we just limit to the - # model instead - lims.append("(n.mid = %s)" % m["id"]) - else: - lims.append("(n.mid = %s and c.ord = %s)" % (m["id"], t["ord"])) - return " or ".join(lims) - - def _findField(self, field: str, val: str) -> Optional[str]: - field = field.lower() - val = val.replace("*", "%") - # find models that have that field - mods = {} - for m in self.col.models.all(): - for f in m["flds"]: - if unicodedata.normalize("NFC", f["name"].lower()) == field: - mods[str(m["id"])] = (m, f["ord"]) - if not mods: - # nothing has that field - return None - # gather nids - regex = re.escape(val).replace("_", ".").replace(re.escape("%"), ".*") - nids = [] - for (id, mid, flds) in self.col.db.execute( - """ -select id, mid, flds from notes -where mid in %s and flds like ? escape '\\'""" - % (ids2str(list(mods.keys()))), - "%" + val + "%", - ): - flds = splitFields(flds) - ord = mods[str(mid)][1] - strg = flds[ord] - try: - if re.search("(?si)^" + regex + "$", strg): - nids.append(id) - except sre_constants.error: - return None - if not nids: - return "0" - return "n.id in %s" % ids2str(nids) - - def _findDupes(self, args) -> Optional[str]: - # caller must call stripHTMLMedia on passed val - (val, __) = args - try: - mid, val = val.split(",", 1) - except OSError: - return None - csum = fieldChecksum(val) - nids = [] - for nid, flds in self.col.db.execute( - "select id, flds from notes where mid=? and csum=?", mid, csum - ): - if stripHTMLMedia(splitFields(flds)[0]) == val: - nids.append(nid) - return "n.id in %s" % ids2str(nids) + def findNotes(self, query): + return self.col.find_notes(query) # Find and replace diff --git a/pylib/anki/hooks.py b/pylib/anki/hooks.py index 912d42fb3..2439eff3c 100644 --- a/pylib/anki/hooks.py +++ b/pylib/anki/hooks.py @@ -492,32 +492,6 @@ class _SchemaWillChangeFilter: schema_will_change = _SchemaWillChangeFilter() -class _SearchTermsPreparedHook: - _hooks: List[Callable[[Dict[str, Callable]], None]] = [] - - def append(self, cb: Callable[[Dict[str, Callable]], None]) -> None: - """(searches: Dict[str, Callable])""" - self._hooks.append(cb) - - def remove(self, cb: Callable[[Dict[str, Callable]], None]) -> None: - if cb in self._hooks: - self._hooks.remove(cb) - - def __call__(self, searches: Dict[str, Callable]) -> None: - for hook in self._hooks: - try: - hook(searches) - except: - # if the hook fails, remove it - self._hooks.remove(hook) - raise - # legacy support - runHook("search", searches) - - -search_terms_prepared = _SearchTermsPreparedHook() - - class _SyncProgressDidChangeHook: _hooks: List[Callable[[str], None]] = [] diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index 2591211e2..16d604b17 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -2,7 +2,6 @@ import pytest from anki.consts import * -from anki.find import Finder from tests.shared import getEmptyCol @@ -11,32 +10,6 @@ class DummyCollection: return None -def test_parse(): - f = Finder(DummyCollection()) - assert f._tokenize("hello world") == ["hello", "world"] - assert f._tokenize("hello world") == ["hello", "world"] - assert f._tokenize("one -two") == ["one", "-", "two"] - assert f._tokenize("one --two") == ["one", "-", "two"] - assert f._tokenize("one - two") == ["one", "-", "two"] - assert f._tokenize("one or -two") == ["one", "or", "-", "two"] - assert f._tokenize("'hello \"world\"'") == ['hello "world"'] - assert f._tokenize('"hello world"') == ["hello world"] - assert f._tokenize("one (two or ( three or four))") == [ - "one", - "(", - "two", - "or", - "(", - "three", - "or", - "four", - ")", - ")", - ] - assert f._tokenize("embedded'string") == ["embedded'string"] - assert f._tokenize("deck:'two words'") == ["deck:two words"] - - def test_findCards(): deck = getEmptyCol() f = deck.newNote() diff --git a/pylib/tools/genhooks.py b/pylib/tools/genhooks.py index 465a064cc..4ab2e90e8 100644 --- a/pylib/tools/genhooks.py +++ b/pylib/tools/genhooks.py @@ -37,11 +37,6 @@ hooks = [ args=["exporters: List[Tuple[str, Any]]"], legacy_hook="exportersList", ), - Hook( - name="search_terms_prepared", - args=["searches: Dict[str, Callable]"], - legacy_hook="search", - ), Hook( name="note_type_added", args=["notetype: Dict[str, Any]"], diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index efcea412d..3b5abeb7e 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -13,7 +13,7 @@ import unicodedata from dataclasses import dataclass from enum import Enum from operator import itemgetter -from typing import Callable, List, Optional, Union +from typing import Callable, List, Optional, Sequence, Union import anki import aqt.forms @@ -69,6 +69,13 @@ class FindDupesDialog: browser: Browser +@dataclass +class SearchContext: + search: str + # if set, provided card ids will be used instead of the regular search + card_ids: Optional[Sequence[int]] = None + + # Data model ########################################################################## @@ -82,7 +89,7 @@ class DataModel(QAbstractTableModel): self.activeCols = self.col.conf.get( "activeCols", ["noteFld", "template", "cardDue", "deck"] ) - self.cards: List[int] = [] + self.cards: Sequence[int] = [] self.cardObjs: Dict[int, Card] = {} def getCard(self, index: QModelIndex) -> Card: @@ -169,23 +176,21 @@ class DataModel(QAbstractTableModel): # Filtering ###################################################################### - def search(self, txt): + def search(self, txt: str) -> None: self.beginReset() - t = time.time() - # the db progress handler may cause a refresh, so we need to zero out - # old data first self.cards = [] invalid = False try: - self.cards = self.col.findCards(txt, order=True) + ctx = SearchContext(search=txt) + gui_hooks.browser_will_search(ctx) + if ctx.card_ids is None: + ctx.card_ids = self.col.find_cards(txt) + gui_hooks.browser_did_search(ctx) + self.cards = ctx.card_ids except Exception as e: - if str(e) == "invalidSearch": - self.cards = [] - invalid = True - else: - raise + print("search failed:", e) + invalid = True finally: - # print "fetch cards in %dms" % ((time.time() - t)*1000) self.endReset() if invalid: diff --git a/qt/aqt/gui_hooks.py b/qt/aqt/gui_hooks.py index 30a9ec0a7..a066504ab 100644 --- a/qt/aqt/gui_hooks.py +++ b/qt/aqt/gui_hooks.py @@ -359,6 +359,32 @@ class _BrowserDidChangeRowHook: browser_did_change_row = _BrowserDidChangeRowHook() +class _BrowserDidSearchHook: + """Allows you to modify the list of returned card ids from a search.""" + + _hooks: List[Callable[["aqt.browser.SearchContext"], None]] = [] + + def append(self, cb: Callable[["aqt.browser.SearchContext"], None]) -> None: + """(context: aqt.browser.SearchContext)""" + self._hooks.append(cb) + + def remove(self, cb: Callable[["aqt.browser.SearchContext"], None]) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__(self, context: aqt.browser.SearchContext) -> None: + for hook in self._hooks: + try: + hook(context) + except: + # if the hook fails, remove it + self._hooks.remove(hook) + raise + + +browser_did_search = _BrowserDidSearchHook() + + class _BrowserMenusDidInitHook: _hooks: List[Callable[["aqt.browser.Browser"], None]] = [] @@ -484,6 +510,42 @@ class _BrowserWillBuildTreeFilter: browser_will_build_tree = _BrowserWillBuildTreeFilter() +class _BrowserWillSearchHook: + """Allows you to modify the search text, or perform your own search. + + You can modify context.search to change the text that is sent to the + searching backend. + + If you set context.card_ids to a list of ids, the regular search will + not be performed, and the provided ids will be used instead. + + Your add-on should check if context.card_ids is not None, and return + without making changes if it has been set. + """ + + _hooks: List[Callable[["aqt.browser.SearchContext"], None]] = [] + + def append(self, cb: Callable[["aqt.browser.SearchContext"], None]) -> None: + """(context: aqt.browser.SearchContext)""" + self._hooks.append(cb) + + def remove(self, cb: Callable[["aqt.browser.SearchContext"], None]) -> None: + if cb in self._hooks: + self._hooks.remove(cb) + + def __call__(self, context: aqt.browser.SearchContext) -> None: + for hook in self._hooks: + try: + hook(context) + except: + # if the hook fails, remove it + self._hooks.remove(hook) + raise + + +browser_will_search = _BrowserWillSearchHook() + + class _BrowserWillShowHook: _hooks: List[Callable[["aqt.browser.Browser"], None]] = [] diff --git a/qt/tools/genhooks_gui.py b/qt/tools/genhooks_gui.py index 21f8de560..be3232a92 100644 --- a/qt/tools/genhooks_gui.py +++ b/qt/tools/genhooks_gui.py @@ -235,6 +235,26 @@ hooks = [ return True """, ), + Hook( + name="browser_will_search", + args=["context: aqt.browser.SearchContext"], + doc="""Allows you to modify the search text, or perform your own search. + + You can modify context.search to change the text that is sent to the + searching backend. + + If you set context.card_ids to a list of ids, the regular search will + not be performed, and the provided ids will be used instead. + + Your add-on should check if context.card_ids is not None, and return + without making changes if it has been set. + """, + ), + Hook( + name="browser_did_search", + args=["context: aqt.browser.SearchContext"], + doc="""Allows you to modify the list of returned card ids from a search.""", + ), # States ################### Hook( From dfa7f5e142509cbea8ce7e4ad604723019f20f7f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 16:57:33 +1000 Subject: [PATCH 122/150] fix reversing sort order --- qt/aqt/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 3b5abeb7e..6fec53a00 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -219,7 +219,7 @@ class DataModel(QAbstractTableModel): def _reverse(self): self.beginReset() - self.cards.reverse() + self.cards = list(reversed(self.cards)) self.endReset() def saveSelection(self): From 99416477acc0ef13d4203e32c450a854725ddcf7 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 17:38:09 +1000 Subject: [PATCH 123/150] allow customizing search order --- qt/aqt/browser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qt/aqt/browser.py b/qt/aqt/browser.py index 6fec53a00..4c38a70ec 100644 --- a/qt/aqt/browser.py +++ b/qt/aqt/browser.py @@ -72,6 +72,7 @@ class FindDupesDialog: @dataclass class SearchContext: search: str + order: Union[bool, str] = True # if set, provided card ids will be used instead of the regular search card_ids: Optional[Sequence[int]] = None @@ -184,7 +185,7 @@ class DataModel(QAbstractTableModel): ctx = SearchContext(search=txt) gui_hooks.browser_will_search(ctx) if ctx.card_ids is None: - ctx.card_ids = self.col.find_cards(txt) + ctx.card_ids = self.col.find_cards(txt, order=ctx.order) gui_hooks.browser_did_search(ctx) self.cards = ctx.card_ids except Exception as e: From cc44523449cda621c20f80e6583040deb57debb1 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 18:29:04 +1000 Subject: [PATCH 124/150] remove debugging line --- rslib/src/search/cards.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index fd1953131..e4d8eaf2a 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -48,7 +48,6 @@ pub(crate) fn search_cards<'a, 'b>( .query_map(&args, |row| row.get(0))? .collect::>()?; - println!("sql {}\nargs {:?} count {}", sql, args, ids.len()); Ok(ids) } From 9dda5cf6ca7610fabfeef1e0e29b49a656e97767 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 21 Mar 2020 21:24:21 +1000 Subject: [PATCH 125/150] fall back on stock json if orjson unavailable --- pylib/anki/rsbackend.py | 14 +++++++++++++- pylib/setup.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 6f7c98628..52bdc0d8f 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -20,7 +20,6 @@ from typing import ( ) import ankirspy # pytype: disable=import-error -import orjson import anki.backend_pb2 as pb import anki.buildinfo @@ -36,6 +35,19 @@ assert ankirspy.buildhash() == anki.buildinfo.buildhash SchedTimingToday = pb.SchedTimingTodayOut +try: + import orjson +except: + # add compat layer for 32 bit builds that can't use orjson + print("reverting to stock json") + import json + + class orjson: # type: ignore + def dumps(obj: Any) -> bytes: + return json.dumps(obj).encode("utf8") + + loads = json.loads + class Interrupted(Exception): pass diff --git a/pylib/setup.py b/pylib/setup.py index b23d783d7..e42b783bf 100644 --- a/pylib/setup.py +++ b/pylib/setup.py @@ -21,7 +21,7 @@ setuptools.setup( "requests", "decorator", "protobuf", - "orjson", + 'orjson; platform_machine == "x86_64"', 'psutil; sys_platform == "win32"', 'distro; sys_platform != "darwin" and sys_platform != "win32"', ], From 199713a39aad1e559cb85d230ae22b64394631bc Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 10:26:09 +1000 Subject: [PATCH 126/150] handle collections with sortBackwards set to 0 instead of a bool --- rslib/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/src/config.rs b/rslib/src/config.rs index 3ee0114d0..c768b1122 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -28,7 +28,7 @@ pub struct Config { pub(crate) local_offset: Option, #[serde(rename = "sortType", deserialize_with = "default_on_invalid")] pub(crate) browser_sort_kind: SortKind, - #[serde(rename = "sortBackwards", default)] + #[serde(rename = "sortBackwards", deserialize_with = "default_on_invalid")] pub(crate) browser_sort_reverse: bool, } From f28e57a367ae873e0b3d7a02a96a2014f3915f8d Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 12:59:24 +1000 Subject: [PATCH 127/150] add enum for controlling sort order eg col.find_cards("", order=BuiltinSortKind.CARD_DUE) --- proto/backend.proto | 22 ++++++++++++++++++++++ pylib/anki/collection.py | 12 ++++++++++-- pylib/anki/rsbackend.py | 17 ++++++++++++++--- pylib/tests/test_find.py | 9 +++++++++ rslib/src/backend/mod.rs | 29 ++++++++++++++++++++++++++++- rslib/src/search/cards.rs | 6 ++++++ 6 files changed, 89 insertions(+), 6 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 638786a5a..3821ad49f 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -331,6 +331,7 @@ message SortOrder { Empty from_config = 1; Empty none = 2; string custom = 3; + BuiltinSearchOrder builtin = 4; } } @@ -341,3 +342,24 @@ message SearchNotesIn { message SearchNotesOut { repeated int64 note_ids = 2; } + +message BuiltinSearchOrder { + BuiltinSortKind kind = 1; + bool reverse = 2; +} + +enum BuiltinSortKind { + NOTE_CREATION = 0; + NOTE_MOD = 1; + NOTE_FIELD = 2; + NOTE_TAGS = 3; + NOTE_TYPE = 4; + CARD_MOD = 5; + CARD_REPS = 6; + CARD_DUE = 7; + CARD_EASE = 8; + CARD_LAPSES = 9; + CARD_INTERVAL = 10; + CARD_DECK = 11; + CARD_TEMPLATE = 12; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index fb03431a3..b8258adb4 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -616,8 +616,16 @@ where c.nid = n.id and c.id in %s group by nid""" # Finding cards ########################################################################## - def find_cards(self, query: str, order: Union[bool, str] = False) -> Sequence[int]: - return self.backend.search_cards(query, order) + # if order=True, use the sort order stored in the collection config + # if order=False, do no ordering + # if order is a string, that text is added after 'order by' in the sql statement + # if order is an int enum, sort using that builtin sort. + # + # the reverse argument only applies when a BuiltinSortKind is provided. + def find_cards( + self, query: str, order: Union[bool, str, int] = False, reverse: bool = False, + ) -> Sequence[int]: + return self.backend.search_cards(query, order, reverse) def find_notes(self, query: str) -> Sequence[int]: return self.backend.search_notes(query) diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 52bdc0d8f..b3d055577 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -34,6 +34,7 @@ from anki.types import assert_impossible_literal assert ankirspy.buildhash() == anki.buildinfo.buildhash SchedTimingToday = pb.SchedTimingTodayOut +BuiltinSortKind = pb.BuiltinSortKind try: import orjson @@ -435,13 +436,23 @@ class RustBackend: def _db_command(self, input: Dict[str, Any]) -> Any: return orjson.loads(self._backend.db_command(orjson.dumps(input))) - def search_cards(self, search: str, order: Union[bool, str]) -> Sequence[int]: + def search_cards( + self, search: str, order: Union[bool, str, int], reverse: bool = False + ) -> Sequence[int]: if isinstance(order, str): mode = pb.SortOrder(custom=order) - elif not order: + elif order is True: + mode = pb.SortOrder(from_config=pb.Empty()) + elif order is False: mode = pb.SortOrder(none=pb.Empty()) else: - mode = pb.SortOrder(from_config=pb.Empty()) + # sadly we can't use the protobuf type in a Union, so we + # have to accept an int and convert it + kind = BuiltinSortKind.Value(BuiltinSortKind.Name(order)) + mode = pb.SortOrder( + builtin=pb.BuiltinSearchOrder(kind=kind, reverse=reverse) + ) + return self._run_command( pb.BackendInput(search_cards=pb.SearchCardsIn(search=search, order=mode)) ).search_cards.card_ids diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index 16d604b17..10758b56e 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -2,6 +2,7 @@ import pytest from anki.consts import * +from anki.rsbackend import BuiltinSortKind from tests.shared import getEmptyCol @@ -120,6 +121,14 @@ def test_findCards(): deck.conf["sortBackwards"] = True deck.flush() assert deck.findCards("", order=True)[0] in latestCardIds + assert ( + deck.find_cards("", order=BuiltinSortKind.CARD_DUE, reverse=False)[0] + == firstCardId + ) + assert ( + deck.find_cards("", order=BuiltinSortKind.CARD_DUE, reverse=True)[0] + != firstCardId + ) # model assert len(deck.findCards("note:basic")) == 5 assert len(deck.findCards("-note:basic")) == 0 diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index cc9815778..8bbba11a7 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -3,8 +3,9 @@ use crate::backend::dbproxy::db_command_bytes; use crate::backend_proto::backend_input::Value; -use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn}; +use crate::backend_proto::{BuiltinSortKind, Empty, RenderedTemplateReplacement, SyncMediaIn}; use crate::collection::{open_collection, Collection}; +use crate::config::SortKind; use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}; use crate::i18n::{tr_args, FString, I18n}; use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}; @@ -588,6 +589,10 @@ impl Backend { Some(V::None(_)) => SortMode::NoOrder, Some(V::Custom(s)) => SortMode::Custom(s), Some(V::FromConfig(_)) => SortMode::FromConfig, + Some(V::Builtin(b)) => SortMode::Builtin { + kind: sort_kind_from_pb(b.kind), + reverse: b.reverse, + }, None => SortMode::FromConfig, } } else { @@ -677,3 +682,25 @@ fn media_sync_progress(p: &MediaSyncProgress, i18n: &I18n) -> pb::MediaSyncProgr ), } } + +fn sort_kind_from_pb(kind: i32) -> SortKind { + use SortKind as SK; + match pb::BuiltinSortKind::from_i32(kind) { + Some(pbkind) => match pbkind { + BuiltinSortKind::NoteCreation => SK::NoteCreation, + BuiltinSortKind::NoteMod => SK::NoteMod, + BuiltinSortKind::NoteField => SK::NoteField, + BuiltinSortKind::NoteTags => SK::NoteTags, + BuiltinSortKind::NoteType => SK::NoteType, + BuiltinSortKind::CardMod => SK::CardMod, + BuiltinSortKind::CardReps => SK::CardReps, + BuiltinSortKind::CardDue => SK::CardDue, + BuiltinSortKind::CardEase => SK::CardEase, + BuiltinSortKind::CardLapses => SK::CardLapses, + BuiltinSortKind::CardInterval => SK::CardInterval, + BuiltinSortKind::CardDeck => SK::CardDeck, + BuiltinSortKind::CardTemplate => SK::CardTemplate, + }, + _ => SortKind::NoteCreation, + } +} diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index e4d8eaf2a..2c10f1e8d 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -13,6 +13,7 @@ use rusqlite::params; pub(crate) enum SortMode { NoOrder, FromConfig, + Builtin { kind: SortKind, reverse: bool }, Custom(String), } @@ -37,6 +38,11 @@ pub(crate) fn search_cards<'a, 'b>( sql.push_str(" order by "); write_order(&mut sql, &conf.browser_sort_kind, conf.browser_sort_reverse)?; } + SortMode::Builtin { kind, reverse } => { + prepare_sort(req, &kind)?; + sql.push_str(" order by "); + write_order(&mut sql, &kind, reverse)?; + } SortMode::Custom(order_clause) => { sql.push_str(" order by "); sql.push_str(&order_clause); From 47fcdd0723173e0008429895a9f5546f095f10a8 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 13:17:00 +1000 Subject: [PATCH 128/150] possible fix for CI failure --- rslib/src/time.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/rslib/src/time.rs b/rslib/src/time.rs index 8d9084c45..e392cbb6e 100644 --- a/rslib/src/time.rs +++ b/rslib/src/time.rs @@ -4,15 +4,35 @@ use std::time; pub(crate) fn i64_unix_secs() -> i64 { - time::SystemTime::now() - .duration_since(time::SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() as i64 + elapsed().as_secs() as i64 } pub(crate) fn i64_unix_millis() -> i64 { + elapsed().as_millis() as i64 +} + +#[cfg(not(test))] +fn elapsed() -> time::Duration { time::SystemTime::now() .duration_since(time::SystemTime::UNIX_EPOCH) .unwrap() - .as_millis() as i64 +} + +// when running in CI, shift the current time away from the cutoff point +// to accomodate unit tests that depend on the current time +#[cfg(test)] +fn elapsed() -> time::Duration { + use chrono::{Local, Timelike}; + + let now = Local::now(); + + let mut elap = time::SystemTime::now() + .duration_since(time::SystemTime::UNIX_EPOCH) + .unwrap(); + + if now.hour() >= 2 && now.hour() < 4 { + elap -= time::Duration::from_secs(60 * 60 * 2); + } + + elap } From c5629e96dfcdc8c7e953a5b908946dd7fb89d6be Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 03:39:38 +0000 Subject: [PATCH 129/150] exclude autogenerated src from build deps prevents unnecessary rebuilds --- rslib/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rslib/Makefile b/rslib/Makefile index 58cbadfbe..2c38d2723 100644 --- a/rslib/Makefile +++ b/rslib/Makefile @@ -25,7 +25,7 @@ develop: .build/vernum ftl/repo ftl/repo: (cd ftl && ./scripts/fetch-latest-translations) -ALL_SOURCE := $(shell ${FIND} src -type f) $(wildcard ftl/*.ftl) +ALL_SOURCE := $(shell ${FIND} src -type f | egrep -v "i18n/autogen|i18n/ftl|_proto.rs") $(wildcard ftl/*.ftl) # nightly currently required for ignoring files in rustfmt.toml RUST_TOOLCHAIN := $(shell cat rust-toolchain) From 69d8cdd9ed1e32683aabe1714ed6402a59fab4cd Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 14:15:02 +1000 Subject: [PATCH 130/150] use backend for v1 and v2 cutoff calculations this should also fix the CI failures, which were happening because the datetime module wasn't matching the shifted time.time() --- proto/backend.proto | 12 +++++++---- pylib/anki/rsbackend.py | 31 +++++++++++++++++++------- pylib/anki/sched.py | 7 +++--- pylib/anki/schedv2.py | 43 ++++++------------------------------- rslib/src/backend/mod.rs | 10 ++++----- rslib/src/sched/cutoff.rs | 14 ++++++------ rslib/src/storage/sqlite.rs | 1 + 7 files changed, 53 insertions(+), 65 deletions(-) diff --git a/proto/backend.proto b/proto/backend.proto index 3821ad49f..66bcebf25 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -6,6 +6,10 @@ package backend_proto; message Empty {} +message OptionalInt32 { + sint32 val = 1; +} + message BackendInit { repeated string preferred_langs = 1; string locale_folder_path = 2; @@ -163,10 +167,10 @@ message TemplateRequirementAny { message SchedTimingTodayIn { int64 created_secs = 1; - sint32 created_mins_west = 2; - int64 now_secs = 3; - sint32 now_mins_west = 4; - sint32 rollover_hour = 5; + int64 now_secs = 2; + OptionalInt32 created_mins_west = 3; + OptionalInt32 now_mins_west = 4; + OptionalInt32 rollover_hour = 5; } message SchedTimingTodayOut { diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index b3d055577..ffad4e277 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -30,6 +30,7 @@ from anki.fluent_pb2 import FluentString as TR from anki.models import AllTemplateReqs from anki.sound import AVTag, SoundOrVideoTag, TTSTag from anki.types import assert_impossible_literal +from anki.utils import intTime assert ankirspy.buildhash() == anki.buildinfo.buildhash @@ -276,19 +277,33 @@ class RustBackend: def sched_timing_today( self, created_secs: int, - created_mins_west: int, - now_secs: int, - now_mins_west: int, - rollover: int, + created_mins_west: Optional[int], + now_mins_west: Optional[int], + rollover: Optional[int], ) -> SchedTimingToday: + if created_mins_west is not None: + crt_west = pb.OptionalInt32(val=created_mins_west) + else: + crt_west = None + + if now_mins_west is not None: + now_west = pb.OptionalInt32(val=now_mins_west) + else: + now_west = None + + if rollover is not None: + roll = pb.OptionalInt32(val=rollover) + else: + roll = None + return self._run_command( pb.BackendInput( sched_timing_today=pb.SchedTimingTodayIn( created_secs=created_secs, - created_mins_west=created_mins_west, - now_secs=now_secs, - now_mins_west=now_mins_west, - rollover_hour=rollover, + now_secs=intTime(), + created_mins_west=crt_west, + now_mins_west=now_west, + rollover_hour=roll, ) ) ).sched_timing_today diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index 0f696934b..7aacb97cd 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -867,10 +867,9 @@ did = ?, queue = %s, due = ?, usn = ? where id = ?""" def _updateCutoff(self) -> None: oldToday = self.today - # days since col created - self.today = int((time.time() - self.col.crt) // 86400) - # end of day cutoff - self.dayCutoff = self.col.crt + (self.today + 1) * 86400 + timing = self._timing_today() + self.today = timing.days_elapsed + self.dayCutoff = timing.next_day_at if oldToday != self.today: self.col.log(self.today, self.dayCutoff) # update all daily counts, but don't save decks to prevent needless diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index ad72381aa..0b8050887 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -3,7 +3,6 @@ from __future__ import annotations -import datetime import itertools import random import time @@ -1353,13 +1352,8 @@ where id = ? def _updateCutoff(self) -> None: oldToday = self.today timing = self._timing_today() - - if self._new_timezone_enabled(): - self.today = timing.days_elapsed - self.dayCutoff = timing.next_day_at - else: - self.today = self._daysSinceCreation() - self.dayCutoff = self._dayCutoff() + self.today = timing.days_elapsed + self.dayCutoff = timing.next_day_at if oldToday != self.today: self.col.log(self.today, self.dayCutoff) @@ -1385,51 +1379,28 @@ where id = ? if time.time() > self.dayCutoff: self.reset() - def _dayCutoff(self) -> int: - rolloverTime = self.col.conf.get("rollover", 4) - if rolloverTime < 0: - rolloverTime = 24 + rolloverTime - date = datetime.datetime.today() - date = date.replace(hour=rolloverTime, minute=0, second=0, microsecond=0) - if date < datetime.datetime.today(): - date = date + datetime.timedelta(days=1) - - stamp = int(time.mktime(date.timetuple())) - return stamp - - def _daysSinceCreation(self) -> int: - startDate = datetime.datetime.fromtimestamp(self.col.crt) - startDate = startDate.replace( - hour=self._rolloverHour(), minute=0, second=0, microsecond=0 - ) - return int((time.time() - time.mktime(startDate.timetuple())) // 86400) - def _rolloverHour(self) -> int: return self.col.conf.get("rollover", 4) # New timezone handling ########################################################################## - def _new_timezone_enabled(self) -> bool: - return self.col.conf.get("creationOffset") is not None - def _timing_today(self) -> SchedTimingToday: return self.col.backend.sched_timing_today( self.col.crt, self._creation_timezone_offset(), - intTime(), self._current_timezone_offset(), self._rolloverHour(), ) - def _current_timezone_offset(self) -> int: + def _current_timezone_offset(self) -> Optional[int]: if self.col.server: - return self.col.conf.get("localOffset", 0) + return self.col.conf.get("localOffset", None) else: - return self.col.backend.local_minutes_west(intTime()) + return None - def _creation_timezone_offset(self) -> int: - return self.col.conf.get("creationOffset", 0) + def _creation_timezone_offset(self) -> Optional[int]: + return self.col.conf.get("creationOffset", None) def set_creation_offset(self): """Save the UTC west offset at the time of creation into the DB. diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 8bbba11a7..0240d8e9d 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -13,7 +13,7 @@ use crate::log::{default_logger, Logger}; use crate::media::check::MediaChecker; use crate::media::sync::MediaSyncProgress; use crate::media::MediaManager; -use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today_v2_new}; +use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today}; use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span}; use crate::search::{search_cards, search_notes, SortMode}; use crate::template::{ @@ -350,12 +350,12 @@ impl Backend { } fn sched_timing_today(&self, input: pb::SchedTimingTodayIn) -> pb::SchedTimingTodayOut { - let today = sched_timing_today_v2_new( + let today = sched_timing_today( input.created_secs as i64, - input.created_mins_west, input.now_secs as i64, - input.now_mins_west, - input.rollover_hour as i8, + input.created_mins_west.map(|v| v.val), + input.now_mins_west.map(|v| v.val), + input.rollover_hour.map(|v| v.val as i8), ); pb::SchedTimingTodayOut { days_elapsed: today.days_elapsed, diff --git a/rslib/src/sched/cutoff.rs b/rslib/src/sched/cutoff.rs index 04bdb60fa..8ba4de722 100644 --- a/rslib/src/sched/cutoff.rs +++ b/rslib/src/sched/cutoff.rs @@ -1,7 +1,6 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::time::i64_unix_secs; use chrono::{Date, Duration, FixedOffset, Local, TimeZone}; #[derive(Debug, PartialEq, Clone, Copy)] @@ -140,26 +139,25 @@ fn sched_timing_today_v2_legacy( /// Based on provided input, get timing info from the relevant function. pub(crate) fn sched_timing_today( created_secs: i64, + now_secs: i64, created_mins_west: Option, now_mins_west: Option, rollover_hour: Option, ) -> SchedTimingToday { - let now = i64_unix_secs(); - match (rollover_hour, created_mins_west) { (None, _) => { // if rollover unset, v1 scheduler - sched_timing_today_v1(created_secs, now) + sched_timing_today_v1(created_secs, now_secs) } (Some(roll), None) => { // if creation offset unset, v2 legacy cutoff using local timezone - let offset = local_minutes_west_for_stamp(now); - sched_timing_today_v2_legacy(created_secs, roll, now, offset) + let offset = local_minutes_west_for_stamp(now_secs); + sched_timing_today_v2_legacy(created_secs, roll, now_secs, offset) } (Some(roll), Some(crt_west)) => { // new cutoff code, using provided current timezone, falling back on local timezone - let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now)); - sched_timing_today_v2_new(created_secs, crt_west, now, now_west, roll) + let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now_secs)); + sched_timing_today_v2_new(created_secs, crt_west, now_secs, now_west, roll) } } } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 23d6eec1b..bbda2fab9 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -324,6 +324,7 @@ impl StorageContext<'_> { self.timing_today = Some(sched_timing_today( crt, + i64_unix_secs(), conf.creation_offset, now_offset, conf.rollover, From ac36fba90f8b362838c7d16c75f21273619723d3 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 14:41:01 +1000 Subject: [PATCH 131/150] handle the two remaining timing issues --- pylib/tests/shared.py | 25 +++++++++++++++++++++ pylib/tests/test_exporting.py | 2 ++ pylib/tests/test_find.py | 41 +++++++++++++++++++---------------- pylib/tests/test_schedv2.py | 11 ---------- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/pylib/tests/shared.py b/pylib/tests/shared.py index df83eac73..a89252ada 100644 --- a/pylib/tests/shared.py +++ b/pylib/tests/shared.py @@ -1,9 +1,22 @@ import os import shutil import tempfile +import time from anki import Collection as aopen +# Between 2-4AM, shift the time back so test assumptions hold. +lt = time.localtime() +if lt.tm_hour >= 2 and lt.tm_hour < 4: + orig_time = time.time + + def adjusted_time(): + return orig_time() - 60 * 60 * 2 + + time.time = adjusted_time +else: + orig_time = None + def assertException(exception, func): found = False @@ -48,3 +61,15 @@ def getUpgradeDeckPath(name="anki12.anki"): testDir = os.path.dirname(__file__) + + +def errorsAfterMidnight(func): + lt = time.localtime() + if lt.tm_hour < 4: + print("test disabled around cutoff", func) + else: + func() + + +def isNearCutoff(): + return orig_time is not None diff --git a/pylib/tests/test_exporting.py b/pylib/tests/test_exporting.py index 1ab6b1ee1..db29d018e 100644 --- a/pylib/tests/test_exporting.py +++ b/pylib/tests/test_exporting.py @@ -6,6 +6,7 @@ import tempfile from anki import Collection as aopen from anki.exporting import * from anki.importing import Anki2Importer +from tests.shared import errorsAfterMidnight from tests.shared import getEmptyCol as getEmptyColOrig @@ -97,6 +98,7 @@ def test_export_ankipkg(): e.exportInto(newname) +@errorsAfterMidnight def test_export_anki_due(): setup1() deck = getEmptyCol() diff --git a/pylib/tests/test_find.py b/pylib/tests/test_find.py index 10758b56e..66f462c84 100644 --- a/pylib/tests/test_find.py +++ b/pylib/tests/test_find.py @@ -3,7 +3,7 @@ import pytest from anki.consts import * from anki.rsbackend import BuiltinSortKind -from tests.shared import getEmptyCol +from tests.shared import getEmptyCol, isNearCutoff class DummyCollection: @@ -189,19 +189,27 @@ def test_findCards(): assert len(deck.findCards("prop:ease>2")) == 1 assert len(deck.findCards("-prop:ease>2")) > 1 # recently failed - assert len(deck.findCards("rated:1:1")) == 0 - assert len(deck.findCards("rated:1:2")) == 0 - c = deck.sched.getCard() - deck.sched.answerCard(c, 2) - assert len(deck.findCards("rated:1:1")) == 0 - assert len(deck.findCards("rated:1:2")) == 1 - c = deck.sched.getCard() - deck.sched.answerCard(c, 1) - assert len(deck.findCards("rated:1:1")) == 1 - assert len(deck.findCards("rated:1:2")) == 1 - assert len(deck.findCards("rated:1")) == 2 - assert len(deck.findCards("rated:0:2")) == 0 - assert len(deck.findCards("rated:2:2")) == 1 + if not isNearCutoff(): + assert len(deck.findCards("rated:1:1")) == 0 + assert len(deck.findCards("rated:1:2")) == 0 + c = deck.sched.getCard() + deck.sched.answerCard(c, 2) + assert len(deck.findCards("rated:1:1")) == 0 + assert len(deck.findCards("rated:1:2")) == 1 + c = deck.sched.getCard() + deck.sched.answerCard(c, 1) + assert len(deck.findCards("rated:1:1")) == 1 + assert len(deck.findCards("rated:1:2")) == 1 + assert len(deck.findCards("rated:1")) == 2 + assert len(deck.findCards("rated:0:2")) == 0 + assert len(deck.findCards("rated:2:2")) == 1 + # added + assert len(deck.findCards("added:0")) == 0 + deck.db.execute("update cards set id = id - 86400*1000 where id = ?", id) + assert len(deck.findCards("added:1")) == deck.cardCount() - 1 + assert len(deck.findCards("added:2")) == deck.cardCount() + else: + print("some find tests disabled near cutoff") # empty field assert len(deck.findCards("front:")) == 0 f = deck.newNote() @@ -215,11 +223,6 @@ def test_findCards(): assert len(deck.findCards("-(tag:monkey OR tag:sheep)")) == 6 assert len(deck.findCards("tag:monkey or (tag:sheep sheep)")) == 2 assert len(deck.findCards("tag:monkey or (tag:sheep octopus)")) == 1 - # added - assert len(deck.findCards("added:0")) == 0 - deck.db.execute("update cards set id = id - 86400*1000 where id = ?", id) - assert len(deck.findCards("added:1")) == deck.cardCount() - 1 - assert len(deck.findCards("added:2")) == deck.cardCount() # flag with pytest.raises(Exception): deck.findCards("flag:12") diff --git a/pylib/tests/test_schedv2.py b/pylib/tests/test_schedv2.py index 0df6c2005..1feb94f4c 100644 --- a/pylib/tests/test_schedv2.py +++ b/pylib/tests/test_schedv2.py @@ -16,17 +16,6 @@ def getEmptyCol(): return col -# Between 2-4AM, shift the time back so test assumptions hold. -lt = time.localtime() -if lt.tm_hour >= 2 and lt.tm_hour < 4: - orig_time = time.time - - def adjusted_time(): - return orig_time() - 60 * 60 * 2 - - time.time = adjusted_time - - def test_clock(): d = getEmptyCol() if (d.sched.dayCutoff - intTime()) < 10 * 60: From 6c6817563e7c09f9bc3682732074a7f0c6f01b03 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 17:32:34 +1000 Subject: [PATCH 132/150] fix sync error introduced by 69d8cdd9ed1e32683aabe1714ed6402a59fab4cd --- pylib/anki/schedv2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 0b8050887..62ddbc852 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1393,11 +1393,14 @@ where id = ? self._rolloverHour(), ) - def _current_timezone_offset(self) -> Optional[int]: + def _current_timezone_offset(self) -> int: if self.col.server: return self.col.conf.get("localOffset", None) else: - return None + # note: while we could return None to sched_timing_today to have + # the backend calculate it, this function is also used to set + # localOffset, so it must not return None + return self.col.backend.local_minutes_west(intTime()) def _creation_timezone_offset(self) -> Optional[int]: return self.col.conf.get("creationOffset", None) From 25ff4642ec489755182003ea9b6801b91ca6a208 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 17:33:14 +1000 Subject: [PATCH 133/150] accept now_mins_west for v2 legacy timing as well --- rslib/src/sched/cutoff.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rslib/src/sched/cutoff.rs b/rslib/src/sched/cutoff.rs index 8ba4de722..ba3ba484e 100644 --- a/rslib/src/sched/cutoff.rs +++ b/rslib/src/sched/cutoff.rs @@ -144,19 +144,18 @@ pub(crate) fn sched_timing_today( now_mins_west: Option, rollover_hour: Option, ) -> SchedTimingToday { + let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now_secs)); match (rollover_hour, created_mins_west) { (None, _) => { // if rollover unset, v1 scheduler sched_timing_today_v1(created_secs, now_secs) } (Some(roll), None) => { - // if creation offset unset, v2 legacy cutoff using local timezone - let offset = local_minutes_west_for_stamp(now_secs); - sched_timing_today_v2_legacy(created_secs, roll, now_secs, offset) + // if creationOffset unset, v2 scheduler with legacy cutoff handling + sched_timing_today_v2_legacy(created_secs, roll, now_secs, now_west) } (Some(roll), Some(crt_west)) => { - // new cutoff code, using provided current timezone, falling back on local timezone - let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now_secs)); + // v2 scheduler, new cutoff handling sched_timing_today_v2_new(created_secs, crt_west, now_secs, now_west, roll) } } From 430f1ad616aa9f78100be19e1390aae541092575 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 18:52:44 +1000 Subject: [PATCH 134/150] handle trailing whitespace inside group --- rslib/src/search/parser.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs index cca8a1817..4018cc56c 100644 --- a/rslib/src/search/parser.rs +++ b/rslib/src/search/parser.rs @@ -162,6 +162,9 @@ fn group_inner(input: &str) -> IResult<&str, Vec> { if nodes.is_empty() { Err(nom::Err::Error((remaining, nom::error::ErrorKind::Many1))) } else { + // chomp any trailing whitespace + let (remaining, _) = whitespace0(remaining)?; + Ok((remaining, nodes)) } } @@ -428,6 +431,16 @@ mod test { ] ); + // including in groups + assert_eq!( + parse("( t t2 )")?, + vec![Group(vec![ + Search(UnqualifiedText("t".into())), + And, + Search(UnqualifiedText("t2".into())) + ])] + ); + assert_eq!( parse(r#"hello -(world and "foo:bar baz") OR test"#)?, vec![ From 0b94eee97ea9e7473d6644c7226b4b40cb840c72 Mon Sep 17 00:00:00 2001 From: zjosua Date: Sun, 22 Mar 2020 11:49:40 +0100 Subject: [PATCH 135/150] Fill _lrnQueue with tuples, not lists --- pylib/anki/schedv2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 62ddbc852..9f14e253b 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -549,6 +549,8 @@ limit %d""" % (self._deckLimit(), self.reportLimit), cutoff, ) + for i in range(len(self._lrnQueue)): + self._lrnQueue[i] = (self._lrnQueue[i][0], self._lrnQueue[i][1]) # as it arrives sorted by did first, we need to sort it self._lrnQueue.sort() return self._lrnQueue From 427bf268fc979a6190b4689f42705f5e17048a20 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 22 Mar 2020 21:56:02 +1000 Subject: [PATCH 136/150] apply same list->tuple fix to v1 sched --- pylib/anki/sched.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylib/anki/sched.py b/pylib/anki/sched.py index 7aacb97cd..0c59773bc 100644 --- a/pylib/anki/sched.py +++ b/pylib/anki/sched.py @@ -291,6 +291,8 @@ limit %d""" % (self._deckLimit(), self.reportLimit), self.dayCutoff, ) + for i in range(len(self._lrnQueue)): + self._lrnQueue[i] = (self._lrnQueue[i][0], self._lrnQueue[i][1]) # as it arrives sorted by did first, we need to sort it self._lrnQueue.sort() return self._lrnQueue From 75b7ebb1565672fca3e4d5b4b61822fc8d5581c4 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 07:40:50 +1000 Subject: [PATCH 137/150] add back new_timezone_enabled(), as it's used in the prefs screen --- pylib/anki/schedv2.py | 9 ++++++--- qt/aqt/preferences.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 9f14e253b..a060bb06a 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1384,9 +1384,6 @@ where id = ? def _rolloverHour(self) -> int: return self.col.conf.get("rollover", 4) - # New timezone handling - ########################################################################## - def _timing_today(self) -> SchedTimingToday: return self.col.backend.sched_timing_today( self.col.crt, @@ -1407,6 +1404,12 @@ where id = ? def _creation_timezone_offset(self) -> Optional[int]: return self.col.conf.get("creationOffset", None) + # New timezone handling - GUI helpers + ########################################################################## + + def new_timezone_enabled(self) -> bool: + return self.col.conf.get("creationOffset") is not None + def set_creation_offset(self): """Save the UTC west offset at the time of creation into the DB. diff --git a/qt/aqt/preferences.py b/qt/aqt/preferences.py index a72849c33..6f5c020c5 100644 --- a/qt/aqt/preferences.py +++ b/qt/aqt/preferences.py @@ -98,7 +98,7 @@ class Preferences(QDialog): f.new_timezone.setVisible(False) else: f.newSched.setChecked(True) - f.new_timezone.setChecked(self.mw.col.sched._new_timezone_enabled()) + f.new_timezone.setChecked(self.mw.col.sched.new_timezone_enabled()) def updateCollection(self): f = self.form @@ -124,7 +124,7 @@ class Preferences(QDialog): qc["dayLearnFirst"] = f.dayLearnFirst.isChecked() self._updateDayCutoff() if self.mw.col.schedVer() != 1: - was_enabled = self.mw.col.sched._new_timezone_enabled() + was_enabled = self.mw.col.sched.new_timezone_enabled() is_enabled = f.new_timezone.isChecked() if was_enabled != is_enabled: if is_enabled: From dc8cf9d554f3fa76dbc62ad7ec5f85a409cd5958 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 09:27:42 +1000 Subject: [PATCH 138/150] release GIL during collection open/close --- pylib/anki/rsbackend.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index ffad4e277..630d74368 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -256,11 +256,14 @@ class RustBackend: media_db_path=media_db_path, log_path=log_path, ) - ) + ), + release_gil=True, ) def close_collection(self): - self._run_command(pb.BackendInput(close_collection=pb.Empty())) + self._run_command( + pb.BackendInput(close_collection=pb.Empty()), release_gil=True + ) def template_requirements( self, template_fronts: List[str], field_map: Dict[str, int] From cd9ceebd5949f7f42df4ebac359fc8aff2f6fe55 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 13:31:53 +1000 Subject: [PATCH 139/150] simplify how the local offset is passed around - no need to store it in conf - move local_minutes_west() call to collection --- pylib/anki/collection.py | 7 ++----- pylib/anki/schedv2.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index b8258adb4..ae42bec0c 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -138,10 +138,6 @@ class _Collection: self.sched = V1Scheduler(self) elif ver == 2: self.sched = V2Scheduler(self) - if not self.server: - self.conf["localOffset"] = self.sched._current_timezone_offset() - elif self.server.minutes_west is not None: - self.conf["localOffset"] = self.server.minutes_west def changeSchedulerVer(self, ver: int) -> None: if ver == self.schedVer(): @@ -164,12 +160,13 @@ class _Collection: self._loadScheduler() + # the sync code uses this to send the local timezone to AnkiWeb def localOffset(self) -> Optional[int]: "Minutes west of UTC. Only applies to V2 scheduler." if isinstance(self.sched, V1Scheduler): return None else: - return self.sched._current_timezone_offset() + return self.backend.local_minutes_west(intTime()) # DB-related ########################################################################## diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index a060bb06a..7cff185aa 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1392,14 +1392,16 @@ where id = ? self._rolloverHour(), ) - def _current_timezone_offset(self) -> int: + def _current_timezone_offset(self) -> Optional[int]: if self.col.server: - return self.col.conf.get("localOffset", None) + mins = self.col.server.minutes_west + if mins is not None: + return mins + # older Anki versions stored the local offset in + # the config + return self.col.conf.get("localOffset", 0) else: - # note: while we could return None to sched_timing_today to have - # the backend calculate it, this function is also used to set - # localOffset, so it must not return None - return self.col.backend.local_minutes_west(intTime()) + return None def _creation_timezone_offset(self) -> Optional[int]: return self.col.conf.get("creationOffset", None) From 4e2e0d1b845dd7c9e722640b2f18f4fa5e8d4cf2 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 13:17:42 +1000 Subject: [PATCH 140/150] fix setting of wal --- pylib/anki/collection.py | 2 ++ rslib/src/storage/sqlite.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index ae42bec0c..920cabfff 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -251,6 +251,8 @@ crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", if self.db: if save: self.save(trx=False) + else: + self.db.rollback() if not self.server: self.db.execute("pragma journal_mode = delete") self.backend.close_collection() diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index bbda2fab9..7e5b2f87a 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -53,7 +53,7 @@ fn open_or_create_collection_db(path: &Path) -> Result { db.pragma_update(None, "page_size", &4096)?; db.pragma_update(None, "cache_size", &(-40 * 1024))?; db.pragma_update(None, "legacy_file_format", &false)?; - db.pragma_update(None, "journal", &"wal")?; + db.pragma_update(None, "journal_mode", &"wal")?; db.pragma_update(None, "temp_store", &"memory")?; db.set_prepared_statement_cache_capacity(50); From 966eb666f00321c98ae8a51afb2bf303ece50297 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 13:18:48 +1000 Subject: [PATCH 141/150] fix v2 timing being returned for v1 users --- pylib/anki/schedv2.py | 5 ++++- pylib/anki/storage.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pylib/anki/schedv2.py b/pylib/anki/schedv2.py index 7cff185aa..76110b686 100644 --- a/pylib/anki/schedv2.py +++ b/pylib/anki/schedv2.py @@ -1385,11 +1385,14 @@ where id = ? return self.col.conf.get("rollover", 4) def _timing_today(self) -> SchedTimingToday: + roll: Optional[int] = None + if self.col.schedVer() > 1: + roll = self._rolloverHour() return self.col.backend.sched_timing_today( self.col.crt, self._creation_timezone_offset(), self._current_timezone_offset(), - self._rolloverHour(), + roll, ) def _current_timezone_offset(self) -> Optional[int]: diff --git a/pylib/anki/storage.py b/pylib/anki/storage.py index f19c4d327..948b45839 100644 --- a/pylib/anki/storage.py +++ b/pylib/anki/storage.py @@ -5,6 +5,7 @@ import copy import json import os import weakref +from dataclasses import dataclass from typing import Any, Dict, Optional, Tuple from anki.collection import _Collection @@ -23,6 +24,7 @@ from anki.stdmodels import ( from anki.utils import intTime +@dataclass class ServerData: minutes_west: Optional[int] = None From c6153421307565165e46993f530a823389346ad3 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 16:08:06 +1000 Subject: [PATCH 142/150] fix English being shown in prefs for fully qualified langs like zh-CN https://anki.tenderapp.com/discussions/ankidesktop/39845-a-new-bug-has-been-found --- qt/aqt/preferences.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qt/aqt/preferences.py b/qt/aqt/preferences.py index 6f5c020c5..8f183030a 100644 --- a/qt/aqt/preferences.py +++ b/qt/aqt/preferences.py @@ -62,6 +62,8 @@ class Preferences(QDialog): lang = anki.lang.currentLang if lang in anki.lang.compatMap: lang = anki.lang.compatMap[lang] + else: + lang = lang.replace("-", "_") try: return codes.index(lang) except: From 14a970e923df618995d8599adc8dd52b04075d81 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 18:38:28 +1000 Subject: [PATCH 143/150] rename long filenames in fields if files renamed in a previous sync --- rslib/src/media/check.rs | 36 ++++++++++++++++++++++++++++++++---- rslib/src/media/files.rs | 2 +- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 843d14493..971747b50 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -8,7 +8,8 @@ use crate::latex::extract_latex_expanding_clozes; use crate::log::debug; use crate::media::database::MediaDatabaseContext; use crate::media::files::{ - data_for_file, filename_if_normalized, trash_folder, MEDIA_SYNC_FILESIZE_LIMIT, + data_for_file, filename_if_normalized, normalize_nfc_filename, trash_folder, + MEDIA_SYNC_FILESIZE_LIMIT, }; use crate::notes::{for_every_note, set_note, Note}; use crate::text::{normalize_to_nfc, MediaRef}; @@ -17,6 +18,7 @@ use coarsetime::Instant; use lazy_static::lazy_static; use regex::Regex; use std::collections::{HashMap, HashSet}; +use std::path::Path; use std::{borrow::Cow, fs, io}; lazy_static! { @@ -393,7 +395,12 @@ where info: "missing note type".to_string(), kind: DBErrorKind::MissingEntity, })?; - if fix_and_extract_media_refs(note, &mut referenced_files, renamed)? { + if fix_and_extract_media_refs( + note, + &mut referenced_files, + renamed, + &self.mgr.media_folder, + )? { // note was modified, needs saving set_note(&self.ctx.storage.db, note, nt)?; collection_modified = true; @@ -417,11 +424,17 @@ fn fix_and_extract_media_refs( note: &mut Note, seen_files: &mut HashSet, renamed: &HashMap, + media_folder: &Path, ) -> Result { let mut updated = false; for idx in 0..note.fields().len() { - let field = normalize_and_maybe_rename_files(¬e.fields()[idx], renamed, seen_files); + let field = normalize_and_maybe_rename_files( + ¬e.fields()[idx], + renamed, + seen_files, + media_folder, + ); if let Cow::Owned(field) = field { // field was modified, need to save note.set_field(idx, field)?; @@ -438,6 +451,7 @@ fn normalize_and_maybe_rename_files<'a>( field: &'a str, renamed: &HashMap, seen_files: &mut HashSet, + media_folder: &Path, ) -> Cow<'a, str> { let refs = extract_media_refs(field); let mut field: Cow = field.into(); @@ -454,7 +468,21 @@ fn normalize_and_maybe_rename_files<'a>( if let Some(new_name) = renamed.get(fname.as_ref()) { fname = new_name.to_owned().into(); } - // if it was not in NFC or was renamed, update the field + // if the filename was in NFC and was not renamed as part of the + // media check, it may have already been renamed during a previous + // sync. If that's the case and the renamed version exists on disk, + // we'll need to update the field to match it. It may be possible + // to remove this check in the future once we can be sure all media + // files stored on AnkiWeb are in normalized form. + if matches!(fname, Cow::Borrowed(_)) { + if let Cow::Owned(normname) = normalize_nfc_filename(fname.as_ref().into()) { + let path = media_folder.join(&normname); + if path.exists() { + fname = normname.into(); + } + } + } + // update the field if the filename was modified if let Cow::Owned(ref new_name) = fname { field = rename_media_ref_in_field(field.as_ref(), &media_ref, new_name).into(); } diff --git a/rslib/src/media/files.rs b/rslib/src/media/files.rs index bd8dfdcb4..791f0ad5e 100644 --- a/rslib/src/media/files.rs +++ b/rslib/src/media/files.rs @@ -84,7 +84,7 @@ pub(crate) fn normalize_filename(fname: &str) -> Cow { } /// See normalize_filename(). This function expects NFC-normalized input. -fn normalize_nfc_filename(mut fname: Cow) -> Cow { +pub(crate) fn normalize_nfc_filename(mut fname: Cow) -> Cow { if fname.chars().any(disallowed_char) { fname = fname.replace(disallowed_char, "").into() } From 7a4f3d031844bcd895b24bf5075381c4d3058e22 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 17:44:26 +1000 Subject: [PATCH 144/150] dump more info in card()/bcard() --- qt/aqt/main.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index efb046f66..1ce9d1a20 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -1368,11 +1368,42 @@ will be lost. Continue?""" sys.stderr = self._oldStderr sys.stdout = self._oldStdout - def _debugCard(self): - return self.reviewer.card.__dict__ + def _card_repr(self, card: anki.cards.Card) -> None: + import pprint, copy - def _debugBrowserCard(self): - return aqt.dialogs._dialogs["Browser"][1].card.__dict__ + if not card: + print("no card") + return + + print("Front:", card.question()) + print("\n") + print("Back:", card.answer()) + + print("\nNote:") + note = copy.copy(card.note()) + for k, v in note.items(): + print(f"- {k}:", v) + + print("\n") + del note.fields + del note._fmap + del note._model + pprint.pprint(note.__dict__) + + print("\nCard:") + c = copy.copy(card) + c._render_output = None + pprint.pprint(c.__dict__) + + def _debugCard(self) -> Optional[anki.cards.Card]: + card = self.reviewer.card + self._card_repr(card) + return card + + def _debugBrowserCard(self) -> Optional[anki.cards.Card]: + card = aqt.dialogs._dialogs["Browser"][1].card + self._card_repr(card) + return card def onDebugPrint(self, frm): cursor = frm.text.textCursor() From 7d94465256c8948d4737a222f038bb0f6f37475b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 18:39:37 +1000 Subject: [PATCH 145/150] cache dark mode value so UI doesn't break when it changes https://anki.tenderapp.com/discussions/ankidesktop/39550-cant-deactivate-night-mode-on-2121-for-mac --- qt/aqt/theme.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qt/aqt/theme.py b/qt/aqt/theme.py index 492538534..b77e248c6 100644 --- a/qt/aqt/theme.py +++ b/qt/aqt/theme.py @@ -4,7 +4,7 @@ import platform import sys -from typing import Dict +from typing import Dict, Optional from anki.utils import isMac from aqt import QApplication, gui_hooks, isWin @@ -17,6 +17,7 @@ class ThemeManager: _icon_cache_light: Dict[str, QIcon] = {} _icon_cache_dark: Dict[str, QIcon] = {} _icon_size = 128 + _macos_dark_mode_cached: Optional[bool] = None def macos_dark_mode(self) -> bool: if not getattr(sys, "frozen", False): @@ -25,9 +26,13 @@ class ThemeManager: return False if qtminor < 13: return False - import darkdetect # pylint: disable=import-error + if self._macos_dark_mode_cached is None: + import darkdetect # pylint: disable=import-error - return darkdetect.isDark() is True + # cache the value, as the interface gets messed up + # if the value changes after starting Anki + self._macos_dark_mode_cached = darkdetect.isDark() is True + return self._macos_dark_mode_cached def get_night_mode(self) -> bool: return self.macos_dark_mode() or self._night_mode_preference From f429986246545c7e14527e9d6a0b36224d3140e0 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 18:57:23 +1000 Subject: [PATCH 146/150] fix collection_did_load() not being called, and remove dead code https://anki.tenderapp.com/discussions/ankidesktop/39765-_colloadingstate-is-never-run-thus-collection_did_load-hook-is-never-triggered --- qt/aqt/main.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 1ce9d1a20..baf5c49f9 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -466,6 +466,7 @@ close the profile or restart Anki.""" self.setEnabled(True) self.maybeEnableUndo() + gui_hooks.collection_did_load(self.col) self.moveToState("deckBrowser") return True @@ -602,14 +603,6 @@ from the profile screen." self.maybe_check_for_addon_updates() self.deckBrowser.show() - def _colLoadingState(self, oldState) -> None: - "Run once, when col is loaded." - self.enableColMenuItems() - # ensure cwd is set if media dir exists - self.col.media.dir() - gui_hooks.collection_did_load(self.col) - self.moveToState("overview") - def _selectedDeck(self) -> Optional[Dict[str, Any]]: did = self.col.decks.selected() if not self.col.decks.nameOrNone(did): From f889616ef14e6aa8303b795f21dc91b49a970f98 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 19:06:13 +1000 Subject: [PATCH 147/150] don't pop up network errors for media sync log them instead --- qt/aqt/mediasync.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qt/aqt/mediasync.py b/qt/aqt/mediasync.py index d105a098d..fd0238254 100644 --- a/qt/aqt/mediasync.py +++ b/qt/aqt/mediasync.py @@ -11,7 +11,8 @@ from typing import List, Union import aqt from anki import hooks from anki.consts import SYNC_BASE -from anki.rsbackend import TR, Interrupted, MediaSyncProgress, Progress, ProgressKind +from anki.rsbackend import TR, Interrupted, MediaSyncProgress, Progress, \ + ProgressKind, NetworkError from anki.types import assert_impossible from anki.utils import intTime from aqt import gui_hooks @@ -100,6 +101,10 @@ class MediaSyncer: if isinstance(exc, Interrupted): self._log_and_notify(tr(TR.SYNC_MEDIA_ABORTED)) return + elif isinstance(exc, NetworkError): + # avoid popups for network errors + self._log_and_notify(str(exc)) + return self._log_and_notify(tr(TR.SYNC_MEDIA_FAILED)) showWarning(str(exc)) From 84eaf43525034f0ba5fd7f63efbdeace74328e97 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 19:15:32 +1000 Subject: [PATCH 148/150] reduce the chances of a race condition in mplayer code Not perfect, it may still happen. https://anki.tenderapp.com/discussions/ankidesktop/39832-an-error-occurred-audio --- qt/aqt/sound.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qt/aqt/sound.py b/qt/aqt/sound.py index 94b71032f..cdd9b65d0 100644 --- a/qt/aqt/sound.py +++ b/qt/aqt/sound.py @@ -393,8 +393,9 @@ class SimpleMplayerSlaveModePlayer(SimpleMplayerPlayer): The trailing newline is automatically added.""" str_args = [str(x) for x in args] - self._process.stdin.write(" ".join(str_args).encode("utf8") + b"\n") - self._process.stdin.flush() + if self._process: + self._process.stdin.write(" ".join(str_args).encode("utf8") + b"\n") + self._process.stdin.flush() def seek_relative(self, secs: int) -> None: self.command("seek", secs, 0) From 11a4d582b4824c4900ae2b1bb040f7b0c0d24a3f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 19:53:57 +1000 Subject: [PATCH 149/150] convert asc to desc instead of appending desc to the end of the order as the latter doesn't work when sorting on more than one column https://anki.tenderapp.com/discussions/beta-testing/1868-anki-2124-beta#comment_48174812 --- pylib/anki/collection.py | 12 +++++++++--- rslib/src/search/cards.rs | 36 +++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index 920cabfff..fedd21f76 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -617,10 +617,16 @@ where c.nid = n.id and c.id in %s group by nid""" # if order=True, use the sort order stored in the collection config # if order=False, do no ordering - # if order is a string, that text is added after 'order by' in the sql statement - # if order is an int enum, sort using that builtin sort. # - # the reverse argument only applies when a BuiltinSortKind is provided. + # if order is a string, that text is added after 'order by' in the sql statement. + # you must add ' asc' or ' desc' to the order, as Anki will replace asc with + # desc and vice versa when reverse is set in the collection config, eg + # order="c.ivl asc, c.due desc" + # + # if order is an int enum, sort using that builtin sort, eg + # col.find_cards("", order=BuiltinSortKind.CARD_DUE) + # the reverse argument only applies when a BuiltinSortKind is provided; + # otherwise the collection config defines whether reverse is set or not def find_cards( self, query: str, order: Union[bool, str, int] = False, reverse: bool = False, ) -> Sequence[int]: diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 2c10f1e8d..8ab045b7b 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -61,29 +61,35 @@ pub(crate) fn search_cards<'a, 'b>( fn write_order(sql: &mut String, kind: &SortKind, reverse: bool) -> Result<()> { let tmp_str; let order = match kind { - SortKind::NoteCreation => "n.id, c.ord", - SortKind::NoteMod => "n.mod, c.ord", - SortKind::NoteField => "n.sfld collate nocase, c.ord", - SortKind::CardMod => "c.mod", - SortKind::CardReps => "c.reps", - SortKind::CardDue => "c.type, c.due", + SortKind::NoteCreation => "n.id asc, c.ord asc", + SortKind::NoteMod => "n.mod asc, c.ord asc", + SortKind::NoteField => "n.sfld collate nocase asc, c.ord asc", + SortKind::CardMod => "c.mod asc", + SortKind::CardReps => "c.reps asc", + SortKind::CardDue => "c.type asc, c.due asc", SortKind::CardEase => { - tmp_str = format!("c.type = {}, c.factor", CardType::New as i8); + tmp_str = format!("c.type = {} asc, c.factor asc", CardType::New as i8); &tmp_str } - SortKind::CardLapses => "c.lapses", - SortKind::CardInterval => "c.ivl", - SortKind::NoteTags => "n.tags", - SortKind::CardDeck => "(select v from sort_order where k = c.did)", - SortKind::NoteType => "(select v from sort_order where k = n.mid)", - SortKind::CardTemplate => "(select v from sort_order where k1 = n.mid and k2 = c.ord)", + SortKind::CardLapses => "c.lapses asc", + SortKind::CardInterval => "c.ivl asc", + SortKind::NoteTags => "n.tags asc", + SortKind::CardDeck => "(select v from sort_order where k = c.did) asc", + SortKind::NoteType => "(select v from sort_order where k = n.mid) asc", + SortKind::CardTemplate => "(select v from sort_order where k1 = n.mid and k2 = c.ord) asc", }; if order.is_empty() { return Ok(()); } - sql.push_str(order); if reverse { - sql.push_str(" desc"); + sql.push_str( + &order + .to_ascii_lowercase() + .replace(" desc", "") + .replace(" asc", " desc"), + ) + } else { + sql.push_str(order); } Ok(()) } From b1a8107aa153bb91ba37b58576035118f7cc0a67 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 23 Mar 2020 19:54:01 +1000 Subject: [PATCH 150/150] formatting --- qt/aqt/mediasync.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qt/aqt/mediasync.py b/qt/aqt/mediasync.py index fd0238254..ebafee8d6 100644 --- a/qt/aqt/mediasync.py +++ b/qt/aqt/mediasync.py @@ -11,8 +11,14 @@ from typing import List, Union import aqt from anki import hooks from anki.consts import SYNC_BASE -from anki.rsbackend import TR, Interrupted, MediaSyncProgress, Progress, \ - ProgressKind, NetworkError +from anki.rsbackend import ( + TR, + Interrupted, + MediaSyncProgress, + NetworkError, + Progress, + ProgressKind, +) from anki.types import assert_impossible from anki.utils import intTime from aqt import gui_hooks