From b9b837e7bd824279650bdd31b6374a7f68c6edb9 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 17 May 2020 14:13:21 +1000 Subject: [PATCH] update before_upload() --- pylib/anki/collection.py | 15 +-------------- pylib/anki/decks.py | 8 -------- pylib/anki/models.py | 10 +--------- rslib/src/backend/mod.rs | 2 +- rslib/src/collection.rs | 21 ++++++++++++++++----- rslib/src/storage/card/mod.rs | 7 +++++++ rslib/src/storage/deck/mod.rs | 7 +++++++ rslib/src/storage/deckconf/mod.rs | 10 +++------- rslib/src/storage/graves/mod.rs | 1 - rslib/src/storage/note/mod.rs | 9 ++++++++- rslib/src/storage/notetype/mod.rs | 7 +++++++ rslib/src/storage/revlog/mod.rs | 7 +++++++ rslib/src/storage/sqlite.rs | 25 ++++++++++++++++++++++++- 13 files changed, 82 insertions(+), 47 deletions(-) diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index e81f47371..833c474b5 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -237,21 +237,8 @@ class _Collection: def beforeUpload(self) -> None: "Called before a full upload." - tbls = "notes", "cards", "revlog" - for t in tbls: - self.db.execute("update %s set usn=0 where usn=-1" % t) - # we can save space by removing the log of deletions - self.db.execute("delete from graves") - self._usn += 1 - self.models.beforeUpload() - self.decks.beforeUpload() - self.backend.before_upload() - self.modSchema(check=False) - self.ls = self.scm - # ensure db is compacted before upload self.save(trx=False) - self.db.execute("vacuum") - self.db.execute("analyze") + self.backend.before_upload() self.close(save=False, downgrade=True) # Object creation helpers diff --git a/pylib/anki/decks.py b/pylib/anki/decks.py index 170bafd9e..e20d5d47f 100644 --- a/pylib/anki/decks.py +++ b/pylib/anki/decks.py @@ -566,14 +566,6 @@ class DeckManager: def nameMap(self) -> dict: return dict((d["name"], d) for d in self.all()) - # Sync handling - ########################################################################## - - def beforeUpload(self) -> None: - for d in self.all(): - d["usn"] = 0 - self.save() - # Dynamic decks ########################################################################## diff --git a/pylib/anki/models.py b/pylib/anki/models.py index a29892a55..8aac12468 100644 --- a/pylib/anki/models.py +++ b/pylib/anki/models.py @@ -20,7 +20,7 @@ NoteType = Dict[str, Any] Field = Dict[str, Any] Template = Dict[str, Union[str, int, None]] -# fixme: syncing, beforeUpload +# fixme: syncing class ModelsDictProxy: @@ -479,11 +479,3 @@ and notes.mid = ? and cards.ord = ?""", print("_availClozeOrds() is deprecated") note = anki.rsbackend.BackendNote(fields=[flds]) return self.col.backend.cloze_numbers_in_note(note) - - # Sync handling - ########################################################################## - - def beforeUpload(self) -> None: - for m in self.all(): - m["usn"] = 0 - self.save() diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 13b77052f..22345697d 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -795,7 +795,7 @@ impl Backend { } fn before_upload(&self) -> Result<()> { - self.with_col(|col| col.transact(None, |col| col.before_upload())) + self.with_col(|col| col.before_upload()) } fn all_tags(&self) -> Result { diff --git a/rslib/src/collection.rs b/rslib/src/collection.rs index c52ba7733..2949c33b7 100644 --- a/rslib/src/collection.rs +++ b/rslib/src/collection.rs @@ -144,10 +144,21 @@ impl Collection { self.storage.usn(self.server) } - pub(crate) fn before_upload(&self) -> Result<()> { - self.storage.clear_tag_usns()?; - self.storage.clear_deck_conf_usns()?; - - Ok(()) + /// Prepare for upload. Caller should not create transaction. + pub(crate) fn before_upload(&mut self) -> Result<()> { + self.transact(None, |col| { + col.storage.clear_all_graves()?; + col.storage.clear_pending_note_usns()?; + col.storage.clear_pending_card_usns()?; + col.storage.clear_pending_revlog_usns()?; + col.storage.clear_tag_usns()?; + col.storage.clear_deck_conf_usns()?; + col.storage.clear_deck_usns()?; + col.storage.clear_notetype_usns()?; + col.storage.increment_usn()?; + col.storage.set_schema_modified()?; + col.storage.set_last_sync(col.storage.get_schema_mtime()?) + })?; + self.storage.optimize() } } diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index 0a498cd84..119c68129 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -184,6 +184,13 @@ impl super::SqliteStorage { .optional() .map_err(Into::into) } + + pub(crate) fn clear_pending_card_usns(&self) -> Result<()> { + self.db + .prepare("update cards set usn = 0 where usn = -1")? + .execute(NO_PARAMS)?; + Ok(()) + } } #[cfg(test)] diff --git a/rslib/src/storage/deck/mod.rs b/rslib/src/storage/deck/mod.rs index de10a74e4..299a84769 100644 --- a/rslib/src/storage/deck/mod.rs +++ b/rslib/src/storage/deck/mod.rs @@ -220,6 +220,13 @@ impl SqliteStorage { .map_err(Into::into) } + pub(crate) fn clear_deck_usns(&self) -> Result<()> { + self.db + .prepare("update decks set usn = 0 where usn != 0")? + .execute(NO_PARAMS)?; + Ok(()) + } + // Upgrading/downgrading/legacy pub(super) fn add_default_deck(&self, i18n: &I18n) -> Result<()> { diff --git a/rslib/src/storage/deckconf/mod.rs b/rslib/src/storage/deckconf/mod.rs index c126947ed..113400506 100644 --- a/rslib/src/storage/deckconf/mod.rs +++ b/rslib/src/storage/deckconf/mod.rs @@ -80,13 +80,9 @@ impl SqliteStorage { } pub(crate) fn clear_deck_conf_usns(&self) -> Result<()> { - for mut conf in self.all_deck_config()? { - if conf.usn.0 != 0 { - conf.usn.0 = 0; - self.update_deck_conf(&conf)?; - } - } - + self.db + .prepare("update deck_config set usn = 0 where usn != 0")? + .execute(NO_PARAMS)?; Ok(()) } diff --git a/rslib/src/storage/graves/mod.rs b/rslib/src/storage/graves/mod.rs index b55fc7155..d78586c2d 100644 --- a/rslib/src/storage/graves/mod.rs +++ b/rslib/src/storage/graves/mod.rs @@ -19,7 +19,6 @@ impl SqliteStorage { Ok(()) } - #[allow(dead_code)] pub(crate) fn clear_all_graves(&self) -> Result<()> { self.db.execute("delete from graves", NO_PARAMS)?; Ok(()) diff --git a/rslib/src/storage/note/mod.rs b/rslib/src/storage/note/mod.rs index da4f86e08..c9fc3d79c 100644 --- a/rslib/src/storage/note/mod.rs +++ b/rslib/src/storage/note/mod.rs @@ -7,7 +7,7 @@ use crate::{ tags::{join_tags, split_tags}, timestamp::TimestampMillis, }; -use rusqlite::{params, OptionalExtension}; +use rusqlite::{params, OptionalExtension, NO_PARAMS}; fn split_fields(fields: &str) -> Vec { fields.split('\x1f').map(Into::into).collect() @@ -88,4 +88,11 @@ impl super::SqliteStorage { .query_row(&[nid], |r| r.get(0)) .map_err(Into::into) } + + pub(crate) fn clear_pending_note_usns(&self) -> Result<()> { + self.db + .prepare("update notes set usn = 0 where usn = -1")? + .execute(NO_PARAMS)?; + Ok(()) + } } diff --git a/rslib/src/storage/notetype/mod.rs b/rslib/src/storage/notetype/mod.rs index fdc0663a5..e9c620e33 100644 --- a/rslib/src/storage/notetype/mod.rs +++ b/rslib/src/storage/notetype/mod.rs @@ -308,6 +308,13 @@ and ord in ", .collect() } + pub(crate) fn clear_notetype_usns(&self) -> Result<()> { + self.db + .prepare("update notetypes set usn = 0 where usn != 0")? + .execute(NO_PARAMS)?; + Ok(()) + } + // Upgrading/downgrading/legacy pub(crate) fn get_all_notetypes_as_schema11( diff --git a/rslib/src/storage/revlog/mod.rs b/rslib/src/storage/revlog/mod.rs index dca2073a8..862bc4745 100644 --- a/rslib/src/storage/revlog/mod.rs +++ b/rslib/src/storage/revlog/mod.rs @@ -12,4 +12,11 @@ impl SqliteStorage { .execute(NO_PARAMS) .map_err(Into::into) } + + pub(crate) fn clear_pending_revlog_usns(&self) -> Result<()> { + self.db + .prepare("update revlog set usn = 0 where usn = -1")? + .execute(NO_PARAMS)?; + Ok(()) + } } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index f7771e47a..59784cafe 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -275,6 +275,13 @@ impl SqliteStorage { } } + pub(crate) fn increment_usn(&self) -> Result<()> { + self.db + .prepare_cached("update col set usn = usn + 1")? + .execute(NO_PARAMS)?; + Ok(()) + } + pub(crate) fn creation_stamp(&self) -> Result { self.db .prepare_cached("select crt from col")? @@ -296,6 +303,22 @@ impl SqliteStorage { Ok(()) } + pub(crate) fn get_schema_mtime(&self) -> Result { + self.db + .prepare_cached("select scm from col")? + .query_and_then(NO_PARAMS, |r| r.get(0))? + .next() + .ok_or_else(|| AnkiError::invalid_input("missing col"))? + .map_err(Into::into) + } + + pub(crate) fn set_last_sync(&self, stamp: TimestampSecs) -> Result<()> { + self.db + .prepare("update col set ls = ?")? + .execute(&[stamp])?; + Ok(()) + } + ////////////////////////////////////////// /// true if corrupt/can't access @@ -312,7 +335,7 @@ impl SqliteStorage { } pub(crate) fn optimize(&self) -> Result<()> { - self.db.execute_batch("vacuum")?; + self.db.execute_batch("vacuum; analyze")?; Ok(()) }