diff --git a/proto/backend.proto b/proto/backend.proto index b125dede8..318132622 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -496,6 +496,7 @@ message Progress { string media_check = 3; FullSyncProgress full_sync = 4; NormalSyncProgress normal_sync = 5; + DatabaseCheckProgress database_check = 6; } } @@ -521,6 +522,12 @@ message NormalSyncProgress { string removed = 3; } +message DatabaseCheckProgress { + string stage = 1; + uint32 stage_total = 2; + uint32 stage_current = 3; +} + // Messages /////////////////////////////////////////////////////////// diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index dca6a55af..edc770e6d 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -153,6 +153,7 @@ def proto_exception_to_native(err: pb.BackendError) -> Exception: MediaSyncProgress = pb.MediaSyncProgress FullSyncProgress = pb.FullSyncProgress NormalSyncProgress = pb.NormalSyncProgress +DatabaseCheckProgress = pb.DatabaseCheckProgress FormatTimeSpanContext = pb.FormatTimespanIn.Context @@ -163,12 +164,19 @@ class ProgressKind(enum.Enum): MediaCheck = 2 FullSync = 3 NormalSync = 4 + DatabaseCheck = 5 @dataclass class Progress: kind: ProgressKind - val: Union[MediaSyncProgress, pb.FullSyncProgress, NormalSyncProgress, str] + val: Union[ + MediaSyncProgress, + pb.FullSyncProgress, + NormalSyncProgress, + DatabaseCheckProgress, + str, + ] @staticmethod def from_proto(proto: pb.Progress) -> Progress: @@ -181,6 +189,8 @@ class Progress: return Progress(kind=ProgressKind.FullSync, val=proto.full_sync) elif kind == "normal_sync": return Progress(kind=ProgressKind.NormalSync, val=proto.normal_sync) + elif kind == "database_check": + return Progress(kind=ProgressKind.DatabaseCheck, val=proto.database_check) else: return Progress(kind=ProgressKind.NoProgress, val="") diff --git a/qt/aqt/dbcheck.py b/qt/aqt/dbcheck.py new file mode 100644 index 000000000..e49232967 --- /dev/null +++ b/qt/aqt/dbcheck.py @@ -0,0 +1,55 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +from __future__ import annotations + +import aqt +from anki.rsbackend import DatabaseCheckProgress, ProgressKind +from aqt.qt import * +from aqt.utils import showText, tooltip + + +def on_progress(mw: aqt.main.AnkiQt): + progress = mw.col.latest_progress() + if progress.kind != ProgressKind.DatabaseCheck: + return + + assert isinstance(progress.val, DatabaseCheckProgress) + mw.progress.update( + process=False, label=progress.val.stage, + value=progress.val.stage_current, + max=progress.val.stage_total, + ) + + +def check_db(mw: aqt.AnkiQt) -> None: + def on_timer(): + on_progress(mw) + + timer = QTimer(mw) + qconnect(timer.timeout, on_timer) + timer.start(100) + + def on_future_done(fut): + timer.stop() + ret, ok = fut.result() + + if not ok: + showText(ret) + else: + tooltip(ret) + + # if an error has directed the user to check the database, + # silently clean up any broken reset hooks which distract from + # the underlying issue + n = 0 + while n < 10: + try: + mw.reset() + break + except Exception as e: + print("swallowed exception in reset hook:", e) + n += 1 + continue + + mw.taskman.with_progress(mw.col.fixIntegrity, on_future_done) diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 34014d36f..639f114d6 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -13,7 +13,6 @@ import time import weakref import zipfile from argparse import Namespace -from concurrent.futures import Future from threading import Thread from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple @@ -35,6 +34,7 @@ from anki.sound import AVTag, SoundOrVideoTag from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields from aqt import gui_hooks from aqt.addons import DownloadLogEntry, check_and_prompt_for_updates, show_log_to_user +from aqt.dbcheck import check_db from aqt.emptycards import show_empty_cards from aqt.legacy import install_pylib_legacy from aqt.mediacheck import check_media_db @@ -59,7 +59,6 @@ from aqt.utils import ( saveGeom, saveSplitter, showInfo, - showText, showWarning, tooltip, tr, @@ -1334,28 +1333,7 @@ will be lost. Continue?""" ########################################################################## def onCheckDB(self): - def on_done(future: Future): - ret, ok = future.result() - - if not ok: - showText(ret) - else: - tooltip(ret) - - # if an error has directed the user to check the database, - # silently clean up any broken reset hooks which distract from - # the underlying issue - n = 0 - while n < 10: - try: - self.reset() - break - except Exception as e: - print("swallowed exception in reset hook:", e) - n += 1 - continue - - self.taskman.with_progress(self.col.fixIntegrity, on_done) + check_db(self) def on_check_media_db(self) -> None: check_media_db(self) diff --git a/qt/aqt/progress.py b/qt/aqt/progress.py index e96b72cf0..0dc73f032 100644 --- a/qt/aqt/progress.py +++ b/qt/aqt/progress.py @@ -107,11 +107,13 @@ class ProgressManager: elapsed = time.time() - self._lastUpdate if label: self._win.form.label.setText(label) - self._max = max + + self._max = max or 0 + self._win.form.progressBar.setMaximum(self._max) if self._max: - self._win.form.progressBar.setMaximum(max) self._counter = value or (self._counter + 1) self._win.form.progressBar.setValue(self._counter) + if process and elapsed >= 0.2: self._updating = True self.app.processEvents() @@ -139,7 +141,6 @@ class ProgressManager: if not self._levels: return if self._shown: - self.update(maybeShow=False) return delta = time.time() - self._firstTime if delta > 0.5: diff --git a/rslib/ftl/database-check.ftl b/rslib/ftl/database-check.ftl index 0d06449c6..0f3e387ad 100644 --- a/rslib/ftl/database-check.ftl +++ b/rslib/ftl/database-check.ftl @@ -41,3 +41,12 @@ database-check-revlog-properties = { $count -> [one] Fixed { $count } review entry with invalid properties. *[other] Fixed { $count } review entries with invalid properties. } + + +## Progress info + +database-check-checking-integrity = Checking collection... +database-check-rebuilding = Rebuilding... +database-check-checking-cards = Checking cards... +database-check-checking-notes = Checking notes... +database-check-checking-history = Checking history... diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 5c5ed2280..4065e7a45 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -14,6 +14,7 @@ use crate::{ cloze::add_cloze_numbers_in_string, collection::{open_collection, Collection}, config::SortKind, + dbcheck::DatabaseCheckProgress, deckconf::{DeckConf, DeckConfID, DeckConfSchema11}, decks::{Deck, DeckID, DeckSchema11}, err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}, @@ -120,6 +121,7 @@ enum Progress { MediaCheck(u32), FullSync(FullSyncProgress), NormalSync(NormalSyncProgress), + DatabaseCheck(DatabaseCheckProgress), } /// Convert an Anki error to a protobuf error. @@ -1008,10 +1010,15 @@ impl BackendService for Backend { //------------------------------------------------------------------- fn check_database(&mut self, _input: pb::Empty) -> BackendResult { + let mut handler = self.new_progress_handler(); + let progress_fn = move |progress, throttle| { + handler.update(Progress::DatabaseCheck(progress), throttle); + }; self.with_col(|col| { - col.check_database().map(|problems| pb::CheckDatabaseOut { - problems: problems.to_i18n_strings(&col.i18n), - }) + col.check_database(progress_fn) + .map(|problems| pb::CheckDatabaseOut { + problems: problems.to_i18n_strings(&col.i18n), + }) }) } @@ -1605,6 +1612,27 @@ fn progress_to_proto(progress: Option, i18n: &I18n) -> pb::Progress { removed, }) } + Progress::DatabaseCheck(p) => { + let mut stage_total = 0; + let mut stage_current = 0; + let stage = match p { + DatabaseCheckProgress::Integrity => i18n.tr(TR::DatabaseCheckCheckingIntegrity), + DatabaseCheckProgress::Optimize => i18n.tr(TR::DatabaseCheckRebuilding), + DatabaseCheckProgress::Cards => i18n.tr(TR::DatabaseCheckCheckingCards), + DatabaseCheckProgress::Notes { current, total } => { + stage_total = total; + stage_current = current; + i18n.tr(TR::DatabaseCheckCheckingNotes) + } + DatabaseCheckProgress::History => i18n.tr(TR::DatabaseCheckCheckingHistory), + } + .to_string(); + pb::progress::Value::DatabaseCheck(pb::DatabaseCheckProgress { + stage, + stage_current, + stage_total, + }) + } } } else { pb::progress::Value::None(pb::Empty {}) diff --git a/rslib/src/dbcheck.rs b/rslib/src/dbcheck.rs index 2af108af3..6b5d72dcd 100644 --- a/rslib/src/dbcheck.rs +++ b/rslib/src/dbcheck.rs @@ -29,6 +29,15 @@ pub struct CheckDatabaseOutput { field_count_mismatch: usize, } +#[derive(Debug, Clone, Copy)] +pub(crate) enum DatabaseCheckProgress { + Integrity, + Optimize, + Cards, + Notes { current: u32, total: u32 }, + History, +} + impl CheckDatabaseOutput { pub fn to_i18n_strings(&self, i18n: &I18n) -> Vec { let mut probs = Vec::new(); @@ -88,7 +97,11 @@ impl CheckDatabaseOutput { impl Collection { /// Check the database, returning a list of problems that were fixed. - pub(crate) fn check_database(&mut self) -> Result { + pub(crate) fn check_database(&mut self, mut progress_fn: F) -> Result + where + F: FnMut(DatabaseCheckProgress, bool), + { + progress_fn(DatabaseCheckProgress::Integrity, false); debug!(self.log, "quick check"); if self.storage.quick_check_corrupt() { debug!(self.log, "quick check failed"); @@ -98,16 +111,21 @@ impl Collection { }); } + progress_fn(DatabaseCheckProgress::Optimize, false); debug!(self.log, "optimize"); self.storage.optimize()?; - self.transact(None, |col| col.check_database_inner()) + self.transact(None, |col| col.check_database_inner(progress_fn)) } - fn check_database_inner(&mut self) -> Result { + fn check_database_inner(&mut self, mut progress_fn: F) -> Result + where + F: FnMut(DatabaseCheckProgress, bool), + { let mut out = CheckDatabaseOutput::default(); // cards first, as we need to be able to read them to process notes + progress_fn(DatabaseCheckProgress::Cards, false); debug!(self.log, "check cards"); self.check_card_properties(&mut out)?; self.check_orphaned_cards(&mut out)?; @@ -117,7 +135,9 @@ impl Collection { self.check_filtered_cards(&mut out)?; debug!(self.log, "check notetypes"); - self.check_notetypes(&mut out)?; + self.check_notetypes(&mut out, &mut progress_fn)?; + + progress_fn(DatabaseCheckProgress::History, false); debug!(self.log, "check review log"); self.check_revlog(&mut out)?; @@ -192,7 +212,14 @@ impl Collection { Ok(()) } - fn check_notetypes(&mut self, out: &mut CheckDatabaseOutput) -> Result<()> { + fn check_notetypes( + &mut self, + out: &mut CheckDatabaseOutput, + mut progress_fn: F, + ) -> Result<()> + where + F: FnMut(DatabaseCheckProgress, bool), + { let nids_by_notetype = self.storage.all_note_ids_by_notetype()?; let norm = self.normalize_note_text(); let usn = self.usn()?; @@ -201,6 +228,9 @@ impl Collection { // will rebuild tag list below self.storage.clear_tags()?; + let total_notes = self.storage.total_notes()?; + let mut checked_notes = 0; + for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) { debug!(self.log, "check notetype: {}", ntid); let mut group = group.peekable(); @@ -214,6 +244,15 @@ impl Collection { let mut genctx = None; for (_, nid) in group { + progress_fn( + DatabaseCheckProgress::Notes { + current: checked_notes, + total: total_notes, + }, + true, + ); + checked_notes += 1; + let mut note = self.storage.get_note(nid)?.unwrap(); let cards = self.storage.existing_cards_for_note(nid)?; @@ -328,6 +367,8 @@ mod test { use super::*; use crate::{collection::open_test_collection, decks::DeckID, search::SortMode}; + fn progress_fn(_progress: DatabaseCheckProgress, _throttle: bool) {} + #[test] fn cards() -> Result<()> { let mut col = open_test_collection(); @@ -340,7 +381,7 @@ mod test { .db .execute_batch("update cards set ivl=1.5,due=2000000,odue=1.5")?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -350,12 +391,12 @@ mod test { } ); // should be idempotent - assert_eq!(col.check_database()?, Default::default()); + assert_eq!(col.check_database(progress_fn)?, Default::default()); // missing deck col.storage.db.execute_batch("update cards set did=123")?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -370,7 +411,7 @@ mod test { // missing note col.storage.remove_note(note.id)?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -396,7 +437,7 @@ mod test { values (0,0,0,0,1.5,1.5,0,0,0)", )?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -426,7 +467,7 @@ mod test { card.id.0 += 1; col.storage.add_card(&mut card)?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -446,7 +487,7 @@ mod test { card.ord = 10; col.storage.add_card(&mut card)?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -473,7 +514,7 @@ mod test { col.storage .db .execute_batch("update notes set flds = 'a\x1fb\x1fc\x1fd'")?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -488,7 +529,7 @@ mod test { col.storage .db .execute_batch("update notes set flds = 'a'")?; - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { @@ -516,7 +557,7 @@ mod test { .execute(&[deck.id])?; assert_eq!(col.storage.get_all_deck_names()?.len(), 2); - let out = col.check_database()?; + let out = col.check_database(progress_fn)?; assert_eq!( out, CheckDatabaseOutput { diff --git a/rslib/src/storage/note/mod.rs b/rslib/src/storage/note/mod.rs index 36d4f4b16..7771839f4 100644 --- a/rslib/src/storage/note/mod.rs +++ b/rslib/src/storage/note/mod.rs @@ -130,4 +130,12 @@ impl super::SqliteStorage { .query_and_then(params![csum, ntid, nid], |r| r.get(0).map_err(Into::into))? .collect() } + + /// Return total number of notes. Slow. + pub(crate) fn total_notes(&self) -> Result { + self.db + .prepare("select count() from notes")? + .query_row(NO_PARAMS, |r| r.get(0)) + .map_err(Into::into) + } }