add progress to db check

proto/backend.proto

@@ -496,6 +496,7 @@ message Progress {
         string media_check = 3;
         FullSyncProgress full_sync = 4;
         NormalSyncProgress normal_sync = 5;
+        DatabaseCheckProgress database_check = 6;
     }
 }
 
@@ -521,6 +522,12 @@ message NormalSyncProgress {
     string removed = 3;
 }
 
+message DatabaseCheckProgress {
+    string stage = 1;
+    uint32 stage_total = 2;
+    uint32 stage_current = 3;
+}
+
 
 // Messages
 ///////////////////////////////////////////////////////////

pylib/anki/rsbackend.py

@@ -153,6 +153,7 @@ def proto_exception_to_native(err: pb.BackendError) -> Exception:
 MediaSyncProgress = pb.MediaSyncProgress
 FullSyncProgress = pb.FullSyncProgress
 NormalSyncProgress = pb.NormalSyncProgress
+DatabaseCheckProgress = pb.DatabaseCheckProgress
 
 FormatTimeSpanContext = pb.FormatTimespanIn.Context
 
@@ -163,12 +164,19 @@ class ProgressKind(enum.Enum):
     MediaCheck = 2
     FullSync = 3
     NormalSync = 4
+    DatabaseCheck = 5
 
 
 @dataclass
 class Progress:
     kind: ProgressKind
-    val: Union[MediaSyncProgress, pb.FullSyncProgress, NormalSyncProgress, str]
+    val: Union[
+        MediaSyncProgress,
+        pb.FullSyncProgress,
+        NormalSyncProgress,
+        DatabaseCheckProgress,
+        str,
+    ]
 
     @staticmethod
     def from_proto(proto: pb.Progress) -> Progress:
@@ -181,6 +189,8 @@ class Progress:
             return Progress(kind=ProgressKind.FullSync, val=proto.full_sync)
         elif kind == "normal_sync":
             return Progress(kind=ProgressKind.NormalSync, val=proto.normal_sync)
+        elif kind == "database_check":
+            return Progress(kind=ProgressKind.DatabaseCheck, val=proto.database_check)
         else:
             return Progress(kind=ProgressKind.NoProgress, val="")
 

qt/aqt/dbcheck.py

@@ -0,0 +1,55 @@
+# Copyright: Ankitects Pty Ltd and contributors
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+from __future__ import annotations
+
+import aqt
+from anki.rsbackend import DatabaseCheckProgress, ProgressKind
+from aqt.qt import *
+from aqt.utils import showText, tooltip
+
+
+def on_progress(mw: aqt.main.AnkiQt):
+    progress = mw.col.latest_progress()
+    if progress.kind != ProgressKind.DatabaseCheck:
+        return
+
+    assert isinstance(progress.val, DatabaseCheckProgress)
+    mw.progress.update(
+        process=False, label=progress.val.stage,
+        value=progress.val.stage_current,
+        max=progress.val.stage_total,
+    )
+
+
+def check_db(mw: aqt.AnkiQt) -> None:
+    def on_timer():
+        on_progress(mw)
+
+    timer = QTimer(mw)
+    qconnect(timer.timeout, on_timer)
+    timer.start(100)
+
+    def on_future_done(fut):
+        timer.stop()
+        ret, ok = fut.result()
+
+        if not ok:
+            showText(ret)
+        else:
+            tooltip(ret)
+
+        # if an error has directed the user to check the database,
+        # silently clean up any broken reset hooks which distract from
+        # the underlying issue
+        n = 0
+        while n < 10:
+            try:
+                mw.reset()
+                break
+            except Exception as e:
+                print("swallowed exception in reset hook:", e)
+                n += 1
+                continue
+
+    mw.taskman.with_progress(mw.col.fixIntegrity, on_future_done)

qt/aqt/main.py

@@ -13,7 +13,6 @@ import time
 import weakref
 import zipfile
 from argparse import Namespace
-from concurrent.futures import Future
 from threading import Thread
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
 
@@ -35,6 +34,7 @@ from anki.sound import AVTag, SoundOrVideoTag
 from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields
 from aqt import gui_hooks
 from aqt.addons import DownloadLogEntry, check_and_prompt_for_updates, show_log_to_user
+from aqt.dbcheck import check_db
 from aqt.emptycards import show_empty_cards
 from aqt.legacy import install_pylib_legacy
 from aqt.mediacheck import check_media_db
@@ -59,7 +59,6 @@ from aqt.utils import (
     saveGeom,
     saveSplitter,
     showInfo,
-    showText,
     showWarning,
     tooltip,
     tr,
@@ -1334,28 +1333,7 @@ will be lost. Continue?"""
     ##########################################################################
 
     def onCheckDB(self):
-        def on_done(future: Future):
+        check_db(self)
-            ret, ok = future.result()
-
-            if not ok:
-                showText(ret)
-            else:
-                tooltip(ret)
-
-            # if an error has directed the user to check the database,
-            # silently clean up any broken reset hooks which distract from
-            # the underlying issue
-            n = 0
-            while n < 10:
-                try:
-                    self.reset()
-                    break
-                except Exception as e:
-                    print("swallowed exception in reset hook:", e)
-                    n += 1
-                    continue
-
-        self.taskman.with_progress(self.col.fixIntegrity, on_done)
 
     def on_check_media_db(self) -> None:
         check_media_db(self)

qt/aqt/progress.py

@@ -107,11 +107,13 @@ class ProgressManager:
         elapsed = time.time() - self._lastUpdate
         if label:
             self._win.form.label.setText(label)
-        self._max = max
+
+        self._max = max or 0
+        self._win.form.progressBar.setMaximum(self._max)
         if self._max:
-            self._win.form.progressBar.setMaximum(max)
             self._counter = value or (self._counter + 1)
             self._win.form.progressBar.setValue(self._counter)
+
         if process and elapsed >= 0.2:
             self._updating = True
             self.app.processEvents()
@@ -139,7 +141,6 @@ class ProgressManager:
         if not self._levels:
             return
         if self._shown:
-            self.update(maybeShow=False)
             return
         delta = time.time() - self._firstTime
         if delta > 0.5:

rslib/ftl/database-check.ftl

@@ -41,3 +41,12 @@ database-check-revlog-properties = { $count ->
   [one] Fixed { $count } review entry with invalid properties.
   *[other] Fixed { $count } review entries with invalid properties.
   }
+
+
+## Progress info
+
+database-check-checking-integrity = Checking collection...
+database-check-rebuilding = Rebuilding...
+database-check-checking-cards = Checking cards...
+database-check-checking-notes = Checking notes...
+database-check-checking-history = Checking history...

rslib/src/backend/mod.rs

@@ -14,6 +14,7 @@ use crate::{
     cloze::add_cloze_numbers_in_string,
     collection::{open_collection, Collection},
     config::SortKind,
+    dbcheck::DatabaseCheckProgress,
     deckconf::{DeckConf, DeckConfID, DeckConfSchema11},
     decks::{Deck, DeckID, DeckSchema11},
     err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind},
@@ -120,6 +121,7 @@ enum Progress {
     MediaCheck(u32),
     FullSync(FullSyncProgress),
     NormalSync(NormalSyncProgress),
+    DatabaseCheck(DatabaseCheckProgress),
 }
 
 /// Convert an Anki error to a protobuf error.
@@ -1008,10 +1010,15 @@ impl BackendService for Backend {
     //-------------------------------------------------------------------
 
     fn check_database(&mut self, _input: pb::Empty) -> BackendResult<pb::CheckDatabaseOut> {
+        let mut handler = self.new_progress_handler();
+        let progress_fn = move |progress, throttle| {
+            handler.update(Progress::DatabaseCheck(progress), throttle);
+        };
         self.with_col(|col| {
-            col.check_database().map(|problems| pb::CheckDatabaseOut {
+            col.check_database(progress_fn)
-                problems: problems.to_i18n_strings(&col.i18n),
+                .map(|problems| pb::CheckDatabaseOut {
-            })
+                    problems: problems.to_i18n_strings(&col.i18n),
+                })
         })
     }
 
@@ -1605,6 +1612,27 @@ fn progress_to_proto(progress: Option<Progress>, i18n: &I18n) -> pb::Progress {
                     removed,
                 })
             }
+            Progress::DatabaseCheck(p) => {
+                let mut stage_total = 0;
+                let mut stage_current = 0;
+                let stage = match p {
+                    DatabaseCheckProgress::Integrity => i18n.tr(TR::DatabaseCheckCheckingIntegrity),
+                    DatabaseCheckProgress::Optimize => i18n.tr(TR::DatabaseCheckRebuilding),
+                    DatabaseCheckProgress::Cards => i18n.tr(TR::DatabaseCheckCheckingCards),
+                    DatabaseCheckProgress::Notes { current, total } => {
+                        stage_total = total;
+                        stage_current = current;
+                        i18n.tr(TR::DatabaseCheckCheckingNotes)
+                    }
+                    DatabaseCheckProgress::History => i18n.tr(TR::DatabaseCheckCheckingHistory),
+                }
+                .to_string();
+                pb::progress::Value::DatabaseCheck(pb::DatabaseCheckProgress {
+                    stage,
+                    stage_current,
+                    stage_total,
+                })
+            }
         }
     } else {
         pb::progress::Value::None(pb::Empty {})

rslib/src/dbcheck.rs

@@ -29,6 +29,15 @@ pub struct CheckDatabaseOutput {
     field_count_mismatch: usize,
 }
 
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum DatabaseCheckProgress {
+    Integrity,
+    Optimize,
+    Cards,
+    Notes { current: u32, total: u32 },
+    History,
+}
+
 impl CheckDatabaseOutput {
     pub fn to_i18n_strings(&self, i18n: &I18n) -> Vec<String> {
         let mut probs = Vec::new();
@@ -88,7 +97,11 @@ impl CheckDatabaseOutput {
 
 impl Collection {
     /// Check the database, returning a list of problems that were fixed.
-    pub(crate) fn check_database(&mut self) -> Result<CheckDatabaseOutput> {
+    pub(crate) fn check_database<F>(&mut self, mut progress_fn: F) -> Result<CheckDatabaseOutput>
+    where
+        F: FnMut(DatabaseCheckProgress, bool),
+    {
+        progress_fn(DatabaseCheckProgress::Integrity, false);
         debug!(self.log, "quick check");
         if self.storage.quick_check_corrupt() {
             debug!(self.log, "quick check failed");
@@ -98,16 +111,21 @@ impl Collection {
             });
         }
 
+        progress_fn(DatabaseCheckProgress::Optimize, false);
         debug!(self.log, "optimize");
         self.storage.optimize()?;
 
-        self.transact(None, |col| col.check_database_inner())
+        self.transact(None, |col| col.check_database_inner(progress_fn))
     }
 
-    fn check_database_inner(&mut self) -> Result<CheckDatabaseOutput> {
+    fn check_database_inner<F>(&mut self, mut progress_fn: F) -> Result<CheckDatabaseOutput>
+    where
+        F: FnMut(DatabaseCheckProgress, bool),
+    {
         let mut out = CheckDatabaseOutput::default();
 
         // cards first, as we need to be able to read them to process notes
+        progress_fn(DatabaseCheckProgress::Cards, false);
         debug!(self.log, "check cards");
         self.check_card_properties(&mut out)?;
         self.check_orphaned_cards(&mut out)?;
@@ -117,7 +135,9 @@ impl Collection {
         self.check_filtered_cards(&mut out)?;
 
         debug!(self.log, "check notetypes");
-        self.check_notetypes(&mut out)?;
+        self.check_notetypes(&mut out, &mut progress_fn)?;
+
+        progress_fn(DatabaseCheckProgress::History, false);
 
         debug!(self.log, "check review log");
         self.check_revlog(&mut out)?;
@@ -192,7 +212,14 @@ impl Collection {
         Ok(())
     }
 
-    fn check_notetypes(&mut self, out: &mut CheckDatabaseOutput) -> Result<()> {
+    fn check_notetypes<F>(
+        &mut self,
+        out: &mut CheckDatabaseOutput,
+        mut progress_fn: F,
+    ) -> Result<()>
+    where
+        F: FnMut(DatabaseCheckProgress, bool),
+    {
         let nids_by_notetype = self.storage.all_note_ids_by_notetype()?;
         let norm = self.normalize_note_text();
         let usn = self.usn()?;
@@ -201,6 +228,9 @@ impl Collection {
         // will rebuild tag list below
         self.storage.clear_tags()?;
 
+        let total_notes = self.storage.total_notes()?;
+        let mut checked_notes = 0;
+
         for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
             debug!(self.log, "check notetype: {}", ntid);
             let mut group = group.peekable();
@@ -214,6 +244,15 @@ impl Collection {
 
             let mut genctx = None;
             for (_, nid) in group {
+                progress_fn(
+                    DatabaseCheckProgress::Notes {
+                        current: checked_notes,
+                        total: total_notes,
+                    },
+                    true,
+                );
+                checked_notes += 1;
+
                 let mut note = self.storage.get_note(nid)?.unwrap();
 
                 let cards = self.storage.existing_cards_for_note(nid)?;
@@ -328,6 +367,8 @@ mod test {
     use super::*;
     use crate::{collection::open_test_collection, decks::DeckID, search::SortMode};
 
+    fn progress_fn(_progress: DatabaseCheckProgress, _throttle: bool) {}
+
     #[test]
     fn cards() -> Result<()> {
         let mut col = open_test_collection();
@@ -340,7 +381,7 @@ mod test {
             .db
             .execute_batch("update cards set ivl=1.5,due=2000000,odue=1.5")?;
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -350,12 +391,12 @@ mod test {
             }
         );
         // should be idempotent
-        assert_eq!(col.check_database()?, Default::default());
+        assert_eq!(col.check_database(progress_fn)?, Default::default());
 
         // missing deck
         col.storage.db.execute_batch("update cards set did=123")?;
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -370,7 +411,7 @@ mod test {
 
         // missing note
         col.storage.remove_note(note.id)?;
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -396,7 +437,7 @@ mod test {
         values (0,0,0,0,1.5,1.5,0,0,0)",
         )?;
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -426,7 +467,7 @@ mod test {
         card.id.0 += 1;
         col.storage.add_card(&mut card)?;
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -446,7 +487,7 @@ mod test {
         card.ord = 10;
         col.storage.add_card(&mut card)?;
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -473,7 +514,7 @@ mod test {
         col.storage
             .db
             .execute_batch("update notes set flds = 'a\x1fb\x1fc\x1fd'")?;
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -488,7 +529,7 @@ mod test {
         col.storage
             .db
             .execute_batch("update notes set flds = 'a'")?;
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {
@@ -516,7 +557,7 @@ mod test {
             .execute(&[deck.id])?;
         assert_eq!(col.storage.get_all_deck_names()?.len(), 2);
 
-        let out = col.check_database()?;
+        let out = col.check_database(progress_fn)?;
         assert_eq!(
             out,
             CheckDatabaseOutput {

rslib/src/storage/note/mod.rs

@@ -130,4 +130,12 @@ impl super::SqliteStorage {
             .query_and_then(params![csum, ntid, nid], |r| r.get(0).map_err(Into::into))?
             .collect()
     }
+
+    /// Return total number of notes. Slow.
+    pub(crate) fn total_notes(&self) -> Result<u32> {
+        self.db
+            .prepare("select count() from notes")?
+            .query_row(NO_PARAMS, |r| r.get(0))
+            .map_err(Into::into)
+    }
 }