GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-24 16:56:36 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

add progress to db check

Browse source
This commit is contained in:
Damien Elmes 2020-06-08 20:28:11 +10:00
parent a0c1b68b86
commit 7c444b4d35
9 changed files with 183 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
proto/backend.proto
View file

@ -496,6 +496,7 @@ message Progress {
string media_check = 3;
FullSyncProgress full_sync = 4;
NormalSyncProgress normal_sync = 5;
DatabaseCheckProgress database_check = 6;
}
}
@ -521,6 +522,12 @@ message NormalSyncProgress {
string removed = 3;
}
message DatabaseCheckProgress {
string stage = 1;
uint32 stage_total = 2;
uint32 stage_current = 3;
}
// Messages
///////////////////////////////////////////////////////////

12
pylib/anki/rsbackend.py
View file

@ -153,6 +153,7 @@ def proto_exception_to_native(err: pb.BackendError) -> Exception:
MediaSyncProgress = pb.MediaSyncProgress
FullSyncProgress = pb.FullSyncProgress
NormalSyncProgress = pb.NormalSyncProgress
DatabaseCheckProgress = pb.DatabaseCheckProgress
FormatTimeSpanContext = pb.FormatTimespanIn.Context
@ -163,12 +164,19 @@ class ProgressKind(enum.Enum):
MediaCheck = 2
FullSync = 3
NormalSync = 4
DatabaseCheck = 5
@dataclass
class Progress:
kind: ProgressKind
val: Union[MediaSyncProgress, pb.FullSyncProgress, NormalSyncProgress, str]
val: Union[
MediaSyncProgress,
pb.FullSyncProgress,
NormalSyncProgress,
DatabaseCheckProgress,
str,
]
@staticmethod
def from_proto(proto: pb.Progress) -> Progress:
@ -181,6 +189,8 @@ class Progress:
return Progress(kind=ProgressKind.FullSync, val=proto.full_sync)
elif kind == "normal_sync":
return Progress(kind=ProgressKind.NormalSync, val=proto.normal_sync)
elif kind == "database_check":
return Progress(kind=ProgressKind.DatabaseCheck, val=proto.database_check)
else:
return Progress(kind=ProgressKind.NoProgress, val="")

55
qt/aqt/dbcheck.py Normal file
View file

@ -0,0 +1,55 @@
# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
from __future__ import annotations
import aqt
from anki.rsbackend import DatabaseCheckProgress, ProgressKind
from aqt.qt import *
from aqt.utils import showText, tooltip
def on_progress(mw: aqt.main.AnkiQt):
progress = mw.col.latest_progress()
if progress.kind != ProgressKind.DatabaseCheck:
return
assert isinstance(progress.val, DatabaseCheckProgress)
mw.progress.update(
process=False, label=progress.val.stage,
value=progress.val.stage_current,
max=progress.val.stage_total,
)
def check_db(mw: aqt.AnkiQt) -> None:
def on_timer():
on_progress(mw)
timer = QTimer(mw)
qconnect(timer.timeout, on_timer)
timer.start(100)
def on_future_done(fut):
timer.stop()
ret, ok = fut.result()
if not ok:
showText(ret)
else:
tooltip(ret)
# if an error has directed the user to check the database,
# silently clean up any broken reset hooks which distract from
# the underlying issue
n = 0
while n < 10:
try:
mw.reset()
break
except Exception as e:
print("swallowed exception in reset hook:", e)
n += 1
continue
mw.taskman.with_progress(mw.col.fixIntegrity, on_future_done)

26
qt/aqt/main.py
View file

@ -13,7 +13,6 @@ import time
import weakref
import zipfile
from argparse import Namespace
from concurrent.futures import Future
from threading import Thread
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
@ -35,6 +34,7 @@ from anki.sound import AVTag, SoundOrVideoTag
from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields
from aqt import gui_hooks
from aqt.addons import DownloadLogEntry, check_and_prompt_for_updates, show_log_to_user
from aqt.dbcheck import check_db
from aqt.emptycards import show_empty_cards
from aqt.legacy import install_pylib_legacy
from aqt.mediacheck import check_media_db
@ -59,7 +59,6 @@ from aqt.utils import (
saveGeom,
saveSplitter,
showInfo,
showText,
showWarning,
tooltip,
tr,
@ -1334,28 +1333,7 @@ will be lost. Continue?"""
##########################################################################
def onCheckDB(self):
def on_done(future: Future):
ret, ok = future.result()
if not ok:
showText(ret)
else:
tooltip(ret)
# if an error has directed the user to check the database,
# silently clean up any broken reset hooks which distract from
# the underlying issue
n = 0
while n < 10:
try:
self.reset()
break
except Exception as e:
print("swallowed exception in reset hook:", e)
n += 1
continue
self.taskman.with_progress(self.col.fixIntegrity, on_done)
check_db(self)
def on_check_media_db(self) -> None:
check_media_db(self)

7
qt/aqt/progress.py
View file

@ -107,11 +107,13 @@ class ProgressManager:
elapsed = time.time() - self._lastUpdate
if label:
self._win.form.label.setText(label)
self._max = max
self._max = max or 0
self._win.form.progressBar.setMaximum(self._max)
if self._max:
self._win.form.progressBar.setMaximum(max)
self._counter = value or (self._counter + 1)
self._win.form.progressBar.setValue(self._counter)
if process and elapsed >= 0.2:
self._updating = True
self.app.processEvents()
@ -139,7 +141,6 @@ class ProgressManager:
if not self._levels:
return
if self._shown:
self.update(maybeShow=False)
return
delta = time.time() - self._firstTime
if delta > 0.5:

9
rslib/ftl/database-check.ftl
View file

@ -41,3 +41,12 @@ database-check-revlog-properties = { $count ->
[one] Fixed { $count } review entry with invalid properties.
*[other] Fixed { $count } review entries with invalid properties.
}
## Progress info
database-check-checking-integrity = Checking collection...
database-check-rebuilding = Rebuilding...
database-check-checking-cards = Checking cards...
database-check-checking-notes = Checking notes...
database-check-checking-history = Checking history...

34
rslib/src/backend/mod.rs
View file

@ -14,6 +14,7 @@ use crate::{
cloze::add_cloze_numbers_in_string,
collection::{open_collection, Collection},
config::SortKind,
dbcheck::DatabaseCheckProgress,
deckconf::{DeckConf, DeckConfID, DeckConfSchema11},
decks::{Deck, DeckID, DeckSchema11},
err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind},
@ -120,6 +121,7 @@ enum Progress {
MediaCheck(u32),
FullSync(FullSyncProgress),
NormalSync(NormalSyncProgress),
DatabaseCheck(DatabaseCheckProgress),
}
/// Convert an Anki error to a protobuf error.
@ -1008,10 +1010,15 @@ impl BackendService for Backend {
//-------------------------------------------------------------------
fn check_database(&mut self, _input: pb::Empty) -> BackendResult<pb::CheckDatabaseOut> {
let mut handler = self.new_progress_handler();
let progress_fn = move |progress, throttle| {
handler.update(Progress::DatabaseCheck(progress), throttle);
};
self.with_col(|col| {
col.check_database().map(|problems| pb::CheckDatabaseOut {
problems: problems.to_i18n_strings(&col.i18n),
})
col.check_database(progress_fn)
.map(|problems| pb::CheckDatabaseOut {
problems: problems.to_i18n_strings(&col.i18n),
})
})
}
@ -1605,6 +1612,27 @@ fn progress_to_proto(progress: Option<Progress>, i18n: &I18n) -> pb::Progress {
removed,
})
}
Progress::DatabaseCheck(p) => {
let mut stage_total = 0;
let mut stage_current = 0;
let stage = match p {
DatabaseCheckProgress::Integrity => i18n.tr(TR::DatabaseCheckCheckingIntegrity),
DatabaseCheckProgress::Optimize => i18n.tr(TR::DatabaseCheckRebuilding),
DatabaseCheckProgress::Cards => i18n.tr(TR::DatabaseCheckCheckingCards),
DatabaseCheckProgress::Notes { current, total } => {
stage_total = total;
stage_current = current;
i18n.tr(TR::DatabaseCheckCheckingNotes)
}
DatabaseCheckProgress::History => i18n.tr(TR::DatabaseCheckCheckingHistory),
}
.to_string();
pb::progress::Value::DatabaseCheck(pb::DatabaseCheckProgress {
stage,
stage_current,
stage_total,
})
}
}
} else {
pb::progress::Value::None(pb::Empty {})

71
rslib/src/dbcheck.rs
View file

@ -29,6 +29,15 @@ pub struct CheckDatabaseOutput {
field_count_mismatch: usize,
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum DatabaseCheckProgress {
Integrity,
Optimize,
Cards,
Notes { current: u32, total: u32 },
History,
}
impl CheckDatabaseOutput {
pub fn to_i18n_strings(&self, i18n: &I18n) -> Vec<String> {
let mut probs = Vec::new();
@ -88,7 +97,11 @@ impl CheckDatabaseOutput {
impl Collection {
/// Check the database, returning a list of problems that were fixed.
pub(crate) fn check_database(&mut self) -> Result<CheckDatabaseOutput> {
pub(crate) fn check_database<F>(&mut self, mut progress_fn: F) -> Result<CheckDatabaseOutput>
where
F: FnMut(DatabaseCheckProgress, bool),
{
progress_fn(DatabaseCheckProgress::Integrity, false);
debug!(self.log, "quick check");
if self.storage.quick_check_corrupt() {
debug!(self.log, "quick check failed");
@ -98,16 +111,21 @@ impl Collection {
});
}
progress_fn(DatabaseCheckProgress::Optimize, false);
debug!(self.log, "optimize");
self.storage.optimize()?;
self.transact(None, |col| col.check_database_inner())
self.transact(None, |col| col.check_database_inner(progress_fn))
}
fn check_database_inner(&mut self) -> Result<CheckDatabaseOutput> {
fn check_database_inner<F>(&mut self, mut progress_fn: F) -> Result<CheckDatabaseOutput>
where
F: FnMut(DatabaseCheckProgress, bool),
{
let mut out = CheckDatabaseOutput::default();
// cards first, as we need to be able to read them to process notes
progress_fn(DatabaseCheckProgress::Cards, false);
debug!(self.log, "check cards");
self.check_card_properties(&mut out)?;
self.check_orphaned_cards(&mut out)?;
@ -117,7 +135,9 @@ impl Collection {
self.check_filtered_cards(&mut out)?;
debug!(self.log, "check notetypes");
self.check_notetypes(&mut out)?;
self.check_notetypes(&mut out, &mut progress_fn)?;
progress_fn(DatabaseCheckProgress::History, false);
debug!(self.log, "check review log");
self.check_revlog(&mut out)?;
@ -192,7 +212,14 @@ impl Collection {
Ok(())
}
fn check_notetypes(&mut self, out: &mut CheckDatabaseOutput) -> Result<()> {
fn check_notetypes<F>(
&mut self,
out: &mut CheckDatabaseOutput,
mut progress_fn: F,
) -> Result<()>
where
F: FnMut(DatabaseCheckProgress, bool),
{
let nids_by_notetype = self.storage.all_note_ids_by_notetype()?;
let norm = self.normalize_note_text();
let usn = self.usn()?;
@ -201,6 +228,9 @@ impl Collection {
// will rebuild tag list below
self.storage.clear_tags()?;
let total_notes = self.storage.total_notes()?;
let mut checked_notes = 0;
for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
debug!(self.log, "check notetype: {}", ntid);
let mut group = group.peekable();
@ -214,6 +244,15 @@ impl Collection {
let mut genctx = None;
for (_, nid) in group {
progress_fn(
DatabaseCheckProgress::Notes {
current: checked_notes,
total: total_notes,
},
true,
);
checked_notes += 1;
let mut note = self.storage.get_note(nid)?.unwrap();
let cards = self.storage.existing_cards_for_note(nid)?;
@ -328,6 +367,8 @@ mod test {
use super::*;
use crate::{collection::open_test_collection, decks::DeckID, search::SortMode};
fn progress_fn(_progress: DatabaseCheckProgress, _throttle: bool) {}
#[test]
fn cards() -> Result<()> {
let mut col = open_test_collection();
@ -340,7 +381,7 @@ mod test {
.db
.execute_batch("update cards set ivl=1.5,due=2000000,odue=1.5")?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -350,12 +391,12 @@ mod test {
}
);
// should be idempotent
assert_eq!(col.check_database()?, Default::default());
assert_eq!(col.check_database(progress_fn)?, Default::default());
// missing deck
col.storage.db.execute_batch("update cards set did=123")?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -370,7 +411,7 @@ mod test {
// missing note
col.storage.remove_note(note.id)?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -396,7 +437,7 @@ mod test {
values (0,0,0,0,1.5,1.5,0,0,0)",
)?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -426,7 +467,7 @@ mod test {
card.id.0 += 1;
col.storage.add_card(&mut card)?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -446,7 +487,7 @@ mod test {
card.ord = 10;
col.storage.add_card(&mut card)?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -473,7 +514,7 @@ mod test {
col.storage
.db
.execute_batch("update notes set flds = 'a\x1fb\x1fc\x1fd'")?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -488,7 +529,7 @@ mod test {
col.storage
.db
.execute_batch("update notes set flds = 'a'")?;
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {
@ -516,7 +557,7 @@ mod test {
.execute(&[deck.id])?;
assert_eq!(col.storage.get_all_deck_names()?.len(), 2);
let out = col.check_database()?;
let out = col.check_database(progress_fn)?;
assert_eq!(
out,
CheckDatabaseOutput {

8
rslib/src/storage/note/mod.rs
View file

@ -130,4 +130,12 @@ impl super::SqliteStorage {
.query_and_then(params![csum, ntid, nid], |r| r.get(0).map_err(Into::into))?
.collect()
}
/// Return total number of notes. Slow.
pub(crate) fn total_notes(&self) -> Result<u32> {
self.db
.prepare("select count() from notes")?
.query_row(NO_PARAMS, |r| r.get(0))
.map_err(Into::into)
}
}