diff --git a/proto/backend.proto b/proto/backend.proto index 96fea75e3..6b918f447 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -144,7 +144,7 @@ message BackendOutput { bytes get_changed_notetypes = 56; int64 add_or_update_notetype = 57; bytes get_all_decks = 58; - Empty check_database = 59; + CheckDatabaseOut check_database = 59; bytes get_notetype_legacy = 61; NoteTypeNames get_notetype_names = 62; NoteTypeUseCounts get_notetype_names_and_counts = 63; @@ -749,3 +749,7 @@ message UpdateNoteTagsIn { string replacement = 3; bool regex = 4; } + +message CheckDatabaseOut { + repeated string problems = 1; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index f5030f112..321222e4f 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -9,7 +9,6 @@ import os import pprint import random import re -import stat import time import traceback import weakref @@ -25,11 +24,11 @@ from anki.consts import * from anki.dbproxy import DBProxy from anki.decks import DeckManager from anki.errors import AnkiError -from anki.lang import _, ngettext +from anki.lang import _ from anki.media import MediaManager from anki.models import ModelManager, NoteType, Template from anki.notes import Note -from anki.rsbackend import TR, RustBackend +from anki.rsbackend import TR, DBError, RustBackend from anki.sched import Scheduler as V1Scheduler from anki.schedv2 import Scheduler as V2Scheduler from anki.tags import TagManager @@ -675,208 +674,20 @@ select id from notes where mid = ?) limit 1""" Returns tuple of (error: str, ok: bool). 'ok' will be true if no problems were found. """ - problems = [] - # problems that don't require a full sync - syncable_problems = [] - self.save() - oldSize = os.stat(self.path)[stat.ST_SIZE] - if self.db.scalar("pragma integrity_check") != "ok": - return (_("Collection is corrupt. Please see the manual."), False) - # note types with a missing model - ids = self.db.list( - """ -select id from notes where mid not in """ - + ids2str(self.models.ids()) - ) - if ids: - problems.append( - ngettext( - "Deleted %d note with missing note type.", - "Deleted %d notes with missing note type.", - len(ids), - ) - % len(ids) - ) - self.remNotes(ids) - # for each model - for m in self.models.all(): - for t in m["tmpls"]: - if t["did"] == "None": - t["did"] = None - problems.append(_("Fixed AnkiDroid deck override bug.")) - self.models.save(m, updateReqs=False) - if m["type"] == MODEL_STD: - # cards with invalid ordinal - ids = self.db.list( - """ -select id from cards where ord not in %s and nid in ( -select id from notes where mid = ?)""" - % ids2str([t["ord"] for t in m["tmpls"]]), - m["id"], - ) - if ids: - problems.append( - ngettext( - "Deleted %d card with missing template.", - "Deleted %d cards with missing template.", - len(ids), - ) - % len(ids) - ) - self.remCards(ids) - # notes with invalid field count - ids = [] - for id, flds in self.db.execute( - "select id, flds from notes where mid = ?", m["id"] - ): - if (flds.count("\x1f") + 1) != len(m["flds"]): - ids.append(id) - if ids: - problems.append( - ngettext( - "Deleted %d note with wrong field count.", - "Deleted %d notes with wrong field count.", - len(ids), - ) - % len(ids) - ) - self.remNotes(ids) - # delete any notes with missing cards - ids = self.db.list( - """ -select id from notes where id not in (select distinct nid from cards)""" - ) - if ids: - cnt = len(ids) - problems.append( - ngettext( - "Deleted %d note with no cards.", - "Deleted %d notes with no cards.", - cnt, - ) - % cnt - ) - self._remNotes(ids) - # cards with missing notes - ids = self.db.list( - """ -select id from cards where nid not in (select id from notes)""" - ) - if ids: - cnt = len(ids) - problems.append( - ngettext( - "Deleted %d card with missing note.", - "Deleted %d cards with missing note.", - cnt, - ) - % cnt - ) - self.remCards(ids) - # cards with odue set when it shouldn't be - ids = self.db.list( - """ -select id from cards where odue > 0 and (type=1 or queue=2) and not odid""" - ) - if ids: - cnt = len(ids) - problems.append( - ngettext( - "Fixed %d card with invalid properties.", - "Fixed %d cards with invalid properties.", - cnt, - ) - % cnt - ) - self.db.execute("update cards set odue=0 where id in " + ids2str(ids)) - # cards with odid set when not in a dyn deck - dids = [id for id in self.decks.allIds() if not self.decks.isDyn(id)] - ids = self.db.list( - """ -select id from cards where odid > 0 and did in %s""" - % ids2str(dids) - ) - if ids: - cnt = len(ids) - problems.append( - ngettext( - "Fixed %d card with invalid properties.", - "Fixed %d cards with invalid properties.", - cnt, - ) - % cnt - ) - self.db.execute( - "update cards set odid=0, odue=0 where id in " + ids2str(ids) - ) - # tags & field cache - self.tags.register([], clear=True) - for m in self.models.all(): - self.after_note_updates( - self.models.nids(m), mark_modified=False, generate_cards=False - ) - # new cards can't have a due position > 32 bits, so wrap items over - # 2 million back to 1 million - self.db.execute( - """ -update cards set due=1000000+due%1000000,mod=?,usn=? where due>=1000000 -and type=0 and queue!=4""", - intTime(), - self.usn(), - ) - rowcount = self.db.scalar("select changes()") - if rowcount: - syncable_problems.append( - "Found %d new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen." - % rowcount - ) - # new card position - self.conf["nextPos"] = ( - self.db.scalar("select max(due)+1 from cards where type = 0") or 0 - ) - # reviews should have a reasonable due # - ids = self.db.list("select id from cards where queue = 2 and due > 100000") - if ids: - problems.append("Reviews had incorrect due date.") - self.db.execute( - "update cards set due = ?, ivl = 1, mod = ?, usn = ? where id in %s" - % ids2str(ids), - self.sched.today, - intTime(), - self.usn(), - ) - # v2 sched had a bug that could create decimal intervals - self.db.execute( - "update cards set ivl=round(ivl),due=round(due) where ivl!=round(ivl) or due!=round(due)" - ) - rowcount = self.db.scalar("select changes()") - if rowcount: - problems.append("Fixed %d cards with v2 scheduler bug." % rowcount) - - self.db.execute( - "update revlog set ivl=round(ivl),lastIvl=round(lastIvl) where ivl!=round(ivl) or lastIvl!=round(lastIvl)" - ) - rowcount = self.db.scalar("select changes()") - if rowcount: - problems.append( - "Fixed %d review history entries with v2 scheduler bug." % rowcount - ) - # models - if self.models.ensureNotEmpty(): - problems.append("Added missing note type.") - # misc other - self.backend.check_database() - # and finally, optimize - self.optimize() - newSize = os.stat(self.path)[stat.ST_SIZE] - txt = _("Database rebuilt and optimized.") - ok = not problems - problems.append(txt) - # if any problems were found, force a full sync - if not ok: - self.modSchema(check=False) - self.save() - problems.extend(syncable_problems) + self.save(trx=False) + try: + problems = self.backend.check_database() + ok = not problems + problems.append(self.tr(TR.DATABASE_CHECK_REBUILT)) + except DBError as e: + problems = [str(e.args[0])] + ok = False + finally: + try: + self.db.begin() + except: + # may fail if the DB is very corrupt + pass return ("\n".join(problems), ok) def optimize(self) -> None: diff --git a/pylib/anki/models.py b/pylib/anki/models.py index f0500b717..4f31e56f3 100644 --- a/pylib/anki/models.py +++ b/pylib/anki/models.py @@ -311,9 +311,12 @@ class ModelManager: # Tools ################################################## - def nids(self, m: NoteType) -> Any: + def nids(self, ntid: int) -> Any: "Note ids for M." - return self.col.db.list("select id from notes where mid = ?", m["id"]) + if isinstance(ntid, dict): + # legacy callers passed in note type + ntid = ntid["id"] + return self.col.db.list("select id from notes where mid = ?", ntid) def useCount(self, m: NoteType) -> Any: "Number of note using M." diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index 10ecc7419..e4986c417 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -736,8 +736,12 @@ class RustBackend: pb.BackendInput(deck_tree=pb.DeckTreeIn(include_counts=include_counts)) ).deck_tree - def check_database(self) -> None: - self._run_command(pb.BackendInput(check_database=pb.Empty())) + def check_database(self) -> List[str]: + return list( + self._run_command( + pb.BackendInput(check_database=pb.Empty()), release_gil=True + ).check_database.problems + ) def legacy_deck_tree(self) -> Sequence: bytes = self._run_command( @@ -819,4 +823,4 @@ def translate_string_in( # temporarily force logging of media handling if "RUST_LOG" not in os.environ: - os.environ["RUST_LOG"] = "warn,anki::media=debug" + os.environ["RUST_LOG"] = "warn,anki::media=debug,anki::dbcheck=debug" diff --git a/qt/aqt/main.py b/qt/aqt/main.py index d520f8d02..15646d080 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -1304,11 +1304,7 @@ will be lost. Continue?""" ########################################################################## def onCheckDB(self): - "True if no problems" - self.progress.start() - - def onDone(future: Future): - self.progress.finish() + def on_done(future: Future): ret, ok = future.result() if not ok: @@ -1319,15 +1315,17 @@ will be lost. Continue?""" # if an error has directed the user to check the database, # silently clean up any broken reset hooks which distract from # the underlying issue - while True: + n = 0 + while n < 10: try: self.reset() break except Exception as e: print("swallowed exception in reset hook:", e) + n += 1 continue - self.taskman.run_in_background(self.col.fixIntegrity, onDone) + self.taskman.with_progress(self.col.fixIntegrity, on_done) def on_check_media_db(self) -> None: check_media_db(self) diff --git a/rslib/ftl/database-check.ftl b/rslib/ftl/database-check.ftl new file mode 100644 index 000000000..ed30321f9 --- /dev/null +++ b/rslib/ftl/database-check.ftl @@ -0,0 +1,43 @@ +database-check-corrupt = Collection file is corrupt. Please restore from an automatic backup. +database-check-rebuilt = Database rebuilt and optimized. +database-check-card-properties = + { $count -> + [one] Fixed { $count } card with invalid properties. + *[other] Fixed { $count } cards with invalid properties. + } + +database-check-missing-templates = + { $count -> + [one] Deleted { $count } card with missing template. + *[other] Deleted { $count } cards with missing template. + } + +database-check-field-count = { $count -> + [one] Fixed { $count } note with wrong field count. + *[other] Fixed { $count } notes with wrong field count. + } + +database-check-new-card-high-due = { $count -> + [one] Found { $count } new card with a due number >= 1,000,000 - consider repositioning them in the Browse screen. + *[other] Found { $count } new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen. + } + +database-check-card-missing-note = { $count -> + [one] Deleted {$count} card with missing note. + *[other] Deleted {$count} cards with missing note. + } + +database-check-duplicate-card-ords = { $count -> + [one] Deleted { $count } card with duplicate template. + *[other] Deleted { $count } cards with duplicate template. + } + +database-check-missing-decks = { $count -> + [one] Fixed { $count } missing deck. + *[other] Fixed { $count } missing decks. + } + +database-check-revlog-properties = { $count -> + [one] Fixed { $count } review entry with invalid properties. + *[other] Fixed { $count } review entries with invalid properties. + } diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 4011c0331..90a12f9e4 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -355,10 +355,7 @@ impl Backend { self.remove_deck(did)?; pb::Empty {} }), - Value::CheckDatabase(_) => { - self.check_database()?; - OValue::CheckDatabase(pb::Empty {}) - } + Value::CheckDatabase(_) => OValue::CheckDatabase(self.check_database()?), Value::DeckTreeLegacy(_) => OValue::DeckTreeLegacy(self.deck_tree_legacy()?), Value::FieldNamesForNotes(input) => { OValue::FieldNamesForNotes(self.field_names_for_notes(input)?) @@ -1042,8 +1039,11 @@ impl Backend { self.with_col(|col| col.remove_deck_and_child_decks(DeckID(did))) } - fn check_database(&self) -> Result<()> { - self.with_col(|col| col.transact(None, |col| col.check_database())) + fn check_database(&self) -> Result { + self.with_col(|col| { + col.check_database() + .map(|problems| pb::CheckDatabaseOut { problems }) + }) } fn deck_tree_legacy(&self) -> Result> { diff --git a/rslib/src/config.rs b/rslib/src/config.rs index deadf7432..94ce3096a 100644 --- a/rslib/src/config.rs +++ b/rslib/src/config.rs @@ -156,6 +156,10 @@ impl Collection { Ok(pos) } + pub(crate) fn set_next_card_position(&self, pos: u32) -> Result<()> { + self.set_config(ConfigKey::NextNewCardPosition, &pos) + } + pub(crate) fn sched_ver(&self) -> SchedulerVersion { self.get_config_optional(ConfigKey::SchedulerVersion) .unwrap_or(SchedulerVersion::V1) diff --git a/rslib/src/dbcheck.rs b/rslib/src/dbcheck.rs index 1d316ae36..4849da8ed 100644 --- a/rslib/src/dbcheck.rs +++ b/rslib/src/dbcheck.rs @@ -1,12 +1,299 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::{collection::Collection, err::Result}; +use crate::{ + collection::Collection, + err::{AnkiError, DBErrorKind, Result}, + i18n::{tr_args, TR}, + notetype::{ + all_stock_notetypes, AlreadyGeneratedCardInfo, CardGenContext, NoteType, NoteTypeKind, + }, + timestamp::{TimestampMillis, TimestampSecs}, +}; +use itertools::Itertools; +use slog::debug; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; impl Collection { - pub(crate) fn check_database(&mut self) -> Result<()> { + /// Check the database, returning a list of problems that were fixed. + pub(crate) fn check_database(&mut self) -> Result> { + debug!(self.log, "quick check"); + if self.storage.quick_check_corrupt() { + debug!(self.log, "quick check failed"); + return Err(AnkiError::DBError { + info: self.i18n.tr(TR::DatabaseCheckCorrupt).into(), + kind: DBErrorKind::Corrupt, + }); + } + + debug!(self.log, "optimize"); + self.storage.optimize()?; + + self.transact(None, |col| col.check_database_inner()) + } + + fn check_database_inner(&mut self) -> Result> { + let mut probs = vec![]; + + // cards first, as we need to be able to read them to process notes + debug!(self.log, "check cards"); + self.check_card_properties(&mut probs)?; + self.check_orphaned_cards(&mut probs)?; + + debug!(self.log, "check decks"); + self.check_missing_deck_ids(&mut probs)?; + self.check_filtered_cards(&mut probs)?; + + debug!(self.log, "check notetypes"); + self.check_notetypes(&mut probs)?; + + debug!(self.log, "check review log"); + self.check_revlog(&mut probs)?; + + debug!(self.log, "missing decks"); let names = self.storage.get_all_deck_names()?; - self.add_missing_decks(&names)?; + self.add_missing_deck_names(&names)?; + + self.update_next_new_position()?; + + debug!(self.log, "db check finished with problems: {:#?}", probs); + + Ok(probs) + } + + fn check_card_properties(&mut self, probs: &mut Vec) -> Result<()> { + let timing = self.timing_today()?; + let (new_cnt, other_cnt) = self.storage.fix_card_properties( + timing.days_elapsed, + TimestampSecs::now(), + self.usn()?, + )?; + if new_cnt > 0 { + probs.push( + self.i18n + .trn(TR::DatabaseCheckNewCardHighDue, tr_args!["count"=>new_cnt]), + ); + } + if other_cnt > 0 { + probs.push(self.i18n.trn( + TR::DatabaseCheckCardProperties, + tr_args!["count"=>other_cnt], + )); + } Ok(()) } + + fn check_orphaned_cards(&mut self, probs: &mut Vec) -> Result<()> { + let orphaned = self.storage.delete_orphaned_cards()?; + if orphaned > 0 { + self.storage.set_schema_modified()?; + probs.push(self.i18n.trn( + TR::DatabaseCheckCardMissingNote, + tr_args!["count"=>orphaned], + )); + } + Ok(()) + } + + fn check_missing_deck_ids(&mut self, probs: &mut Vec) -> Result<()> { + let mut cnt = 0; + for did in self.storage.missing_decks()? { + self.recover_missing_deck(did)?; + cnt += 1; + } + + if cnt > 0 { + probs.push( + self.i18n + .trn(TR::DatabaseCheckMissingDecks, tr_args!["count"=>cnt]), + ); + } + + Ok(()) + } + + fn check_filtered_cards(&mut self, probs: &mut Vec) -> Result<()> { + let decks: HashMap<_, _> = self + .storage + .get_all_decks()? + .into_iter() + .map(|d| (d.id, d)) + .collect(); + + let mut wrong = 0; + + for (cid, did) in self.storage.all_filtered_cards_by_deck()? { + // we expect calling code to ensure all decks already exist + if let Some(deck) = decks.get(&did) { + if !deck.is_filtered() { + let mut card = self.storage.get_card(cid)?.unwrap(); + card.odid.0 = 0; + card.odue = 0; + self.storage.update_card(&card)?; + wrong += 1; + } + } + } + + if wrong > 0 { + self.storage.set_schema_modified()?; + probs.push( + self.i18n + .trn(TR::DatabaseCheckCardProperties, tr_args!["count"=>wrong]), + ); + } + + Ok(()) + } + + fn check_notetypes(&mut self, probs: &mut Vec) -> Result<()> { + let nids_by_notetype = self.storage.all_note_ids_by_notetype()?; + let norm = self.normalize_note_text(); + let usn = self.usn()?; + let stamp = TimestampMillis::now(); + + // will rebuild tag list below + self.storage.clear_tags()?; + + let mut note_fields = 0; + let mut dupe_ords = 0; + let mut missing_template_ords = 0; + + for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) { + debug!(self.log, "check notetype: {}", ntid); + let mut group = group.peekable(); + let nt = match self.get_notetype(ntid)? { + None => { + let first_note = self.storage.get_note(group.peek().unwrap().1)?.unwrap(); + self.recover_notetype(stamp, first_note.fields.len())? + } + Some(nt) => nt, + }; + + let mut genctx = None; + for (_, nid) in group { + let mut note = self.storage.get_note(nid)?.unwrap(); + + let cards = self.storage.existing_cards_for_note(nid)?; + dupe_ords += self.remove_duplicate_card_ordinals(&cards)?; + missing_template_ords += self.remove_cards_without_template(&nt, &cards)?; + + // fix fields + if note.fields.len() != nt.fields.len() { + note.fix_field_count(&nt); + note_fields += 1; + note.tags.push("db-check".into()); + } + + // write note, updating tags and generating missing cards + let ctx = genctx.get_or_insert_with(|| CardGenContext::new(&nt, usn)); + self.update_note_inner_generating_cards(&ctx, &mut note, false, norm)?; + } + } + + // if the collection is empty and the user has deleted all note types, ensure at least + // one note type exists + if self.storage.get_all_notetype_names()?.is_empty() { + let mut nt = all_stock_notetypes(&self.i18n).remove(0); + self.add_notetype_inner(&mut nt)?; + } + + if note_fields > 0 { + self.storage.set_schema_modified()?; + probs.push( + self.i18n + .trn(TR::DatabaseCheckFieldCount, tr_args!["count"=>note_fields]), + ); + } + if dupe_ords > 0 { + self.storage.set_schema_modified()?; + probs.push(self.i18n.trn( + TR::DatabaseCheckDuplicateCardOrds, + tr_args!["count"=>dupe_ords], + )); + } + if missing_template_ords > 0 { + self.storage.set_schema_modified()?; + probs.push(self.i18n.trn( + TR::DatabaseCheckMissingTemplates, + tr_args!["count"=>missing_template_ords], + )); + } + + Ok(()) + } + + fn remove_duplicate_card_ordinals( + &mut self, + cards: &[AlreadyGeneratedCardInfo], + ) -> Result { + let mut ords = HashSet::new(); + let mut removed = 0; + for card in cards { + if !ords.insert(card.ord) { + self.storage.remove_card(card.id)?; + removed += 1; + } + } + + Ok(removed) + } + + fn remove_cards_without_template( + &mut self, + nt: &NoteType, + cards: &[AlreadyGeneratedCardInfo], + ) -> Result { + if nt.config.kind() == NoteTypeKind::Cloze { + return Ok(0); + } + let mut removed = 0; + for card in cards { + if card.ord as usize >= nt.templates.len() { + self.storage.remove_card(card.id)?; + removed += 1; + } + } + + Ok(removed) + } + + fn recover_notetype( + &mut self, + stamp: TimestampMillis, + field_count: usize, + ) -> Result> { + debug!(self.log, "create recovery notetype"); + let mut basic = all_stock_notetypes(&self.i18n).remove(0); + let mut field = 3; + while basic.fields.len() < field_count { + basic.add_field(format!("{}", field)); + field += 1; + } + basic.name = format!("db-check-{}-{}", stamp, field_count); + self.add_notetype(&mut basic)?; + Ok(Arc::new(basic)) + } + + fn check_revlog(&self, probs: &mut Vec) -> Result<()> { + let cnt = self.storage.fix_revlog_properties()?; + + if cnt > 0 { + self.storage.set_schema_modified()?; + probs.push( + self.i18n + .trn(TR::DatabaseCheckRevlogProperties, tr_args!["count"=>cnt]), + ); + } + + Ok(()) + } + + fn update_next_new_position(&self) -> Result<()> { + let pos = self.storage.max_new_card_position().unwrap_or(0); + self.set_next_card_position(pos) + } } diff --git a/rslib/src/decks/mod.rs b/rslib/src/decks/mod.rs index 9d207b200..a9eca38ef 100644 --- a/rslib/src/decks/mod.rs +++ b/rslib/src/decks/mod.rs @@ -209,6 +209,12 @@ impl Collection { }) } + pub(crate) fn recover_missing_deck(&mut self, _did: DeckID) -> Result<()> { + // todo + + Ok(()) + } + pub fn get_or_create_normal_deck(&mut self, human_name: &str) -> Result { let native_name = human_deck_name_to_native(human_name); if let Some(did) = self.storage.get_deck_id(&native_name)? { diff --git a/rslib/src/decks/tree.rs b/rslib/src/decks/tree.rs index b08b725b9..119bd739f 100644 --- a/rslib/src/decks/tree.rs +++ b/rslib/src/decks/tree.rs @@ -215,7 +215,7 @@ impl Collection { Ok(LegacyDueCounts::from(tree)) } - pub(crate) fn add_missing_decks(&mut self, names: &[(DeckID, String)]) -> Result<()> { + pub(crate) fn add_missing_deck_names(&mut self, names: &[(DeckID, String)]) -> Result<()> { let mut parents = HashSet::new(); for (_id, name) in names { parents.insert(UniCase::new(name.as_str())); diff --git a/rslib/src/err.rs b/rslib/src/err.rs index 32389afc7..e463dc692 100644 --- a/rslib/src/err.rs +++ b/rslib/src/err.rs @@ -102,6 +102,10 @@ impl AnkiError { // already localized info.into() } + AnkiError::DBError { info, kind } => match kind { + DBErrorKind::Corrupt => info.clone(), + _ => format!("{:?}", self), + }, _ => format!("{:?}", self), } } @@ -262,5 +266,6 @@ pub enum DBErrorKind { FileTooNew, FileTooOld, MissingEntity, + Corrupt, Other, } diff --git a/rslib/src/notes.rs b/rslib/src/notes.rs index 540c20cf9..f9db8eb2f 100644 --- a/rslib/src/notes.rs +++ b/rslib/src/notes.rs @@ -135,6 +135,20 @@ impl Note { } changed } + + /// Pad or merge fields to match note type. + pub(crate) fn fix_field_count(&mut self, nt: &NoteType) { + while self.fields.len() < nt.fields.len() { + self.fields.push("".into()) + } + while self.fields.len() > nt.fields.len() && self.fields.len() > 1 { + let last = self.fields.pop().unwrap(); + self.fields + .last_mut() + .unwrap() + .push_str(&format!("; {}", last)); + } + } } impl From for pb::Note { diff --git a/rslib/src/notetype/mod.rs b/rslib/src/notetype/mod.rs index af5b39e4d..95559614b 100644 --- a/rslib/src/notetype/mod.rs +++ b/rslib/src/notetype/mod.rs @@ -329,8 +329,12 @@ impl From for NoteTypeProto { impl Collection { /// Add a new notetype, and allocate it an ID. pub fn add_notetype(&mut self, nt: &mut NoteType) -> Result<()> { + self.transact(None, |col| col.add_notetype_inner(nt)) + } + + pub(crate) fn add_notetype_inner(&mut self, nt: &mut NoteType) -> Result<()> { nt.prepare_for_adding()?; - self.transact(None, |col| col.storage.add_new_notetype(nt)) + self.storage.add_new_notetype(nt) } /// Saves changes to a note type. This will force a full sync if templates diff --git a/rslib/src/storage/card/fix_due_new.sql b/rslib/src/storage/card/fix_due_new.sql new file mode 100644 index 000000000..8190f2fec --- /dev/null +++ b/rslib/src/storage/card/fix_due_new.sql @@ -0,0 +1,19 @@ +update cards +set + due = ( + case + when type = 0 + and queue != 4 then 1000000 + due % 1000000 + else due + end + ), + mod = ?1, + usn = ?2 +where + due != ( + case + when type = 0 + and queue != 4 then 1000000 + due % 1000000 + else due + end + ); \ No newline at end of file diff --git a/rslib/src/storage/card/fix_due_other.sql b/rslib/src/storage/card/fix_due_other.sql new file mode 100644 index 000000000..8d71c1941 --- /dev/null +++ b/rslib/src/storage/card/fix_due_other.sql @@ -0,0 +1,19 @@ +update cards +set + due = ( + case + when queue = 2 + and due > 100000 then ?1 + else min(max(round(due), -2147483648), 2147483647) + end + ), + mod = ?2, + usn = ?3 +where + due != ( + case + when queue = 2 + and due > 100000 then ?1 + else min(max(round(due), -2147483648), 2147483647) + end + ); \ No newline at end of file diff --git a/rslib/src/storage/card/fix_ivl.sql b/rslib/src/storage/card/fix_ivl.sql new file mode 100644 index 000000000..a77c1793b --- /dev/null +++ b/rslib/src/storage/card/fix_ivl.sql @@ -0,0 +1,7 @@ +update cards +set + ivl = min(max(round(ivl), 0), 2147483647), + mod = ?1, + usn = ?2 +where + ivl != min(max(round(ivl), 0), 2147483647) \ No newline at end of file diff --git a/rslib/src/storage/card/fix_odue.sql b/rslib/src/storage/card/fix_odue.sql new file mode 100644 index 000000000..dc121333e --- /dev/null +++ b/rslib/src/storage/card/fix_odue.sql @@ -0,0 +1,27 @@ +update cards +set + odue = ( + case + when odue > 0 + and ( + type = 1 + or queue = 2 + ) + and not odid then 0 + else min(max(round(odue), -2147483648), 2147483647) + end + ), + mod = ?1, + usn = ?2 +where + odue != ( + case + when odue > 0 + and ( + type = 1 + or queue = 2 + ) + and not odid then 0 + else min(max(round(odue), -2147483648), 2147483647) + end + ); \ No newline at end of file diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index 3631f16e8..d4e88278f 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -1,13 +1,17 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::card::{Card, CardID, CardQueue, CardType}; -use crate::err::Result; -use crate::timestamp::TimestampMillis; +use crate::{ + card::{Card, CardID, CardQueue, CardType}, + decks::DeckID, + err::Result, + timestamp::{TimestampMillis, TimestampSecs}, + types::Usn, +}; use rusqlite::params; use rusqlite::{ types::{FromSql, FromSqlError, ValueRef}, - OptionalExtension, + OptionalExtension, NO_PARAMS, }; use std::convert::TryFrom; @@ -50,7 +54,7 @@ impl super::SqliteStorage { reps: row.get(10)?, lapses: row.get(11)?, left: row.get(12)?, - odue: row.get(13)?, + odue: row.get(13).ok().unwrap_or_default(), odid: row.get(14)?, flags: row.get(15)?, data: row.get(16)?, @@ -118,6 +122,53 @@ impl super::SqliteStorage { .execute(&[cid])?; Ok(()) } + + /// Fix some invalid card properties, and return number of changed cards. + pub(crate) fn fix_card_properties( + &self, + today: u32, + mtime: TimestampSecs, + usn: Usn, + ) -> Result<(usize, usize)> { + let new_cnt = self + .db + .prepare(include_str!("fix_due_new.sql"))? + .execute(params![mtime, usn])?; + let mut other_cnt = self + .db + .prepare(include_str!("fix_due_other.sql"))? + .execute(params![mtime, usn, today])?; + other_cnt += self + .db + .prepare(include_str!("fix_odue.sql"))? + .execute(params![mtime, usn])?; + other_cnt += self + .db + .prepare(include_str!("fix_ivl.sql"))? + .execute(params![mtime, usn])?; + Ok((new_cnt, other_cnt)) + } + + pub(crate) fn delete_orphaned_cards(&self) -> Result { + self.db + .prepare("delete from cards where nid not in (select id from notes)")? + .execute(NO_PARAMS) + .map_err(Into::into) + } + + pub(crate) fn all_filtered_cards_by_deck(&self) -> Result> { + self.db + .prepare("select id, did from cards where odid > 0")? + .query_and_then(NO_PARAMS, |r| -> Result<_> { Ok((r.get(0)?, r.get(1)?)) })? + .collect() + } + + pub(crate) fn max_new_card_position(&self) -> Result { + self.db + .prepare("select max(due)+1 from cards where type=0")? + .query_row(NO_PARAMS, |r| r.get(0)) + .map_err(Into::into) + } } #[cfg(test)] diff --git a/rslib/src/storage/deck/missing-decks.sql b/rslib/src/storage/deck/missing-decks.sql new file mode 100644 index 000000000..dfba53afc --- /dev/null +++ b/rslib/src/storage/deck/missing-decks.sql @@ -0,0 +1,9 @@ +select + distinct did +from cards +where + did not in ( + select + id + from decks + ); \ No newline at end of file diff --git a/rslib/src/storage/deck/mod.rs b/rslib/src/storage/deck/mod.rs index d0f52b6be..93ece69c9 100644 --- a/rslib/src/storage/deck/mod.rs +++ b/rslib/src/storage/deck/mod.rs @@ -170,6 +170,14 @@ impl SqliteStorage { .collect() } + /// Decks referenced by cards but missing. + pub(crate) fn missing_decks(&self) -> Result> { + self.db + .prepare(include_str!("missing-decks.sql"))? + .query_and_then(NO_PARAMS, |r| r.get(0).map_err(Into::into))? + .collect() + } + // Upgrading/downgrading/legacy pub(super) fn add_default_deck(&self, i18n: &I18n) -> Result<()> { diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs index ce2fb8df6..2cd8da66e 100644 --- a/rslib/src/storage/mod.rs +++ b/rslib/src/storage/mod.rs @@ -8,6 +8,7 @@ mod deckconf; mod graves; mod note; mod notetype; +mod revlog; mod sqlite; mod tag; mod upgrades; diff --git a/rslib/src/storage/notetype/mod.rs b/rslib/src/storage/notetype/mod.rs index ff76c92c0..fdc0663a5 100644 --- a/rslib/src/storage/notetype/mod.rs +++ b/rslib/src/storage/notetype/mod.rs @@ -36,7 +36,7 @@ fn row_to_existing_card(row: &Row) -> Result { nid: row.get(1)?, ord: row.get(2)?, original_deck_id: row.get(3)?, - position_if_new: row.get(4)?, + position_if_new: row.get(4).ok().unwrap_or_default(), }) } @@ -157,6 +157,14 @@ impl SqliteStorage { .collect() } + pub(crate) fn all_note_ids_by_notetype(&self) -> Result> { + let sql = String::from("select mid, id from notes order by mid, id"); + self.db + .prepare(&sql)? + .query_and_then(NO_PARAMS, |r| Ok((r.get(0)?, r.get(1)?)))? + .collect() + } + pub(crate) fn update_notetype_templates( &self, ntid: NoteTypeID, diff --git a/rslib/src/storage/revlog/fix_props.sql b/rslib/src/storage/revlog/fix_props.sql new file mode 100644 index 000000000..ddd950dc7 --- /dev/null +++ b/rslib/src/storage/revlog/fix_props.sql @@ -0,0 +1,7 @@ +update revlog +set + ivl = min(max(round(ivl), -2147483648), 2147483647), + lastIvl = min(max(round(lastIvl), -2147483648), 2147483647) +where + ivl != min(max(round(ivl), -2147483648), 2147483647) + or lastIVl != min(max(round(lastIvl), -2147483648), 2147483647) \ No newline at end of file diff --git a/rslib/src/storage/revlog/mod.rs b/rslib/src/storage/revlog/mod.rs new file mode 100644 index 000000000..dca2073a8 --- /dev/null +++ b/rslib/src/storage/revlog/mod.rs @@ -0,0 +1,15 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::SqliteStorage; +use crate::err::Result; +use rusqlite::NO_PARAMS; + +impl SqliteStorage { + pub(crate) fn fix_revlog_properties(&self) -> Result { + self.db + .prepare(include_str!("fix_props.sql"))? + .execute(NO_PARAMS) + .map_err(Into::into) + } +} diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 25e92fdd2..a7dc1bee6 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -304,6 +304,24 @@ impl SqliteStorage { ////////////////////////////////////////// + /// true if corrupt/can't access + pub(crate) fn quick_check_corrupt(&self) -> bool { + match self.db.pragma_query_value(None, "quick_check", |row| { + row.get(0).map(|v: String| v != "ok") + }) { + Ok(corrupt) => corrupt, + Err(e) => { + println!("error: {:?}", e); + true + } + } + } + + pub(crate) fn optimize(&self) -> Result<()> { + self.db.execute_batch("vacuum")?; + Ok(()) + } + #[cfg(test)] pub(crate) fn db_scalar(&self, sql: &str) -> Result { self.db