rewrite DB check

- notes with wrong field count are now recovered instead of
being deleted
- notes with missing note types are now recovered
- notes with missing cards are now recovered
- recover_missing_deck() still needs implementing
- checks required

proto/backend.proto

@@ -144,7 +144,7 @@ message BackendOutput {
         bytes get_changed_notetypes = 56;
         int64 add_or_update_notetype = 57;
         bytes get_all_decks = 58;
-        Empty check_database = 59;
+        CheckDatabaseOut check_database = 59;
         bytes get_notetype_legacy = 61;
         NoteTypeNames get_notetype_names = 62;
         NoteTypeUseCounts get_notetype_names_and_counts = 63;
@@ -749,3 +749,7 @@ message UpdateNoteTagsIn {
     string replacement = 3;
     bool regex = 4;
 }
+
+message CheckDatabaseOut {
+    repeated string problems = 1;
+}

pylib/anki/collection.py

@@ -9,7 +9,6 @@ import os
 import pprint
 import random
 import re
-import stat
 import time
 import traceback
 import weakref
@@ -25,11 +24,11 @@ from anki.consts import *
 from anki.dbproxy import DBProxy
 from anki.decks import DeckManager
 from anki.errors import AnkiError
-from anki.lang import _, ngettext
+from anki.lang import _
 from anki.media import MediaManager
 from anki.models import ModelManager, NoteType, Template
 from anki.notes import Note
-from anki.rsbackend import TR, RustBackend
+from anki.rsbackend import TR, DBError, RustBackend
 from anki.sched import Scheduler as V1Scheduler
 from anki.schedv2 import Scheduler as V2Scheduler
 from anki.tags import TagManager
@@ -675,208 +674,20 @@ select id from notes where mid = ?) limit 1"""
         Returns tuple of (error: str, ok: bool). 'ok' will be true if no
         problems were found.
         """
-        problems = []
+        self.save(trx=False)
-        # problems that don't require a full sync
+        try:
-        syncable_problems = []
+            problems = self.backend.check_database()
-        self.save()
+            ok = not problems
-        oldSize = os.stat(self.path)[stat.ST_SIZE]
+            problems.append(self.tr(TR.DATABASE_CHECK_REBUILT))
-        if self.db.scalar("pragma integrity_check") != "ok":
+        except DBError as e:
-            return (_("Collection is corrupt. Please see the manual."), False)
+            problems = [str(e.args[0])]
-        # note types with a missing model
+            ok = False
-        ids = self.db.list(
+        finally:
-            """
+            try:
-select id from notes where mid not in """
+                self.db.begin()
-            + ids2str(self.models.ids())
+            except:
-        )
+                # may fail if the DB is very corrupt
-        if ids:
+                pass
-            problems.append(
-                ngettext(
-                    "Deleted %d note with missing note type.",
-                    "Deleted %d notes with missing note type.",
-                    len(ids),
-                )
-                % len(ids)
-            )
-            self.remNotes(ids)
-        # for each model
-        for m in self.models.all():
-            for t in m["tmpls"]:
-                if t["did"] == "None":
-                    t["did"] = None
-                    problems.append(_("Fixed AnkiDroid deck override bug."))
-                    self.models.save(m, updateReqs=False)
-            if m["type"] == MODEL_STD:
-                # cards with invalid ordinal
-                ids = self.db.list(
-                    """
-select id from cards where ord not in %s and nid in (
-select id from notes where mid = ?)"""
-                    % ids2str([t["ord"] for t in m["tmpls"]]),
-                    m["id"],
-                )
-                if ids:
-                    problems.append(
-                        ngettext(
-                            "Deleted %d card with missing template.",
-                            "Deleted %d cards with missing template.",
-                            len(ids),
-                        )
-                        % len(ids)
-                    )
-                    self.remCards(ids)
-            # notes with invalid field count
-            ids = []
-            for id, flds in self.db.execute(
-                "select id, flds from notes where mid = ?", m["id"]
-            ):
-                if (flds.count("\x1f") + 1) != len(m["flds"]):
-                    ids.append(id)
-            if ids:
-                problems.append(
-                    ngettext(
-                        "Deleted %d note with wrong field count.",
-                        "Deleted %d notes with wrong field count.",
-                        len(ids),
-                    )
-                    % len(ids)
-                )
-                self.remNotes(ids)
-        # delete any notes with missing cards
-        ids = self.db.list(
-            """
-select id from notes where id not in (select distinct nid from cards)"""
-        )
-        if ids:
-            cnt = len(ids)
-            problems.append(
-                ngettext(
-                    "Deleted %d note with no cards.",
-                    "Deleted %d notes with no cards.",
-                    cnt,
-                )
-                % cnt
-            )
-            self._remNotes(ids)
-        # cards with missing notes
-        ids = self.db.list(
-            """
-select id from cards where nid not in (select id from notes)"""
-        )
-        if ids:
-            cnt = len(ids)
-            problems.append(
-                ngettext(
-                    "Deleted %d card with missing note.",
-                    "Deleted %d cards with missing note.",
-                    cnt,
-                )
-                % cnt
-            )
-            self.remCards(ids)
-        # cards with odue set when it shouldn't be
-        ids = self.db.list(
-            """
-select id from cards where odue > 0 and (type=1 or queue=2) and not odid"""
-        )
-        if ids:
-            cnt = len(ids)
-            problems.append(
-                ngettext(
-                    "Fixed %d card with invalid properties.",
-                    "Fixed %d cards with invalid properties.",
-                    cnt,
-                )
-                % cnt
-            )
-            self.db.execute("update cards set odue=0 where id in " + ids2str(ids))
-        # cards with odid set when not in a dyn deck
-        dids = [id for id in self.decks.allIds() if not self.decks.isDyn(id)]
-        ids = self.db.list(
-            """
-select id from cards where odid > 0 and did in %s"""
-            % ids2str(dids)
-        )
-        if ids:
-            cnt = len(ids)
-            problems.append(
-                ngettext(
-                    "Fixed %d card with invalid properties.",
-                    "Fixed %d cards with invalid properties.",
-                    cnt,
-                )
-                % cnt
-            )
-            self.db.execute(
-                "update cards set odid=0, odue=0 where id in " + ids2str(ids)
-            )
-        # tags & field cache
-        self.tags.register([], clear=True)
-        for m in self.models.all():
-            self.after_note_updates(
-                self.models.nids(m), mark_modified=False, generate_cards=False
-            )
-        # new cards can't have a due position > 32 bits, so wrap items over
-        # 2 million back to 1 million
-        self.db.execute(
-            """
-update cards set due=1000000+due%1000000,mod=?,usn=? where due>=1000000
-and type=0 and queue!=4""",
-            intTime(),
-            self.usn(),
-        )
-        rowcount = self.db.scalar("select changes()")
-        if rowcount:
-            syncable_problems.append(
-                "Found %d new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen."
-                % rowcount
-            )
-        # new card position
-        self.conf["nextPos"] = (
-            self.db.scalar("select max(due)+1 from cards where type = 0") or 0
-        )
-        # reviews should have a reasonable due #
-        ids = self.db.list("select id from cards where queue = 2 and due > 100000")
-        if ids:
-            problems.append("Reviews had incorrect due date.")
-            self.db.execute(
-                "update cards set due = ?, ivl = 1, mod = ?, usn = ? where id in %s"
-                % ids2str(ids),
-                self.sched.today,
-                intTime(),
-                self.usn(),
-            )
-        # v2 sched had a bug that could create decimal intervals
-        self.db.execute(
-            "update cards set ivl=round(ivl),due=round(due) where ivl!=round(ivl) or due!=round(due)"
-        )
-        rowcount = self.db.scalar("select changes()")
-        if rowcount:
-            problems.append("Fixed %d cards with v2 scheduler bug." % rowcount)
-
-        self.db.execute(
-            "update revlog set ivl=round(ivl),lastIvl=round(lastIvl) where ivl!=round(ivl) or lastIvl!=round(lastIvl)"
-        )
-        rowcount = self.db.scalar("select changes()")
-        if rowcount:
-            problems.append(
-                "Fixed %d review history entries with v2 scheduler bug." % rowcount
-            )
-        # models
-        if self.models.ensureNotEmpty():
-            problems.append("Added missing note type.")
-        # misc other
-        self.backend.check_database()
-        # and finally, optimize
-        self.optimize()
-        newSize = os.stat(self.path)[stat.ST_SIZE]
-        txt = _("Database rebuilt and optimized.")
-        ok = not problems
-        problems.append(txt)
-        # if any problems were found, force a full sync
-        if not ok:
-            self.modSchema(check=False)
-        self.save()
-        problems.extend(syncable_problems)
         return ("\n".join(problems), ok)
 
     def optimize(self) -> None:

pylib/anki/models.py

@@ -311,9 +311,12 @@ class ModelManager:
     # Tools
     ##################################################
 
-    def nids(self, m: NoteType) -> Any:
+    def nids(self, ntid: int) -> Any:
         "Note ids for M."
-        return self.col.db.list("select id from notes where mid = ?", m["id"])
+        if isinstance(ntid, dict):
+            # legacy callers passed in note type
+            ntid = ntid["id"]
+        return self.col.db.list("select id from notes where mid = ?", ntid)
 
     def useCount(self, m: NoteType) -> Any:
         "Number of note using M."

pylib/anki/rsbackend.py

@@ -736,8 +736,12 @@ class RustBackend:
             pb.BackendInput(deck_tree=pb.DeckTreeIn(include_counts=include_counts))
         ).deck_tree
 
-    def check_database(self) -> None:
+    def check_database(self) -> List[str]:
-        self._run_command(pb.BackendInput(check_database=pb.Empty()))
+        return list(
+            self._run_command(
+                pb.BackendInput(check_database=pb.Empty()), release_gil=True
+            ).check_database.problems
+        )
 
     def legacy_deck_tree(self) -> Sequence:
         bytes = self._run_command(
@@ -819,4 +823,4 @@ def translate_string_in(
 
 # temporarily force logging of media handling
 if "RUST_LOG" not in os.environ:
-    os.environ["RUST_LOG"] = "warn,anki::media=debug"
+    os.environ["RUST_LOG"] = "warn,anki::media=debug,anki::dbcheck=debug"

qt/aqt/main.py

@@ -1304,11 +1304,7 @@ will be lost. Continue?"""
     ##########################################################################
 
     def onCheckDB(self):
-        "True if no problems"
+        def on_done(future: Future):
-        self.progress.start()
-
-        def onDone(future: Future):
-            self.progress.finish()
             ret, ok = future.result()
 
             if not ok:
@@ -1319,15 +1315,17 @@ will be lost. Continue?"""
             # if an error has directed the user to check the database,
             # silently clean up any broken reset hooks which distract from
             # the underlying issue
-            while True:
+            n = 0
+            while n < 10:
                 try:
                     self.reset()
                     break
                 except Exception as e:
                     print("swallowed exception in reset hook:", e)
+                    n += 1
                     continue
 
-        self.taskman.run_in_background(self.col.fixIntegrity, onDone)
+        self.taskman.with_progress(self.col.fixIntegrity, on_done)
 
     def on_check_media_db(self) -> None:
         check_media_db(self)

rslib/ftl/database-check.ftl

@@ -0,0 +1,43 @@
+database-check-corrupt = Collection file is corrupt. Please restore from an automatic backup.
+database-check-rebuilt = Database rebuilt and optimized.
+database-check-card-properties =
+    { $count ->
+        [one] Fixed { $count } card with invalid properties.
+       *[other] Fixed { $count } cards with invalid properties.
+    }
+
+database-check-missing-templates =
+    { $count ->
+        [one] Deleted { $count } card with missing template.
+       *[other] Deleted { $count } cards with missing template.
+    }
+
+database-check-field-count = { $count ->
+  [one] Fixed { $count } note with wrong field count.
+  *[other] Fixed { $count } notes with wrong field count.
+  }
+
+database-check-new-card-high-due = { $count ->
+  [one] Found { $count } new card with a due number >= 1,000,000 - consider repositioning them in the Browse screen.
+  *[other] Found { $count } new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen.
+  }
+
+database-check-card-missing-note = { $count ->
+    [one] Deleted {$count} card with missing note.
+   *[other] Deleted {$count} cards with missing note.
+  }
+
+database-check-duplicate-card-ords = { $count ->
+  [one] Deleted { $count } card with duplicate template.
+  *[other] Deleted { $count } cards with duplicate template.
+  }
+
+database-check-missing-decks = { $count ->
+  [one] Fixed { $count } missing deck.
+  *[other] Fixed { $count } missing decks.
+  }
+
+database-check-revlog-properties = { $count ->
+  [one] Fixed { $count } review entry with invalid properties.
+  *[other] Fixed { $count } review entries with invalid properties.
+  }

rslib/src/backend/mod.rs

@@ -355,10 +355,7 @@ impl Backend {
                 self.remove_deck(did)?;
                 pb::Empty {}
             }),
-            Value::CheckDatabase(_) => {
+            Value::CheckDatabase(_) => OValue::CheckDatabase(self.check_database()?),
-                self.check_database()?;
-                OValue::CheckDatabase(pb::Empty {})
-            }
             Value::DeckTreeLegacy(_) => OValue::DeckTreeLegacy(self.deck_tree_legacy()?),
             Value::FieldNamesForNotes(input) => {
                 OValue::FieldNamesForNotes(self.field_names_for_notes(input)?)
@@ -1042,8 +1039,11 @@ impl Backend {
         self.with_col(|col| col.remove_deck_and_child_decks(DeckID(did)))
     }
 
-    fn check_database(&self) -> Result<()> {
+    fn check_database(&self) -> Result<pb::CheckDatabaseOut> {
-        self.with_col(|col| col.transact(None, |col| col.check_database()))
+        self.with_col(|col| {
+            col.check_database()
+                .map(|problems| pb::CheckDatabaseOut { problems })
+        })
     }
 
     fn deck_tree_legacy(&self) -> Result<Vec<u8>> {

rslib/src/config.rs

@@ -156,6 +156,10 @@ impl Collection {
         Ok(pos)
     }
 
+    pub(crate) fn set_next_card_position(&self, pos: u32) -> Result<()> {
+        self.set_config(ConfigKey::NextNewCardPosition, &pos)
+    }
+
     pub(crate) fn sched_ver(&self) -> SchedulerVersion {
         self.get_config_optional(ConfigKey::SchedulerVersion)
             .unwrap_or(SchedulerVersion::V1)

rslib/src/dbcheck.rs

@@ -1,12 +1,299 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::{collection::Collection, err::Result};
+use crate::{
+    collection::Collection,
+    err::{AnkiError, DBErrorKind, Result},
+    i18n::{tr_args, TR},
+    notetype::{
+        all_stock_notetypes, AlreadyGeneratedCardInfo, CardGenContext, NoteType, NoteTypeKind,
+    },
+    timestamp::{TimestampMillis, TimestampSecs},
+};
+use itertools::Itertools;
+use slog::debug;
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
 
 impl Collection {
-    pub(crate) fn check_database(&mut self) -> Result<()> {
+    /// Check the database, returning a list of problems that were fixed.
+    pub(crate) fn check_database(&mut self) -> Result<Vec<String>> {
+        debug!(self.log, "quick check");
+        if self.storage.quick_check_corrupt() {
+            debug!(self.log, "quick check failed");
+            return Err(AnkiError::DBError {
+                info: self.i18n.tr(TR::DatabaseCheckCorrupt).into(),
+                kind: DBErrorKind::Corrupt,
+            });
+        }
+
+        debug!(self.log, "optimize");
+        self.storage.optimize()?;
+
+        self.transact(None, |col| col.check_database_inner())
+    }
+
+    fn check_database_inner(&mut self) -> Result<Vec<String>> {
+        let mut probs = vec![];
+
+        // cards first, as we need to be able to read them to process notes
+        debug!(self.log, "check cards");
+        self.check_card_properties(&mut probs)?;
+        self.check_orphaned_cards(&mut probs)?;
+
+        debug!(self.log, "check decks");
+        self.check_missing_deck_ids(&mut probs)?;
+        self.check_filtered_cards(&mut probs)?;
+
+        debug!(self.log, "check notetypes");
+        self.check_notetypes(&mut probs)?;
+
+        debug!(self.log, "check review log");
+        self.check_revlog(&mut probs)?;
+
+        debug!(self.log, "missing decks");
         let names = self.storage.get_all_deck_names()?;
-        self.add_missing_decks(&names)?;
+        self.add_missing_deck_names(&names)?;
+
+        self.update_next_new_position()?;
+
+        debug!(self.log, "db check finished with problems: {:#?}", probs);
+
+        Ok(probs)
+    }
+
+    fn check_card_properties(&mut self, probs: &mut Vec<String>) -> Result<()> {
+        let timing = self.timing_today()?;
+        let (new_cnt, other_cnt) = self.storage.fix_card_properties(
+            timing.days_elapsed,
+            TimestampSecs::now(),
+            self.usn()?,
+        )?;
+        if new_cnt > 0 {
+            probs.push(
+                self.i18n
+                    .trn(TR::DatabaseCheckNewCardHighDue, tr_args!["count"=>new_cnt]),
+            );
+        }
+        if other_cnt > 0 {
+            probs.push(self.i18n.trn(
+                TR::DatabaseCheckCardProperties,
+                tr_args!["count"=>other_cnt],
+            ));
+        }
         Ok(())
     }
+
+    fn check_orphaned_cards(&mut self, probs: &mut Vec<String>) -> Result<()> {
+        let orphaned = self.storage.delete_orphaned_cards()?;
+        if orphaned > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(self.i18n.trn(
+                TR::DatabaseCheckCardMissingNote,
+                tr_args!["count"=>orphaned],
+            ));
+        }
+        Ok(())
+    }
+
+    fn check_missing_deck_ids(&mut self, probs: &mut Vec<String>) -> Result<()> {
+        let mut cnt = 0;
+        for did in self.storage.missing_decks()? {
+            self.recover_missing_deck(did)?;
+            cnt += 1;
+        }
+
+        if cnt > 0 {
+            probs.push(
+                self.i18n
+                    .trn(TR::DatabaseCheckMissingDecks, tr_args!["count"=>cnt]),
+            );
+        }
+
+        Ok(())
+    }
+
+    fn check_filtered_cards(&mut self, probs: &mut Vec<String>) -> Result<()> {
+        let decks: HashMap<_, _> = self
+            .storage
+            .get_all_decks()?
+            .into_iter()
+            .map(|d| (d.id, d))
+            .collect();
+
+        let mut wrong = 0;
+
+        for (cid, did) in self.storage.all_filtered_cards_by_deck()? {
+            // we expect calling code to ensure all decks already exist
+            if let Some(deck) = decks.get(&did) {
+                if !deck.is_filtered() {
+                    let mut card = self.storage.get_card(cid)?.unwrap();
+                    card.odid.0 = 0;
+                    card.odue = 0;
+                    self.storage.update_card(&card)?;
+                    wrong += 1;
+                }
+            }
+        }
+
+        if wrong > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(
+                self.i18n
+                    .trn(TR::DatabaseCheckCardProperties, tr_args!["count"=>wrong]),
+            );
+        }
+
+        Ok(())
+    }
+
+    fn check_notetypes(&mut self, probs: &mut Vec<String>) -> Result<()> {
+        let nids_by_notetype = self.storage.all_note_ids_by_notetype()?;
+        let norm = self.normalize_note_text();
+        let usn = self.usn()?;
+        let stamp = TimestampMillis::now();
+
+        // will rebuild tag list below
+        self.storage.clear_tags()?;
+
+        let mut note_fields = 0;
+        let mut dupe_ords = 0;
+        let mut missing_template_ords = 0;
+
+        for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
+            debug!(self.log, "check notetype: {}", ntid);
+            let mut group = group.peekable();
+            let nt = match self.get_notetype(ntid)? {
+                None => {
+                    let first_note = self.storage.get_note(group.peek().unwrap().1)?.unwrap();
+                    self.recover_notetype(stamp, first_note.fields.len())?
+                }
+                Some(nt) => nt,
+            };
+
+            let mut genctx = None;
+            for (_, nid) in group {
+                let mut note = self.storage.get_note(nid)?.unwrap();
+
+                let cards = self.storage.existing_cards_for_note(nid)?;
+                dupe_ords += self.remove_duplicate_card_ordinals(&cards)?;
+                missing_template_ords += self.remove_cards_without_template(&nt, &cards)?;
+
+                // fix fields
+                if note.fields.len() != nt.fields.len() {
+                    note.fix_field_count(&nt);
+                    note_fields += 1;
+                    note.tags.push("db-check".into());
+                }
+
+                // write note, updating tags and generating missing cards
+                let ctx = genctx.get_or_insert_with(|| CardGenContext::new(&nt, usn));
+                self.update_note_inner_generating_cards(&ctx, &mut note, false, norm)?;
+            }
+        }
+
+        // if the collection is empty and the user has deleted all note types, ensure at least
+        // one note type exists
+        if self.storage.get_all_notetype_names()?.is_empty() {
+            let mut nt = all_stock_notetypes(&self.i18n).remove(0);
+            self.add_notetype_inner(&mut nt)?;
+        }
+
+        if note_fields > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(
+                self.i18n
+                    .trn(TR::DatabaseCheckFieldCount, tr_args!["count"=>note_fields]),
+            );
+        }
+        if dupe_ords > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(self.i18n.trn(
+                TR::DatabaseCheckDuplicateCardOrds,
+                tr_args!["count"=>dupe_ords],
+            ));
+        }
+        if missing_template_ords > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(self.i18n.trn(
+                TR::DatabaseCheckMissingTemplates,
+                tr_args!["count"=>missing_template_ords],
+            ));
+        }
+
+        Ok(())
+    }
+
+    fn remove_duplicate_card_ordinals(
+        &mut self,
+        cards: &[AlreadyGeneratedCardInfo],
+    ) -> Result<usize> {
+        let mut ords = HashSet::new();
+        let mut removed = 0;
+        for card in cards {
+            if !ords.insert(card.ord) {
+                self.storage.remove_card(card.id)?;
+                removed += 1;
+            }
+        }
+
+        Ok(removed)
+    }
+
+    fn remove_cards_without_template(
+        &mut self,
+        nt: &NoteType,
+        cards: &[AlreadyGeneratedCardInfo],
+    ) -> Result<usize> {
+        if nt.config.kind() == NoteTypeKind::Cloze {
+            return Ok(0);
+        }
+        let mut removed = 0;
+        for card in cards {
+            if card.ord as usize >= nt.templates.len() {
+                self.storage.remove_card(card.id)?;
+                removed += 1;
+            }
+        }
+
+        Ok(removed)
+    }
+
+    fn recover_notetype(
+        &mut self,
+        stamp: TimestampMillis,
+        field_count: usize,
+    ) -> Result<Arc<NoteType>> {
+        debug!(self.log, "create recovery notetype");
+        let mut basic = all_stock_notetypes(&self.i18n).remove(0);
+        let mut field = 3;
+        while basic.fields.len() < field_count {
+            basic.add_field(format!("{}", field));
+            field += 1;
+        }
+        basic.name = format!("db-check-{}-{}", stamp, field_count);
+        self.add_notetype(&mut basic)?;
+        Ok(Arc::new(basic))
+    }
+
+    fn check_revlog(&self, probs: &mut Vec<String>) -> Result<()> {
+        let cnt = self.storage.fix_revlog_properties()?;
+
+        if cnt > 0 {
+            self.storage.set_schema_modified()?;
+            probs.push(
+                self.i18n
+                    .trn(TR::DatabaseCheckRevlogProperties, tr_args!["count"=>cnt]),
+            );
+        }
+
+        Ok(())
+    }
+
+    fn update_next_new_position(&self) -> Result<()> {
+        let pos = self.storage.max_new_card_position().unwrap_or(0);
+        self.set_next_card_position(pos)
+    }
 }

rslib/src/decks/mod.rs

@@ -209,6 +209,12 @@ impl Collection {
         })
     }
 
+    pub(crate) fn recover_missing_deck(&mut self, _did: DeckID) -> Result<()> {
+        // todo
+
+        Ok(())
+    }
+
     pub fn get_or_create_normal_deck(&mut self, human_name: &str) -> Result<Deck> {
         let native_name = human_deck_name_to_native(human_name);
         if let Some(did) = self.storage.get_deck_id(&native_name)? {

rslib/src/decks/tree.rs

@@ -215,7 +215,7 @@ impl Collection {
         Ok(LegacyDueCounts::from(tree))
     }
 
-    pub(crate) fn add_missing_decks(&mut self, names: &[(DeckID, String)]) -> Result<()> {
+    pub(crate) fn add_missing_deck_names(&mut self, names: &[(DeckID, String)]) -> Result<()> {
         let mut parents = HashSet::new();
         for (_id, name) in names {
             parents.insert(UniCase::new(name.as_str()));

rslib/src/err.rs

@@ -102,6 +102,10 @@ impl AnkiError {
                 // already localized
                 info.into()
             }
+            AnkiError::DBError { info, kind } => match kind {
+                DBErrorKind::Corrupt => info.clone(),
+                _ => format!("{:?}", self),
+            },
             _ => format!("{:?}", self),
         }
     }
@@ -262,5 +266,6 @@ pub enum DBErrorKind {
     FileTooNew,
     FileTooOld,
     MissingEntity,
+    Corrupt,
     Other,
 }

rslib/src/notes.rs

@@ -135,6 +135,20 @@ impl Note {
         }
         changed
     }
+
+    /// Pad or merge fields to match note type.
+    pub(crate) fn fix_field_count(&mut self, nt: &NoteType) {
+        while self.fields.len() < nt.fields.len() {
+            self.fields.push("".into())
+        }
+        while self.fields.len() > nt.fields.len() && self.fields.len() > 1 {
+            let last = self.fields.pop().unwrap();
+            self.fields
+                .last_mut()
+                .unwrap()
+                .push_str(&format!("; {}", last));
+        }
+    }
 }
 
 impl From<Note> for pb::Note {

rslib/src/notetype/mod.rs

@@ -329,8 +329,12 @@ impl From<NoteType> for NoteTypeProto {
 impl Collection {
     /// Add a new notetype, and allocate it an ID.
     pub fn add_notetype(&mut self, nt: &mut NoteType) -> Result<()> {
+        self.transact(None, |col| col.add_notetype_inner(nt))
+    }
+
+    pub(crate) fn add_notetype_inner(&mut self, nt: &mut NoteType) -> Result<()> {
         nt.prepare_for_adding()?;
-        self.transact(None, |col| col.storage.add_new_notetype(nt))
+        self.storage.add_new_notetype(nt)
     }
 
     /// Saves changes to a note type. This will force a full sync if templates

rslib/src/storage/card/fix_due_new.sql

@@ -0,0 +1,19 @@
+update cards
+set
+  due = (
+    case
+      when type = 0
+      and queue != 4 then 1000000 + due % 1000000
+      else due
+    end
+  ),
+  mod = ?1,
+  usn = ?2
+where
+  due != (
+    case
+      when type = 0
+      and queue != 4 then 1000000 + due % 1000000
+      else due
+    end
+  );

rslib/src/storage/card/fix_due_other.sql

@@ -0,0 +1,19 @@
+update cards
+set
+  due = (
+    case
+      when queue = 2
+      and due > 100000 then ?1
+      else min(max(round(due), -2147483648), 2147483647)
+    end
+  ),
+  mod = ?2,
+  usn = ?3
+where
+  due != (
+    case
+      when queue = 2
+      and due > 100000 then ?1
+      else min(max(round(due), -2147483648), 2147483647)
+    end
+  );

rslib/src/storage/card/fix_ivl.sql

@@ -0,0 +1,7 @@
+update cards
+set
+  ivl = min(max(round(ivl), 0), 2147483647),
+  mod = ?1,
+  usn = ?2
+where
+  ivl != min(max(round(ivl), 0), 2147483647)

rslib/src/storage/card/fix_odue.sql

@@ -0,0 +1,27 @@
+update cards
+set
+  odue = (
+    case
+      when odue > 0
+      and (
+        type = 1
+        or queue = 2
+      )
+      and not odid then 0
+      else min(max(round(odue), -2147483648), 2147483647)
+    end
+  ),
+  mod = ?1,
+  usn = ?2
+where
+  odue != (
+    case
+      when odue > 0
+      and (
+        type = 1
+        or queue = 2
+      )
+      and not odid then 0
+      else min(max(round(odue), -2147483648), 2147483647)
+    end
+  );

rslib/src/storage/card/mod.rs

@@ -1,13 +1,17 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::card::{Card, CardID, CardQueue, CardType};
+use crate::{
-use crate::err::Result;
+    card::{Card, CardID, CardQueue, CardType},
-use crate::timestamp::TimestampMillis;
+    decks::DeckID,
+    err::Result,
+    timestamp::{TimestampMillis, TimestampSecs},
+    types::Usn,
+};
 use rusqlite::params;
 use rusqlite::{
     types::{FromSql, FromSqlError, ValueRef},
-    OptionalExtension,
+    OptionalExtension, NO_PARAMS,
 };
 use std::convert::TryFrom;
 
@@ -50,7 +54,7 @@ impl super::SqliteStorage {
                 reps: row.get(10)?,
                 lapses: row.get(11)?,
                 left: row.get(12)?,
-                odue: row.get(13)?,
+                odue: row.get(13).ok().unwrap_or_default(),
                 odid: row.get(14)?,
                 flags: row.get(15)?,
                 data: row.get(16)?,
@@ -118,6 +122,53 @@ impl super::SqliteStorage {
             .execute(&[cid])?;
         Ok(())
     }
+
+    /// Fix some invalid card properties, and return number of changed cards.
+    pub(crate) fn fix_card_properties(
+        &self,
+        today: u32,
+        mtime: TimestampSecs,
+        usn: Usn,
+    ) -> Result<(usize, usize)> {
+        let new_cnt = self
+            .db
+            .prepare(include_str!("fix_due_new.sql"))?
+            .execute(params![mtime, usn])?;
+        let mut other_cnt = self
+            .db
+            .prepare(include_str!("fix_due_other.sql"))?
+            .execute(params![mtime, usn, today])?;
+        other_cnt += self
+            .db
+            .prepare(include_str!("fix_odue.sql"))?
+            .execute(params![mtime, usn])?;
+        other_cnt += self
+            .db
+            .prepare(include_str!("fix_ivl.sql"))?
+            .execute(params![mtime, usn])?;
+        Ok((new_cnt, other_cnt))
+    }
+
+    pub(crate) fn delete_orphaned_cards(&self) -> Result<usize> {
+        self.db
+            .prepare("delete from cards where nid not in (select id from notes)")?
+            .execute(NO_PARAMS)
+            .map_err(Into::into)
+    }
+
+    pub(crate) fn all_filtered_cards_by_deck(&self) -> Result<Vec<(CardID, DeckID)>> {
+        self.db
+            .prepare("select id, did from cards where odid > 0")?
+            .query_and_then(NO_PARAMS, |r| -> Result<_> { Ok((r.get(0)?, r.get(1)?)) })?
+            .collect()
+    }
+
+    pub(crate) fn max_new_card_position(&self) -> Result<u32> {
+        self.db
+            .prepare("select max(due)+1 from cards where type=0")?
+            .query_row(NO_PARAMS, |r| r.get(0))
+            .map_err(Into::into)
+    }
 }
 
 #[cfg(test)]

rslib/src/storage/deck/missing-decks.sql

@@ -0,0 +1,9 @@
+select
+  distinct did
+from cards
+where
+  did not in (
+    select
+      id
+    from decks
+  );

rslib/src/storage/deck/mod.rs

@@ -170,6 +170,14 @@ impl SqliteStorage {
             .collect()
     }
 
+    /// Decks referenced by cards but missing.
+    pub(crate) fn missing_decks(&self) -> Result<Vec<DeckID>> {
+        self.db
+            .prepare(include_str!("missing-decks.sql"))?
+            .query_and_then(NO_PARAMS, |r| r.get(0).map_err(Into::into))?
+            .collect()
+    }
+
     // Upgrading/downgrading/legacy
 
     pub(super) fn add_default_deck(&self, i18n: &I18n) -> Result<()> {

rslib/src/storage/mod.rs

@@ -8,6 +8,7 @@ mod deckconf;
 mod graves;
 mod note;
 mod notetype;
+mod revlog;
 mod sqlite;
 mod tag;
 mod upgrades;

rslib/src/storage/notetype/mod.rs

@@ -36,7 +36,7 @@ fn row_to_existing_card(row: &Row) -> Result<AlreadyGeneratedCardInfo> {
         nid: row.get(1)?,
         ord: row.get(2)?,
         original_deck_id: row.get(3)?,
-        position_if_new: row.get(4)?,
+        position_if_new: row.get(4).ok().unwrap_or_default(),
     })
 }
 
@@ -157,6 +157,14 @@ impl SqliteStorage {
             .collect()
     }
 
+    pub(crate) fn all_note_ids_by_notetype(&self) -> Result<Vec<(NoteTypeID, NoteID)>> {
+        let sql = String::from("select mid, id from notes order by mid, id");
+        self.db
+            .prepare(&sql)?
+            .query_and_then(NO_PARAMS, |r| Ok((r.get(0)?, r.get(1)?)))?
+            .collect()
+    }
+
     pub(crate) fn update_notetype_templates(
         &self,
         ntid: NoteTypeID,

rslib/src/storage/revlog/fix_props.sql

@@ -0,0 +1,7 @@
+update revlog
+set
+  ivl = min(max(round(ivl), -2147483648), 2147483647),
+  lastIvl = min(max(round(lastIvl), -2147483648), 2147483647)
+where
+  ivl != min(max(round(ivl), -2147483648), 2147483647)
+  or lastIVl != min(max(round(lastIvl), -2147483648), 2147483647)

rslib/src/storage/revlog/mod.rs

@@ -0,0 +1,15 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::SqliteStorage;
+use crate::err::Result;
+use rusqlite::NO_PARAMS;
+
+impl SqliteStorage {
+    pub(crate) fn fix_revlog_properties(&self) -> Result<usize> {
+        self.db
+            .prepare(include_str!("fix_props.sql"))?
+            .execute(NO_PARAMS)
+            .map_err(Into::into)
+    }
+}

rslib/src/storage/sqlite.rs

@@ -304,6 +304,24 @@ impl SqliteStorage {
 
     //////////////////////////////////////////
 
+    /// true if corrupt/can't access
+    pub(crate) fn quick_check_corrupt(&self) -> bool {
+        match self.db.pragma_query_value(None, "quick_check", |row| {
+            row.get(0).map(|v: String| v != "ok")
+        }) {
+            Ok(corrupt) => corrupt,
+            Err(e) => {
+                println!("error: {:?}", e);
+                true
+            }
+        }
+    }
+
+    pub(crate) fn optimize(&self) -> Result<()> {
+        self.db.execute_batch("vacuum")?;
+        Ok(())
+    }
+
     #[cfg(test)]
     pub(crate) fn db_scalar<T: rusqlite::types::FromSql>(&self, sql: &str) -> Result<T> {
         self.db