GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-24 16:56:36 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

rewrite DB check

Browse source 
- notes with wrong field count are now recovered instead of
being deleted
- notes with missing note types are now recovered
- notes with missing cards are now recovered
- recover_missing_deck() still needs implementing
- checks required
This commit is contained in:
Damien Elmes 2020-05-10 13:50:04 +10:00
parent 4c9b6be832
commit 70cc1699a6
26 changed files with 607 additions and 235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
proto/backend.proto
View file

@ -144,7 +144,7 @@ message BackendOutput {
bytes get_changed_notetypes = 56;
int64 add_or_update_notetype = 57;
bytes get_all_decks = 58;
Empty check_database = 59;
CheckDatabaseOut check_database = 59;
bytes get_notetype_legacy = 61;
NoteTypeNames get_notetype_names = 62;
NoteTypeUseCounts get_notetype_names_and_counts = 63;
@ -749,3 +749,7 @@ message UpdateNoteTagsIn {
string replacement = 3;
bool regex = 4;
}
message CheckDatabaseOut {
repeated string problems = 1;
}

221
pylib/anki/collection.py
View file

@ -9,7 +9,6 @@ import os
import pprint
import random
import re
import stat
import time
import traceback
import weakref
@ -25,11 +24,11 @@ from anki.consts import *
from anki.dbproxy import DBProxy
from anki.decks import DeckManager
from anki.errors import AnkiError
from anki.lang import _, ngettext
from anki.lang import _
from anki.media import MediaManager
from anki.models import ModelManager, NoteType, Template
from anki.notes import Note
from anki.rsbackend import TR, RustBackend
from anki.rsbackend import TR, DBError, RustBackend
from anki.sched import Scheduler as V1Scheduler
from anki.schedv2 import Scheduler as V2Scheduler
from anki.tags import TagManager
@ -675,208 +674,20 @@ select id from notes where mid = ?) limit 1"""
Returns tuple of (error: str, ok: bool). 'ok' will be true if no
problems were found.
"""
problems = []
# problems that don't require a full sync
syncable_problems = []
self.save()
oldSize = os.stat(self.path)[stat.ST_SIZE]
if self.db.scalar("pragma integrity_check") != "ok":
return (_("Collection is corrupt. Please see the manual."), False)
# note types with a missing model
ids = self.db.list(
"""
select id from notes where mid not in """
+ ids2str(self.models.ids())
)
if ids:
problems.append(
ngettext(
"Deleted %d note with missing note type.",
"Deleted %d notes with missing note type.",
len(ids),
)
% len(ids)
)
self.remNotes(ids)
# for each model
for m in self.models.all():
for t in m["tmpls"]:
if t["did"] == "None":
t["did"] = None
problems.append(_("Fixed AnkiDroid deck override bug."))
self.models.save(m, updateReqs=False)
if m["type"] == MODEL_STD:
# cards with invalid ordinal
ids = self.db.list(
"""
select id from cards where ord not in %s and nid in (
select id from notes where mid = ?)"""
% ids2str([t["ord"] for t in m["tmpls"]]),
m["id"],
)
if ids:
problems.append(
ngettext(
"Deleted %d card with missing template.",
"Deleted %d cards with missing template.",
len(ids),
)
% len(ids)
)
self.remCards(ids)
# notes with invalid field count
ids = []
for id, flds in self.db.execute(
"select id, flds from notes where mid = ?", m["id"]
):
if (flds.count("\x1f") + 1) != len(m["flds"]):
ids.append(id)
if ids:
problems.append(
ngettext(
"Deleted %d note with wrong field count.",
"Deleted %d notes with wrong field count.",
len(ids),
)
% len(ids)
)
self.remNotes(ids)
# delete any notes with missing cards
ids = self.db.list(
"""
select id from notes where id not in (select distinct nid from cards)"""
)
if ids:
cnt = len(ids)
problems.append(
ngettext(
"Deleted %d note with no cards.",
"Deleted %d notes with no cards.",
cnt,
)
% cnt
)
self._remNotes(ids)
# cards with missing notes
ids = self.db.list(
"""
select id from cards where nid not in (select id from notes)"""
)
if ids:
cnt = len(ids)
problems.append(
ngettext(
"Deleted %d card with missing note.",
"Deleted %d cards with missing note.",
cnt,
)
% cnt
)
self.remCards(ids)
# cards with odue set when it shouldn't be
ids = self.db.list(
"""
select id from cards where odue > 0 and (type=1 or queue=2) and not odid"""
)
if ids:
cnt = len(ids)
problems.append(
ngettext(
"Fixed %d card with invalid properties.",
"Fixed %d cards with invalid properties.",
cnt,
)
% cnt
)
self.db.execute("update cards set odue=0 where id in " + ids2str(ids))
# cards with odid set when not in a dyn deck
dids = [id for id in self.decks.allIds() if not self.decks.isDyn(id)]
ids = self.db.list(
"""
select id from cards where odid > 0 and did in %s"""
% ids2str(dids)
)
if ids:
cnt = len(ids)
problems.append(
ngettext(
"Fixed %d card with invalid properties.",
"Fixed %d cards with invalid properties.",
cnt,
)
% cnt
)
self.db.execute(
"update cards set odid=0, odue=0 where id in " + ids2str(ids)
)
# tags & field cache
self.tags.register([], clear=True)
for m in self.models.all():
self.after_note_updates(
self.models.nids(m), mark_modified=False, generate_cards=False
)
# new cards can't have a due position > 32 bits, so wrap items over
# 2 million back to 1 million
self.db.execute(
"""
update cards set due=1000000+due%1000000,mod=?,usn=? where due>=1000000
and type=0 and queue!=4""",
intTime(),
self.usn(),
)
rowcount = self.db.scalar("select changes()")
if rowcount:
syncable_problems.append(
"Found %d new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen."
% rowcount
)
# new card position
self.conf["nextPos"] = (
self.db.scalar("select max(due)+1 from cards where type = 0") or 0
)
# reviews should have a reasonable due #
ids = self.db.list("select id from cards where queue = 2 and due > 100000")
if ids:
problems.append("Reviews had incorrect due date.")
self.db.execute(
"update cards set due = ?, ivl = 1, mod = ?, usn = ? where id in %s"
% ids2str(ids),
self.sched.today,
intTime(),
self.usn(),
)
# v2 sched had a bug that could create decimal intervals
self.db.execute(
"update cards set ivl=round(ivl),due=round(due) where ivl!=round(ivl) or due!=round(due)"
)
rowcount = self.db.scalar("select changes()")
if rowcount:
problems.append("Fixed %d cards with v2 scheduler bug." % rowcount)
self.db.execute(
"update revlog set ivl=round(ivl),lastIvl=round(lastIvl) where ivl!=round(ivl) or lastIvl!=round(lastIvl)"
)
rowcount = self.db.scalar("select changes()")
if rowcount:
problems.append(
"Fixed %d review history entries with v2 scheduler bug." % rowcount
)
# models
if self.models.ensureNotEmpty():
problems.append("Added missing note type.")
# misc other
self.backend.check_database()
# and finally, optimize
self.optimize()
newSize = os.stat(self.path)[stat.ST_SIZE]
txt = _("Database rebuilt and optimized.")
ok = not problems
problems.append(txt)
# if any problems were found, force a full sync
if not ok:
self.modSchema(check=False)
self.save()
problems.extend(syncable_problems)
self.save(trx=False)
try:
problems = self.backend.check_database()
ok = not problems
problems.append(self.tr(TR.DATABASE_CHECK_REBUILT))
except DBError as e:
problems = [str(e.args[0])]
ok = False
finally:
try:
self.db.begin()
except:
# may fail if the DB is very corrupt
pass
return ("\n".join(problems), ok)
def optimize(self) -> None:

7
pylib/anki/models.py
View file

@ -311,9 +311,12 @@ class ModelManager:
# Tools
##################################################
def nids(self, m: NoteType) -> Any:
def nids(self, ntid: int) -> Any:
"Note ids for M."
return self.col.db.list("select id from notes where mid = ?", m["id"])
if isinstance(ntid, dict):
# legacy callers passed in note type
ntid = ntid["id"]
return self.col.db.list("select id from notes where mid = ?", ntid)
def useCount(self, m: NoteType) -> Any:
"Number of note using M."

10
pylib/anki/rsbackend.py
View file

@ -736,8 +736,12 @@ class RustBackend:
pb.BackendInput(deck_tree=pb.DeckTreeIn(include_counts=include_counts))
).deck_tree
def check_database(self) -> None:
self._run_command(pb.BackendInput(check_database=pb.Empty()))
def check_database(self) -> List[str]:
return list(
self._run_command(
pb.BackendInput(check_database=pb.Empty()), release_gil=True
).check_database.problems
)
def legacy_deck_tree(self) -> Sequence:
bytes = self._run_command(
@ -819,4 +823,4 @@ def translate_string_in(
# temporarily force logging of media handling
if "RUST_LOG" not in os.environ:
os.environ["RUST_LOG"] = "warn,anki::media=debug"
os.environ["RUST_LOG"] = "warn,anki::media=debug,anki::dbcheck=debug"

12
qt/aqt/main.py
View file

@ -1304,11 +1304,7 @@ will be lost. Continue?"""
##########################################################################
def onCheckDB(self):
"True if no problems"
self.progress.start()
def onDone(future: Future):
self.progress.finish()
def on_done(future: Future):
ret, ok = future.result()
if not ok:
@ -1319,15 +1315,17 @@ will be lost. Continue?"""
# if an error has directed the user to check the database,
# silently clean up any broken reset hooks which distract from
# the underlying issue
while True:
n = 0
while n < 10:
try:
self.reset()
break
except Exception as e:
print("swallowed exception in reset hook:", e)
n += 1
continue
self.taskman.run_in_background(self.col.fixIntegrity, onDone)
self.taskman.with_progress(self.col.fixIntegrity, on_done)
def on_check_media_db(self) -> None:
check_media_db(self)

43
rslib/ftl/database-check.ftl Normal file
View file

@ -0,0 +1,43 @@
database-check-corrupt = Collection file is corrupt. Please restore from an automatic backup.
database-check-rebuilt = Database rebuilt and optimized.
database-check-card-properties =
{ $count ->
[one] Fixed { $count } card with invalid properties.
*[other] Fixed { $count } cards with invalid properties.
}
database-check-missing-templates =
{ $count ->
[one] Deleted { $count } card with missing template.
*[other] Deleted { $count } cards with missing template.
}
database-check-field-count = { $count ->
[one] Fixed { $count } note with wrong field count.
*[other] Fixed { $count } notes with wrong field count.
}
database-check-new-card-high-due = { $count ->
[one] Found { $count } new card with a due number >= 1,000,000 - consider repositioning them in the Browse screen.
*[other] Found { $count } new cards with a due number >= 1,000,000 - consider repositioning them in the Browse screen.
}
database-check-card-missing-note = { $count ->
[one] Deleted {$count} card with missing note.
*[other] Deleted {$count} cards with missing note.
}
database-check-duplicate-card-ords = { $count ->
[one] Deleted { $count } card with duplicate template.
*[other] Deleted { $count } cards with duplicate template.
}
database-check-missing-decks = { $count ->
[one] Fixed { $count } missing deck.
*[other] Fixed { $count } missing decks.
}
database-check-revlog-properties = { $count ->
[one] Fixed { $count } review entry with invalid properties.
*[other] Fixed { $count } review entries with invalid properties.
}

12
rslib/src/backend/mod.rs
View file

@ -355,10 +355,7 @@ impl Backend {
self.remove_deck(did)?;
pb::Empty {}
}),
Value::CheckDatabase(_) => {
self.check_database()?;
OValue::CheckDatabase(pb::Empty {})
}
Value::CheckDatabase(_) => OValue::CheckDatabase(self.check_database()?),
Value::DeckTreeLegacy(_) => OValue::DeckTreeLegacy(self.deck_tree_legacy()?),
Value::FieldNamesForNotes(input) => {
OValue::FieldNamesForNotes(self.field_names_for_notes(input)?)
@ -1042,8 +1039,11 @@ impl Backend {
self.with_col(|col| col.remove_deck_and_child_decks(DeckID(did)))
}
fn check_database(&self) -> Result<()> {
self.with_col(|col| col.transact(None, |col| col.check_database()))
fn check_database(&self) -> Result<pb::CheckDatabaseOut> {
self.with_col(|col| {
col.check_database()
.map(|problems| pb::CheckDatabaseOut { problems })
})
}
fn deck_tree_legacy(&self) -> Result<Vec<u8>> {

4
rslib/src/config.rs
View file

@ -156,6 +156,10 @@ impl Collection {
Ok(pos)
}
pub(crate) fn set_next_card_position(&self, pos: u32) -> Result<()> {
self.set_config(ConfigKey::NextNewCardPosition, &pos)
}
pub(crate) fn sched_ver(&self) -> SchedulerVersion {
self.get_config_optional(ConfigKey::SchedulerVersion)
.unwrap_or(SchedulerVersion::V1)

293
rslib/src/dbcheck.rs
View file

@ -1,12 +1,299 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
use crate::{collection::Collection, err::Result};
use crate::{
collection::Collection,
err::{AnkiError, DBErrorKind, Result},
i18n::{tr_args, TR},
notetype::{
all_stock_notetypes, AlreadyGeneratedCardInfo, CardGenContext, NoteType, NoteTypeKind,
},
timestamp::{TimestampMillis, TimestampSecs},
};
use itertools::Itertools;
use slog::debug;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
impl Collection {
pub(crate) fn check_database(&mut self) -> Result<()> {
/// Check the database, returning a list of problems that were fixed.
pub(crate) fn check_database(&mut self) -> Result<Vec<String>> {
debug!(self.log, "quick check");
if self.storage.quick_check_corrupt() {
debug!(self.log, "quick check failed");
return Err(AnkiError::DBError {
info: self.i18n.tr(TR::DatabaseCheckCorrupt).into(),
kind: DBErrorKind::Corrupt,
});
}
debug!(self.log, "optimize");
self.storage.optimize()?;
self.transact(None, |col| col.check_database_inner())
}
fn check_database_inner(&mut self) -> Result<Vec<String>> {
let mut probs = vec![];
// cards first, as we need to be able to read them to process notes
debug!(self.log, "check cards");
self.check_card_properties(&mut probs)?;
self.check_orphaned_cards(&mut probs)?;
debug!(self.log, "check decks");
self.check_missing_deck_ids(&mut probs)?;
self.check_filtered_cards(&mut probs)?;
debug!(self.log, "check notetypes");
self.check_notetypes(&mut probs)?;
debug!(self.log, "check review log");
self.check_revlog(&mut probs)?;
debug!(self.log, "missing decks");
let names = self.storage.get_all_deck_names()?;
self.add_missing_decks(&names)?;
self.add_missing_deck_names(&names)?;
self.update_next_new_position()?;
debug!(self.log, "db check finished with problems: {:#?}", probs);
Ok(probs)
}
fn check_card_properties(&mut self, probs: &mut Vec<String>) -> Result<()> {
let timing = self.timing_today()?;
let (new_cnt, other_cnt) = self.storage.fix_card_properties(
timing.days_elapsed,
TimestampSecs::now(),
self.usn()?,
)?;
if new_cnt > 0 {
probs.push(
self.i18n
.trn(TR::DatabaseCheckNewCardHighDue, tr_args!["count"=>new_cnt]),
);
}
if other_cnt > 0 {
probs.push(self.i18n.trn(
TR::DatabaseCheckCardProperties,
tr_args!["count"=>other_cnt],
));
}
Ok(())
}
fn check_orphaned_cards(&mut self, probs: &mut Vec<String>) -> Result<()> {
let orphaned = self.storage.delete_orphaned_cards()?;
if orphaned > 0 {
self.storage.set_schema_modified()?;
probs.push(self.i18n.trn(
TR::DatabaseCheckCardMissingNote,
tr_args!["count"=>orphaned],
));
}
Ok(())
}
fn check_missing_deck_ids(&mut self, probs: &mut Vec<String>) -> Result<()> {
let mut cnt = 0;
for did in self.storage.missing_decks()? {
self.recover_missing_deck(did)?;
cnt += 1;
}
if cnt > 0 {
probs.push(
self.i18n
.trn(TR::DatabaseCheckMissingDecks, tr_args!["count"=>cnt]),
);
}
Ok(())
}
fn check_filtered_cards(&mut self, probs: &mut Vec<String>) -> Result<()> {
let decks: HashMap<_, _> = self
.storage
.get_all_decks()?
.into_iter()
.map(|d| (d.id, d))
.collect();
let mut wrong = 0;
for (cid, did) in self.storage.all_filtered_cards_by_deck()? {
// we expect calling code to ensure all decks already exist
if let Some(deck) = decks.get(&did) {
if !deck.is_filtered() {
let mut card = self.storage.get_card(cid)?.unwrap();
card.odid.0 = 0;
card.odue = 0;
self.storage.update_card(&card)?;
wrong += 1;
}
}
}
if wrong > 0 {
self.storage.set_schema_modified()?;
probs.push(
self.i18n
.trn(TR::DatabaseCheckCardProperties, tr_args!["count"=>wrong]),
);
}
Ok(())
}
fn check_notetypes(&mut self, probs: &mut Vec<String>) -> Result<()> {
let nids_by_notetype = self.storage.all_note_ids_by_notetype()?;
let norm = self.normalize_note_text();
let usn = self.usn()?;
let stamp = TimestampMillis::now();
// will rebuild tag list below
self.storage.clear_tags()?;
let mut note_fields = 0;
let mut dupe_ords = 0;
let mut missing_template_ords = 0;
for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
debug!(self.log, "check notetype: {}", ntid);
let mut group = group.peekable();
let nt = match self.get_notetype(ntid)? {
None => {
let first_note = self.storage.get_note(group.peek().unwrap().1)?.unwrap();
self.recover_notetype(stamp, first_note.fields.len())?
}
Some(nt) => nt,
};
let mut genctx = None;
for (_, nid) in group {
let mut note = self.storage.get_note(nid)?.unwrap();
let cards = self.storage.existing_cards_for_note(nid)?;
dupe_ords += self.remove_duplicate_card_ordinals(&cards)?;
missing_template_ords += self.remove_cards_without_template(&nt, &cards)?;
// fix fields
if note.fields.len() != nt.fields.len() {
note.fix_field_count(&nt);
note_fields += 1;
note.tags.push("db-check".into());
}
// write note, updating tags and generating missing cards
let ctx = genctx.get_or_insert_with(|| CardGenContext::new(&nt, usn));
self.update_note_inner_generating_cards(&ctx, &mut note, false, norm)?;
}
}
// if the collection is empty and the user has deleted all note types, ensure at least
// one note type exists
if self.storage.get_all_notetype_names()?.is_empty() {
let mut nt = all_stock_notetypes(&self.i18n).remove(0);
self.add_notetype_inner(&mut nt)?;
}
if note_fields > 0 {
self.storage.set_schema_modified()?;
probs.push(
self.i18n
.trn(TR::DatabaseCheckFieldCount, tr_args!["count"=>note_fields]),
);
}
if dupe_ords > 0 {
self.storage.set_schema_modified()?;
probs.push(self.i18n.trn(
TR::DatabaseCheckDuplicateCardOrds,
tr_args!["count"=>dupe_ords],
));
}
if missing_template_ords > 0 {
self.storage.set_schema_modified()?;
probs.push(self.i18n.trn(
TR::DatabaseCheckMissingTemplates,
tr_args!["count"=>missing_template_ords],
));
}
Ok(())
}
fn remove_duplicate_card_ordinals(
&mut self,
cards: &[AlreadyGeneratedCardInfo],
) -> Result<usize> {
let mut ords = HashSet::new();
let mut removed = 0;
for card in cards {
if !ords.insert(card.ord) {
self.storage.remove_card(card.id)?;
removed += 1;
}
}
Ok(removed)
}
fn remove_cards_without_template(
&mut self,
nt: &NoteType,
cards: &[AlreadyGeneratedCardInfo],
) -> Result<usize> {
if nt.config.kind() == NoteTypeKind::Cloze {
return Ok(0);
}
let mut removed = 0;
for card in cards {
if card.ord as usize >= nt.templates.len() {
self.storage.remove_card(card.id)?;
removed += 1;
}
}
Ok(removed)
}
fn recover_notetype(
&mut self,
stamp: TimestampMillis,
field_count: usize,
) -> Result<Arc<NoteType>> {
debug!(self.log, "create recovery notetype");
let mut basic = all_stock_notetypes(&self.i18n).remove(0);
let mut field = 3;
while basic.fields.len() < field_count {
basic.add_field(format!("{}", field));
field += 1;
}
basic.name = format!("db-check-{}-{}", stamp, field_count);
self.add_notetype(&mut basic)?;
Ok(Arc::new(basic))
}
fn check_revlog(&self, probs: &mut Vec<String>) -> Result<()> {
let cnt = self.storage.fix_revlog_properties()?;
if cnt > 0 {
self.storage.set_schema_modified()?;
probs.push(
self.i18n
.trn(TR::DatabaseCheckRevlogProperties, tr_args!["count"=>cnt]),
);
}
Ok(())
}
fn update_next_new_position(&self) -> Result<()> {
let pos = self.storage.max_new_card_position().unwrap_or(0);
self.set_next_card_position(pos)
}
}

6
rslib/src/decks/mod.rs
View file

@ -209,6 +209,12 @@ impl Collection {
})
}
pub(crate) fn recover_missing_deck(&mut self, _did: DeckID) -> Result<()> {
// todo
Ok(())
}
pub fn get_or_create_normal_deck(&mut self, human_name: &str) -> Result<Deck> {
let native_name = human_deck_name_to_native(human_name);
if let Some(did) = self.storage.get_deck_id(&native_name)? {

2
rslib/src/decks/tree.rs
View file

@ -215,7 +215,7 @@ impl Collection {
Ok(LegacyDueCounts::from(tree))
}
pub(crate) fn add_missing_decks(&mut self, names: &[(DeckID, String)]) -> Result<()> {
pub(crate) fn add_missing_deck_names(&mut self, names: &[(DeckID, String)]) -> Result<()> {
let mut parents = HashSet::new();
for (_id, name) in names {
parents.insert(UniCase::new(name.as_str()));

5
rslib/src/err.rs
View file

@ -102,6 +102,10 @@ impl AnkiError {
// already localized
info.into()
}
AnkiError::DBError { info, kind } => match kind {
DBErrorKind::Corrupt => info.clone(),
_ => format!("{:?}", self),
},
_ => format!("{:?}", self),
}
}
@ -262,5 +266,6 @@ pub enum DBErrorKind {
FileTooNew,
FileTooOld,
MissingEntity,
Corrupt,
Other,
}

14
rslib/src/notes.rs
View file

@ -135,6 +135,20 @@ impl Note {
}
changed
}
/// Pad or merge fields to match note type.
pub(crate) fn fix_field_count(&mut self, nt: &NoteType) {
while self.fields.len() < nt.fields.len() {
self.fields.push("".into())
}
while self.fields.len() > nt.fields.len() && self.fields.len() > 1 {
let last = self.fields.pop().unwrap();
self.fields
.last_mut()
.unwrap()
.push_str(&format!("; {}", last));
}
}
}
impl From<Note> for pb::Note {

6
rslib/src/notetype/mod.rs
View file

@ -329,8 +329,12 @@ impl From<NoteType> for NoteTypeProto {
impl Collection {
/// Add a new notetype, and allocate it an ID.
pub fn add_notetype(&mut self, nt: &mut NoteType) -> Result<()> {
self.transact(None, |col| col.add_notetype_inner(nt))
}
pub(crate) fn add_notetype_inner(&mut self, nt: &mut NoteType) -> Result<()> {
nt.prepare_for_adding()?;
self.transact(None, |col| col.storage.add_new_notetype(nt))
self.storage.add_new_notetype(nt)
}
/// Saves changes to a note type. This will force a full sync if templates

19
rslib/src/storage/card/fix_due_new.sql Normal file
View file

@ -0,0 +1,19 @@
update cards
set
due = (
case
when type = 0
and queue != 4 then 1000000 + due % 1000000
else due
end
),
mod = ?1,
usn = ?2
where
due != (
case
when type = 0
and queue != 4 then 1000000 + due % 1000000
else due
end
);

19
rslib/src/storage/card/fix_due_other.sql Normal file
View file

@ -0,0 +1,19 @@
update cards
set
due = (
case
when queue = 2
and due > 100000 then ?1
else min(max(round(due), -2147483648), 2147483647)
end
),
mod = ?2,
usn = ?3
where
due != (
case
when queue = 2
and due > 100000 then ?1
else min(max(round(due), -2147483648), 2147483647)
end
);

7
rslib/src/storage/card/fix_ivl.sql Normal file
View file

@ -0,0 +1,7 @@
update cards
set
ivl = min(max(round(ivl), 0), 2147483647),
mod = ?1,
usn = ?2
where
ivl != min(max(round(ivl), 0), 2147483647)

27
rslib/src/storage/card/fix_odue.sql Normal file
View file

@ -0,0 +1,27 @@
update cards
set
odue = (
case
when odue > 0
and (
type = 1
or queue = 2
)
and not odid then 0
else min(max(round(odue), -2147483648), 2147483647)
end
),
mod = ?1,
usn = ?2
where
odue != (
case
when odue > 0
and (
type = 1
or queue = 2
)
and not odid then 0
else min(max(round(odue), -2147483648), 2147483647)
end
);

61
rslib/src/storage/card/mod.rs
View file

@ -1,13 +1,17 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
use crate::card::{Card, CardID, CardQueue, CardType};
use crate::err::Result;
use crate::timestamp::TimestampMillis;
use crate::{
card::{Card, CardID, CardQueue, CardType},
decks::DeckID,
err::Result,
timestamp::{TimestampMillis, TimestampSecs},
types::Usn,
};
use rusqlite::params;
use rusqlite::{
types::{FromSql, FromSqlError, ValueRef},
OptionalExtension,
OptionalExtension, NO_PARAMS,
};
use std::convert::TryFrom;
@ -50,7 +54,7 @@ impl super::SqliteStorage {
reps: row.get(10)?,
lapses: row.get(11)?,
left: row.get(12)?,
odue: row.get(13)?,
odue: row.get(13).ok().unwrap_or_default(),
odid: row.get(14)?,
flags: row.get(15)?,
data: row.get(16)?,
@ -118,6 +122,53 @@ impl super::SqliteStorage {
.execute(&[cid])?;
Ok(())
}
/// Fix some invalid card properties, and return number of changed cards.
pub(crate) fn fix_card_properties(
&self,
today: u32,
mtime: TimestampSecs,
usn: Usn,
) -> Result<(usize, usize)> {
let new_cnt = self
.db
.prepare(include_str!("fix_due_new.sql"))?
.execute(params![mtime, usn])?;
let mut other_cnt = self
.db
.prepare(include_str!("fix_due_other.sql"))?
.execute(params![mtime, usn, today])?;
other_cnt += self
.db
.prepare(include_str!("fix_odue.sql"))?
.execute(params![mtime, usn])?;
other_cnt += self
.db
.prepare(include_str!("fix_ivl.sql"))?
.execute(params![mtime, usn])?;
Ok((new_cnt, other_cnt))
}
pub(crate) fn delete_orphaned_cards(&self) -> Result<usize> {
self.db
.prepare("delete from cards where nid not in (select id from notes)")?
.execute(NO_PARAMS)
.map_err(Into::into)
}
pub(crate) fn all_filtered_cards_by_deck(&self) -> Result<Vec<(CardID, DeckID)>> {
self.db
.prepare("select id, did from cards where odid > 0")?
.query_and_then(NO_PARAMS, |r| -> Result<_> { Ok((r.get(0)?, r.get(1)?)) })?
.collect()
}
pub(crate) fn max_new_card_position(&self) -> Result<u32> {
self.db
.prepare("select max(due)+1 from cards where type=0")?
.query_row(NO_PARAMS, |r| r.get(0))
.map_err(Into::into)
}
}
#[cfg(test)]

9
rslib/src/storage/deck/missing-decks.sql Normal file
View file

@ -0,0 +1,9 @@
select
distinct did
from cards
where
did not in (
select
id
from decks
);

8
rslib/src/storage/deck/mod.rs
View file

@ -170,6 +170,14 @@ impl SqliteStorage {
.collect()
}
/// Decks referenced by cards but missing.
pub(crate) fn missing_decks(&self) -> Result<Vec<DeckID>> {
self.db
.prepare(include_str!("missing-decks.sql"))?
.query_and_then(NO_PARAMS, |r| r.get(0).map_err(Into::into))?
.collect()
}
// Upgrading/downgrading/legacy
pub(super) fn add_default_deck(&self, i18n: &I18n) -> Result<()> {

1
rslib/src/storage/mod.rs
View file

@ -8,6 +8,7 @@ mod deckconf;
mod graves;
mod note;
mod notetype;
mod revlog;
mod sqlite;
mod tag;
mod upgrades;

10
rslib/src/storage/notetype/mod.rs
View file

@ -36,7 +36,7 @@ fn row_to_existing_card(row: &Row) -> Result<AlreadyGeneratedCardInfo> {
nid: row.get(1)?,
ord: row.get(2)?,
original_deck_id: row.get(3)?,
position_if_new: row.get(4)?,
position_if_new: row.get(4).ok().unwrap_or_default(),
})
}
@ -157,6 +157,14 @@ impl SqliteStorage {
.collect()
}
pub(crate) fn all_note_ids_by_notetype(&self) -> Result<Vec<(NoteTypeID, NoteID)>> {
let sql = String::from("select mid, id from notes order by mid, id");
self.db
.prepare(&sql)?
.query_and_then(NO_PARAMS, |r| Ok((r.get(0)?, r.get(1)?)))?
.collect()
}
pub(crate) fn update_notetype_templates(
&self,
ntid: NoteTypeID,

7
rslib/src/storage/revlog/fix_props.sql Normal file
View file

@ -0,0 +1,7 @@
update revlog
set
ivl = min(max(round(ivl), -2147483648), 2147483647),
lastIvl = min(max(round(lastIvl), -2147483648), 2147483647)
where
ivl != min(max(round(ivl), -2147483648), 2147483647)
or lastIVl != min(max(round(lastIvl), -2147483648), 2147483647)

15
rslib/src/storage/revlog/mod.rs Normal file
View file

@ -0,0 +1,15 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
use super::SqliteStorage;
use crate::err::Result;
use rusqlite::NO_PARAMS;
impl SqliteStorage {
pub(crate) fn fix_revlog_properties(&self) -> Result<usize> {
self.db
.prepare(include_str!("fix_props.sql"))?
.execute(NO_PARAMS)
.map_err(Into::into)
}
}

18
rslib/src/storage/sqlite.rs
View file

@ -304,6 +304,24 @@ impl SqliteStorage {
//////////////////////////////////////////
/// true if corrupt/can't access
pub(crate) fn quick_check_corrupt(&self) -> bool {
match self.db.pragma_query_value(None, "quick_check", |row| {
row.get(0).map(|v: String| v != "ok")
}) {
Ok(corrupt) => corrupt,
Err(e) => {
println!("error: {:?}", e);
true
}
}
}
pub(crate) fn optimize(&self) -> Result<()> {
self.db.execute_batch("vacuum")?;
Ok(())
}
#[cfg(test)]
pub(crate) fn db_scalar<T: rusqlite::types::FromSql>(&self, sql: &str) -> Result<T> {
self.db