diff --git a/anki/deck.py b/anki/deck.py index 9bafbc06e..dc4153c01 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -1584,7 +1584,7 @@ seq > :s and seq <= :e order by seq desc""", s=start, e=end) required.append("ord") if self.qconf['revCardOrder'] in (REV_CARDS_OLD_FIRST, REV_CARDS_NEW_FIRST): required.append("interval") - cols = ["queue", "due", "groupId"] + required + cols = ["queue", "due", "gid"] + required # update if changed if self.db.scalar( "select 1 from sqlite_master where name = 'ix_cards_multi'"): diff --git a/anki/graves.py b/anki/graves.py index 473242368..8a9b214ed 100644 --- a/anki/graves.py +++ b/anki/graves.py @@ -17,12 +17,12 @@ GROUP = 4 GROUPCONFIG = 5 def registerOne(db, type, id): - db.execute("insert into gravestones values (:t, :id, :ty)", + db.execute("insert into graves values (:t, :id, :ty)", t=intTime(), id=id, ty=type) def registerMany(db, type, ids): - db.executemany("insert into gravestones values (:t, :id, :ty)", + db.executemany("insert into graves values (:t, :id, :ty)", [{'t':intTime(), 'id':x, 'ty':type} for x in ids]) def forgetAll(db): - db.execute("delete from gravestones") + db.execute("delete from graves") diff --git a/anki/models.py b/anki/models.py index c272c6888..809c022f7 100644 --- a/anki/models.py +++ b/anki/models.py @@ -116,8 +116,8 @@ defaultFieldConf = { 'required': False, 'unique': False, 'font': "Arial", - 'editSize': 20, 'quizSize': 20, + 'editSize': 20, 'quizColour': "#fff", 'pre': True, } diff --git a/anki/storage.py b/anki/storage.py index 3674fd747..af23ffb7d 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -2,16 +2,15 @@ # Copyright: Damien Elmes # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html -DECK_VERSION = 100 +CURRENT_VERSION = 100 import os, time, simplejson from anki.lang import _ -#from anki.media import rebuildMediaDir from anki.utils import intTime from anki.db import DB from anki.deck import _Deck -import anki.groups from anki.stdmodels import BasicModel +from anki.errors import AnkiError def Deck(path, queue=True): "Open a new or existing deck. Path must be unicode." @@ -26,7 +25,7 @@ def Deck(path, queue=True): db.execute("pragma cache_size = 20000") # add db to deck and do any remaining upgrades deck = _Deck(db) - if ver < DECK_VERSION: + if ver < CURRENT_VERSION: _upgradeDeck(deck, ver) elif create: deck.addModel(BasicModel(deck)) @@ -44,230 +43,9 @@ def _createDB(db): _addSchema(db) _updateIndices(db) db.execute("analyze") - return DECK_VERSION + return CURRENT_VERSION -def moveTable(s, table): - sql = s.scalar( - "select sql from sqlite_master where name = '%s'" % table) - sql = sql.replace("TABLE "+table, "temporary table %s2" % table) - s.execute(sql) - s.execute("insert into %s2 select * from %s" % (table, table)) - s.execute("drop table "+table) - -def _upgradeSchema(db): - "Alter tables prior to ORM initialization." - try: - ver = db.scalar("select version from deck") - except: - ver = db.scalar("select version from decks") - if ver < 65: - raise Exception("oldDeckVersion") - if ver < 99: - raise "upgrade" - # cards - ########### - moveTable(s, "cards") - import cards - metadata.create_all(engine, tables=[cards.cardsTable]) - s.execute(""" -insert into cards select id, factId, 1, cardModelId, cast(modified as int), -question, answer, ordinal, 0, relativeDelay, type, due, cast(interval as int), -cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") - s.execute("drop table cards2") - # tags - ########### - moveTable(s, "tags") - import deck - deck.DeckStorage._addTables(engine) - s.execute("insert or ignore into tags select id, :t, tag from tags2", - {'t':intTime()}) - # tags should have a leading and trailing space if not empty, and not - # use commas - s.execute(""" -update facts set tags = (case -when trim(tags) == "" then "" -else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " -end) -""") - s.execute("drop table tags2") - s.execute("drop table cardTags") - # facts - ########### - s.execute(""" -create table facts2 -(id, modelId, modified, tags, cache)""") - # use the rowid to give them an integer order - s.execute(""" -insert into facts2 select id, modelId, modified, tags, spaceUntil from -facts order by created""") - s.execute("drop table facts") - import facts - metadata.create_all(engine, tables=[facts.factsTable]) - s.execute(""" -insert or ignore into facts select id, modelId, rowid, -cast(modified as int), tags, cache from facts2""") - s.execute("drop table facts2") - # media - ########### - moveTable(s, "media") - import media - metadata.create_all(engine, tables=[media.mediaTable]) - s.execute(""" -insert or ignore into media select id, filename, size, cast(created as int), -originalPath from media2""") - s.execute("drop table media2") - # longer migrations - ########### - - - - migrateDeck(s, engine) - migrateFields(s, engine) - # # fields - # ########### - # db.execute( - # "alter table fields add column csum text not null default ''") - - - - # models - ########### - moveTable(s, "models") - import models - metadata.create_all(engine, tables=[models.modelsTable]) - s.execute(""" -insert or ignore into models select id, cast(modified as int), name, "" from models2""") - s.execute("drop table models2") - - return ver - -def migrateDeck(s, engine): - import deck - metadata.create_all(engine, tables=[deck.deckTable]) - s.execute(""" -insert into deck select id, cast(created as int), cast(modified as int), -0, 99, ifnull(syncName, ""), cast(lastSync as int), -utcOffset, "", "", "" from decks""") - # update selective study - qconf = deck.defaultQconf.copy() - # delete old selective study settings, which we can't auto-upgrade easily - keys = ("newActive", "newInactive", "revActive", "revInactive") - for k in keys: - s.execute("delete from deckVars where key=:k", {'k':k}) - # copy other settings, ignoring deck order as there's a new default - keys = ("newCardOrder", "newCardSpacing") - for k in keys: - qconf[k] = s.execute("select %s from decks" % k).scalar() - qconf['newPerDay'] = s.execute( - "select newCardsPerDay from decks").scalar() - # fetch remaining settings from decks table - conf = deck.defaultConf.copy() - data = {} - keys = ("sessionRepLimit", "sessionTimeLimit") - for k in keys: - conf[k] = s.execute("select %s from decks" % k).scalar() - # random and due options merged - qconf['revCardOrder'] = min(2, qconf['revCardOrder']) - # no reverse option anymore - qconf['newCardOrder'] = min(1, qconf['newCardOrder']) - # add any deck vars and save - dkeys = ("hexCache", "cssCache") - for (k, v) in s.execute("select * from deckVars").fetchall(): - if k in dkeys: - data[k] = v - else: - conf[k] = v - s.execute("update deck set qconf = :l, config = :c, data = :d", - {'l':simplejson.dumps(qconf), - 'c':simplejson.dumps(conf), - 'd':simplejson.dumps(data)}) - # clean up - s.execute("drop table decks") - s.execute("drop table deckVars") - -def _upgradeDeck(deck, version): - "Upgrade deck to the latest version." - print version, DECK_VERSION - if version < DECK_VERSION: - prog = True - deck.startProgress() - deck.updateProgress(_("Upgrading Deck...")) - oldmod = deck.modified - else: - prog = False - if version < 100: - # update dynamic indices given we don't use priority anymore - for d in ("intervalDesc", "intervalAsc", "randomOrder", - "dueAsc", "dueDesc"): - deck.db.execute("drop index if exists ix_cards_%s2" % d) - execute.db.statement("drop index if exists ix_cards_%s" % d) - # remove old views - for v in ("failedCards", "revCardsOld", "revCardsNew", - "revCardsDue", "revCardsRandom", "acqCardsRandom", - "acqCardsOld", "acqCardsNew"): - deck.db.execute("drop view if exists %s" % v) - # add checksums and index - deck.updateAllFieldChecksums() - # this was only used for calculating average factor - deck.db.execute("drop index if exists ix_cards_factor") - # remove stats, as it's all in the revlog now - deck.db.execute("drop table if exists stats") - # migrate revlog data to new table - deck.db.execute(""" -insert or ignore into revlog select -cast(time*1000 as int), cardId, ease, reps, -cast(lastInterval as int), cast(nextInterval as int), -cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), -0 from reviewHistory""") - deck.db.execute("drop table reviewHistory") - # convert old ease0 into ease1 - deck.db.execute("update revlog set ease = 1 where ease = 0") - # remove priority index - deck.db.execute("drop index if exists ix_cards_priority") - # suspended cards don't use ranges anymore - deck.db.execute("update cards set queue=-1 where queue between -3 and -1") - deck.db.execute("update cards set queue=-2 where queue between 3 and 5") - deck.db.execute("update cards set queue=-3 where queue between 6 and 8") - # update schema time - deck.db.execute("update deck set schemaMod = :t", t=intTime()) - # remove queueDue as it's become dynamic, and type index - deck.db.execute("drop index if exists ix_cards_queueDue") - deck.db.execute("drop index if exists ix_cards_type") - # remove old deleted tables - for t in ("cards", "facts", "models", "media"): - deck.db.execute("drop table if exists %sDeleted" % t) - # finally, update indices & optimize - updateIndices(deck.db) - # rewrite due times for new cards - deck.db.execute(""" -update cards set due = (select pos from facts where factId = facts.id) where type=2""") - # convert due cards into day-based due - deck.db.execute(""" -update cards set due = cast( -(case when due < :stamp then 0 else 1 end) + -((due-:stamp)/86400) as int)+:today where type -between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today) - print "today", deck.sched.today - print "cut", deck.sched.dayCutoff - # setup qconf & config for dynamicIndices() - deck.qconf = simplejson.loads(deck._qconf) - deck.config = simplejson.loads(deck._config) - deck.data = simplejson.loads(deck._data) - # update factPos - deck.config['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1 - deck.flushConfig() - # add default config - - deck.updateDynamicIndices() - deck.db.execute("vacuum") - deck.db.execute("analyze") - deck.db.execute("update deck set version = ?", DECK_VERSION) - deck.db.commit() - if prog: - assert deck.modified == oldmod - deck.finishProgress() - -def _addSchema(db): +def _addSchema(db, addObjs=True): db.executescript(""" create table if not exists deck ( id integer primary key, @@ -348,7 +126,7 @@ create table if not exists fdata ( csum text not null ); -create table if not exists gravestones ( +create table if not exists graves ( delTime integer not null, objectId integer not null, type integer not null @@ -394,19 +172,22 @@ create table if not exists tags ( insert or ignore into deck values(1,%(t)s,%(t)s,%(t)s,%(v)s,'',0,-2,'', '', ''); -""" % ({'t': intTime(), 'v':DECK_VERSION})) - import anki.deck - db.execute("update deck set qconf = ?, conf = ?, data = ?", - simplejson.dumps(anki.deck.defaultQconf), - simplejson.dumps(anki.deck.defaultConf), - "{}") - db.execute( - "insert or ignore into gconf values (1, ?, ?, ?)""", - intTime(), _("Default Config"), - simplejson.dumps(anki.groups.defaultConf)) - db.execute( - "insert or ignore into groups values (1, ?, ?, 1)", - intTime(), _("Default Group")) +""" % ({'t': intTime(), 'v':CURRENT_VERSION})) + # if not upgrading + if addObjs: + import anki.deck + import anki.groups + db.execute("update deck set qconf = ?, conf = ?, data = ?", + simplejson.dumps(anki.deck.defaultQconf), + simplejson.dumps(anki.deck.defaultConf), + "{}") + db.execute( + "insert or ignore into gconf values (1, ?, ?, ?)""", + intTime(), _("Default Config"), + simplejson.dumps(anki.groups.defaultConf)) + db.execute( + "insert or ignore into groups values (1, ?, ?, 1)", + intTime(), _("Default Group")) def _updateIndices(db): "Add indices to the DB." @@ -422,5 +203,262 @@ create index if not exists ix_fdata_csum on fdata (csum); -- media create index if not exists ix_media_csum on media (csum); -- deletion tracking -create index if not exists ix_gravestones_delTime on gravestones (delTime); +create index if not exists ix_graves_delTime on graves (delTime); """) + +# 2.0 schema migration +###################################################################### +# we don't have access to the progress handler at this point, so the GUI code +# will need to set up a progress handling window before opening a deck. + +def _moveTable(db, table): + sql = db.scalar( + "select sql from sqlite_master where name = '%s'" % table) + sql = sql.replace("TABLE "+table, "temporary table %s2" % table) + db.execute(sql) + db.execute("insert into %s2 select * from %s" % (table, table)) + db.execute("drop table "+table) + _addSchema(db, False) + +def _upgradeSchema(db): + "Alter tables prior to ORM initialization." + try: + ver = db.scalar("select version from deck") + except: + ver = db.scalar("select version from decks") + # latest 1.2 is 65 + if ver < 65: + raise AnkiError("oldDeckVersion") + if ver > 99: + return ver + + # cards + ########### + _moveTable(db, "cards") + db.execute(""" +insert into cards select id, factId, cardModelId, 1, cast(modified as int), +question, answer, ordinal, relativeDelay, type, due, cast(interval as int), +cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") + db.execute("drop table cards2") + + # tags + ########### + _moveTable(db, "tags") + db.execute("insert or ignore into tags select id, ?, tag from tags2", + intTime()) + # tags should have a leading and trailing space if not empty, and not + # use commas + db.execute(""" +update facts set tags = (case +when trim(tags) == "" then "" +else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " +end) +""") + db.execute("drop table tags2") + db.execute("drop table cardTags") + + # facts + ########### + db.execute(""" +create table facts2 +(id, modelId, modified, tags, cache)""") + # use the rowid to give them an integer order + db.execute(""" +insert into facts2 select id, modelId, modified, tags, spaceUntil from +facts order by created""") + db.execute("drop table facts") + _addSchema(db, False) + db.execute(""" +insert or ignore into facts select id, modelId, rowid, +cast(modified as int), tags, cache from facts2""") + db.execute("drop table facts2") + + # media + ########### + _moveTable(db, "media") + db.execute(""" +insert or ignore into media select filename, cast(created as int), +originalPath from media2""") + db.execute("drop table media2") + + # fields -> fdata + ########### + db.execute(""" +insert or ignore into fdata select factId, fieldModelId, ordinal, value, '' +from fields""") + db.execute("drop table fields") + + # models + ########### + _moveTable(db, "models") + db.execute(""" +insert or ignore into models select id, cast(modified as int), +name, "{}" from models2""") + db.execute("drop table models2") + + # reviewHistory -> revlog + ########### + db.execute(""" +insert or ignore into revlog select +cast(time*1000 as int), cardId, ease, reps, +cast(lastInterval as int), cast(nextInterval as int), +cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), +0 from reviewHistory""") + db.execute("drop table reviewHistory") + # convert old ease0 into ease1 + db.execute("update revlog set ease = 1 where ease = 0") + + # longer migrations + ########### + _migrateDeckTbl(db) + _migrateFieldsTbl(db) + _migrateTemplatesTbl(db) + + _updateIndices(db) + return ver + +def _migrateDeckTbl(db): + import anki.deck + db.execute("delete from deck") + db.execute(""" +insert or replace into deck select id, cast(created as int), :t, +:t, 99, ifnull(syncName, ""), cast(lastSync as int), +utcOffset, "", "", "" from decks""", t=intTime()) + # update selective study + qconf = anki.deck.defaultQconf.copy() + # delete old selective study settings, which we can't auto-upgrade easily + keys = ("newActive", "newInactive", "revActive", "revInactive") + for k in keys: + db.execute("delete from deckVars where key=:k", k=k) + # copy other settings, ignoring deck order as there's a new default + keys = ("newCardOrder", "newCardSpacing") + for k in keys: + qconf[k] = db.scalar("select %s from decks" % k) + qconf['newPerDay'] = db.scalar( + "select newCardsPerDay from decks") + # fetch remaining settings from decks table + conf = anki.deck.defaultConf.copy() + data = {} + keys = ("sessionRepLimit", "sessionTimeLimit") + for k in keys: + conf[k] = db.scalar("select %s from decks" % k) + # random and due options merged + qconf['revCardOrder'] = min(2, qconf['revCardOrder']) + # no reverse option anymore + qconf['newCardOrder'] = min(1, qconf['newCardOrder']) + # add any deck vars and save + dkeys = ("hexCache", "cssCache") + for (k, v) in db.execute("select * from deckVars").fetchall(): + if k in dkeys: + data[k] = v + else: + conf[k] = v + db.execute("update deck set qconf = :l, conf = :c, data = :d", + l=simplejson.dumps(qconf), + c=simplejson.dumps(conf), + d=simplejson.dumps(data)) + # clean up + db.execute("drop table decks") + db.execute("drop table deckVars") + +def _migrateFieldsTbl(db): + import anki.models + db.execute(""" +insert into fields select id, modelId, ordinal, name, numeric, '' +from fieldModels""") + dconf = anki.models.defaultFieldConf + for row in db.all(""" +select id, features, required, "unique", quizFontFamily, quizFontSize, +quizFontColour, editFontSize from fieldModels"""): + conf = dconf.copy() + (conf['rtl'], + conf['required'], + conf['unique'], + conf['font'], + conf['quizSize'], + conf['quizColour'], + conf['editSize']) = row[1:] + # setup bools + conf['rtl'] = not not conf['rtl'] + conf['pre'] = True + # save + db.execute("update fields set conf = ? where id = ?", + simplejson.dumps(conf), row[0]) + # clean up + db.execute("drop table fieldModels") + +def _migrateTemplatesTbl(db): + # do this after fieldModel migration + import anki.models + db.execute(""" +insert into templates select id, modelId, ordinal, name, active, qformat, +aformat, '' from cardModels""") + dconf = anki.models.defaultTemplateConf + for row in db.all(""" +select id, modelId, questionInAnswer, questionAlign, lastFontColour, +allowEmptyAnswer, typeAnswer from cardModels"""): + conf = dconf.copy() + (conf['hideQ'], + conf['align'], + conf['bg'], + conf['allowEmptyAns'], + fname) = row[2:] + # convert the field name to an id + conf['typeAnswer'] = db.scalar( + "select id from fields where name = ? and mid = ?", + fname, row[1]) + # save + db.execute("update templates set conf = ? where id = ?", + simplejson.dumps(conf), row[0]) + # clean up + db.execute("drop table cardModels") + +def _postSchemaUpgrade(deck): + "Handle the rest of the upgrade to 2.0." + import anki.deck + # remove old views + for v in ("failedCards", "revCardsOld", "revCardsNew", + "revCardsDue", "revCardsRandom", "acqCardsRandom", + "acqCardsOld", "acqCardsNew"): + deck.db.execute("drop view if exists %s" % v) + # update caches + for m in deck.allModels(): + m.updateCache() + # remove stats, as it's all in the revlog now + deck.db.execute("drop table if exists stats") + # suspended cards don't use ranges anymore + deck.db.execute("update cards set queue=-1 where queue between -3 and -1") + deck.db.execute("update cards set queue=-2 where queue between 3 and 5") + deck.db.execute("update cards set queue=-3 where queue between 6 and 8") + # remove old deleted tables + for t in ("cards", "facts", "models", "media"): + deck.db.execute("drop table if exists %sDeleted" % t) + # rewrite due times for new cards + deck.db.execute(""" +update cards set due = (select pos from facts where fid = facts.id) where type=2""") + # convert due cards into day-based due + deck.db.execute(""" +update cards set due = cast( +(case when due < :stamp then 0 else 1 end) + +((due-:stamp)/86400) as int)+:today where type +between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today) + # update factPos + deck.conf['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1 + deck.save() + + # optimize and finish + deck.updateDynamicIndices() + deck.db.execute("vacuum") + deck.db.execute("analyze") + deck.db.execute("update deck set version = ?", CURRENT_VERSION) + deck.save() + +# Post-init upgrade +###################################################################### + +def _upgradeDeck(deck, version): + "Upgrade deck to the latest version." + if version >= CURRENT_VERSION: + return + if version < 100: + _postSchemaUpgrade(deck)