fix upgrade. GUI code should take care of progress handler now

anki/deck.py

@@ -1584,7 +1584,7 @@ seq > :s and seq <= :e order by seq desc""", s=start, e=end)
             required.append("ord")
         if self.qconf['revCardOrder'] in (REV_CARDS_OLD_FIRST, REV_CARDS_NEW_FIRST):
             required.append("interval")
-        cols = ["queue", "due", "groupId"] + required
+        cols = ["queue", "due", "gid"] + required
         # update if changed
         if self.db.scalar(
             "select 1 from sqlite_master where name = 'ix_cards_multi'"):

anki/graves.py

@@ -17,12 +17,12 @@ GROUP = 4
 GROUPCONFIG = 5
 
 def registerOne(db, type, id):
-    db.execute("insert into gravestones values (:t, :id, :ty)",
+    db.execute("insert into graves values (:t, :id, :ty)",
                t=intTime(), id=id, ty=type)
 
 def registerMany(db, type, ids):
-    db.executemany("insert into gravestones values (:t, :id, :ty)",
+    db.executemany("insert into graves values (:t, :id, :ty)",
                    [{'t':intTime(), 'id':x, 'ty':type} for x in ids])
 
 def forgetAll(db):
-    db.execute("delete from gravestones")
+    db.execute("delete from graves")

anki/models.py

@@ -116,8 +116,8 @@ defaultFieldConf = {
     'required': False,
     'unique': False,
     'font': "Arial",
-    'editSize': 20,
     'quizSize': 20,
+    'editSize': 20,
     'quizColour': "#fff",
     'pre': True,
 }

anki/storage.py

@@ -2,16 +2,15 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
-DECK_VERSION = 100
+CURRENT_VERSION = 100
 
 import os, time, simplejson
 from anki.lang import _
-#from anki.media import rebuildMediaDir
 from anki.utils import intTime
 from anki.db import DB
 from anki.deck import _Deck
-import anki.groups
 from anki.stdmodels import BasicModel
+from anki.errors import AnkiError
 
 def Deck(path, queue=True):
     "Open a new or existing deck. Path must be unicode."
@@ -26,7 +25,7 @@ def Deck(path, queue=True):
     db.execute("pragma cache_size = 20000")
     # add db to deck and do any remaining upgrades
     deck = _Deck(db)
-    if ver < DECK_VERSION:
+    if ver < CURRENT_VERSION:
         _upgradeDeck(deck, ver)
     elif create:
         deck.addModel(BasicModel(deck))
@@ -44,230 +43,9 @@ def _createDB(db):
     _addSchema(db)
     _updateIndices(db)
     db.execute("analyze")
-    return DECK_VERSION
+    return CURRENT_VERSION
 
-def moveTable(s, table):
+def _addSchema(db, addObjs=True):
-    sql = s.scalar(
-        "select sql from sqlite_master where name = '%s'" % table)
-    sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
-    s.execute(sql)
-    s.execute("insert into %s2 select * from %s" % (table, table))
-    s.execute("drop table "+table)
-
-def _upgradeSchema(db):
-    "Alter tables prior to ORM initialization."
-    try:
-        ver = db.scalar("select version from deck")
-    except:
-        ver = db.scalar("select version from decks")
-    if ver < 65:
-        raise Exception("oldDeckVersion")
-    if ver < 99:
-        raise "upgrade"
-        # cards
-        ###########
-        moveTable(s, "cards")
-        import cards
-        metadata.create_all(engine, tables=[cards.cardsTable])
-        s.execute("""
-insert into cards select id, factId, 1, cardModelId, cast(modified as int),
-question, answer, ordinal, 0, relativeDelay, type, due, cast(interval as int),
-cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""")
-        s.execute("drop table cards2")
-        # tags
-        ###########
-        moveTable(s, "tags")
-        import deck
-        deck.DeckStorage._addTables(engine)
-        s.execute("insert or ignore into tags select id, :t, tag from tags2",
-                  {'t':intTime()})
-        # tags should have a leading and trailing space if not empty, and not
-        # use commas
-        s.execute("""
-update facts set tags = (case
-when trim(tags) == "" then ""
-else " " || replace(replace(trim(tags), ",", " "), "  ", " ") || " "
-end)
-""")
-        s.execute("drop table tags2")
-        s.execute("drop table cardTags")
-        # facts
-        ###########
-        s.execute("""
-create table facts2
-(id, modelId, modified, tags, cache)""")
-        # use the rowid to give them an integer order
-        s.execute("""
-insert into facts2 select id, modelId, modified, tags, spaceUntil from
-facts order by created""")
-        s.execute("drop table facts")
-        import facts
-        metadata.create_all(engine, tables=[facts.factsTable])
-        s.execute("""
-insert or ignore into facts select id, modelId, rowid,
-cast(modified as int), tags, cache from facts2""")
-        s.execute("drop table facts2")
-        # media
-        ###########
-        moveTable(s, "media")
-        import media
-        metadata.create_all(engine, tables=[media.mediaTable])
-        s.execute("""
-insert or ignore into media select id, filename, size, cast(created as int),
-originalPath from media2""")
-        s.execute("drop table media2")
-        # longer migrations
-        ###########
-
-
-
-        migrateDeck(s, engine)
-        migrateFields(s, engine)
-        # # fields
-        # ###########
-        # db.execute(
-        #     "alter table fields add column csum text not null default ''")
-
-
-
-        # models
-        ###########
-        moveTable(s, "models")
-        import models
-        metadata.create_all(engine, tables=[models.modelsTable])
-        s.execute("""
-insert or ignore into models select id, cast(modified as int), name, "" from models2""")
-        s.execute("drop table models2")
-
-    return ver
-
-def migrateDeck(s, engine):
-    import deck
-    metadata.create_all(engine, tables=[deck.deckTable])
-    s.execute("""
-insert into deck select id, cast(created as int), cast(modified as int),
-0, 99, ifnull(syncName, ""), cast(lastSync as int),
-utcOffset, "", "", "" from decks""")
-    # update selective study
-    qconf = deck.defaultQconf.copy()
-    # delete old selective study settings, which we can't auto-upgrade easily
-    keys = ("newActive", "newInactive", "revActive", "revInactive")
-    for k in keys:
-        s.execute("delete from deckVars where key=:k", {'k':k})
-    # copy other settings, ignoring deck order as there's a new default
-    keys = ("newCardOrder", "newCardSpacing")
-    for k in keys:
-        qconf[k] = s.execute("select %s from decks" % k).scalar()
-    qconf['newPerDay'] = s.execute(
-        "select newCardsPerDay from decks").scalar()
-    # fetch remaining settings from decks table
-    conf = deck.defaultConf.copy()
-    data = {}
-    keys = ("sessionRepLimit", "sessionTimeLimit")
-    for k in keys:
-        conf[k] = s.execute("select %s from decks" % k).scalar()
-    # random and due options merged
-    qconf['revCardOrder'] = min(2, qconf['revCardOrder'])
-    # no reverse option anymore
-    qconf['newCardOrder'] = min(1, qconf['newCardOrder'])
-    # add any deck vars and save
-    dkeys = ("hexCache", "cssCache")
-    for (k, v) in s.execute("select * from deckVars").fetchall():
-        if k in dkeys:
-            data[k] = v
-        else:
-            conf[k] = v
-    s.execute("update deck set qconf = :l, config = :c, data = :d",
-              {'l':simplejson.dumps(qconf),
-               'c':simplejson.dumps(conf),
-               'd':simplejson.dumps(data)})
-    # clean up
-    s.execute("drop table decks")
-    s.execute("drop table deckVars")
-
-def _upgradeDeck(deck, version):
-    "Upgrade deck to the latest version."
-    print version, DECK_VERSION
-    if version < DECK_VERSION:
-        prog = True
-        deck.startProgress()
-        deck.updateProgress(_("Upgrading Deck..."))
-        oldmod = deck.modified
-    else:
-        prog = False
-    if version < 100:
-        # update dynamic indices given we don't use priority anymore
-        for d in ("intervalDesc", "intervalAsc", "randomOrder",
-                  "dueAsc", "dueDesc"):
-            deck.db.execute("drop index if exists ix_cards_%s2" % d)
-            execute.db.statement("drop index if exists ix_cards_%s" % d)
-        # remove old views
-        for v in ("failedCards", "revCardsOld", "revCardsNew",
-                  "revCardsDue", "revCardsRandom", "acqCardsRandom",
-                  "acqCardsOld", "acqCardsNew"):
-            deck.db.execute("drop view if exists %s" % v)
-        # add checksums and index
-        deck.updateAllFieldChecksums()
-        # this was only used for calculating average factor
-        deck.db.execute("drop index if exists ix_cards_factor")
-        # remove stats, as it's all in the revlog now
-        deck.db.execute("drop table if exists stats")
-        # migrate revlog data to new table
-        deck.db.execute("""
-insert or ignore into revlog select
-cast(time*1000 as int), cardId, ease, reps,
-cast(lastInterval as int), cast(nextInterval as int),
-cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
-0 from reviewHistory""")
-        deck.db.execute("drop table reviewHistory")
-        # convert old ease0 into ease1
-        deck.db.execute("update revlog set ease = 1 where ease = 0")
-        # remove priority index
-        deck.db.execute("drop index if exists ix_cards_priority")
-        # suspended cards don't use ranges anymore
-        deck.db.execute("update cards set queue=-1 where queue between -3 and -1")
-        deck.db.execute("update cards set queue=-2 where queue between 3 and 5")
-        deck.db.execute("update cards set queue=-3 where queue between 6 and 8")
-        # update schema time
-        deck.db.execute("update deck set schemaMod = :t", t=intTime())
-        # remove queueDue as it's become dynamic, and type index
-        deck.db.execute("drop index if exists ix_cards_queueDue")
-        deck.db.execute("drop index if exists ix_cards_type")
-        # remove old deleted tables
-        for t in ("cards", "facts", "models", "media"):
-            deck.db.execute("drop table if exists %sDeleted" % t)
-        # finally, update indices & optimize
-        updateIndices(deck.db)
-        # rewrite due times for new cards
-        deck.db.execute("""
-update cards set due = (select pos from facts where factId = facts.id) where type=2""")
-        # convert due cards into day-based due
-        deck.db.execute("""
-update cards set due = cast(
-(case when due < :stamp then 0 else 1 end) +
-((due-:stamp)/86400) as int)+:today where type
-between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
-        print "today", deck.sched.today
-        print "cut", deck.sched.dayCutoff
-        # setup qconf & config for dynamicIndices()
-        deck.qconf = simplejson.loads(deck._qconf)
-        deck.config = simplejson.loads(deck._config)
-        deck.data = simplejson.loads(deck._data)
-        # update factPos
-        deck.config['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1
-        deck.flushConfig()
-        # add default config
-
-        deck.updateDynamicIndices()
-        deck.db.execute("vacuum")
-        deck.db.execute("analyze")
-        deck.db.execute("update deck set version = ?", DECK_VERSION)
-        deck.db.commit()
-    if prog:
-        assert deck.modified == oldmod
-        deck.finishProgress()
-
-def _addSchema(db):
     db.executescript("""
 create table if not exists deck (
     id              integer primary key,
@@ -348,7 +126,7 @@ create table if not exists fdata (
     csum            text not null
 );
 
-create table if not exists gravestones (
+create table if not exists graves (
     delTime         integer not null,
     objectId        integer not null,
     type            integer not null
@@ -394,19 +172,22 @@ create table if not exists tags (
 
 insert or ignore into deck
 values(1,%(t)s,%(t)s,%(t)s,%(v)s,'',0,-2,'', '', '');
-""" % ({'t': intTime(), 'v':DECK_VERSION}))
+""" % ({'t': intTime(), 'v':CURRENT_VERSION}))
-    import anki.deck
+    # if not upgrading
-    db.execute("update deck set qconf = ?, conf = ?, data = ?",
+    if addObjs:
-               simplejson.dumps(anki.deck.defaultQconf),
+        import anki.deck
-               simplejson.dumps(anki.deck.defaultConf),
+        import anki.groups
-               "{}")
+        db.execute("update deck set qconf = ?, conf = ?, data = ?",
-    db.execute(
+                   simplejson.dumps(anki.deck.defaultQconf),
-        "insert or ignore into gconf values (1, ?, ?, ?)""",
+                   simplejson.dumps(anki.deck.defaultConf),
-        intTime(), _("Default Config"),
+                   "{}")
-        simplejson.dumps(anki.groups.defaultConf))
+        db.execute(
-    db.execute(
+            "insert or ignore into gconf values (1, ?, ?, ?)""",
-        "insert or ignore into groups values (1, ?, ?, 1)",
+            intTime(), _("Default Config"),
-        intTime(), _("Default Group"))
+            simplejson.dumps(anki.groups.defaultConf))
+        db.execute(
+            "insert or ignore into groups values (1, ?, ?, 1)",
+            intTime(), _("Default Group"))
 
 def _updateIndices(db):
     "Add indices to the DB."
@@ -422,5 +203,262 @@ create index if not exists ix_fdata_csum on fdata (csum);
 -- media
 create index if not exists ix_media_csum on media (csum);
 -- deletion tracking
-create index if not exists ix_gravestones_delTime on gravestones (delTime);
+create index if not exists ix_graves_delTime on graves (delTime);
 """)
+
+# 2.0 schema migration
+######################################################################
+# we don't have access to the progress handler at this point, so the GUI code
+# will need to set up a progress handling window before opening a deck.
+
+def _moveTable(db, table):
+    sql = db.scalar(
+        "select sql from sqlite_master where name = '%s'" % table)
+    sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
+    db.execute(sql)
+    db.execute("insert into %s2 select * from %s" % (table, table))
+    db.execute("drop table "+table)
+    _addSchema(db, False)
+
+def _upgradeSchema(db):
+    "Alter tables prior to ORM initialization."
+    try:
+        ver = db.scalar("select version from deck")
+    except:
+        ver = db.scalar("select version from decks")
+    # latest 1.2 is 65
+    if ver < 65:
+        raise AnkiError("oldDeckVersion")
+    if ver > 99:
+        return ver
+
+    # cards
+    ###########
+    _moveTable(db, "cards")
+    db.execute("""
+insert into cards select id, factId, cardModelId, 1, cast(modified as int),
+question, answer, ordinal, relativeDelay, type, due, cast(interval as int),
+cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""")
+    db.execute("drop table cards2")
+
+    # tags
+    ###########
+    _moveTable(db, "tags")
+    db.execute("insert or ignore into tags select id, ?, tag from tags2",
+               intTime())
+    # tags should have a leading and trailing space if not empty, and not
+    # use commas
+    db.execute("""
+update facts set tags = (case
+when trim(tags) == "" then ""
+else " " || replace(replace(trim(tags), ",", " "), "  ", " ") || " "
+end)
+""")
+    db.execute("drop table tags2")
+    db.execute("drop table cardTags")
+
+    # facts
+    ###########
+    db.execute("""
+create table facts2
+(id, modelId, modified, tags, cache)""")
+    # use the rowid to give them an integer order
+    db.execute("""
+insert into facts2 select id, modelId, modified, tags, spaceUntil from
+facts order by created""")
+    db.execute("drop table facts")
+    _addSchema(db, False)
+    db.execute("""
+insert or ignore into facts select id, modelId, rowid,
+cast(modified as int), tags, cache from facts2""")
+    db.execute("drop table facts2")
+
+    # media
+    ###########
+    _moveTable(db, "media")
+    db.execute("""
+insert or ignore into media select filename, cast(created as int),
+originalPath from media2""")
+    db.execute("drop table media2")
+
+    # fields -> fdata
+    ###########
+    db.execute("""
+insert or ignore into fdata select factId, fieldModelId, ordinal, value, ''
+from fields""")
+    db.execute("drop table fields")
+
+    # models
+    ###########
+    _moveTable(db, "models")
+    db.execute("""
+insert or ignore into models select id, cast(modified as int),
+name, "{}" from models2""")
+    db.execute("drop table models2")
+
+    # reviewHistory -> revlog
+    ###########
+    db.execute("""
+insert or ignore into revlog select
+cast(time*1000 as int), cardId, ease, reps,
+cast(lastInterval as int), cast(nextInterval as int),
+cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
+0 from reviewHistory""")
+    db.execute("drop table reviewHistory")
+    # convert old ease0 into ease1
+    db.execute("update revlog set ease = 1 where ease = 0")
+
+    # longer migrations
+    ###########
+    _migrateDeckTbl(db)
+    _migrateFieldsTbl(db)
+    _migrateTemplatesTbl(db)
+
+    _updateIndices(db)
+    return ver
+
+def _migrateDeckTbl(db):
+    import anki.deck
+    db.execute("delete from deck")
+    db.execute("""
+insert or replace into deck select id, cast(created as int), :t,
+:t, 99, ifnull(syncName, ""), cast(lastSync as int),
+utcOffset, "", "", "" from decks""", t=intTime())
+    # update selective study
+    qconf = anki.deck.defaultQconf.copy()
+    # delete old selective study settings, which we can't auto-upgrade easily
+    keys = ("newActive", "newInactive", "revActive", "revInactive")
+    for k in keys:
+        db.execute("delete from deckVars where key=:k", k=k)
+    # copy other settings, ignoring deck order as there's a new default
+    keys = ("newCardOrder", "newCardSpacing")
+    for k in keys:
+        qconf[k] = db.scalar("select %s from decks" % k)
+    qconf['newPerDay'] = db.scalar(
+        "select newCardsPerDay from decks")
+    # fetch remaining settings from decks table
+    conf = anki.deck.defaultConf.copy()
+    data = {}
+    keys = ("sessionRepLimit", "sessionTimeLimit")
+    for k in keys:
+        conf[k] = db.scalar("select %s from decks" % k)
+    # random and due options merged
+    qconf['revCardOrder'] = min(2, qconf['revCardOrder'])
+    # no reverse option anymore
+    qconf['newCardOrder'] = min(1, qconf['newCardOrder'])
+    # add any deck vars and save
+    dkeys = ("hexCache", "cssCache")
+    for (k, v) in db.execute("select * from deckVars").fetchall():
+        if k in dkeys:
+            data[k] = v
+        else:
+            conf[k] = v
+    db.execute("update deck set qconf = :l, conf = :c, data = :d",
+               l=simplejson.dumps(qconf),
+               c=simplejson.dumps(conf),
+               d=simplejson.dumps(data))
+    # clean up
+    db.execute("drop table decks")
+    db.execute("drop table deckVars")
+
+def _migrateFieldsTbl(db):
+    import anki.models
+    db.execute("""
+insert into fields select id, modelId, ordinal, name, numeric, ''
+from fieldModels""")
+    dconf = anki.models.defaultFieldConf
+    for row in db.all("""
+select id, features, required, "unique", quizFontFamily, quizFontSize,
+quizFontColour, editFontSize from fieldModels"""):
+        conf = dconf.copy()
+        (conf['rtl'],
+         conf['required'],
+         conf['unique'],
+         conf['font'],
+         conf['quizSize'],
+         conf['quizColour'],
+         conf['editSize']) = row[1:]
+        # setup bools
+        conf['rtl'] = not not conf['rtl']
+        conf['pre'] = True
+        # save
+        db.execute("update fields set conf = ? where id = ?",
+                   simplejson.dumps(conf), row[0])
+    # clean up
+    db.execute("drop table fieldModels")
+
+def _migrateTemplatesTbl(db):
+    # do this after fieldModel migration
+    import anki.models
+    db.execute("""
+insert into templates select id, modelId, ordinal, name, active, qformat,
+aformat, '' from cardModels""")
+    dconf = anki.models.defaultTemplateConf
+    for row in db.all("""
+select id, modelId, questionInAnswer, questionAlign, lastFontColour,
+allowEmptyAnswer, typeAnswer from cardModels"""):
+        conf = dconf.copy()
+        (conf['hideQ'],
+         conf['align'],
+         conf['bg'],
+         conf['allowEmptyAns'],
+         fname) = row[2:]
+        # convert the field name to an id
+        conf['typeAnswer'] = db.scalar(
+            "select id from fields where name = ? and mid = ?",
+            fname, row[1])
+        # save
+        db.execute("update templates set conf = ? where id = ?",
+                   simplejson.dumps(conf), row[0])
+    # clean up
+    db.execute("drop table cardModels")
+
+def _postSchemaUpgrade(deck):
+    "Handle the rest of the upgrade to 2.0."
+    import anki.deck
+    # remove old views
+    for v in ("failedCards", "revCardsOld", "revCardsNew",
+              "revCardsDue", "revCardsRandom", "acqCardsRandom",
+              "acqCardsOld", "acqCardsNew"):
+        deck.db.execute("drop view if exists %s" % v)
+    # update caches
+    for m in deck.allModels():
+        m.updateCache()
+    # remove stats, as it's all in the revlog now
+    deck.db.execute("drop table if exists stats")
+    # suspended cards don't use ranges anymore
+    deck.db.execute("update cards set queue=-1 where queue between -3 and -1")
+    deck.db.execute("update cards set queue=-2 where queue between 3 and 5")
+    deck.db.execute("update cards set queue=-3 where queue between 6 and 8")
+    # remove old deleted tables
+    for t in ("cards", "facts", "models", "media"):
+        deck.db.execute("drop table if exists %sDeleted" % t)
+    # rewrite due times for new cards
+    deck.db.execute("""
+update cards set due = (select pos from facts where fid = facts.id) where type=2""")
+    # convert due cards into day-based due
+    deck.db.execute("""
+update cards set due = cast(
+(case when due < :stamp then 0 else 1 end) +
+((due-:stamp)/86400) as int)+:today where type
+between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
+    # update factPos
+    deck.conf['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1
+    deck.save()
+
+    # optimize and finish
+    deck.updateDynamicIndices()
+    deck.db.execute("vacuum")
+    deck.db.execute("analyze")
+    deck.db.execute("update deck set version = ?", CURRENT_VERSION)
+    deck.save()
+
+# Post-init upgrade
+######################################################################
+
+def _upgradeDeck(deck, version):
+    "Upgrade deck to the latest version."
+    if version >= CURRENT_VERSION:
+        return
+    if version < 100:
+        _postSchemaUpgrade(deck)