diff --git a/anki/media.py b/anki/media.py index 2473e5298..233d64abe 100644 --- a/anki/media.py +++ b/anki/media.py @@ -19,7 +19,7 @@ class MediaManager(object): def __init__(self, deck): self.deck = deck # media directory - self._dir = re.sub("(?i)\.(anki)$", ".media", self.deck.path) + self._dir = re.sub("(?i)\.(anki2)$", ".media", self.deck.path) if not os.path.exists(self._dir): os.makedirs(self._dir) os.chdir(self._dir) diff --git a/anki/migration/__init__.py b/anki/migration/__init__.py new file mode 100644 index 000000000..ea8ce5786 --- /dev/null +++ b/anki/migration/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# Copyright: Damien Elmes +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +class Migrator(object): + + def __init__(self, deck): + pass diff --git a/anki/migration/checker.py b/anki/migration/checker.py new file mode 100644 index 000000000..43bb98f96 --- /dev/null +++ b/anki/migration/checker.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright: Damien Elmes +# License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html + +from anki.db import DB + +def check(path): + "True if deck looks ok." + db = DB(path) + # corrupt? + try: + if db.scalar("pragma integrity_check") != "ok": + return + except: + return + # old version? + if db.scalar("select version from decks") != 65: + return + # fields missing a field model? + if db.list(""" +select id from fields where fieldModelId not in ( +select distinct id from fieldModels)"""): + return + # facts missing a field? + if db.list(""" +select distinct facts.id from facts, fieldModels where +facts.modelId = fieldModels.modelId and fieldModels.id not in +(select fieldModelId from fields where factId = facts.id)"""): + return + # cards missing a fact? + if db.list(""" +select id from cards where factId not in (select id from facts)"""): + return + # cards missing a card model? + if db.list(""" +select id from cards where cardModelId not in +(select id from cardModels)"""): + return + # cards with a card model from the wrong model? + if db.list(""" +select id from cards where cardModelId not in (select cm.id from +cardModels cm, facts f where cm.modelId = f.modelId and +f.id = cards.factId)"""): + return + # facts missing a card? + if db.list(""" +select facts.id from facts +where facts.id not in (select distinct factId from cards)"""): + return + # dangling fields? + if db.list(""" +select id from fields where factId not in (select id from facts)"""): + return + # fields without matching interval + if db.list(""" +select id from fields where ordinal != (select ordinal from fieldModels +where id = fieldModelId)"""): + return + # incorrect types + if db.list(""" +select id from cards where relativeDelay != (case +when successive then 1 when reps then 0 else 2 end)"""): + return + if db.list(""" +select id from cards where type != (case +when type >= 0 then relativeDelay else relativeDelay - 3 end)"""): + return + return True diff --git a/anki/migration/upgrader.py b/anki/migration/upgrader.py new file mode 100644 index 000000000..8a2553c3f --- /dev/null +++ b/anki/migration/upgrader.py @@ -0,0 +1,434 @@ +# -*- coding: utf-8 -*- +# Copyright: Damien Elmes +# License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html + +import os, time, simplejson, re, datetime +from anki.lang import _ +from anki.utils import intTime +from anki.db import DB +from anki.deck import _Deck + +def Deck(path, queue=True, lock=True, server=False): + "Open a new or existing deck. Path must be unicode." + path = os.path.abspath(path) + create = not os.path.exists(path) + if create: + base = os.path.basename(path) + for c in ("/", ":", "\\"): + assert c not in base + # connect + db = DB(path) + if create: + ver = _createDB(db) + else: + ver = _upgradeSchema(db) + db.execute("pragma temp_store = memory") + db.execute("pragma cache_size = 10000") + # add db to deck and do any remaining upgrades + deck = _Deck(db, server) + if ver < CURRENT_VERSION: + _upgradeDeck(deck, ver) + elif create: + # add in reverse order so basic is default + addClozeModel(deck) + addBasicModel(deck) + deck.save() + if lock: + deck.lock() + if not queue: + return deck + # rebuild queue + deck.reset() + return deck + +# 2.0 schema migration +###################################################################### + +def _moveTable(db, table, cards=False): + if cards: + insExtra = " order by created" + else: + insExtra = "" + sql = db.scalar( + "select sql from sqlite_master where name = '%s'" % table) + sql = sql.replace("TABLE "+table, "temporary table %s2" % table) + if cards: + sql = sql.replace("PRIMARY KEY (id),", "") + db.execute(sql) + db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra)) + db.execute("drop table "+table) + _addSchema(db, False) + +def _upgradeSchema(db): + "Alter tables prior to ORM initialization." + try: + ver = db.scalar("select ver from deck") + except: + ver = db.scalar("select version from decks") + # latest 1.2 is 65 + if ver < 65: + raise AnkiError("oldDeckVersion") + if ver > 99: + # anki 2.0 + if ver > CURRENT_VERSION: + # refuse to load decks created with a future version + raise AnkiError("newDeckVersion") + return ver + runHook("1.x upgrade", db) + + # these weren't always correctly set + db.execute("pragma page_size = 4096") + db.execute("pragma legacy_file_format = 0") + + # facts + ########### + # tags should have a leading and trailing space if not empty, and not + # use commas + db.execute(""" +update facts set tags = (case +when trim(tags) == "" then "" +else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " +end) +""") + # pull facts into memory, so we can merge them with fields efficiently + facts = db.all(""" +select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags +from facts order by created""") + # build field hash + fields = {} + for (fid, ord, val) in db.execute( + "select factId, ordinal, value from fields order by factId, ordinal"): + if fid not in fields: + fields[fid] = [] + fields[fid].append((ord, val)) + # build insert data and transform ids, and minimize qt's + # bold/italics/underline cruft. + map = {} + data = [] + factidmap = {} + times = {} + from anki.utils import minimizeHTML + for c, row in enumerate(facts): + oldid = row[0] + row = list(row) + # get rid of old created column and update id + while row[4] in times: + row[4] += 1 + times[row[4]] = True + factidmap[row[0]] = row[4] + row[0] = row[4] + del row[4] + map[oldid] = row[0] + row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])]))) + data.append(row) + # and put the facts into the new table + db.execute("drop table facts") + _addSchema(db, False) + db.executemany("insert into facts values (?,?,?,?,?,?,?,?,'',0,'')", data) + db.execute("drop table fields") + + # cards + ########### + # we need to pull this into memory, to rewrite the creation time if + # it's not unique and update the fact id + times = {} + rows = [] + cardidmap = {} + for row in db.execute(""" +select id, cast(created*1000 as int), factId, ordinal, +cast(modified as int), 0, +(case relativeDelay +when 0 then 1 +when 1 then 2 +when 2 then 0 end), +(case type +when 0 then 1 +when 1 then 2 +when 2 then 0 +else type end), +cast(due as int), cast(interval as int), +cast(factor*1000 as int), reps, noCount from cards +order by created"""): + # find an unused time + row = list(row) + while row[1] in times: + row[1] += 1 + times[row[1]] = True + # rewrite fact id + row[2] = factidmap[row[2]] + # note id change and save all but old id + cardidmap[row[0]] = row[1] + rows.append(row[1:]) + # drop old table and rewrite + db.execute("drop table cards") + _addSchema(db, False) + db.executemany(""" +insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""", + rows) + + # reviewHistory -> revlog + ########### + # fetch the data so we can rewrite ids quickly + r = [] + for row in db.execute(""" +select +cast(time*1000 as int), cardId, 0, ease, +cast(nextInterval as int), cast(lastInterval as int), +cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), +yesCount from reviewHistory"""): + row = list(row) + # new card ids + try: + row[1] = cardidmap[row[1]] + except: + # id doesn't exist + continue + # no ease 0 anymore + row[2] = row[2] or 1 + # determine type, overwriting yesCount + newInt = row[3] + oldInt = row[4] + yesCnt = row[7] + # yesCnt included the current answer + if row[2] > 1: + yesCnt -= 1 + if oldInt < 1: + # new or failed + if yesCnt: + # type=relrn + row[7] = 2 + else: + # type=lrn + row[7] = 0 + else: + # type=rev + row[7] = 1 + r.append(row) + db.executemany( + "insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r) + db.execute("drop table reviewHistory") + + # deck + ########### + _migrateDeckTbl(db) + + # tags + ########### + tags = {} + for t in db.list("select tag from tags"): + tags[t] = intTime() + db.execute("update deck set tags = ?", simplejson.dumps(tags)) + db.execute("drop table tags") + db.execute("drop table cardTags") + + # the rest + ########### + db.execute("drop table media") + db.execute("drop table sources") + _migrateModels(db) + _updateIndices(db) + return ver + +def _migrateDeckTbl(db): + import anki.deck + db.execute("delete from deck") + db.execute(""" +insert or replace into deck select id, cast(created as int), :t, +:t, 99, 0, 0, cast(lastSync as int), +"", "", "", "", "" from decks""", t=intTime()) + # prepare a group to store the old deck options + g, gc, conf = _getDeckVars(db) + # delete old selective study settings, which we can't auto-upgrade easily + keys = ("newActive", "newInactive", "revActive", "revInactive") + for k in keys: + db.execute("delete from deckVars where key=:k", k=k) + # copy other settings, ignoring deck order as there's a new default + g['newSpread'] = db.scalar("select newCardSpacing from decks") + g['newPerDay'] = db.scalar("select newCardsPerDay from decks") + g['repLim'] = db.scalar("select sessionRepLimit from decks") + g['timeLim'] = db.scalar("select sessionTimeLimit from decks") + # this needs to be placed in the model later on + conf['oldNewOrder'] = db.scalar("select newCardOrder from decks") + # no reverse option anymore + conf['oldNewOrder'] = min(1, conf['oldNewOrder']) + # add any deck vars and save + dkeys = ("hexCache", "cssCache") + for (k, v) in db.execute("select * from deckVars").fetchall(): + if k in dkeys: + pass + else: + conf[k] = v + _addDeckVars(db, g, gc, conf) + # clean up + db.execute("drop table decks") + db.execute("drop table deckVars") + +def _migrateModels(db): + import anki.models + times = {} + mods = {} + for row in db.all( + "select id, name from models"): + while 1: + t = intTime(1000) + if t not in times: + times[t] = True + break + m = anki.models.defaultModel.copy() + m['id'] = t + m['name'] = row[1] + m['mod'] = intTime() + m['tags'] = [] + m['flds'] = _fieldsForModel(db, row[0]) + m['tmpls'] = _templatesForModel(db, row[0], m['flds']) + mods[m['id']] = m + db.execute("update facts set mid = ? where mid = ?", t, row[0]) + # save and clean up + db.execute("update deck set models = ?", simplejson.dumps(mods)) + db.execute("drop table fieldModels") + db.execute("drop table cardModels") + db.execute("drop table models") + +def _fieldsForModel(db, mid): + import anki.models + dconf = anki.models.defaultField + flds = [] + for c, row in enumerate(db.all(""" +select name, features, required, "unique", +quizFontFamily, quizFontSize, quizFontColour, editFontSize from fieldModels +where modelId = ? +order by ordinal""", mid)): + conf = dconf.copy() + (conf['name'], + conf['rtl'], + conf['req'], + conf['uniq'], + conf['font'], + conf['qsize'], + conf['qcol'], + conf['esize']) = row + conf['ord'] = c + # ensure data is good + conf['rtl'] = not not conf['rtl'] + conf['pre'] = True + conf['font'] = conf['font'] or "Arial" + conf['qcol'] = conf['qcol'] or "#000" + conf['qsize'] = conf['qsize'] or 20 + conf['esize'] = conf['esize'] or 20 + flds.append(conf) + return flds + +def _templatesForModel(db, mid, flds): + import anki.models + dconf = anki.models.defaultTemplate + tmpls = [] + for c, row in enumerate(db.all(""" +select name, active, qformat, aformat, questionInAnswer, +questionAlign, lastFontColour, allowEmptyAnswer, typeAnswer from cardModels +where modelId = ? +order by ordinal""", mid)): + conf = dconf.copy() + (conf['name'], + conf['actv'], + conf['qfmt'], + conf['afmt'], + conf['hideQ'], + conf['align'], + conf['bg'], + conf['emptyAns'], + conf['typeAns']) = row + conf['ord'] = c + # convert the field name to an ordinal + ordN = None + for (ord, fm) in enumerate(flds): + if fm['name'] == conf['typeAns']: + ordN = ord + break + if ordN is not None: + conf['typeAns'] = ordN + else: + conf['typeAns'] = None + for type in ("qfmt", "afmt"): + # ensure the new style field format + conf[type] = re.sub("%\((.+?)\)s", "{{\\1}}", conf[type]) + # some special names have changed + conf[type] = re.sub( + "(?i){{tags}}", "{{Tags}}", conf[type]) + conf[type] = re.sub( + "(?i){{cardModel}}", "{{Template}}", conf[type]) + conf[type] = re.sub( + "(?i){{modelTags}}", "{{Model}}", conf[type]) + tmpls.append(conf) + return tmpls + +def _postSchemaUpgrade(deck): + "Handle the rest of the upgrade to 2.0." + import anki.deck + # make sure we have a current model id + deck.models.setCurrent(deck.models.models.values()[0]) + # regenerate css, and set new card order + for m in deck.models.all(): + m['newOrder'] = deck.conf['oldNewOrder'] + deck.models.save(m) + del deck.conf['oldNewOrder'] + # fix creation time + deck.sched._updateCutoff() + d = datetime.datetime.today() + d -= datetime.timedelta(hours=4) + d = datetime.datetime(d.year, d.month, d.day) + d += datetime.timedelta(hours=4) + d -= datetime.timedelta(days=1+int((time.time()-deck.crt)/86400)) + deck.crt = int(time.mktime(d.timetuple())) + deck.sched._updateCutoff() + # update uniq cache + deck.updateFieldCache(deck.db.list("select id from facts")) + # remove old views + for v in ("failedCards", "revCardsOld", "revCardsNew", + "revCardsDue", "revCardsRandom", "acqCardsRandom", + "acqCardsOld", "acqCardsNew"): + deck.db.execute("drop view if exists %s" % v) + # remove stats, as it's all in the revlog now + deck.db.execute("drop table if exists stats") + # suspended cards don't use ranges anymore + deck.db.execute("update cards set queue=-1 where queue between -3 and -1") + deck.db.execute("update cards set queue=-2 where queue between 3 and 5") + deck.db.execute("update cards set queue=-3 where queue between 6 and 8") + # remove old deleted tables + for t in ("cards", "facts", "models", "media"): + deck.db.execute("drop table if exists %sDeleted" % t) + # rewrite due times for new cards + deck.db.execute(""" +update cards set due = fid where type=0""") + # and failed cards + left = len(deck.groups.conf(1)['new']['delays']) + deck.db.execute("update cards set edue = ?, left=? where type = 1", + deck.sched.today+1, left) + # and due cards + deck.db.execute(""" +update cards set due = cast( +(case when due < :stamp then 0 else 1 end) + +((due-:stamp)/86400) as int)+:today where type = 2 +""", stamp=deck.sched.dayCutoff, today=deck.sched.today) + # possibly re-randomize + if deck.models.randomNew(): + deck.sched.randomizeCards() + # update insertion id + deck.conf['nextPos'] = deck.db.scalar("select max(id) from facts")+1 + deck.save() + + # optimize and finish + deck.db.commit() + deck.db.execute("vacuum") + deck.db.execute("analyze") + deck.db.execute("update deck set ver = ?", CURRENT_VERSION) + deck.save() + +# Post-init upgrade +###################################################################### + +def _upgradeDeck(deck, version): + "Upgrade deck to the latest version." + if version >= CURRENT_VERSION: + return + if version < 100: + _postSchemaUpgrade(deck) diff --git a/anki/storage.py b/anki/storage.py index 25dc4b349..40e30b666 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -2,19 +2,18 @@ # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -CURRENT_VERSION = 100 +CURRENT_VERSION = 1 -import os, time, simplejson, re, datetime +import os, simplejson from anki.lang import _ from anki.utils import intTime from anki.db import DB from anki.deck import _Deck from anki.stdmodels import addBasicModel, addClozeModel -from anki.errors import AnkiError -from anki.hooks import runHook def Deck(path, queue=True, lock=True, server=False): "Open a new or existing deck. Path must be unicode." + assert path.endswith(".anki2") path = os.path.abspath(path) create = not os.path.exists(path) if create: @@ -46,6 +45,15 @@ def Deck(path, queue=True, lock=True, server=False): deck.reset() return deck +# no upgrades necessary at the moment +def _upgradeSchema(db): + return CURRENT_VERSION +def _upgradeDeck(deck, ver): + return + +# Creating a new deck +###################################################################### + def _createDB(db): db.execute("pragma page_size = 4096") db.execute("pragma legacy_file_format = 0") @@ -174,391 +182,3 @@ create index if not exists ix_revlog_cid on revlog (cid); create index if not exists ix_fsums_fid on fsums (fid); create index if not exists ix_fsums_csum on fsums (csum); """) - -# 2.0 schema migration -###################################################################### - -def _moveTable(db, table, cards=False): - if cards: - insExtra = " order by created" - else: - insExtra = "" - sql = db.scalar( - "select sql from sqlite_master where name = '%s'" % table) - sql = sql.replace("TABLE "+table, "temporary table %s2" % table) - if cards: - sql = sql.replace("PRIMARY KEY (id),", "") - db.execute(sql) - db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra)) - db.execute("drop table "+table) - _addSchema(db, False) - -def _upgradeSchema(db): - "Alter tables prior to ORM initialization." - try: - ver = db.scalar("select ver from deck") - except: - ver = db.scalar("select version from decks") - # latest 1.2 is 65 - if ver < 65: - raise AnkiError("oldDeckVersion") - if ver > 99: - # anki 2.0 - if ver > CURRENT_VERSION: - # refuse to load decks created with a future version - raise AnkiError("newDeckVersion") - return ver - runHook("1.x upgrade", db) - - # facts - ########### - # tags should have a leading and trailing space if not empty, and not - # use commas - db.execute(""" -update facts set tags = (case -when trim(tags) == "" then "" -else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " -end) -""") - # pull facts into memory, so we can merge them with fields efficiently - facts = db.all(""" -select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags -from facts order by created""") - # build field hash - fields = {} - for (fid, ord, val) in db.execute( - "select factId, ordinal, value from fields order by factId, ordinal"): - if fid not in fields: - fields[fid] = [] - fields[fid].append((ord, val)) - # build insert data and transform ids, and minimize qt's - # bold/italics/underline cruft. - map = {} - data = [] - factidmap = {} - times = {} - from anki.utils import minimizeHTML - for c, row in enumerate(facts): - oldid = row[0] - row = list(row) - # get rid of old created column and update id - while row[4] in times: - row[4] += 1 - times[row[4]] = True - factidmap[row[0]] = row[4] - row[0] = row[4] - del row[4] - map[oldid] = row[0] - row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])]))) - data.append(row) - # and put the facts into the new table - db.execute("drop table facts") - _addSchema(db, False) - db.executemany("insert into facts values (?,?,?,?,?,?,?,?,'',0,'')", data) - db.execute("drop table fields") - - # cards - ########### - # we need to pull this into memory, to rewrite the creation time if - # it's not unique and update the fact id - times = {} - rows = [] - cardidmap = {} - for row in db.execute(""" -select id, cast(created*1000 as int), factId, ordinal, -cast(modified as int), 0, -(case relativeDelay -when 0 then 1 -when 1 then 2 -when 2 then 0 end), -(case type -when 0 then 1 -when 1 then 2 -when 2 then 0 -else type end), -cast(due as int), cast(interval as int), -cast(factor*1000 as int), reps, noCount from cards -order by created"""): - # find an unused time - row = list(row) - while row[1] in times: - row[1] += 1 - times[row[1]] = True - # rewrite fact id - row[2] = factidmap[row[2]] - # note id change and save all but old id - cardidmap[row[0]] = row[1] - rows.append(row[1:]) - # drop old table and rewrite - db.execute("drop table cards") - _addSchema(db, False) - db.executemany(""" -insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""", - rows) - - # reviewHistory -> revlog - ########### - # fetch the data so we can rewrite ids quickly - r = [] - for row in db.execute(""" -select -cast(time*1000 as int), cardId, 0, ease, -cast(nextInterval as int), cast(lastInterval as int), -cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), -yesCount from reviewHistory"""): - row = list(row) - # new card ids - try: - row[1] = cardidmap[row[1]] - except: - # id doesn't exist - continue - # no ease 0 anymore - row[2] = row[2] or 1 - # determine type, overwriting yesCount - newInt = row[3] - oldInt = row[4] - yesCnt = row[7] - # yesCnt included the current answer - if row[2] > 1: - yesCnt -= 1 - if oldInt < 1: - # new or failed - if yesCnt: - # type=relrn - row[7] = 2 - else: - # type=lrn - row[7] = 0 - else: - # type=rev - row[7] = 1 - r.append(row) - db.executemany( - "insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r) - db.execute("drop table reviewHistory") - - # deck - ########### - _migrateDeckTbl(db) - - # tags - ########### - tags = {} - for t in db.list("select tag from tags"): - tags[t] = intTime() - db.execute("update deck set tags = ?", simplejson.dumps(tags)) - db.execute("drop table tags") - db.execute("drop table cardTags") - - # the rest - ########### - db.execute("drop table media") - db.execute("drop table sources") - _migrateModels(db) - _updateIndices(db) - return ver - -def _migrateDeckTbl(db): - import anki.deck - db.execute("delete from deck") - db.execute(""" -insert or replace into deck select id, cast(created as int), :t, -:t, 99, 0, 0, cast(lastSync as int), -"", "", "", "", "" from decks""", t=intTime()) - # prepare a group to store the old deck options - g, gc, conf = _getDeckVars(db) - # delete old selective study settings, which we can't auto-upgrade easily - keys = ("newActive", "newInactive", "revActive", "revInactive") - for k in keys: - db.execute("delete from deckVars where key=:k", k=k) - # copy other settings, ignoring deck order as there's a new default - g['newSpread'] = db.scalar("select newCardSpacing from decks") - g['newPerDay'] = db.scalar("select newCardsPerDay from decks") - g['repLim'] = db.scalar("select sessionRepLimit from decks") - g['timeLim'] = db.scalar("select sessionTimeLimit from decks") - # this needs to be placed in the model later on - conf['oldNewOrder'] = db.scalar("select newCardOrder from decks") - # no reverse option anymore - conf['oldNewOrder'] = min(1, conf['oldNewOrder']) - # add any deck vars and save - dkeys = ("hexCache", "cssCache") - for (k, v) in db.execute("select * from deckVars").fetchall(): - if k in dkeys: - pass - else: - conf[k] = v - _addDeckVars(db, g, gc, conf) - # clean up - db.execute("drop table decks") - db.execute("drop table deckVars") - -def _migrateModels(db): - import anki.models - times = {} - mods = {} - for row in db.all( - "select id, name from models"): - while 1: - t = intTime(1000) - if t not in times: - times[t] = True - break - m = anki.models.defaultModel.copy() - m['id'] = t - m['name'] = row[1] - m['mod'] = intTime() - m['tags'] = [] - m['flds'] = _fieldsForModel(db, row[0]) - m['tmpls'] = _templatesForModel(db, row[0], m['flds']) - mods[m['id']] = m - db.execute("update facts set mid = ? where mid = ?", t, row[0]) - # save and clean up - db.execute("update deck set models = ?", simplejson.dumps(mods)) - db.execute("drop table fieldModels") - db.execute("drop table cardModels") - db.execute("drop table models") - -def _fieldsForModel(db, mid): - import anki.models - dconf = anki.models.defaultField - flds = [] - for c, row in enumerate(db.all(""" -select name, features, required, "unique", -quizFontFamily, quizFontSize, quizFontColour, editFontSize from fieldModels -where modelId = ? -order by ordinal""", mid)): - conf = dconf.copy() - (conf['name'], - conf['rtl'], - conf['req'], - conf['uniq'], - conf['font'], - conf['qsize'], - conf['qcol'], - conf['esize']) = row - conf['ord'] = c - # ensure data is good - conf['rtl'] = not not conf['rtl'] - conf['pre'] = True - conf['font'] = conf['font'] or "Arial" - conf['qcol'] = conf['qcol'] or "#000" - conf['qsize'] = conf['qsize'] or 20 - conf['esize'] = conf['esize'] or 20 - flds.append(conf) - return flds - -def _templatesForModel(db, mid, flds): - import anki.models - dconf = anki.models.defaultTemplate - tmpls = [] - for c, row in enumerate(db.all(""" -select name, active, qformat, aformat, questionInAnswer, -questionAlign, lastFontColour, allowEmptyAnswer, typeAnswer from cardModels -where modelId = ? -order by ordinal""", mid)): - conf = dconf.copy() - (conf['name'], - conf['actv'], - conf['qfmt'], - conf['afmt'], - conf['hideQ'], - conf['align'], - conf['bg'], - conf['emptyAns'], - conf['typeAns']) = row - conf['ord'] = c - # convert the field name to an ordinal - ordN = None - for (ord, fm) in enumerate(flds): - if fm['name'] == conf['typeAns']: - ordN = ord - break - if ordN is not None: - conf['typeAns'] = ordN - else: - conf['typeAns'] = None - for type in ("qfmt", "afmt"): - # ensure the new style field format - conf[type] = re.sub("%\((.+?)\)s", "{{\\1}}", conf[type]) - # some special names have changed - conf[type] = re.sub( - "(?i){{tags}}", "{{Tags}}", conf[type]) - conf[type] = re.sub( - "(?i){{cardModel}}", "{{Template}}", conf[type]) - conf[type] = re.sub( - "(?i){{modelTags}}", "{{Model}}", conf[type]) - tmpls.append(conf) - return tmpls - -def _postSchemaUpgrade(deck): - "Handle the rest of the upgrade to 2.0." - import anki.deck - # make sure we have a current model id - deck.models.setCurrent(deck.models.models.values()[0]) - # regenerate css, and set new card order - for m in deck.models.all(): - m['newOrder'] = deck.conf['oldNewOrder'] - deck.models.save(m) - del deck.conf['oldNewOrder'] - # fix creation time - deck.sched._updateCutoff() - d = datetime.datetime.today() - d -= datetime.timedelta(hours=4) - d = datetime.datetime(d.year, d.month, d.day) - d += datetime.timedelta(hours=4) - d -= datetime.timedelta(days=1+int((time.time()-deck.crt)/86400)) - deck.crt = int(time.mktime(d.timetuple())) - deck.sched._updateCutoff() - # update uniq cache - deck.updateFieldCache(deck.db.list("select id from facts")) - # remove old views - for v in ("failedCards", "revCardsOld", "revCardsNew", - "revCardsDue", "revCardsRandom", "acqCardsRandom", - "acqCardsOld", "acqCardsNew"): - deck.db.execute("drop view if exists %s" % v) - # remove stats, as it's all in the revlog now - deck.db.execute("drop table if exists stats") - # suspended cards don't use ranges anymore - deck.db.execute("update cards set queue=-1 where queue between -3 and -1") - deck.db.execute("update cards set queue=-2 where queue between 3 and 5") - deck.db.execute("update cards set queue=-3 where queue between 6 and 8") - # remove old deleted tables - for t in ("cards", "facts", "models", "media"): - deck.db.execute("drop table if exists %sDeleted" % t) - # rewrite due times for new cards - deck.db.execute(""" -update cards set due = fid where type=0""") - # and failed cards - left = len(deck.groups.conf(1)['new']['delays']) - deck.db.execute("update cards set edue = ?, left=? where type = 1", - deck.sched.today+1, left) - # and due cards - deck.db.execute(""" -update cards set due = cast( -(case when due < :stamp then 0 else 1 end) + -((due-:stamp)/86400) as int)+:today where type = 2 -""", stamp=deck.sched.dayCutoff, today=deck.sched.today) - # possibly re-randomize - if deck.models.randomNew(): - deck.sched.randomizeCards() - # update insertion id - deck.conf['nextPos'] = deck.db.scalar("select max(id) from facts")+1 - deck.save() - - # optimize and finish - deck.db.commit() - deck.db.execute("vacuum") - deck.db.execute("analyze") - deck.db.execute("update deck set ver = ?", CURRENT_VERSION) - deck.save() - -# Post-init upgrade -###################################################################### - -def _upgradeDeck(deck, version): - "Upgrade deck to the latest version." - if version >= CURRENT_VERSION: - return - if version < 100: - _postSchemaUpgrade(deck) diff --git a/tests/shared.py b/tests/shared.py index d91957a7f..8fe3f8d73 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -1,4 +1,4 @@ -import tempfile, os +import tempfile, os, shutil from anki import Deck def assertException(exception, func): @@ -10,8 +10,14 @@ def assertException(exception, func): assert found def getEmptyDeck(**kwargs): - (fd, nam) = tempfile.mkstemp(suffix=".anki") + (fd, nam) = tempfile.mkstemp(suffix=".anki2") os.unlink(nam) return Deck(nam, **kwargs) +def getUpgradeDeckPath(): + src = os.path.join(testDir, "support", "anki12.anki") + (fd, dst) = tempfile.mkstemp(suffix=".anki2") + shutil.copy(src, dst) + return dst + testDir = os.path.dirname(__file__) diff --git a/tests/test_deck.py b/tests/test_deck.py index 9ccbfd25b..63047b31d 100644 --- a/tests/test_deck.py +++ b/tests/test_deck.py @@ -1,7 +1,8 @@ # coding: utf-8 import os, re, datetime -from tests.shared import assertException, getEmptyDeck, testDir +from tests.shared import assertException, getEmptyDeck, testDir, \ + getUpgradeDeckPath from anki.stdmodels import addBasicModel from anki.consts import * @@ -12,7 +13,7 @@ newMod = None def test_create(): global newPath, newMod - path = "/tmp/test_attachNew.anki" + path = "/tmp/test_attachNew.anki2" try: os.unlink(path) except OSError: @@ -32,7 +33,7 @@ def test_open(): def test_openReadOnly(): # non-writeable dir assertException(Exception, - lambda: Deck("/attachroot")) + lambda: Deck("/attachroot.anki2")) # reuse tmp file from before, test non-writeable file os.chmod(newPath, 0) assertException(Exception, @@ -116,11 +117,8 @@ def test_fieldChecksum(): "select count() from fsums") == 2 def test_upgrade(): - import tempfile, shutil - src = os.path.join(testDir, "support", "anki12.anki") - (fd, dst) = tempfile.mkstemp(suffix=".anki") + dst = getUpgradeDeckPath() print "upgrade to", dst - shutil.copy(src, dst) deck = Deck(dst) # creation time should have been adjusted d = datetime.datetime.fromtimestamp(deck.crt) diff --git a/tests/test_migration.py b/tests/test_migration.py new file mode 100644 index 000000000..adedf0402 --- /dev/null +++ b/tests/test_migration.py @@ -0,0 +1,12 @@ +# coding: utf-8 + +from shared import getUpgradeDeckPath +from anki.migration.checker import check + +def test_checker(): + dst = getUpgradeDeckPath() + assert check(dst) + # if it's corrupted, will fail + open(dst, "w+").write("foo") + assert not check(dst) +