fix upgrading; drop old mnemosyne 1 importer

anki/consts.py

@@ -44,6 +44,9 @@ SYNC_PORT = int(os.environ.get("SYNC_PORT") or 80)
 SYNC_URL = "http://%s:%d/sync/" % (SYNC_HOST, SYNC_PORT)
 SYNC_VER = 0
 
+# deck schema
+SCHEMA_VERSION = 1
+
 # Labels
 ##########################################################################
 

anki/importing/mnemosyne10.py

@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright: Damien Elmes <anki@ichi2.net>
-# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-
-import sys, pickle, time, re
-from anki.importing import Importer, ForeignCard
-from anki.errors import *
-
-class Mnemosyne10Importer(Importer):
-
-    multipleCardsAllowed = False
-
-    def foreignCards(self):
-        # empty objects so we can load the native mnemosyne file
-        class MnemosyneModule(object):
-            class StartTime:
-                pass
-            class Category:
-                pass
-            class Item:
-                pass
-        for module in ('mnemosyne',
-                       'mnemosyne.core',
-                       'mnemosyne.core.mnemosyne_core'):
-            sys.modules[module] = MnemosyneModule()
-        try:
-            file = open(self.file, "rb")
-        except (IOError, OSError), e:
-            raise ImportFormatError(type="systemError",
-                                    info=str(e))
-        header = file.readline().strip()
-        # read the structure in
-        try:
-            struct = pickle.load(file)
-        except (EOFError, KeyError):
-            raise ImportFormatError(type="invalidFile")
-        startTime = struct[0].time
-        daysPassed = (time.time() - startTime) / 86400.0
-        # gather cards
-        cards = []
-        for item in struct[2]:
-            card = ForeignCard()
-            card.fields.append(self.fudgeText(item.q))
-            card.fields.append(self.fudgeText(item.a))
-            # scheduling data
-            card.interval = item.next_rep - item.last_rep
-            secDelta = (item.next_rep - daysPassed) * 86400.0
-            card.due = card.nextTime = time.time() + secDelta
-            card.factor = item.easiness
-            # for some reason mnemosyne starts cards off on 1 instead of 0
-            card.successive = max(
-                (item.acq_reps_since_lapse + item.ret_reps_since_lapse -1), 0)
-            card.yesCount = max((item.acq_reps + item.ret_reps) - 1, 0)
-            card.noCount = item.lapses
-            card.reps = card.yesCount + card.noCount
-            if item.cat.name != u"<default>":
-                card.tags = item.cat.name.replace(" ", "_")
-            cards.append(card)
-        return cards
-
-    def fields(self):
-        return 2
-
-    def fudgeText(self, text):
-        text = text.replace("\n", "<br>")
-        text = re.sub('<sound src="(.*?)">', '[sound:\\1]', text)
-        text = re.sub('<(/?latex)>', '[\\1]', text)
-        text = re.sub('<(/?\$)>', '[\\1]', text)
-        text = re.sub('<(/?\$\$)>', '[\\1]', text)
-        return text

anki/migration/upgrader.py

@@ -2,80 +2,53 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html
 
-import os, time, simplejson, re, datetime
+import os, time, simplejson, re, datetime, shutil
 from anki.lang import _
-from anki.utils import intTime
+from anki.utils import intTime, namedtmp
 from anki.db import DB
 from anki.deck import _Deck
+from anki.consts import *
+from anki.storage import _addSchema, _getDeckVars, _addDeckVars, \
+    _updateIndices
 
-def Deck(path, queue=True, lock=True, server=False):
+#
-    "Open a new or existing deck. Path must be unicode."
+# Upgrading is the first step in migrating to 2.0. The ids are temporary and
-    path = os.path.abspath(path)
+# may not be unique across multiple decks. After each of a user's v1.2 decks
-    create = not os.path.exists(path)
+# are upgraded, they need to be merged.
-    if create:
+#
-        base = os.path.basename(path)
+# Caller should have called check() on path before using this.
-        for c in ("/", ":", "\\"):
+#
-            assert c not in base
+
-    # connect
+class Upgrader(object):
-    db = DB(path)
+
-    if create:
+    def __init__(self):
-        ver = _createDB(db)
+        pass
-    else:
+
-        ver = _upgradeSchema(db)
+    def upgrade(self, path):
+        self.path = path
+        self._openDB(path)
+        self._upgradeSchema()
+        self._openDeck()
+        self._upgradeDeck()
+        return self.deck
+
+    def _openDB(self, path):
+        self.tmppath = namedtmp(os.path.basename(path))
+        shutil.copy(path, self.tmppath)
+        self.db = DB(self.tmppath)
+
+    def _openDeck(self):
+        self.deck = _Deck(self.db)
+
+    # Schema upgrade
+    ######################################################################
+
+    def _upgradeSchema(self):
+        "Alter tables prior to ORM initialization."
+        db = self.db
+        # speed up the upgrade
         db.execute("pragma temp_store = memory")
         db.execute("pragma cache_size = 10000")
-    # add db to deck and do any remaining upgrades
-    deck = _Deck(db, server)
-    if ver < CURRENT_VERSION:
-        _upgradeDeck(deck, ver)
-    elif create:
-        # add in reverse order so basic is default
-        addClozeModel(deck)
-        addBasicModel(deck)
-        deck.save()
-    if lock:
-        deck.lock()
-    if not queue:
-        return deck
-    # rebuild queue
-    deck.reset()
-    return deck
-
-# 2.0 schema migration
-######################################################################
-
-def _moveTable(db, table, cards=False):
-    if cards:
-        insExtra = " order by created"
-    else:
-        insExtra = ""
-    sql = db.scalar(
-        "select sql from sqlite_master where name = '%s'" % table)
-    sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
-    if cards:
-        sql = sql.replace("PRIMARY KEY (id),", "")
-    db.execute(sql)
-    db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra))
-    db.execute("drop table "+table)
-    _addSchema(db, False)
-
-def _upgradeSchema(db):
-    "Alter tables prior to ORM initialization."
-    try:
-        ver = db.scalar("select ver from deck")
-    except:
-        ver = db.scalar("select version from decks")
-    # latest 1.2 is 65
-    if ver < 65:
-        raise AnkiError("oldDeckVersion")
-    if ver > 99:
-        # anki 2.0
-        if ver > CURRENT_VERSION:
-            # refuse to load decks created with a future version
-            raise AnkiError("newDeckVersion")
-        return ver
-    runHook("1.x upgrade", db)
-
         # these weren't always correctly set
         db.execute("pragma page_size = 4096")
         db.execute("pragma legacy_file_format = 0")
@@ -92,8 +65,8 @@ end)
 """)
         # pull facts into memory, so we can merge them with fields efficiently
         facts = db.all("""
-select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
+select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int),
-from facts order by created""")
+0, tags from facts order by created""")
         # build field hash
         fields = {}
         for (fid, ord, val) in db.execute(
@@ -210,7 +183,7 @@ yesCount from reviewHistory"""):
 
         # deck
         ###########
-    _migrateDeckTbl(db)
+        self._migrateDeckTbl()
 
         # tags
         ###########
@@ -225,12 +198,12 @@ yesCount from reviewHistory"""):
         ###########
         db.execute("drop table media")
         db.execute("drop table sources")
-    _migrateModels(db)
+        self._migrateModels()
         _updateIndices(db)
-    return ver
 
-def _migrateDeckTbl(db):
+    def _migrateDeckTbl(self):
         import anki.deck
+        db = self.db
         db.execute("delete from deck")
         db.execute("""
 insert or replace into deck select id, cast(created as int), :t,
@@ -263,8 +236,9 @@ insert or replace into deck select id, cast(created as int), :t,
         db.execute("drop table decks")
         db.execute("drop table deckVars")
 
-def _migrateModels(db):
+    def _migrateModels(self):
         import anki.models
+        db = self.db
         times = {}
         mods = {}
         for row in db.all(
@@ -279,8 +253,8 @@ def _migrateModels(db):
             m['name'] = row[1]
             m['mod'] = intTime()
             m['tags'] = []
-        m['flds'] = _fieldsForModel(db, row[0])
+            m['flds'] = self._fieldsForModel(row[0])
-        m['tmpls'] = _templatesForModel(db, row[0], m['flds'])
+            m['tmpls'] = self._templatesForModel(row[0], m['flds'])
             mods[m['id']] = m
             db.execute("update facts set mid = ? where mid = ?", t, row[0])
         # save and clean up
@@ -289,8 +263,9 @@ def _migrateModels(db):
         db.execute("drop table cardModels")
         db.execute("drop table models")
 
-def _fieldsForModel(db, mid):
+    def _fieldsForModel(self, mid):
         import anki.models
+        db = self.db
         dconf = anki.models.defaultField
         flds = []
         for c, row in enumerate(db.all("""
@@ -318,8 +293,9 @@ order by ordinal""", mid)):
             flds.append(conf)
         return flds
 
-def _templatesForModel(db, mid, flds):
+    def _templatesForModel(self, mid, flds):
         import anki.models
+        db = self.db
         dconf = anki.models.defaultTemplate
         tmpls = []
         for c, row in enumerate(db.all("""
@@ -361,9 +337,13 @@ order by ordinal""", mid)):
             tmpls.append(conf)
         return tmpls
 
-def _postSchemaUpgrade(deck):
+    # Upgrading deck
+    ######################################################################
+
+    def _upgradeDeck(self):
         "Handle the rest of the upgrade to 2.0."
         import anki.deck
+        deck = self.deck
         # make sure we have a current model id
         deck.models.setCurrent(deck.models.models.values()[0])
         # regenerate css, and set new card order
@@ -420,15 +400,5 @@ update cards set due = cast(
         deck.db.commit()
         deck.db.execute("vacuum")
         deck.db.execute("analyze")
-    deck.db.execute("update deck set ver = ?", CURRENT_VERSION)
+        deck.db.execute("update deck set ver = ?", SCHEMA_VERSION)
         deck.save()
-
-# Post-init upgrade
-######################################################################
-
-def _upgradeDeck(deck, version):
-    "Upgrade deck to the latest version."
-    if version >= CURRENT_VERSION:
-        return
-    if version < 100:
-        _postSchemaUpgrade(deck)

anki/storage.py

@@ -2,13 +2,12 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-CURRENT_VERSION = 1
-
 import os, simplejson
 from anki.lang import _
 from anki.utils import intTime
 from anki.db import DB
 from anki.deck import _Deck
+from anki.consts import *
 from anki.stdmodels import addBasicModel, addClozeModel
 
 def Deck(path, queue=True, lock=True, server=False):
@@ -30,7 +29,7 @@ def Deck(path, queue=True, lock=True, server=False):
     db.execute("pragma cache_size = 10000")
     # add db to deck and do any remaining upgrades
     deck = _Deck(db, server)
-    if ver < CURRENT_VERSION:
+    if ver < SCHEMA_VERSION:
         _upgradeDeck(deck, ver)
     elif create:
         # add in reverse order so basic is default
@@ -47,7 +46,7 @@ def Deck(path, queue=True, lock=True, server=False):
 
 # no upgrades necessary at the moment
 def _upgradeSchema(db):
-    return CURRENT_VERSION
+    return SCHEMA_VERSION
 def _upgradeDeck(deck, ver):
     return
 
@@ -61,7 +60,7 @@ def _createDB(db):
     _addSchema(db)
     _updateIndices(db)
     db.execute("analyze")
-    return CURRENT_VERSION
+    return SCHEMA_VERSION
 
 def _addSchema(db, setDeckConf=True):
     db.executescript("""
@@ -140,7 +139,7 @@ create table if not exists graves (
 
 insert or ignore into deck
 values(1,0,0,0,%(v)s,0,0,0,'','{}','','','{}');
-""" % ({'v':CURRENT_VERSION}))
+""" % ({'v':SCHEMA_VERSION}))
     import anki.deck
     if setDeckConf:
         _addDeckVars(db, *_getDeckVars(db))

tests/test_deck.py

@@ -116,19 +116,6 @@ def test_fieldChecksum():
     assert deck.db.scalar(
         "select count() from fsums") == 2
 
-def test_upgrade():
-    dst = getUpgradeDeckPath()
-    print "upgrade to", dst
-    deck = Deck(dst)
-    # creation time should have been adjusted
-    d = datetime.datetime.fromtimestamp(deck.crt)
-    assert d.hour == 4 and d.minute == 0
-    # 3 new, 2 failed, 1 due
-    deck.conf['counts'] = COUNT_REMAINING
-    assert deck.sched.cardCounts() == (3,2,1)
-    # now's a good time to test the integrity check too
-    deck.fixIntegrity()
-
 def test_selective():
     deck = getEmptyDeck()
     f = deck.newFact()

tests/test_migration.py

@@ -1,7 +1,10 @@
 # coding: utf-8
 
+import datetime
+from anki.consts import *
 from shared import getUpgradeDeckPath
 from anki.migration.checker import check
+from anki.migration.upgrader import Upgrader
 
 def test_checker():
     dst = getUpgradeDeckPath()
@@ -10,3 +13,20 @@ def test_checker():
     open(dst, "w+").write("foo")
     assert not check(dst)
 
+def test_upgrade():
+    dst = getUpgradeDeckPath()
+    u = Upgrader()
+    print "upgrade to", dst
+    deck = u.upgrade(dst)
+    # creation time should have been adjusted
+    d = datetime.datetime.fromtimestamp(deck.crt)
+    assert d.hour == 4 and d.minute == 0
+    # 3 new, 2 failed, 1 due
+    deck.reset()
+    deck.conf['counts'] = COUNT_REMAINING
+    assert deck.sched.cardCounts() == (3,2,1)
+    # now's a good time to test the integrity check too
+    deck.fixIntegrity()
+
+
+