mirror of
https://github.com/ankitects/anki.git
synced 2025-09-25 01:06:35 -04:00
fix upgrading; drop old mnemosyne 1 importer
This commit is contained in:
Damien Elmes
parent
cf4abcb403
commit
76960abd75
6 changed files with 366 additions and 457 deletions
|
|
@ -44,6 +44,9 @@ SYNC_PORT = int(os.environ.get("SYNC_PORT") or 80)
|
|||
SYNC_URL = "http://%s:%d/sync/" % (SYNC_HOST, SYNC_PORT)
|
||||
SYNC_VER = 0
|
||||
|
||||
# deck schema
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
# Labels
|
||||
##########################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
import sys, pickle, time, re
|
||||
from anki.importing import Importer, ForeignCard
|
||||
from anki.errors import *
|
||||
|
||||
class Mnemosyne10Importer(Importer):
|
||||
|
||||
multipleCardsAllowed = False
|
||||
|
||||
def foreignCards(self):
|
||||
# empty objects so we can load the native mnemosyne file
|
||||
class MnemosyneModule(object):
|
||||
class StartTime:
|
||||
pass
|
||||
class Category:
|
||||
pass
|
||||
class Item:
|
||||
pass
|
||||
for module in ('mnemosyne',
|
||||
'mnemosyne.core',
|
||||
'mnemosyne.core.mnemosyne_core'):
|
||||
sys.modules[module] = MnemosyneModule()
|
||||
try:
|
||||
file = open(self.file, "rb")
|
||||
except (IOError, OSError), e:
|
||||
raise ImportFormatError(type="systemError",
|
||||
info=str(e))
|
||||
header = file.readline().strip()
|
||||
# read the structure in
|
||||
try:
|
||||
struct = pickle.load(file)
|
||||
except (EOFError, KeyError):
|
||||
raise ImportFormatError(type="invalidFile")
|
||||
startTime = struct[0].time
|
||||
daysPassed = (time.time() - startTime) / 86400.0
|
||||
# gather cards
|
||||
cards = []
|
||||
for item in struct[2]:
|
||||
card = ForeignCard()
|
||||
card.fields.append(self.fudgeText(item.q))
|
||||
card.fields.append(self.fudgeText(item.a))
|
||||
# scheduling data
|
||||
card.interval = item.next_rep - item.last_rep
|
||||
secDelta = (item.next_rep - daysPassed) * 86400.0
|
||||
card.due = card.nextTime = time.time() + secDelta
|
||||
card.factor = item.easiness
|
||||
# for some reason mnemosyne starts cards off on 1 instead of 0
|
||||
card.successive = max(
|
||||
(item.acq_reps_since_lapse + item.ret_reps_since_lapse -1), 0)
|
||||
card.yesCount = max((item.acq_reps + item.ret_reps) - 1, 0)
|
||||
card.noCount = item.lapses
|
||||
card.reps = card.yesCount + card.noCount
|
||||
if item.cat.name != u"<default>":
|
||||
card.tags = item.cat.name.replace(" ", "_")
|
||||
cards.append(card)
|
||||
return cards
|
||||
|
||||
def fields(self):
|
||||
return 2
|
||||
|
||||
def fudgeText(self, text):
|
||||
text = text.replace("\n", "<br>")
|
||||
text = re.sub('<sound src="(.*?)">', '[sound:\\1]', text)
|
||||
text = re.sub('<(/?latex)>', '[\\1]', text)
|
||||
text = re.sub('<(/?\$)>', '[\\1]', text)
|
||||
text = re.sub('<(/?\$\$)>', '[\\1]', text)
|
||||
return text
|
||||
|
|
@ -2,80 +2,53 @@
|
|||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html
|
||||
|
||||
import os, time, simplejson, re, datetime
|
||||
import os, time, simplejson, re, datetime, shutil
|
||||
from anki.lang import _
|
||||
from anki.utils import intTime
|
||||
from anki.utils import intTime, namedtmp
|
||||
from anki.db import DB
|
||||
from anki.deck import _Deck
|
||||
from anki.consts import *
|
||||
from anki.storage import _addSchema, _getDeckVars, _addDeckVars, \
|
||||
_updateIndices
|
||||
|
||||
def Deck(path, queue=True, lock=True, server=False):
|
||||
"Open a new or existing deck. Path must be unicode."
|
||||
path = os.path.abspath(path)
|
||||
create = not os.path.exists(path)
|
||||
if create:
|
||||
base = os.path.basename(path)
|
||||
for c in ("/", ":", "\\"):
|
||||
assert c not in base
|
||||
# connect
|
||||
db = DB(path)
|
||||
if create:
|
||||
ver = _createDB(db)
|
||||
else:
|
||||
ver = _upgradeSchema(db)
|
||||
#
|
||||
# Upgrading is the first step in migrating to 2.0. The ids are temporary and
|
||||
# may not be unique across multiple decks. After each of a user's v1.2 decks
|
||||
# are upgraded, they need to be merged.
|
||||
#
|
||||
# Caller should have called check() on path before using this.
|
||||
#
|
||||
|
||||
class Upgrader(object):
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def upgrade(self, path):
|
||||
self.path = path
|
||||
self._openDB(path)
|
||||
self._upgradeSchema()
|
||||
self._openDeck()
|
||||
self._upgradeDeck()
|
||||
return self.deck
|
||||
|
||||
def _openDB(self, path):
|
||||
self.tmppath = namedtmp(os.path.basename(path))
|
||||
shutil.copy(path, self.tmppath)
|
||||
self.db = DB(self.tmppath)
|
||||
|
||||
def _openDeck(self):
|
||||
self.deck = _Deck(self.db)
|
||||
|
||||
# Schema upgrade
|
||||
######################################################################
|
||||
|
||||
def _upgradeSchema(self):
|
||||
"Alter tables prior to ORM initialization."
|
||||
db = self.db
|
||||
# speed up the upgrade
|
||||
db.execute("pragma temp_store = memory")
|
||||
db.execute("pragma cache_size = 10000")
|
||||
# add db to deck and do any remaining upgrades
|
||||
deck = _Deck(db, server)
|
||||
if ver < CURRENT_VERSION:
|
||||
_upgradeDeck(deck, ver)
|
||||
elif create:
|
||||
# add in reverse order so basic is default
|
||||
addClozeModel(deck)
|
||||
addBasicModel(deck)
|
||||
deck.save()
|
||||
if lock:
|
||||
deck.lock()
|
||||
if not queue:
|
||||
return deck
|
||||
# rebuild queue
|
||||
deck.reset()
|
||||
return deck
|
||||
|
||||
# 2.0 schema migration
|
||||
######################################################################
|
||||
|
||||
def _moveTable(db, table, cards=False):
|
||||
if cards:
|
||||
insExtra = " order by created"
|
||||
else:
|
||||
insExtra = ""
|
||||
sql = db.scalar(
|
||||
"select sql from sqlite_master where name = '%s'" % table)
|
||||
sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
|
||||
if cards:
|
||||
sql = sql.replace("PRIMARY KEY (id),", "")
|
||||
db.execute(sql)
|
||||
db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra))
|
||||
db.execute("drop table "+table)
|
||||
_addSchema(db, False)
|
||||
|
||||
def _upgradeSchema(db):
|
||||
"Alter tables prior to ORM initialization."
|
||||
try:
|
||||
ver = db.scalar("select ver from deck")
|
||||
except:
|
||||
ver = db.scalar("select version from decks")
|
||||
# latest 1.2 is 65
|
||||
if ver < 65:
|
||||
raise AnkiError("oldDeckVersion")
|
||||
if ver > 99:
|
||||
# anki 2.0
|
||||
if ver > CURRENT_VERSION:
|
||||
# refuse to load decks created with a future version
|
||||
raise AnkiError("newDeckVersion")
|
||||
return ver
|
||||
runHook("1.x upgrade", db)
|
||||
|
||||
# these weren't always correctly set
|
||||
db.execute("pragma page_size = 4096")
|
||||
db.execute("pragma legacy_file_format = 0")
|
||||
|
|
@ -92,8 +65,8 @@ end)
|
|||
""")
|
||||
# pull facts into memory, so we can merge them with fields efficiently
|
||||
facts = db.all("""
|
||||
select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
|
||||
from facts order by created""")
|
||||
select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int),
|
||||
0, tags from facts order by created""")
|
||||
# build field hash
|
||||
fields = {}
|
||||
for (fid, ord, val) in db.execute(
|
||||
|
|
@ -210,7 +183,7 @@ yesCount from reviewHistory"""):
|
|||
|
||||
# deck
|
||||
###########
|
||||
_migrateDeckTbl(db)
|
||||
self._migrateDeckTbl()
|
||||
|
||||
# tags
|
||||
###########
|
||||
|
|
@ -225,12 +198,12 @@ yesCount from reviewHistory"""):
|
|||
###########
|
||||
db.execute("drop table media")
|
||||
db.execute("drop table sources")
|
||||
_migrateModels(db)
|
||||
self._migrateModels()
|
||||
_updateIndices(db)
|
||||
return ver
|
||||
|
||||
def _migrateDeckTbl(db):
|
||||
def _migrateDeckTbl(self):
|
||||
import anki.deck
|
||||
db = self.db
|
||||
db.execute("delete from deck")
|
||||
db.execute("""
|
||||
insert or replace into deck select id, cast(created as int), :t,
|
||||
|
|
@ -263,8 +236,9 @@ insert or replace into deck select id, cast(created as int), :t,
|
|||
db.execute("drop table decks")
|
||||
db.execute("drop table deckVars")
|
||||
|
||||
def _migrateModels(db):
|
||||
def _migrateModels(self):
|
||||
import anki.models
|
||||
db = self.db
|
||||
times = {}
|
||||
mods = {}
|
||||
for row in db.all(
|
||||
|
|
@ -279,8 +253,8 @@ def _migrateModels(db):
|
|||
m['name'] = row[1]
|
||||
m['mod'] = intTime()
|
||||
m['tags'] = []
|
||||
m['flds'] = _fieldsForModel(db, row[0])
|
||||
m['tmpls'] = _templatesForModel(db, row[0], m['flds'])
|
||||
m['flds'] = self._fieldsForModel(row[0])
|
||||
m['tmpls'] = self._templatesForModel(row[0], m['flds'])
|
||||
mods[m['id']] = m
|
||||
db.execute("update facts set mid = ? where mid = ?", t, row[0])
|
||||
# save and clean up
|
||||
|
|
@ -289,8 +263,9 @@ def _migrateModels(db):
|
|||
db.execute("drop table cardModels")
|
||||
db.execute("drop table models")
|
||||
|
||||
def _fieldsForModel(db, mid):
|
||||
def _fieldsForModel(self, mid):
|
||||
import anki.models
|
||||
db = self.db
|
||||
dconf = anki.models.defaultField
|
||||
flds = []
|
||||
for c, row in enumerate(db.all("""
|
||||
|
|
@ -318,8 +293,9 @@ order by ordinal""", mid)):
|
|||
flds.append(conf)
|
||||
return flds
|
||||
|
||||
def _templatesForModel(db, mid, flds):
|
||||
def _templatesForModel(self, mid, flds):
|
||||
import anki.models
|
||||
db = self.db
|
||||
dconf = anki.models.defaultTemplate
|
||||
tmpls = []
|
||||
for c, row in enumerate(db.all("""
|
||||
|
|
@ -361,9 +337,13 @@ order by ordinal""", mid)):
|
|||
tmpls.append(conf)
|
||||
return tmpls
|
||||
|
||||
def _postSchemaUpgrade(deck):
|
||||
# Upgrading deck
|
||||
######################################################################
|
||||
|
||||
def _upgradeDeck(self):
|
||||
"Handle the rest of the upgrade to 2.0."
|
||||
import anki.deck
|
||||
deck = self.deck
|
||||
# make sure we have a current model id
|
||||
deck.models.setCurrent(deck.models.models.values()[0])
|
||||
# regenerate css, and set new card order
|
||||
|
|
@ -420,15 +400,5 @@ update cards set due = cast(
|
|||
deck.db.commit()
|
||||
deck.db.execute("vacuum")
|
||||
deck.db.execute("analyze")
|
||||
deck.db.execute("update deck set ver = ?", CURRENT_VERSION)
|
||||
deck.db.execute("update deck set ver = ?", SCHEMA_VERSION)
|
||||
deck.save()
|
||||
|
||||
# Post-init upgrade
|
||||
######################################################################
|
||||
|
||||
def _upgradeDeck(deck, version):
|
||||
"Upgrade deck to the latest version."
|
||||
if version >= CURRENT_VERSION:
|
||||
return
|
||||
if version < 100:
|
||||
_postSchemaUpgrade(deck)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,12 @@
|
|||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
CURRENT_VERSION = 1
|
||||
|
||||
import os, simplejson
|
||||
from anki.lang import _
|
||||
from anki.utils import intTime
|
||||
from anki.db import DB
|
||||
from anki.deck import _Deck
|
||||
from anki.consts import *
|
||||
from anki.stdmodels import addBasicModel, addClozeModel
|
||||
|
||||
def Deck(path, queue=True, lock=True, server=False):
|
||||
|
|
@ -30,7 +29,7 @@ def Deck(path, queue=True, lock=True, server=False):
|
|||
db.execute("pragma cache_size = 10000")
|
||||
# add db to deck and do any remaining upgrades
|
||||
deck = _Deck(db, server)
|
||||
if ver < CURRENT_VERSION:
|
||||
if ver < SCHEMA_VERSION:
|
||||
_upgradeDeck(deck, ver)
|
||||
elif create:
|
||||
# add in reverse order so basic is default
|
||||
|
|
@ -47,7 +46,7 @@ def Deck(path, queue=True, lock=True, server=False):
|
|||
|
||||
# no upgrades necessary at the moment
|
||||
def _upgradeSchema(db):
|
||||
return CURRENT_VERSION
|
||||
return SCHEMA_VERSION
|
||||
def _upgradeDeck(deck, ver):
|
||||
return
|
||||
|
||||
|
|
@ -61,7 +60,7 @@ def _createDB(db):
|
|||
_addSchema(db)
|
||||
_updateIndices(db)
|
||||
db.execute("analyze")
|
||||
return CURRENT_VERSION
|
||||
return SCHEMA_VERSION
|
||||
|
||||
def _addSchema(db, setDeckConf=True):
|
||||
db.executescript("""
|
||||
|
|
@ -140,7 +139,7 @@ create table if not exists graves (
|
|||
|
||||
insert or ignore into deck
|
||||
values(1,0,0,0,%(v)s,0,0,0,'','{}','','','{}');
|
||||
""" % ({'v':CURRENT_VERSION}))
|
||||
""" % ({'v':SCHEMA_VERSION}))
|
||||
import anki.deck
|
||||
if setDeckConf:
|
||||
_addDeckVars(db, *_getDeckVars(db))
|
||||
|
|
|
|||
|
|
@ -116,19 +116,6 @@ def test_fieldChecksum():
|
|||
assert deck.db.scalar(
|
||||
"select count() from fsums") == 2
|
||||
|
||||
def test_upgrade():
|
||||
dst = getUpgradeDeckPath()
|
||||
print "upgrade to", dst
|
||||
deck = Deck(dst)
|
||||
# creation time should have been adjusted
|
||||
d = datetime.datetime.fromtimestamp(deck.crt)
|
||||
assert d.hour == 4 and d.minute == 0
|
||||
# 3 new, 2 failed, 1 due
|
||||
deck.conf['counts'] = COUNT_REMAINING
|
||||
assert deck.sched.cardCounts() == (3,2,1)
|
||||
# now's a good time to test the integrity check too
|
||||
deck.fixIntegrity()
|
||||
|
||||
def test_selective():
|
||||
deck = getEmptyDeck()
|
||||
f = deck.newFact()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
# coding: utf-8
|
||||
|
||||
import datetime
|
||||
from anki.consts import *
|
||||
from shared import getUpgradeDeckPath
|
||||
from anki.migration.checker import check
|
||||
from anki.migration.upgrader import Upgrader
|
||||
|
||||
def test_checker():
|
||||
dst = getUpgradeDeckPath()
|
||||
|
|
@ -10,3 +13,20 @@ def test_checker():
|
|||
open(dst, "w+").write("foo")
|
||||
assert not check(dst)
|
||||
|
||||
def test_upgrade():
|
||||
dst = getUpgradeDeckPath()
|
||||
u = Upgrader()
|
||||
print "upgrade to", dst
|
||||
deck = u.upgrade(dst)
|
||||
# creation time should have been adjusted
|
||||
d = datetime.datetime.fromtimestamp(deck.crt)
|
||||
assert d.hour == 4 and d.minute == 0
|
||||
# 3 new, 2 failed, 1 due
|
||||
deck.reset()
|
||||
deck.conf['counts'] = COUNT_REMAINING
|
||||
assert deck.sched.cardCounts() == (3,2,1)
|
||||
# now's a good time to test the integrity check too
|
||||
deck.fixIntegrity()
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue