mirror of
https://github.com/ankitects/anki.git
synced 2025-09-22 07:52:24 -04:00
split upgrade code into separate file; use .anki2 now
This commit is contained in:
Damien Elmes
parent
050afa57ad
commit
cf4abcb403
8 changed files with 548 additions and 402 deletions
|
|
@ -19,7 +19,7 @@ class MediaManager(object):
|
|||
def __init__(self, deck):
|
||||
self.deck = deck
|
||||
# media directory
|
||||
self._dir = re.sub("(?i)\.(anki)$", ".media", self.deck.path)
|
||||
self._dir = re.sub("(?i)\.(anki2)$", ".media", self.deck.path)
|
||||
if not os.path.exists(self._dir):
|
||||
os.makedirs(self._dir)
|
||||
os.chdir(self._dir)
|
||||
|
|
|
|||
8
anki/migration/__init__.py
Normal file
8
anki/migration/__init__.py
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
class Migrator(object):
|
||||
|
||||
def __init__(self, deck):
|
||||
pass
|
||||
68
anki/migration/checker.py
Normal file
68
anki/migration/checker.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html
|
||||
|
||||
from anki.db import DB
|
||||
|
||||
def check(path):
|
||||
"True if deck looks ok."
|
||||
db = DB(path)
|
||||
# corrupt?
|
||||
try:
|
||||
if db.scalar("pragma integrity_check") != "ok":
|
||||
return
|
||||
except:
|
||||
return
|
||||
# old version?
|
||||
if db.scalar("select version from decks") != 65:
|
||||
return
|
||||
# fields missing a field model?
|
||||
if db.list("""
|
||||
select id from fields where fieldModelId not in (
|
||||
select distinct id from fieldModels)"""):
|
||||
return
|
||||
# facts missing a field?
|
||||
if db.list("""
|
||||
select distinct facts.id from facts, fieldModels where
|
||||
facts.modelId = fieldModels.modelId and fieldModels.id not in
|
||||
(select fieldModelId from fields where factId = facts.id)"""):
|
||||
return
|
||||
# cards missing a fact?
|
||||
if db.list("""
|
||||
select id from cards where factId not in (select id from facts)"""):
|
||||
return
|
||||
# cards missing a card model?
|
||||
if db.list("""
|
||||
select id from cards where cardModelId not in
|
||||
(select id from cardModels)"""):
|
||||
return
|
||||
# cards with a card model from the wrong model?
|
||||
if db.list("""
|
||||
select id from cards where cardModelId not in (select cm.id from
|
||||
cardModels cm, facts f where cm.modelId = f.modelId and
|
||||
f.id = cards.factId)"""):
|
||||
return
|
||||
# facts missing a card?
|
||||
if db.list("""
|
||||
select facts.id from facts
|
||||
where facts.id not in (select distinct factId from cards)"""):
|
||||
return
|
||||
# dangling fields?
|
||||
if db.list("""
|
||||
select id from fields where factId not in (select id from facts)"""):
|
||||
return
|
||||
# fields without matching interval
|
||||
if db.list("""
|
||||
select id from fields where ordinal != (select ordinal from fieldModels
|
||||
where id = fieldModelId)"""):
|
||||
return
|
||||
# incorrect types
|
||||
if db.list("""
|
||||
select id from cards where relativeDelay != (case
|
||||
when successive then 1 when reps then 0 else 2 end)"""):
|
||||
return
|
||||
if db.list("""
|
||||
select id from cards where type != (case
|
||||
when type >= 0 then relativeDelay else relativeDelay - 3 end)"""):
|
||||
return
|
||||
return True
|
||||
434
anki/migration/upgrader.py
Normal file
434
anki/migration/upgrader.py
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/copyleft/agpl.html
|
||||
|
||||
import os, time, simplejson, re, datetime
|
||||
from anki.lang import _
|
||||
from anki.utils import intTime
|
||||
from anki.db import DB
|
||||
from anki.deck import _Deck
|
||||
|
||||
def Deck(path, queue=True, lock=True, server=False):
|
||||
"Open a new or existing deck. Path must be unicode."
|
||||
path = os.path.abspath(path)
|
||||
create = not os.path.exists(path)
|
||||
if create:
|
||||
base = os.path.basename(path)
|
||||
for c in ("/", ":", "\\"):
|
||||
assert c not in base
|
||||
# connect
|
||||
db = DB(path)
|
||||
if create:
|
||||
ver = _createDB(db)
|
||||
else:
|
||||
ver = _upgradeSchema(db)
|
||||
db.execute("pragma temp_store = memory")
|
||||
db.execute("pragma cache_size = 10000")
|
||||
# add db to deck and do any remaining upgrades
|
||||
deck = _Deck(db, server)
|
||||
if ver < CURRENT_VERSION:
|
||||
_upgradeDeck(deck, ver)
|
||||
elif create:
|
||||
# add in reverse order so basic is default
|
||||
addClozeModel(deck)
|
||||
addBasicModel(deck)
|
||||
deck.save()
|
||||
if lock:
|
||||
deck.lock()
|
||||
if not queue:
|
||||
return deck
|
||||
# rebuild queue
|
||||
deck.reset()
|
||||
return deck
|
||||
|
||||
# 2.0 schema migration
|
||||
######################################################################
|
||||
|
||||
def _moveTable(db, table, cards=False):
|
||||
if cards:
|
||||
insExtra = " order by created"
|
||||
else:
|
||||
insExtra = ""
|
||||
sql = db.scalar(
|
||||
"select sql from sqlite_master where name = '%s'" % table)
|
||||
sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
|
||||
if cards:
|
||||
sql = sql.replace("PRIMARY KEY (id),", "")
|
||||
db.execute(sql)
|
||||
db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra))
|
||||
db.execute("drop table "+table)
|
||||
_addSchema(db, False)
|
||||
|
||||
def _upgradeSchema(db):
|
||||
"Alter tables prior to ORM initialization."
|
||||
try:
|
||||
ver = db.scalar("select ver from deck")
|
||||
except:
|
||||
ver = db.scalar("select version from decks")
|
||||
# latest 1.2 is 65
|
||||
if ver < 65:
|
||||
raise AnkiError("oldDeckVersion")
|
||||
if ver > 99:
|
||||
# anki 2.0
|
||||
if ver > CURRENT_VERSION:
|
||||
# refuse to load decks created with a future version
|
||||
raise AnkiError("newDeckVersion")
|
||||
return ver
|
||||
runHook("1.x upgrade", db)
|
||||
|
||||
# these weren't always correctly set
|
||||
db.execute("pragma page_size = 4096")
|
||||
db.execute("pragma legacy_file_format = 0")
|
||||
|
||||
# facts
|
||||
###########
|
||||
# tags should have a leading and trailing space if not empty, and not
|
||||
# use commas
|
||||
db.execute("""
|
||||
update facts set tags = (case
|
||||
when trim(tags) == "" then ""
|
||||
else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " "
|
||||
end)
|
||||
""")
|
||||
# pull facts into memory, so we can merge them with fields efficiently
|
||||
facts = db.all("""
|
||||
select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
|
||||
from facts order by created""")
|
||||
# build field hash
|
||||
fields = {}
|
||||
for (fid, ord, val) in db.execute(
|
||||
"select factId, ordinal, value from fields order by factId, ordinal"):
|
||||
if fid not in fields:
|
||||
fields[fid] = []
|
||||
fields[fid].append((ord, val))
|
||||
# build insert data and transform ids, and minimize qt's
|
||||
# bold/italics/underline cruft.
|
||||
map = {}
|
||||
data = []
|
||||
factidmap = {}
|
||||
times = {}
|
||||
from anki.utils import minimizeHTML
|
||||
for c, row in enumerate(facts):
|
||||
oldid = row[0]
|
||||
row = list(row)
|
||||
# get rid of old created column and update id
|
||||
while row[4] in times:
|
||||
row[4] += 1
|
||||
times[row[4]] = True
|
||||
factidmap[row[0]] = row[4]
|
||||
row[0] = row[4]
|
||||
del row[4]
|
||||
map[oldid] = row[0]
|
||||
row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])])))
|
||||
data.append(row)
|
||||
# and put the facts into the new table
|
||||
db.execute("drop table facts")
|
||||
_addSchema(db, False)
|
||||
db.executemany("insert into facts values (?,?,?,?,?,?,?,?,'',0,'')", data)
|
||||
db.execute("drop table fields")
|
||||
|
||||
# cards
|
||||
###########
|
||||
# we need to pull this into memory, to rewrite the creation time if
|
||||
# it's not unique and update the fact id
|
||||
times = {}
|
||||
rows = []
|
||||
cardidmap = {}
|
||||
for row in db.execute("""
|
||||
select id, cast(created*1000 as int), factId, ordinal,
|
||||
cast(modified as int), 0,
|
||||
(case relativeDelay
|
||||
when 0 then 1
|
||||
when 1 then 2
|
||||
when 2 then 0 end),
|
||||
(case type
|
||||
when 0 then 1
|
||||
when 1 then 2
|
||||
when 2 then 0
|
||||
else type end),
|
||||
cast(due as int), cast(interval as int),
|
||||
cast(factor*1000 as int), reps, noCount from cards
|
||||
order by created"""):
|
||||
# find an unused time
|
||||
row = list(row)
|
||||
while row[1] in times:
|
||||
row[1] += 1
|
||||
times[row[1]] = True
|
||||
# rewrite fact id
|
||||
row[2] = factidmap[row[2]]
|
||||
# note id change and save all but old id
|
||||
cardidmap[row[0]] = row[1]
|
||||
rows.append(row[1:])
|
||||
# drop old table and rewrite
|
||||
db.execute("drop table cards")
|
||||
_addSchema(db, False)
|
||||
db.executemany("""
|
||||
insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""",
|
||||
rows)
|
||||
|
||||
# reviewHistory -> revlog
|
||||
###########
|
||||
# fetch the data so we can rewrite ids quickly
|
||||
r = []
|
||||
for row in db.execute("""
|
||||
select
|
||||
cast(time*1000 as int), cardId, 0, ease,
|
||||
cast(nextInterval as int), cast(lastInterval as int),
|
||||
cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
|
||||
yesCount from reviewHistory"""):
|
||||
row = list(row)
|
||||
# new card ids
|
||||
try:
|
||||
row[1] = cardidmap[row[1]]
|
||||
except:
|
||||
# id doesn't exist
|
||||
continue
|
||||
# no ease 0 anymore
|
||||
row[2] = row[2] or 1
|
||||
# determine type, overwriting yesCount
|
||||
newInt = row[3]
|
||||
oldInt = row[4]
|
||||
yesCnt = row[7]
|
||||
# yesCnt included the current answer
|
||||
if row[2] > 1:
|
||||
yesCnt -= 1
|
||||
if oldInt < 1:
|
||||
# new or failed
|
||||
if yesCnt:
|
||||
# type=relrn
|
||||
row[7] = 2
|
||||
else:
|
||||
# type=lrn
|
||||
row[7] = 0
|
||||
else:
|
||||
# type=rev
|
||||
row[7] = 1
|
||||
r.append(row)
|
||||
db.executemany(
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r)
|
||||
db.execute("drop table reviewHistory")
|
||||
|
||||
# deck
|
||||
###########
|
||||
_migrateDeckTbl(db)
|
||||
|
||||
# tags
|
||||
###########
|
||||
tags = {}
|
||||
for t in db.list("select tag from tags"):
|
||||
tags[t] = intTime()
|
||||
db.execute("update deck set tags = ?", simplejson.dumps(tags))
|
||||
db.execute("drop table tags")
|
||||
db.execute("drop table cardTags")
|
||||
|
||||
# the rest
|
||||
###########
|
||||
db.execute("drop table media")
|
||||
db.execute("drop table sources")
|
||||
_migrateModels(db)
|
||||
_updateIndices(db)
|
||||
return ver
|
||||
|
||||
def _migrateDeckTbl(db):
|
||||
import anki.deck
|
||||
db.execute("delete from deck")
|
||||
db.execute("""
|
||||
insert or replace into deck select id, cast(created as int), :t,
|
||||
:t, 99, 0, 0, cast(lastSync as int),
|
||||
"", "", "", "", "" from decks""", t=intTime())
|
||||
# prepare a group to store the old deck options
|
||||
g, gc, conf = _getDeckVars(db)
|
||||
# delete old selective study settings, which we can't auto-upgrade easily
|
||||
keys = ("newActive", "newInactive", "revActive", "revInactive")
|
||||
for k in keys:
|
||||
db.execute("delete from deckVars where key=:k", k=k)
|
||||
# copy other settings, ignoring deck order as there's a new default
|
||||
g['newSpread'] = db.scalar("select newCardSpacing from decks")
|
||||
g['newPerDay'] = db.scalar("select newCardsPerDay from decks")
|
||||
g['repLim'] = db.scalar("select sessionRepLimit from decks")
|
||||
g['timeLim'] = db.scalar("select sessionTimeLimit from decks")
|
||||
# this needs to be placed in the model later on
|
||||
conf['oldNewOrder'] = db.scalar("select newCardOrder from decks")
|
||||
# no reverse option anymore
|
||||
conf['oldNewOrder'] = min(1, conf['oldNewOrder'])
|
||||
# add any deck vars and save
|
||||
dkeys = ("hexCache", "cssCache")
|
||||
for (k, v) in db.execute("select * from deckVars").fetchall():
|
||||
if k in dkeys:
|
||||
pass
|
||||
else:
|
||||
conf[k] = v
|
||||
_addDeckVars(db, g, gc, conf)
|
||||
# clean up
|
||||
db.execute("drop table decks")
|
||||
db.execute("drop table deckVars")
|
||||
|
||||
def _migrateModels(db):
|
||||
import anki.models
|
||||
times = {}
|
||||
mods = {}
|
||||
for row in db.all(
|
||||
"select id, name from models"):
|
||||
while 1:
|
||||
t = intTime(1000)
|
||||
if t not in times:
|
||||
times[t] = True
|
||||
break
|
||||
m = anki.models.defaultModel.copy()
|
||||
m['id'] = t
|
||||
m['name'] = row[1]
|
||||
m['mod'] = intTime()
|
||||
m['tags'] = []
|
||||
m['flds'] = _fieldsForModel(db, row[0])
|
||||
m['tmpls'] = _templatesForModel(db, row[0], m['flds'])
|
||||
mods[m['id']] = m
|
||||
db.execute("update facts set mid = ? where mid = ?", t, row[0])
|
||||
# save and clean up
|
||||
db.execute("update deck set models = ?", simplejson.dumps(mods))
|
||||
db.execute("drop table fieldModels")
|
||||
db.execute("drop table cardModels")
|
||||
db.execute("drop table models")
|
||||
|
||||
def _fieldsForModel(db, mid):
|
||||
import anki.models
|
||||
dconf = anki.models.defaultField
|
||||
flds = []
|
||||
for c, row in enumerate(db.all("""
|
||||
select name, features, required, "unique",
|
||||
quizFontFamily, quizFontSize, quizFontColour, editFontSize from fieldModels
|
||||
where modelId = ?
|
||||
order by ordinal""", mid)):
|
||||
conf = dconf.copy()
|
||||
(conf['name'],
|
||||
conf['rtl'],
|
||||
conf['req'],
|
||||
conf['uniq'],
|
||||
conf['font'],
|
||||
conf['qsize'],
|
||||
conf['qcol'],
|
||||
conf['esize']) = row
|
||||
conf['ord'] = c
|
||||
# ensure data is good
|
||||
conf['rtl'] = not not conf['rtl']
|
||||
conf['pre'] = True
|
||||
conf['font'] = conf['font'] or "Arial"
|
||||
conf['qcol'] = conf['qcol'] or "#000"
|
||||
conf['qsize'] = conf['qsize'] or 20
|
||||
conf['esize'] = conf['esize'] or 20
|
||||
flds.append(conf)
|
||||
return flds
|
||||
|
||||
def _templatesForModel(db, mid, flds):
|
||||
import anki.models
|
||||
dconf = anki.models.defaultTemplate
|
||||
tmpls = []
|
||||
for c, row in enumerate(db.all("""
|
||||
select name, active, qformat, aformat, questionInAnswer,
|
||||
questionAlign, lastFontColour, allowEmptyAnswer, typeAnswer from cardModels
|
||||
where modelId = ?
|
||||
order by ordinal""", mid)):
|
||||
conf = dconf.copy()
|
||||
(conf['name'],
|
||||
conf['actv'],
|
||||
conf['qfmt'],
|
||||
conf['afmt'],
|
||||
conf['hideQ'],
|
||||
conf['align'],
|
||||
conf['bg'],
|
||||
conf['emptyAns'],
|
||||
conf['typeAns']) = row
|
||||
conf['ord'] = c
|
||||
# convert the field name to an ordinal
|
||||
ordN = None
|
||||
for (ord, fm) in enumerate(flds):
|
||||
if fm['name'] == conf['typeAns']:
|
||||
ordN = ord
|
||||
break
|
||||
if ordN is not None:
|
||||
conf['typeAns'] = ordN
|
||||
else:
|
||||
conf['typeAns'] = None
|
||||
for type in ("qfmt", "afmt"):
|
||||
# ensure the new style field format
|
||||
conf[type] = re.sub("%\((.+?)\)s", "{{\\1}}", conf[type])
|
||||
# some special names have changed
|
||||
conf[type] = re.sub(
|
||||
"(?i){{tags}}", "{{Tags}}", conf[type])
|
||||
conf[type] = re.sub(
|
||||
"(?i){{cardModel}}", "{{Template}}", conf[type])
|
||||
conf[type] = re.sub(
|
||||
"(?i){{modelTags}}", "{{Model}}", conf[type])
|
||||
tmpls.append(conf)
|
||||
return tmpls
|
||||
|
||||
def _postSchemaUpgrade(deck):
|
||||
"Handle the rest of the upgrade to 2.0."
|
||||
import anki.deck
|
||||
# make sure we have a current model id
|
||||
deck.models.setCurrent(deck.models.models.values()[0])
|
||||
# regenerate css, and set new card order
|
||||
for m in deck.models.all():
|
||||
m['newOrder'] = deck.conf['oldNewOrder']
|
||||
deck.models.save(m)
|
||||
del deck.conf['oldNewOrder']
|
||||
# fix creation time
|
||||
deck.sched._updateCutoff()
|
||||
d = datetime.datetime.today()
|
||||
d -= datetime.timedelta(hours=4)
|
||||
d = datetime.datetime(d.year, d.month, d.day)
|
||||
d += datetime.timedelta(hours=4)
|
||||
d -= datetime.timedelta(days=1+int((time.time()-deck.crt)/86400))
|
||||
deck.crt = int(time.mktime(d.timetuple()))
|
||||
deck.sched._updateCutoff()
|
||||
# update uniq cache
|
||||
deck.updateFieldCache(deck.db.list("select id from facts"))
|
||||
# remove old views
|
||||
for v in ("failedCards", "revCardsOld", "revCardsNew",
|
||||
"revCardsDue", "revCardsRandom", "acqCardsRandom",
|
||||
"acqCardsOld", "acqCardsNew"):
|
||||
deck.db.execute("drop view if exists %s" % v)
|
||||
# remove stats, as it's all in the revlog now
|
||||
deck.db.execute("drop table if exists stats")
|
||||
# suspended cards don't use ranges anymore
|
||||
deck.db.execute("update cards set queue=-1 where queue between -3 and -1")
|
||||
deck.db.execute("update cards set queue=-2 where queue between 3 and 5")
|
||||
deck.db.execute("update cards set queue=-3 where queue between 6 and 8")
|
||||
# remove old deleted tables
|
||||
for t in ("cards", "facts", "models", "media"):
|
||||
deck.db.execute("drop table if exists %sDeleted" % t)
|
||||
# rewrite due times for new cards
|
||||
deck.db.execute("""
|
||||
update cards set due = fid where type=0""")
|
||||
# and failed cards
|
||||
left = len(deck.groups.conf(1)['new']['delays'])
|
||||
deck.db.execute("update cards set edue = ?, left=? where type = 1",
|
||||
deck.sched.today+1, left)
|
||||
# and due cards
|
||||
deck.db.execute("""
|
||||
update cards set due = cast(
|
||||
(case when due < :stamp then 0 else 1 end) +
|
||||
((due-:stamp)/86400) as int)+:today where type = 2
|
||||
""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
|
||||
# possibly re-randomize
|
||||
if deck.models.randomNew():
|
||||
deck.sched.randomizeCards()
|
||||
# update insertion id
|
||||
deck.conf['nextPos'] = deck.db.scalar("select max(id) from facts")+1
|
||||
deck.save()
|
||||
|
||||
# optimize and finish
|
||||
deck.db.commit()
|
||||
deck.db.execute("vacuum")
|
||||
deck.db.execute("analyze")
|
||||
deck.db.execute("update deck set ver = ?", CURRENT_VERSION)
|
||||
deck.save()
|
||||
|
||||
# Post-init upgrade
|
||||
######################################################################
|
||||
|
||||
def _upgradeDeck(deck, version):
|
||||
"Upgrade deck to the latest version."
|
||||
if version >= CURRENT_VERSION:
|
||||
return
|
||||
if version < 100:
|
||||
_postSchemaUpgrade(deck)
|
||||
404
anki/storage.py
404
anki/storage.py
|
|
@ -2,19 +2,18 @@
|
|||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
CURRENT_VERSION = 100
|
||||
CURRENT_VERSION = 1
|
||||
|
||||
import os, time, simplejson, re, datetime
|
||||
import os, simplejson
|
||||
from anki.lang import _
|
||||
from anki.utils import intTime
|
||||
from anki.db import DB
|
||||
from anki.deck import _Deck
|
||||
from anki.stdmodels import addBasicModel, addClozeModel
|
||||
from anki.errors import AnkiError
|
||||
from anki.hooks import runHook
|
||||
|
||||
def Deck(path, queue=True, lock=True, server=False):
|
||||
"Open a new or existing deck. Path must be unicode."
|
||||
assert path.endswith(".anki2")
|
||||
path = os.path.abspath(path)
|
||||
create = not os.path.exists(path)
|
||||
if create:
|
||||
|
|
@ -46,6 +45,15 @@ def Deck(path, queue=True, lock=True, server=False):
|
|||
deck.reset()
|
||||
return deck
|
||||
|
||||
# no upgrades necessary at the moment
|
||||
def _upgradeSchema(db):
|
||||
return CURRENT_VERSION
|
||||
def _upgradeDeck(deck, ver):
|
||||
return
|
||||
|
||||
# Creating a new deck
|
||||
######################################################################
|
||||
|
||||
def _createDB(db):
|
||||
db.execute("pragma page_size = 4096")
|
||||
db.execute("pragma legacy_file_format = 0")
|
||||
|
|
@ -174,391 +182,3 @@ create index if not exists ix_revlog_cid on revlog (cid);
|
|||
create index if not exists ix_fsums_fid on fsums (fid);
|
||||
create index if not exists ix_fsums_csum on fsums (csum);
|
||||
""")
|
||||
|
||||
# 2.0 schema migration
|
||||
######################################################################
|
||||
|
||||
def _moveTable(db, table, cards=False):
|
||||
if cards:
|
||||
insExtra = " order by created"
|
||||
else:
|
||||
insExtra = ""
|
||||
sql = db.scalar(
|
||||
"select sql from sqlite_master where name = '%s'" % table)
|
||||
sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
|
||||
if cards:
|
||||
sql = sql.replace("PRIMARY KEY (id),", "")
|
||||
db.execute(sql)
|
||||
db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra))
|
||||
db.execute("drop table "+table)
|
||||
_addSchema(db, False)
|
||||
|
||||
def _upgradeSchema(db):
|
||||
"Alter tables prior to ORM initialization."
|
||||
try:
|
||||
ver = db.scalar("select ver from deck")
|
||||
except:
|
||||
ver = db.scalar("select version from decks")
|
||||
# latest 1.2 is 65
|
||||
if ver < 65:
|
||||
raise AnkiError("oldDeckVersion")
|
||||
if ver > 99:
|
||||
# anki 2.0
|
||||
if ver > CURRENT_VERSION:
|
||||
# refuse to load decks created with a future version
|
||||
raise AnkiError("newDeckVersion")
|
||||
return ver
|
||||
runHook("1.x upgrade", db)
|
||||
|
||||
# facts
|
||||
###########
|
||||
# tags should have a leading and trailing space if not empty, and not
|
||||
# use commas
|
||||
db.execute("""
|
||||
update facts set tags = (case
|
||||
when trim(tags) == "" then ""
|
||||
else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " "
|
||||
end)
|
||||
""")
|
||||
# pull facts into memory, so we can merge them with fields efficiently
|
||||
facts = db.all("""
|
||||
select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
|
||||
from facts order by created""")
|
||||
# build field hash
|
||||
fields = {}
|
||||
for (fid, ord, val) in db.execute(
|
||||
"select factId, ordinal, value from fields order by factId, ordinal"):
|
||||
if fid not in fields:
|
||||
fields[fid] = []
|
||||
fields[fid].append((ord, val))
|
||||
# build insert data and transform ids, and minimize qt's
|
||||
# bold/italics/underline cruft.
|
||||
map = {}
|
||||
data = []
|
||||
factidmap = {}
|
||||
times = {}
|
||||
from anki.utils import minimizeHTML
|
||||
for c, row in enumerate(facts):
|
||||
oldid = row[0]
|
||||
row = list(row)
|
||||
# get rid of old created column and update id
|
||||
while row[4] in times:
|
||||
row[4] += 1
|
||||
times[row[4]] = True
|
||||
factidmap[row[0]] = row[4]
|
||||
row[0] = row[4]
|
||||
del row[4]
|
||||
map[oldid] = row[0]
|
||||
row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])])))
|
||||
data.append(row)
|
||||
# and put the facts into the new table
|
||||
db.execute("drop table facts")
|
||||
_addSchema(db, False)
|
||||
db.executemany("insert into facts values (?,?,?,?,?,?,?,?,'',0,'')", data)
|
||||
db.execute("drop table fields")
|
||||
|
||||
# cards
|
||||
###########
|
||||
# we need to pull this into memory, to rewrite the creation time if
|
||||
# it's not unique and update the fact id
|
||||
times = {}
|
||||
rows = []
|
||||
cardidmap = {}
|
||||
for row in db.execute("""
|
||||
select id, cast(created*1000 as int), factId, ordinal,
|
||||
cast(modified as int), 0,
|
||||
(case relativeDelay
|
||||
when 0 then 1
|
||||
when 1 then 2
|
||||
when 2 then 0 end),
|
||||
(case type
|
||||
when 0 then 1
|
||||
when 1 then 2
|
||||
when 2 then 0
|
||||
else type end),
|
||||
cast(due as int), cast(interval as int),
|
||||
cast(factor*1000 as int), reps, noCount from cards
|
||||
order by created"""):
|
||||
# find an unused time
|
||||
row = list(row)
|
||||
while row[1] in times:
|
||||
row[1] += 1
|
||||
times[row[1]] = True
|
||||
# rewrite fact id
|
||||
row[2] = factidmap[row[2]]
|
||||
# note id change and save all but old id
|
||||
cardidmap[row[0]] = row[1]
|
||||
rows.append(row[1:])
|
||||
# drop old table and rewrite
|
||||
db.execute("drop table cards")
|
||||
_addSchema(db, False)
|
||||
db.executemany("""
|
||||
insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""",
|
||||
rows)
|
||||
|
||||
# reviewHistory -> revlog
|
||||
###########
|
||||
# fetch the data so we can rewrite ids quickly
|
||||
r = []
|
||||
for row in db.execute("""
|
||||
select
|
||||
cast(time*1000 as int), cardId, 0, ease,
|
||||
cast(nextInterval as int), cast(lastInterval as int),
|
||||
cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
|
||||
yesCount from reviewHistory"""):
|
||||
row = list(row)
|
||||
# new card ids
|
||||
try:
|
||||
row[1] = cardidmap[row[1]]
|
||||
except:
|
||||
# id doesn't exist
|
||||
continue
|
||||
# no ease 0 anymore
|
||||
row[2] = row[2] or 1
|
||||
# determine type, overwriting yesCount
|
||||
newInt = row[3]
|
||||
oldInt = row[4]
|
||||
yesCnt = row[7]
|
||||
# yesCnt included the current answer
|
||||
if row[2] > 1:
|
||||
yesCnt -= 1
|
||||
if oldInt < 1:
|
||||
# new or failed
|
||||
if yesCnt:
|
||||
# type=relrn
|
||||
row[7] = 2
|
||||
else:
|
||||
# type=lrn
|
||||
row[7] = 0
|
||||
else:
|
||||
# type=rev
|
||||
row[7] = 1
|
||||
r.append(row)
|
||||
db.executemany(
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r)
|
||||
db.execute("drop table reviewHistory")
|
||||
|
||||
# deck
|
||||
###########
|
||||
_migrateDeckTbl(db)
|
||||
|
||||
# tags
|
||||
###########
|
||||
tags = {}
|
||||
for t in db.list("select tag from tags"):
|
||||
tags[t] = intTime()
|
||||
db.execute("update deck set tags = ?", simplejson.dumps(tags))
|
||||
db.execute("drop table tags")
|
||||
db.execute("drop table cardTags")
|
||||
|
||||
# the rest
|
||||
###########
|
||||
db.execute("drop table media")
|
||||
db.execute("drop table sources")
|
||||
_migrateModels(db)
|
||||
_updateIndices(db)
|
||||
return ver
|
||||
|
||||
def _migrateDeckTbl(db):
|
||||
import anki.deck
|
||||
db.execute("delete from deck")
|
||||
db.execute("""
|
||||
insert or replace into deck select id, cast(created as int), :t,
|
||||
:t, 99, 0, 0, cast(lastSync as int),
|
||||
"", "", "", "", "" from decks""", t=intTime())
|
||||
# prepare a group to store the old deck options
|
||||
g, gc, conf = _getDeckVars(db)
|
||||
# delete old selective study settings, which we can't auto-upgrade easily
|
||||
keys = ("newActive", "newInactive", "revActive", "revInactive")
|
||||
for k in keys:
|
||||
db.execute("delete from deckVars where key=:k", k=k)
|
||||
# copy other settings, ignoring deck order as there's a new default
|
||||
g['newSpread'] = db.scalar("select newCardSpacing from decks")
|
||||
g['newPerDay'] = db.scalar("select newCardsPerDay from decks")
|
||||
g['repLim'] = db.scalar("select sessionRepLimit from decks")
|
||||
g['timeLim'] = db.scalar("select sessionTimeLimit from decks")
|
||||
# this needs to be placed in the model later on
|
||||
conf['oldNewOrder'] = db.scalar("select newCardOrder from decks")
|
||||
# no reverse option anymore
|
||||
conf['oldNewOrder'] = min(1, conf['oldNewOrder'])
|
||||
# add any deck vars and save
|
||||
dkeys = ("hexCache", "cssCache")
|
||||
for (k, v) in db.execute("select * from deckVars").fetchall():
|
||||
if k in dkeys:
|
||||
pass
|
||||
else:
|
||||
conf[k] = v
|
||||
_addDeckVars(db, g, gc, conf)
|
||||
# clean up
|
||||
db.execute("drop table decks")
|
||||
db.execute("drop table deckVars")
|
||||
|
||||
def _migrateModels(db):
|
||||
import anki.models
|
||||
times = {}
|
||||
mods = {}
|
||||
for row in db.all(
|
||||
"select id, name from models"):
|
||||
while 1:
|
||||
t = intTime(1000)
|
||||
if t not in times:
|
||||
times[t] = True
|
||||
break
|
||||
m = anki.models.defaultModel.copy()
|
||||
m['id'] = t
|
||||
m['name'] = row[1]
|
||||
m['mod'] = intTime()
|
||||
m['tags'] = []
|
||||
m['flds'] = _fieldsForModel(db, row[0])
|
||||
m['tmpls'] = _templatesForModel(db, row[0], m['flds'])
|
||||
mods[m['id']] = m
|
||||
db.execute("update facts set mid = ? where mid = ?", t, row[0])
|
||||
# save and clean up
|
||||
db.execute("update deck set models = ?", simplejson.dumps(mods))
|
||||
db.execute("drop table fieldModels")
|
||||
db.execute("drop table cardModels")
|
||||
db.execute("drop table models")
|
||||
|
||||
def _fieldsForModel(db, mid):
|
||||
import anki.models
|
||||
dconf = anki.models.defaultField
|
||||
flds = []
|
||||
for c, row in enumerate(db.all("""
|
||||
select name, features, required, "unique",
|
||||
quizFontFamily, quizFontSize, quizFontColour, editFontSize from fieldModels
|
||||
where modelId = ?
|
||||
order by ordinal""", mid)):
|
||||
conf = dconf.copy()
|
||||
(conf['name'],
|
||||
conf['rtl'],
|
||||
conf['req'],
|
||||
conf['uniq'],
|
||||
conf['font'],
|
||||
conf['qsize'],
|
||||
conf['qcol'],
|
||||
conf['esize']) = row
|
||||
conf['ord'] = c
|
||||
# ensure data is good
|
||||
conf['rtl'] = not not conf['rtl']
|
||||
conf['pre'] = True
|
||||
conf['font'] = conf['font'] or "Arial"
|
||||
conf['qcol'] = conf['qcol'] or "#000"
|
||||
conf['qsize'] = conf['qsize'] or 20
|
||||
conf['esize'] = conf['esize'] or 20
|
||||
flds.append(conf)
|
||||
return flds
|
||||
|
||||
def _templatesForModel(db, mid, flds):
|
||||
import anki.models
|
||||
dconf = anki.models.defaultTemplate
|
||||
tmpls = []
|
||||
for c, row in enumerate(db.all("""
|
||||
select name, active, qformat, aformat, questionInAnswer,
|
||||
questionAlign, lastFontColour, allowEmptyAnswer, typeAnswer from cardModels
|
||||
where modelId = ?
|
||||
order by ordinal""", mid)):
|
||||
conf = dconf.copy()
|
||||
(conf['name'],
|
||||
conf['actv'],
|
||||
conf['qfmt'],
|
||||
conf['afmt'],
|
||||
conf['hideQ'],
|
||||
conf['align'],
|
||||
conf['bg'],
|
||||
conf['emptyAns'],
|
||||
conf['typeAns']) = row
|
||||
conf['ord'] = c
|
||||
# convert the field name to an ordinal
|
||||
ordN = None
|
||||
for (ord, fm) in enumerate(flds):
|
||||
if fm['name'] == conf['typeAns']:
|
||||
ordN = ord
|
||||
break
|
||||
if ordN is not None:
|
||||
conf['typeAns'] = ordN
|
||||
else:
|
||||
conf['typeAns'] = None
|
||||
for type in ("qfmt", "afmt"):
|
||||
# ensure the new style field format
|
||||
conf[type] = re.sub("%\((.+?)\)s", "{{\\1}}", conf[type])
|
||||
# some special names have changed
|
||||
conf[type] = re.sub(
|
||||
"(?i){{tags}}", "{{Tags}}", conf[type])
|
||||
conf[type] = re.sub(
|
||||
"(?i){{cardModel}}", "{{Template}}", conf[type])
|
||||
conf[type] = re.sub(
|
||||
"(?i){{modelTags}}", "{{Model}}", conf[type])
|
||||
tmpls.append(conf)
|
||||
return tmpls
|
||||
|
||||
def _postSchemaUpgrade(deck):
|
||||
"Handle the rest of the upgrade to 2.0."
|
||||
import anki.deck
|
||||
# make sure we have a current model id
|
||||
deck.models.setCurrent(deck.models.models.values()[0])
|
||||
# regenerate css, and set new card order
|
||||
for m in deck.models.all():
|
||||
m['newOrder'] = deck.conf['oldNewOrder']
|
||||
deck.models.save(m)
|
||||
del deck.conf['oldNewOrder']
|
||||
# fix creation time
|
||||
deck.sched._updateCutoff()
|
||||
d = datetime.datetime.today()
|
||||
d -= datetime.timedelta(hours=4)
|
||||
d = datetime.datetime(d.year, d.month, d.day)
|
||||
d += datetime.timedelta(hours=4)
|
||||
d -= datetime.timedelta(days=1+int((time.time()-deck.crt)/86400))
|
||||
deck.crt = int(time.mktime(d.timetuple()))
|
||||
deck.sched._updateCutoff()
|
||||
# update uniq cache
|
||||
deck.updateFieldCache(deck.db.list("select id from facts"))
|
||||
# remove old views
|
||||
for v in ("failedCards", "revCardsOld", "revCardsNew",
|
||||
"revCardsDue", "revCardsRandom", "acqCardsRandom",
|
||||
"acqCardsOld", "acqCardsNew"):
|
||||
deck.db.execute("drop view if exists %s" % v)
|
||||
# remove stats, as it's all in the revlog now
|
||||
deck.db.execute("drop table if exists stats")
|
||||
# suspended cards don't use ranges anymore
|
||||
deck.db.execute("update cards set queue=-1 where queue between -3 and -1")
|
||||
deck.db.execute("update cards set queue=-2 where queue between 3 and 5")
|
||||
deck.db.execute("update cards set queue=-3 where queue between 6 and 8")
|
||||
# remove old deleted tables
|
||||
for t in ("cards", "facts", "models", "media"):
|
||||
deck.db.execute("drop table if exists %sDeleted" % t)
|
||||
# rewrite due times for new cards
|
||||
deck.db.execute("""
|
||||
update cards set due = fid where type=0""")
|
||||
# and failed cards
|
||||
left = len(deck.groups.conf(1)['new']['delays'])
|
||||
deck.db.execute("update cards set edue = ?, left=? where type = 1",
|
||||
deck.sched.today+1, left)
|
||||
# and due cards
|
||||
deck.db.execute("""
|
||||
update cards set due = cast(
|
||||
(case when due < :stamp then 0 else 1 end) +
|
||||
((due-:stamp)/86400) as int)+:today where type = 2
|
||||
""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
|
||||
# possibly re-randomize
|
||||
if deck.models.randomNew():
|
||||
deck.sched.randomizeCards()
|
||||
# update insertion id
|
||||
deck.conf['nextPos'] = deck.db.scalar("select max(id) from facts")+1
|
||||
deck.save()
|
||||
|
||||
# optimize and finish
|
||||
deck.db.commit()
|
||||
deck.db.execute("vacuum")
|
||||
deck.db.execute("analyze")
|
||||
deck.db.execute("update deck set ver = ?", CURRENT_VERSION)
|
||||
deck.save()
|
||||
|
||||
# Post-init upgrade
|
||||
######################################################################
|
||||
|
||||
def _upgradeDeck(deck, version):
|
||||
"Upgrade deck to the latest version."
|
||||
if version >= CURRENT_VERSION:
|
||||
return
|
||||
if version < 100:
|
||||
_postSchemaUpgrade(deck)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import tempfile, os
|
||||
import tempfile, os, shutil
|
||||
from anki import Deck
|
||||
|
||||
def assertException(exception, func):
|
||||
|
|
@ -10,8 +10,14 @@ def assertException(exception, func):
|
|||
assert found
|
||||
|
||||
def getEmptyDeck(**kwargs):
|
||||
(fd, nam) = tempfile.mkstemp(suffix=".anki")
|
||||
(fd, nam) = tempfile.mkstemp(suffix=".anki2")
|
||||
os.unlink(nam)
|
||||
return Deck(nam, **kwargs)
|
||||
|
||||
def getUpgradeDeckPath():
|
||||
src = os.path.join(testDir, "support", "anki12.anki")
|
||||
(fd, dst) = tempfile.mkstemp(suffix=".anki2")
|
||||
shutil.copy(src, dst)
|
||||
return dst
|
||||
|
||||
testDir = os.path.dirname(__file__)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# coding: utf-8
|
||||
|
||||
import os, re, datetime
|
||||
from tests.shared import assertException, getEmptyDeck, testDir
|
||||
from tests.shared import assertException, getEmptyDeck, testDir, \
|
||||
getUpgradeDeckPath
|
||||
from anki.stdmodels import addBasicModel
|
||||
from anki.consts import *
|
||||
|
||||
|
|
@ -12,7 +13,7 @@ newMod = None
|
|||
|
||||
def test_create():
|
||||
global newPath, newMod
|
||||
path = "/tmp/test_attachNew.anki"
|
||||
path = "/tmp/test_attachNew.anki2"
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
|
|
@ -32,7 +33,7 @@ def test_open():
|
|||
def test_openReadOnly():
|
||||
# non-writeable dir
|
||||
assertException(Exception,
|
||||
lambda: Deck("/attachroot"))
|
||||
lambda: Deck("/attachroot.anki2"))
|
||||
# reuse tmp file from before, test non-writeable file
|
||||
os.chmod(newPath, 0)
|
||||
assertException(Exception,
|
||||
|
|
@ -116,11 +117,8 @@ def test_fieldChecksum():
|
|||
"select count() from fsums") == 2
|
||||
|
||||
def test_upgrade():
|
||||
import tempfile, shutil
|
||||
src = os.path.join(testDir, "support", "anki12.anki")
|
||||
(fd, dst) = tempfile.mkstemp(suffix=".anki")
|
||||
dst = getUpgradeDeckPath()
|
||||
print "upgrade to", dst
|
||||
shutil.copy(src, dst)
|
||||
deck = Deck(dst)
|
||||
# creation time should have been adjusted
|
||||
d = datetime.datetime.fromtimestamp(deck.crt)
|
||||
|
|
|
|||
12
tests/test_migration.py
Normal file
12
tests/test_migration.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# coding: utf-8
|
||||
|
||||
from shared import getUpgradeDeckPath
|
||||
from anki.migration.checker import check
|
||||
|
||||
def test_checker():
|
||||
dst = getUpgradeDeckPath()
|
||||
assert check(dst)
|
||||
# if it's corrupted, will fail
|
||||
open(dst, "w+").write("foo")
|
||||
assert not check(dst)
|
||||
|
||||
Loading…
Reference in a new issue