diff --git a/anki/cards.py b/anki/cards.py index a287c3c11..c94cd14ee 100644 --- a/anki/cards.py +++ b/anki/cards.py @@ -3,7 +3,7 @@ # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html import time -from anki.utils import genID, intTime, hexifyID +from anki.utils import intTime, hexifyID MAX_TIMER = 60 @@ -18,7 +18,7 @@ MAX_TIMER = 60 # Flags: unused; reserved for future use # Due is used differently for different queues. -# - new queue: fact.pos +# - new queue: fact.id # - rev queue: integer day # - lrn queue: integer timestamp @@ -26,46 +26,46 @@ class Card(object): def __init__(self, deck, id=None): self.deck = deck + self.timerStarted = None + self._qa = None if id: self.id = id self.load() else: # to flush, set fid, tid, due and ord - self.id = genID() + self.id = None self.gid = 1 - self.q = "" - self.a = "" - self.flags = 0 + self.crt = intTime() self.type = 2 self.queue = 2 - self.interval = 0 + self.ivl = 0 self.factor = 0 self.reps = 0 self.streak = 0 self.lapses = 0 self.grade = 0 self.cycles = 0 - self.timerStarted = None + self.data = "" def load(self): (self.id, self.fid, self.tid, self.gid, - self.mod, - self.q, - self.a, self.ord, + self.crt, + self.mod, self.type, self.queue, self.due, - self.interval, + self.ivl, self.factor, self.reps, self.streak, self.lapses, self.grade, - self.cycles) = self.deck.db.first( + self.cycles, + self.data) = self.deck.db.first( "select * from cards where id = ?", self.id) def flush(self): @@ -78,38 +78,56 @@ insert or replace into cards values self.fid, self.tid, self.gid, - self.mod, - self.q, - self.a, self.ord, + self.crt, + self.mod, self.type, self.queue, self.due, - self.interval, + self.ivl, self.factor, self.reps, self.streak, self.lapses, self.grade, - self.cycles) + self.cycles, + self.data) def flushSched(self): self.mod = intTime() self.deck.db.execute( """update cards set -mod=?, type=?, queue=?, due=?, interval=?, factor=?, reps=?, +mod=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?, streak=?, lapses=?, grade=?, cycles=? where id = ?""", - self.mod, self.type, self.queue, self.due, self.interval, + self.mod, self.type, self.queue, self.due, self.ivl, self.factor, self.reps, self.streak, self.lapses, self.grade, self.cycles, self.id) + def q(self): + return self._getQA()['q'] + + def a(self): + return self._getQA()['a'] + + def _getQA(self, reload=False): + # this is a hack at the moment + if not self._qa or reload: + self._qa = self.deck.formatQA( + self.id, + self.deck._cacheFacts([self.fid])[self.fid], + self.deck._cacheMeta("and c.id = %d" % self.id)[2][self.id]) + return self._qa + def fact(self): - return self.deck.getFact(self.deck, self.fid) + return self.deck.getFact(self.fid) + + def template(self): + return self.deck.getTemplate(self.tid) def startTimer(self): self.timerStarted = time.time() - def userTime(self): + def timeTaken(self): return min(time.time() - self.timerStarted, MAX_TIMER) # Questions and answers diff --git a/anki/deck.py b/anki/deck.py index 7d096a6eb..84801de8d 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -8,7 +8,7 @@ from operator import itemgetter from itertools import groupby from anki.lang import _, ngettext -from anki.utils import parseTags, tidyHTML, genID, ids2str, hexifyID, \ +from anki.utils import parseTags, tidyHTML, ids2str, hexifyID, \ canonifyTags, joinTags, addTags, deleteTags, checksum, fieldChecksum, \ stripHTML, intTime @@ -21,7 +21,7 @@ from anki.media import MediaRegistry from anki.consts import * import anki.latex # sets up hook -import anki.cards, anki.facts, anki.models, anki.graves, anki.template +import anki.cards, anki.facts, anki.models, anki.template # Settings related to queue building. These may be loaded without the rest of # the config to check due counts faster on mobile clients. @@ -70,8 +70,7 @@ class _Deck(object): if self.utcOffset == -2: # shared deck; reset timezone and creation date self.utcOffset = time.timezone + 60*60*4 - self.created = intTime() - self.mod = self.created + self.crt = intTime() self.undoEnabled = False self.sessionStartReps = 0 self.sessionStartTime = 0 @@ -85,7 +84,7 @@ class _Deck(object): ########################################################################## def load(self): - (self.created, + (self.crt, self.mod, self.schema, self.syncName, @@ -94,7 +93,7 @@ class _Deck(object): self.qconf, self.conf, self.data) = self.db.first(""" -select created, mod, schema, syncName, lastSync, +select crt, mod, schema, syncName, lastSync, utcOffset, qconf, conf, data from deck""") self.qconf = simplejson.loads(self.qconf) self.conf = simplejson.loads(self.conf) @@ -137,9 +136,14 @@ qconf=?, conf=?, data=?""", self.db.rollback() def modSchema(self): + if not self.schemaDirty(): + # next sync will be full + self.emptyTrash() self.schema = intTime() - # next sync will be full, so we can forget old gravestones - anki.graves.forgetAll(self.db) + + def schemaDirty(self): + "True if schema changed since last sync, or syncing off." + return self.schema > self.lastSync # unsorted ########################################################################## @@ -152,6 +156,13 @@ qconf=?, conf=?, data=?""", def getCard(self, id): return anki.cards.Card(self, id) + def getFact(self, id): + return anki.facts.Fact(self, id=id) + + def getTemplate(self, id): + return anki.models.Template(self, self.deck.db.first( + "select * from templates where id = ?", id)) + # if card: # return card # if sched.name == "main": @@ -448,7 +459,7 @@ due > :now and due < :now""", now=time.time()) def addFact(self, fact): "Add a fact to the deck. Return number of new cards." # check we have card models available - cms = self.availableCardModels(fact) + cms = self.findTemplates(fact) if not cms: return None # set pos @@ -458,6 +469,8 @@ due > :now and due < :now""", now=time.time()) isRandom = self.qconf['newCardOrder'] == NEW_CARDS_RANDOM if isRandom: due = random.randrange(0, 10000) + # flush the fact so we get its id + fact.flush(cache=False) for template in cms: print "fixme:specify group on fact add" group = self.groupForTemplate(template) @@ -482,8 +495,8 @@ due > :now and due < :now""", now=time.time()) id = self.conf['currentGroupId'] return self.db.query(anki.groups.GroupConf).get(id).load() - def availableCardModels(self, fact, checkActive=True): - "List of active card models that aren't empty for FACT." + def findTemplates(self, fact, checkActive=True): + "Return active, non-empty templates." ok = [] for template in fact.model.templates: if template.active or not checkActive: @@ -505,7 +518,7 @@ due > :now and due < :now""", now=time.time()) def addCards(self, fact, tids): ids = [] - for template in self.availableCardModels(fact, False): + for template in self.findTemplates(fact, False): if template.id not in tids: continue if self.db.scalar(""" @@ -538,40 +551,26 @@ where fid = :fid and tid = :cmid""", return self.db.scalar("select count(id) from cards where fid = :id", id=fid) - def deleteFact(self, fid): - "Delete a fact. Removes any associated cards. Don't flush." - # remove any remaining cards - self.db.execute("insert into cardsDeleted select id, :time " - "from cards where fid = :fid", - time=time.time(), fid=fid) - self.db.execute( - "delete from cards where fid = :id", id=fid) - # and then the fact - self.deleteFacts([fid]) - - def deleteFacts(self, ids): - "Bulk delete facts by ID; don't touch cards." + def _deleteFacts(self, ids): + "Bulk delete facts by ID. Don't call this directly." if not ids: return - now = time.time() strids = ids2str(ids) self.db.execute("delete from facts where id in %s" % strids) self.db.execute("delete from fdata where fid in %s" % strids) - anki.graves.registerMany(self.db, anki.graves.FACT, ids) - def deleteDanglingFacts(self): - "Delete any facts without cards. Return deleted ids." + def _deleteDanglingFacts(self): + "Delete any facts without cards. Don't call this directly." ids = self.db.list(""" -select facts.id from facts -where facts.id not in (select distinct fid from cards)""") - self.deleteFacts(ids) +select id from facts where id not in (select distinct fid from cards)""") + self._deleteFacts(ids) return ids def previewFact(self, oldFact, cms=None): "Duplicate fact and generate cards for preview. Don't add to deck." # check we have card models available if cms is None: - cms = self.availableCardModels(oldFact, checkActive=True) + cms = self.findTemplates(oldFact, checkActive=True) if not cms: return [] fact = self.cloneFact(oldFact) @@ -596,30 +595,42 @@ where facts.id not in (select distinct fid from cards)""") ########################################################################## def cardCount(self): - return self.db.scalar("select count() from cards") + all = self.db.scalar("select count() from cards") + trash = self.db.scalar("select count() from cards where queue = -4") + return all - trash def deleteCard(self, id): - "Delete a card given its id. Delete any unused facts. Don't flush." + "Delete a card given its id. Delete any unused facts." self.deleteCards([id]) def deleteCards(self, ids): "Bulk delete cards by ID." if not ids: return - now = time.time() - strids = ids2str(ids) + sids = ids2str(ids) self.startProgress() - # grab fact ids - fids = self.db.list("select fid from cards where id in %s" - % strids) - # drop from cards - self.db.execute("delete from cards where id in %s" % strids) - # note deleted - anki.graves.registerMany(self.db, anki.graves.CARD, ids) - # remove any dangling facts - self.deleteDanglingFacts() + if self.schemaDirty(): + # immediate delete? + self.db.execute("delete from cards where id in %s" % sids) + # remove any dangling facts + self._deleteDanglingFacts() + else: + # trash + sfids = ids2str( + self.db.list("select fid from cards where id in "+sids)) + self.db.execute("delete from revlog where cid in "+sids) + self.db.execute("update cards set crt = 0 where id in "+sids) + self.db.execute("update facts set crt = 0 where id in "+sfids) + self.db.execute("delete from fdata where fid in "+sfids) self.finishProgress() + def emptyTrash(self): + self.db.executescript(""" +delete from facts where id in (select fid from cards where queue = -4); +delete from fdata where fid in (select fid from cards where queue = -4); +delete from revlog where cid in (select id from cards where queue = -4); +delete from cards where queue = -4;""") + # Models ########################################################################## @@ -639,6 +650,7 @@ where facts.id not in (select distinct fid from cards)""") def deleteModel(self, mid): "Delete MODEL, and all its cards/facts." + # do a direct delete self.modSchema() # delete facts/cards self.deleteCards(self.db.list(""" @@ -648,7 +660,6 @@ select id from cards where fid in (select id from facts where mid = ?)""", self.db.execute("delete from models where id = ?", mid) self.db.execute("delete from templates where mid = ?", mid) self.db.execute("delete from fields where mid = ?", mid) - anki.graves.registerOne(self.db, anki.graves.MODEL, mid) # GUI should ensure last model is not deleted if self.conf['currentModelId'] == mid: self.conf['currentModelId'] = self.db.scalar( @@ -732,16 +743,15 @@ and fmid = :id""" % sfids, id=old.id) # new for field in newModel.fields: if field not in seen: - d = [{'id': genID(), - 'fid': f, + d = [{'fid': f, 'fmid': field.id, 'ord': field.ord} for f in fids] self.db.executemany(''' insert into fdata -(id, fid, fmid, ord, value) +(fid, fmid, ord, value) values -(:id, :fid, :fmid, :ord, "")''', d) +(:fid, :fmid, :ord, "")''', d) # fact modtime self.db.execute(""" update facts set @@ -895,7 +905,7 @@ where tid in %s""" % strids, now=time.time()) ########################################################################## def updateCache(self, ids, type="card"): - "Update cache after cards, facts or models changed." + "Update cache after facts or models changed." # gather metadata if type == "card": where = "and c.id in " + ids2str(ids) @@ -911,10 +921,6 @@ where tid in %s""" % strids, now=time.time()) # generate q/a pend = [self.formatQA(cids[n], facts[fids[n]], meta[cids[n]]) for n in range(len(cids))] - # update q/a - self.db.executemany( - "update cards set q = :q, a = :a, mod = %d where id = :id" % - intTime(), pend) for p in pend: self.media.registerText(p['q']) self.media.registerText(p['a']) @@ -927,17 +933,21 @@ where tid in %s""" % strids, now=time.time()) "Returns hash of id, question, answer." d = {'id': cardId} fields = {} + tags = None for (k, v) in fact.items(): + if k == None: + tags = v[1] + continue fields["text:"+k] = stripHTML(v[1]) if v[1]: fields[k] = '%s' % ( hexifyID(v[0]), v[1]) else: fields[k] = u"" - fields['Tags'] = meta[3] - fields['Model'] = meta[4] - fields['Template'] = meta[5] - fields['Group'] = meta[6] + fields['Tags'] = tags + fields['Model'] = meta[3] + fields['Template'] = meta[4] + fields['Group'] = meta[5] # render q & a for (type, format) in (("q", meta[1]), ("a", meta[2])): if filters: @@ -950,12 +960,12 @@ where tid in %s""" % strids, now=time.time()) def _cacheMeta(self, where=""): "Return cids, fids, and cid -> data hash." - # data is [fid, qfmt, afmt, tags, model, template, group] + # data is [fid, qfmt, afmt, model, template, group] meta = {} cids = [] fids = [] for r in self.db.execute(""" -select c.id, f.id, t.qfmt, t.afmt, f.tags, m.name, t.name, g.name +select c.id, f.id, t.qfmt, t.afmt, m.name, t.name, g.name from cards c, facts f, models m, templates t, groups g where c.fid == f.id and f.mid == m.id and c.tid = t.id and c.gid = g.id @@ -970,9 +980,8 @@ c.tid = t.id and c.gid = g.id facts = {} for id, fields in groupby(self.db.all(""" select fdata.fid, fields.name, fields.id, fdata.val -from fdata, fields where fdata.fid in %s and -fdata.fmid = fields.id -order by fdata.fid""" % ids2str(ids)), itemgetter(0)): +from fdata left outer join fields on fdata.fmid = fields.id +where fdata.fid in %s order by fdata.fid""" % ids2str(ids)), itemgetter(0)): facts[id] = dict([(f[1], f[2:]) for f in fields]) return facts @@ -992,7 +1001,7 @@ order by fdata.fid""" % ids2str(ids)), itemgetter(0)): r = [] for (fid, map) in facts.items(): for (fmid, val) in map.values(): - if fmid not in confs: + if fmid and fmid not in confs: confs[fmid] = simplejson.loads(self.db.scalar( "select conf from fields where id = ?", fmid)) @@ -1046,15 +1055,15 @@ insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(), self.registerTags(newTags) # find facts missing the tags if add: - l = "tags not " + l = "val not " fn = addTags else: - l = "tags " + l = "val " fn = deleteTags lim = " or ".join( [l+"like :_%d" % c for c, t in enumerate(newTags)]) res = self.db.all( - "select id, tags from facts where " + lim, + "select fid, val from fdata where ord = -1 and " + lim, **dict([("_%d" % x, '%% %s %%' % y) for x, y in enumerate(newTags)])) # update tags fids = [] @@ -1062,8 +1071,10 @@ insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(), fids.append(row[0]) return {'id': row[0], 't': fn(tags, row[1])} self.db.executemany(""" -update facts set tags = :t, mod = %d -where id = :id""" % intTime(), [fix(row) for row in res]) +update fdata set val = :t +where fid = :id""", [fix(row) for row in res]) + self.db.execute("update facts set mod = ? where id in " + + ids2str(fids), intTime()) # update q/a cache self.updateCache(fids, type="fact") self.finishProgress() @@ -1235,41 +1246,6 @@ where id = :id""" % intTime(), [fix(row) for row in res]) # DB maintenance ########################################################################## - def recoverCards(self, ids): - "Put cards with damaged facts into new facts." - # create a new model in case the user has mod a previous one - from anki.stdmodels import RecoveryModel - m = RecoveryModel() - last = self.currentModel - self.addModel(m) - def repl(s): - # strip field model text - return re.sub("(.*?)", "\\1", s) - # add new facts, pointing old card at new fact - for (id, q, a) in self.db.all(""" -select id, question, answer from cards -where id in %s""" % ids2str(ids)): - f = self.newFact() - f['Question'] = repl(q) - f['Answer'] = repl(a) - try: - f.tags = self.db.scalar(""" -select group_concat(name, " ") from tags t, cardTags ct -where cardId = :cid and ct.tagId = t.id""", cid=id) or u"" - if f.tags: - f.tags = " " + f.tags + " " - except: - raise Exception("Your sqlite is too old.") - cards = self.addFact(f) - # delete the freshly created card and point old card to this fact - self.db.execute("delete from cards where id = :id", - id=f.cards[0].id) - self.db.execute(""" -update cards set fid = :fid, tid = :cmid, ord = 0 -where id = :id""", fid=f.id, cmid=m.templates[0].id, id=id) - # restore old model - self.currentModel = last - def fixIntegrity(self, quick=False): "Fix possible problems and rebuild caches." self.save() diff --git a/anki/facts.py b/anki/facts.py index 1a5c9a964..605fbf08d 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -4,7 +4,7 @@ import time from anki.errors import AnkiError -from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime, \ +from anki.utils import stripHTMLMedia, fieldChecksum, intTime, \ addTags, deleteTags, parseTags class Fact(object): @@ -16,10 +16,11 @@ class Fact(object): self.id = id self.load() else: - self.id = genID() + self.id = None self.model = model self.mid = model.id - self.mod = intTime() + self.crt = intTime() + self.mod = self.crt self.tags = "" self.cache = "" self._fields = [""] * len(self.model.fields) @@ -27,22 +28,24 @@ class Fact(object): def load(self): (self.mid, - self.mod, - self.pos, - self.tags) = self.deck.db.first(""" -select mid, mod, pos, tags from facts where id = ?""", self.id) + self.crt, + self.mod) = self.deck.db.first(""" +select mid, crt, mod from facts where id = ?""", self.id) self._fields = self.deck.db.list(""" -select value from fdata where fid = ? order by ordinal""", self.id) +select val from fdata where fid = ? and fmid order by ord""", self.id) + self.tags = self.deck.db.scalar(""" +select val from fdata where fid = ? and ord = -1""", self.id) self.model = self.deck.getModel(self.mid) - def flush(self): + def flush(self, cache=True): self.mod = intTime() # facts table self.cache = stripHTMLMedia(u" ".join(self._fields)) - self.deck.db.execute(""" -insert or replace into facts values (?, ?, ?, ?, ?, ?)""", - self.id, self.mid, self.mod, - self.pos, self.tags, self.cache) + res = self.deck.db.execute(""" +insert or replace into facts values (?, ?, ?, ?, ?)""", + self.id, self.mid, self.crt, + self.mod, self.cache) + self.id = res.lastrowid # fdata table self.deck.db.execute("delete from fdata where fid = ?", self.id) d = [] @@ -50,6 +53,7 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?)""", val = self._fields[ord] d.append(dict(fid=self.id, fmid=fmid, ord=ord, val=val)) + d.append(dict(fid=self.id, fmid=0, ord=-1, val=self.tags)) self.deck.db.executemany(""" insert into fdata values (:fid, :fmid, :ord, :val, '')""", d) # media and caches @@ -106,9 +110,14 @@ insert into fdata values (:fid, :fmid, :ord, :val, '')""", d) return True val = self[name] csum = fieldChecksum(val) + print "in check, ", self.id + if self.id: + lim = "and fid != :fid" + else: + lim = "" return not self.deck.db.scalar( - "select 1 from fdata where csum = ? and fid != ? and val = ?", - csum, self.id, val) + "select 1 from fdata where csum = :c %s and val = :v" % lim, + c=csum, v=val, fid=self.id) def fieldComplete(self, name, text=None): (fmid, ord, conf) = self._fmap[name] diff --git a/anki/find.py b/anki/find.py index 61f52dfa9..4950ef322 100644 --- a/anki/find.py +++ b/anki/find.py @@ -400,7 +400,8 @@ def _findCards(deck, query): tquery += "select id from facts except " if token == "none": tquery += """ -select cards.id from cards, facts where facts.tags = '' and cards.fid = facts.id """ +select id from cards where fid in (select fid from fdata where ord = -1 and +val = ''""" else: token = token.replace("*", "%") if not token.startswith("%"): @@ -409,7 +410,7 @@ select cards.id from cards, facts where facts.tags = '' and cards.fid = facts.id token += " %" args["_tag_%d" % c] = token tquery += """ -select id from facts where tags like :_tag_%d""" % c +select fid from fdata where ord = -1 and val like :_tag_%d""" % c elif type == SEARCH_TYPE: if qquery: if isNeg: diff --git a/anki/graves.py b/anki/graves.py deleted file mode 100644 index 8a9b214ed..000000000 --- a/anki/graves.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright: Damien Elmes -# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html - -# FIXME: -# - check if we have to int(time) -# - port all the code referencing the old tables - -import time -from anki.utils import intTime - -FACT = 0 -CARD = 1 -MODEL = 2 -MEDIA = 3 -GROUP = 4 -GROUPCONFIG = 5 - -def registerOne(db, type, id): - db.execute("insert into graves values (:t, :id, :ty)", - t=intTime(), id=id, ty=type) - -def registerMany(db, type, ids): - db.executemany("insert into graves values (:t, :id, :ty)", - [{'t':intTime(), 'id':x, 'ty':type} for x in ids]) - -def forgetAll(db): - db.execute("delete from graves") diff --git a/anki/groups.py b/anki/groups.py index c9eadf24a..78281ebdd 100644 --- a/anki/groups.py +++ b/anki/groups.py @@ -27,7 +27,7 @@ defaultConf = { class GroupConfig(object): def __init__(self, name): self.name = name - self.id = genID() + self.id = None self.config = defaultConf def load(self): diff --git a/anki/importing/__init__.py b/anki/importing/__init__.py index c650e23b7..82be6f8f4 100644 --- a/anki/importing/__init__.py +++ b/anki/importing/__init__.py @@ -15,7 +15,7 @@ import time #from anki.cards import cardsTable #from anki.facts import factsTable, fieldsTable from anki.lang import _ -from anki.utils import genID, canonifyTags, fieldChecksum +from anki.utils import canonifyTags, fieldChecksum from anki.utils import canonifyTags, ids2str from anki.errors import * #from anki.deck import NEW_CARDS_RANDOM diff --git a/anki/latex.py b/anki/latex.py index 5e1039309..dabbadfb0 100644 --- a/anki/latex.py +++ b/anki/latex.py @@ -3,7 +3,7 @@ # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html import re, tempfile, os, sys, shutil, cgi, subprocess -from anki.utils import genID, checksum, call +from anki.utils import checksum, call from anki.hooks import addHook from htmlentitydefs import entitydefs from anki.lang import _ diff --git a/anki/media.py b/anki/media.py index 58a95fa90..6472dfeff 100644 --- a/anki/media.py +++ b/anki/media.py @@ -3,7 +3,7 @@ # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html import os, shutil, re, urllib2, time, tempfile, unicodedata, urllib -from anki.utils import checksum, genID, intTime +from anki.utils import checksum, intTime from anki.lang import _ class MediaRegistry(object): @@ -176,10 +176,14 @@ If a file with the same name exists, return a unique name.""" if isinstance(s, unicode): return unicodedata.normalize('NFD', s) return s - for (question, answer) in self.deck.db.all( - "select q, a from cards"): - for txt in (question, answer): - for f in self.mediaFiles(txt): + # generate q/a and look through all references + (cids, fids, meta) = self.deck._cacheMeta() + facts = self.deck._cacheFacts(fids) + pend = [self.deck.formatQA(cids[n], facts[fids[n]], meta[cids[n]]) + for n in range(len(cids))] + for p in pend: + for type in ("q", "a"): + for f in self.mediaFiles(p[type]): normrefs[norm(f)] = True self.registerFile(f) # find unused media diff --git a/anki/models.py b/anki/models.py index 7c04d2244..ecafdc01e 100644 --- a/anki/models.py +++ b/anki/models.py @@ -8,14 +8,9 @@ template or field, you should call model.flush(), rather than trying to save the subobject directly. """ -import time, re, simplejson, copy as copyMod -from anki.utils import genID, canonifyTags, intTime -from anki.fonts import toPlatformFont -from anki.utils import parseTags, hexifyID, checksum, stripHTML, intTime +import simplejson +from anki.utils import intTime from anki.lang import _ -from anki.hooks import runFilter -from anki.template import render -from copy import copy # Models ########################################################################## @@ -104,15 +99,23 @@ insert or replace into models values (?, ?, ?, ?)""", def copy(self): "Copy, flush and return." new = Model(self.deck, self.id) - new.id = genID() + new.id = None new.name += _(" copy") - for f in new.fields: - f.id = genID() - f.mid = new.id - for t in new.templates: - t.id = genID() - t.mid = new.id + # get new id + f = new.fields; new.fields = [] + t = new.templates; new.templates = [] new.flush() + # then put back + new.fields = f + new.templates = t + for f in new.fields: + f.id = None + f.mid = new.id + f._flush() + for t in new.templates: + t.id = None + t.mid = new.id + t._flush() return new # Field model object @@ -175,7 +178,7 @@ class Template(object): if data: self.initFromData(data) else: - self.id = genID() + self.id = None self.active = True self.conf = defaultTemplateConf.copy() diff --git a/anki/sched.py b/anki/sched.py index c80dee555..14e9c4175 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -495,7 +495,7 @@ and queue between 1 and 2""", # cutoff must not be more than 24 hours in the future cutoff = min(time.time() + 86400, cutoff) self.dayCutoff = cutoff - self.today = int(cutoff/86400 - self.deck.created/86400) + self.today = int(cutoff/86400 - self.deck.crt/86400) def checkDay(self): # check if the day has rolled over diff --git a/anki/storage.py b/anki/storage.py index 5d1884dde..e8d3f56bd 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -49,10 +49,10 @@ def _addSchema(db, setDeckConf=True): db.executescript(""" create table if not exists deck ( id integer primary key, - created integer not null, + crt integer not null, mod integer not null, + ver integer not null, schema integer not null, - version integer not null, syncName text not null, lastSync integer not null, utcOffset integer not null, @@ -66,28 +66,27 @@ create table if not exists cards ( fid integer not null, tid integer not null, gid integer not null, - mod integer not null, - q text not null, - a text not null, ord integer not null, + crt integer not null, + mod integer not null, type integer not null, queue integer not null, due integer not null, - interval integer not null, + ivl integer not null, factor integer not null, reps integer not null, streak integer not null, lapses integer not null, grade integer not null, - cycles integer not null + cycles integer not null, + data text not null ); create table if not exists facts ( id integer primary key, mid integer not null, + crt integer not null, mod integer not null, - pos integer not null, - tags text not null, cache text not null ); @@ -112,7 +111,7 @@ create table if not exists templates ( mid integer not null, ord integer not null, name text not null, - active integer not null, + actv integer not null, qfmt text not null, afmt text not null, conf text not null @@ -126,12 +125,6 @@ create table if not exists fdata ( csum text not null ); -create table if not exists graves ( - delTime integer not null, - objectId integer not null, - type integer not null -); - create table if not exists gconf ( id integer primary key, mod integer not null, @@ -140,7 +133,7 @@ create table if not exists gconf ( ); create table if not exists groups ( - id integer primary key autoincrement, + id integer primary key, mod integer not null, name text not null, gcid integer not null @@ -157,10 +150,10 @@ create table if not exists revlog ( cid integer not null, ease integer not null, rep integer not null, + int integer not null, lastInt integer not null, - interval integer not null, factor integer not null, - userTime integer not null, + taken integer not null, flags integer not null ); @@ -175,6 +168,7 @@ values(1,%(t)s,%(t)s,%(t)s,%(v)s,'',0,-2,'', '', ''); """ % ({'t': intTime(), 'v':CURRENT_VERSION})) import anki.deck import anki.groups + # create a default group/configuration, which should not be removed db.execute( "insert or ignore into gconf values (1, ?, ?, ?)""", intTime(), _("Default Config"), @@ -194,15 +188,15 @@ def _updateIndices(db): -- sync summaries create index if not exists ix_cards_mod on cards (mod); create index if not exists ix_facts_mod on facts (mod); --- card spacing +-- card spacing, etc create index if not exists ix_cards_fid on cards (fid); -- fact data create index if not exists ix_fdata_fid on fdata (fid); create index if not exists ix_fdata_csum on fdata (csum); +-- revlog by card +create index if not exists ix_revlog_cid on revlog (cid); -- media create index if not exists ix_media_csum on media (csum); --- deletion tracking -create index if not exists ix_graves_delTime on graves (delTime); """) # 2.0 schema migration @@ -210,19 +204,19 @@ create index if not exists ix_graves_delTime on graves (delTime); # we don't have access to the progress handler at this point, so the GUI code # will need to set up a progress handling window before opening a deck. -def _moveTable(db, table): +def _moveTable(db, table, insExtra=""): sql = db.scalar( "select sql from sqlite_master where name = '%s'" % table) sql = sql.replace("TABLE "+table, "temporary table %s2" % table) db.execute(sql) - db.execute("insert into %s2 select * from %s" % (table, table)) + db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra)) db.execute("drop table "+table) _addSchema(db, False) def _upgradeSchema(db): "Alter tables prior to ORM initialization." try: - ver = db.scalar("select version from deck") + ver = db.scalar("select ver from deck") except: ver = db.scalar("select version from decks") # latest 1.2 is 65 @@ -233,11 +227,18 @@ def _upgradeSchema(db): # cards ########### - _moveTable(db, "cards") + # move into temp table + _moveTable(db, "cards", " order by created") + # use the new order to rewrite card ids + for (old, new) in db.all("select id, rowid from cards2"): + db.execute( + "update reviewHistory set cardId = ? where cardId = ?", new, old) + # move back, preserving new ids db.execute(""" -insert into cards select id, factId, cardModelId, 1, cast(modified as int), -question, answer, ordinal, relativeDelay, type, due, cast(interval as int), -cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") +insert into cards select rowid, factId, cardModelId, 1, ordinal, +cast(created as int), cast(modified as int), relativeDelay, type, due, +cast(interval as int), cast(factor*1000 as int), reps, successive, noCount, +0, 0, "" from cards2 order by created""") db.execute("drop table cards2") # tags @@ -245,6 +246,11 @@ cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") _moveTable(db, "tags") db.execute("insert or ignore into tags select id, ?, tag from tags2", intTime()) + db.execute("drop table tags2") + db.execute("drop table cardTags") + + # facts + ########### # tags should have a leading and trailing space if not empty, and not # use commas db.execute(""" @@ -253,23 +259,26 @@ when trim(tags) == "" then "" else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " end) """) - db.execute("drop table tags2") - db.execute("drop table cardTags") - - # facts - ########### + # we store them as fields now + db.execute("insert into fields select null, id, 0, -1, tags from facts") + # put facts in a temporary table, sorted by created db.execute(""" create table facts2 -(id, modelId, modified, tags, cache)""") - # use the rowid to give them an integer order +(id, modelId, created, modified, cache)""") db.execute(""" -insert into facts2 select id, modelId, modified, tags, spaceUntil from -facts order by created""") +insert into facts2 select id, modelId, created, modified, spaceUntil +from facts order by created""") + # use the new order to rewrite fact ids + for (old, new) in db.all("select id, rowid from facts2"): + db.execute("update fields set factId = ? where factId = ?", + new, old) + db.execute("update cards set fid = ? where fid = ?", new, old) + # and put the facts into the new table db.execute("drop table facts") _addSchema(db, False) db.execute(""" -insert or ignore into facts select id, modelId, rowid, -cast(modified as int), tags, cache from facts2""") +insert or ignore into facts select rowid, modelId, +cast(created as int), cast(modified as int), cache from facts2""") db.execute("drop table facts2") # media @@ -283,15 +292,15 @@ originalPath from media2""") # fields -> fdata ########### db.execute(""" -insert or ignore into fdata select factId, fieldModelId, ordinal, value, '' -from fields""") +insert into fdata select factId, fieldModelId, ordinal, value, '' +from fields order by factId, ordinal""") db.execute("drop table fields") # models ########### _moveTable(db, "models") db.execute(""" -insert or ignore into models select id, cast(modified as int), +insert into models select id, cast(modified as int), name, "{}" from models2""") db.execute("drop table models2") @@ -349,7 +358,7 @@ utcOffset, "", "", "" from decks""", t=intTime()) dkeys = ("hexCache", "cssCache") for (k, v) in db.execute("select * from deckVars").fetchall(): if k in dkeys: - data[k] = v + pass else: conf[k] = v db.execute("update deck set qconf = :l, conf = :c, data = :d", @@ -412,7 +421,7 @@ allowEmptyAnswer, typeAnswer from cardModels"""): # clean up db.execute("drop table cardModels") -def _rewriteIds(deck): +def _rewriteModelIds(deck): # rewrite model/template/field ids models = deck.allModels() deck.db.execute("delete from models") @@ -441,7 +450,7 @@ def _rewriteIds(deck): def _postSchemaUpgrade(deck): "Handle the rest of the upgrade to 2.0." import anki.deck - _rewriteIds(deck) + _rewriteModelIds(deck) # remove old views for v in ("failedCards", "revCardsOld", "revCardsNew", "revCardsDue", "revCardsRandom", "acqCardsRandom", @@ -472,22 +481,23 @@ def _postSchemaUpgrade(deck): deck.db.execute("drop table if exists %sDeleted" % t) # rewrite due times for new cards deck.db.execute(""" -update cards set due = (select pos from facts where fid = facts.id) where type=2""") +update cards set due = fid where type=2""") # convert due cards into day-based due deck.db.execute(""" update cards set due = cast( (case when due < :stamp then 0 else 1 end) + ((due-:stamp)/86400) as int)+:today where type between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today) - # update factPos - deck.conf['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1 + # track ids + #deck.conf['nextFact'] = deck.db.scalar("select max(id) from facts")+1 + #deck.conf['nextCard'] = deck.db.scalar("select max(id) from cards")+1 deck.save() # optimize and finish deck.updateDynamicIndices() deck.db.execute("vacuum") deck.db.execute("analyze") - deck.db.execute("update deck set version = ?", CURRENT_VERSION) + deck.db.execute("update deck set ver = ?", CURRENT_VERSION) deck.save() # Post-init upgrade diff --git a/anki/sync.py b/anki/sync.py index 16c1b87f8..a0b8bd81c 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -10,7 +10,7 @@ from anki.errors import * #from anki.models import Model, Field, Template #from anki.facts import Fact #from anki.cards import Card -from anki.utils import ids2str, hexifyID, checksum +from anki.utils import ids2str, checksum #from anki.media import mediaFiles from anki.lang import _ from hooks import runHook diff --git a/anki/utils.py b/anki/utils.py index 4daa82440..ee5d1b146 100644 --- a/anki/utils.py +++ b/anki/utils.py @@ -197,28 +197,6 @@ def entsToTxt(html): # IDs ############################################################################## -def genID(static=[]): - "Generate a random, unique 64bit ID." - # 23 bits of randomness, 41 bits of current time - # random rather than a counter to ensure efficient btree - t = long(time.time()*1000) - if not static: - static.extend([t, {}]) - else: - if static[0] != t: - static[0] = t - static[1] = {} - while 1: - rand = random.getrandbits(23) - if rand not in static[1]: - static[1][rand] = True - break - x = rand << 41 | t - # turn into a signed long - if x >= 9223372036854775808L: - x -= 18446744073709551616L - return x - def hexifyID(id): if id < 0: id += 18446744073709551616L @@ -231,11 +209,7 @@ def dehexifyID(id): return id def ids2str(ids): - """Given a list of integers, return a string '(int1,int2,.)' - -The caller is responsible for ensuring only integers are provided. -This is safe if you use sqlite primary key columns, which are guaranteed -to be integers.""" + """Given a list of integers, return a string '(int1,int2,...)'.""" return "(%s)" % ",".join([str(i) for i in ids]) # Tags diff --git a/tests/test_deck.py b/tests/test_deck.py index ad75118a2..b3eb6187e 100644 --- a/tests/test_deck.py +++ b/tests/test_deck.py @@ -58,7 +58,7 @@ def test_factAddDelete(): assert n == 2 # check q/a generation c0 = f.cards()[0] - assert re.sub("", "", c0.q) == u"one" + assert re.sub("", "", c0.q()) == u"one" # it should not be a duplicate for p in f.problems(): assert not p diff --git a/tests/test_models.py b/tests/test_models.py index eb6a12070..a93abbcb7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -90,8 +90,6 @@ def test_modelChange(): assert deck.modelUseCount(m2) == 1 assert deck.cardCount() == 3 assert deck.factCount() == 2 - (q, a) = deck.db.first(""" -select q, a from cards where fid = :id""", - id=f.id) - assert stripHTML(q) == u"e" - assert stripHTML(a) == u"r" + c = deck.getCard(deck.db.scalar("select id from cards where fid = ?", f.id)) + assert stripHTML(c.q()) == u"e" + assert stripHTML(c.a()) == u"r"