remove q/a cache, tags in fields, rewrite remaining ids, more

Anki used random 64bit IDs for cards, facts and fields. This had some nice
properties:
- merging data in syncs and imports was simply a matter of copying each way,
  as conflicts were astronomically unlikely
- it made it easy to identify identical cards and prevent them from being
  reimported
But there were some negatives too:
- they're more expensive to store
- javascript can't handle numbers > 2**53, which means AnkiMobile, iAnki and
  so on have to treat the ids as strings, which is slow
- simply copying data in a sync or import can lead to corruption, as while a
  duplicate id indicates the data was originally the same, it may have
  diverged. A more intelligent approach is necessary.
- sqlite was sorting the fields table based on the id, which meant the fields
  were spread across the table, and costly to fetch

So instead, we'll move to incremental ids. In the case of model changes we'll
declare that a schema change and force a full sync to avoid having to deal
with conflicts, and in the case of cards and facts, we'll need to update the
ids on one end to merge. Identical cards can be detected by checking to see if
their id is the same and their creation time is the same.

Creation time has been added back to cards and facts because it's necessary
for sync conflict merging. That means facts.pos is not required.

The graves table has been removed. It's not necessary for schema related
changes, and dead cards/facts can be represented as a card with queue=-4 and
created=0. Because we will record schema modification time and can ensure a
full sync propagates to all endpoints, it means we can remove the dead
cards/facts on schema change.

Tags have been removed from the facts table and are represented as a field
with ord=-1 and fmid=0. Combined with the locality improvement for fields, it
means that fetching fields is not much more expensive than using the q/a
cache.

Because of the above, removing the q/a cache is a possibility now. The q and a
columns on cards has been dropped. It will still be necessary to render the
q/a on fact add/edit, since we need to record media references. It would be
nice to avoid this in the future. Perhaps one way would be the ability to
assign a type to fields, like "image", "audio", or "latex". LaTeX needs
special consider anyway, as it was being rendered into the q/a cache.

anki/cards.py

@@ -3,7 +3,7 @@
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
 import time
-from anki.utils import genID, intTime, hexifyID
+from anki.utils import intTime, hexifyID
 
 MAX_TIMER = 60
 
@@ -18,7 +18,7 @@ MAX_TIMER = 60
 # Flags: unused; reserved for future use
 
 # Due is used differently for different queues.
-# - new queue: fact.pos
+# - new queue: fact.id
 # - rev queue: integer day
 # - lrn queue: integer timestamp
 
@@ -26,46 +26,46 @@ class Card(object):
 
     def __init__(self, deck, id=None):
         self.deck = deck
+        self.timerStarted = None
+        self._qa = None
         if id:
             self.id = id
             self.load()
         else:
             # to flush, set fid, tid, due and ord
-            self.id = genID()
+            self.id = None
             self.gid = 1
-            self.q = ""
+            self.crt = intTime()
-            self.a = ""
-            self.flags = 0
             self.type = 2
             self.queue = 2
-            self.interval = 0
+            self.ivl = 0
             self.factor = 0
             self.reps = 0
             self.streak = 0
             self.lapses = 0
             self.grade = 0
             self.cycles = 0
-        self.timerStarted = None
+            self.data = ""
 
     def load(self):
         (self.id,
          self.fid,
          self.tid,
          self.gid,
-         self.mod,
-         self.q,
-         self.a,
          self.ord,
+         self.crt,
+         self.mod,
          self.type,
          self.queue,
          self.due,
-         self.interval,
+         self.ivl,
          self.factor,
          self.reps,
          self.streak,
          self.lapses,
          self.grade,
-         self.cycles) = self.deck.db.first(
+         self.cycles,
+         self.data) = self.deck.db.first(
              "select * from cards where id = ?", self.id)
 
     def flush(self):
@@ -78,38 +78,56 @@ insert or replace into cards values
             self.fid,
             self.tid,
             self.gid,
-            self.mod,
-            self.q,
-            self.a,
             self.ord,
+            self.crt,
+            self.mod,
             self.type,
             self.queue,
             self.due,
-            self.interval,
+            self.ivl,
             self.factor,
             self.reps,
             self.streak,
             self.lapses,
             self.grade,
-            self.cycles)
+            self.cycles,
+            self.data)
 
     def flushSched(self):
         self.mod = intTime()
         self.deck.db.execute(
             """update cards set
-mod=?, type=?, queue=?, due=?, interval=?, factor=?, reps=?,
+mod=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?,
 streak=?, lapses=?, grade=?, cycles=? where id = ?""",
-            self.mod, self.type, self.queue, self.due, self.interval,
+            self.mod, self.type, self.queue, self.due, self.ivl,
             self.factor, self.reps, self.streak, self.lapses,
             self.grade, self.cycles, self.id)
 
+    def q(self):
+        return self._getQA()['q']
+
+    def a(self):
+        return self._getQA()['a']
+
+    def _getQA(self, reload=False):
+        # this is a hack at the moment
+        if not self._qa or reload:
+            self._qa = self.deck.formatQA(
+                self.id,
+                self.deck._cacheFacts([self.fid])[self.fid],
+                self.deck._cacheMeta("and c.id = %d" % self.id)[2][self.id])
+        return self._qa
+
     def fact(self):
-        return self.deck.getFact(self.deck, self.fid)
+        return self.deck.getFact(self.fid)
+
+    def template(self):
+        return self.deck.getTemplate(self.tid)
 
     def startTimer(self):
         self.timerStarted = time.time()
 
-    def userTime(self):
+    def timeTaken(self):
         return min(time.time() - self.timerStarted, MAX_TIMER)
 
     # Questions and answers

anki/deck.py

@@ -8,7 +8,7 @@ from operator import itemgetter
 from itertools import groupby
 
 from anki.lang import _, ngettext
-from anki.utils import parseTags, tidyHTML, genID, ids2str, hexifyID, \
+from anki.utils import parseTags, tidyHTML, ids2str, hexifyID, \
      canonifyTags, joinTags, addTags, deleteTags, checksum, fieldChecksum, \
      stripHTML, intTime
 
@@ -21,7 +21,7 @@ from anki.media import MediaRegistry
 from anki.consts import *
 import anki.latex # sets up hook
 
-import anki.cards, anki.facts, anki.models, anki.graves, anki.template
+import anki.cards, anki.facts, anki.models, anki.template
 
 # Settings related to queue building. These may be loaded without the rest of
 # the config to check due counts faster on mobile clients.
@@ -70,8 +70,7 @@ class _Deck(object):
         if self.utcOffset == -2:
             # shared deck; reset timezone and creation date
             self.utcOffset = time.timezone + 60*60*4
-            self.created = intTime()
+            self.crt = intTime()
-            self.mod = self.created
         self.undoEnabled = False
         self.sessionStartReps = 0
         self.sessionStartTime = 0
@@ -85,7 +84,7 @@ class _Deck(object):
     ##########################################################################
 
     def load(self):
-        (self.created,
+        (self.crt,
          self.mod,
          self.schema,
          self.syncName,
@@ -94,7 +93,7 @@ class _Deck(object):
          self.qconf,
          self.conf,
          self.data) = self.db.first("""
-select created, mod, schema, syncName, lastSync,
+select crt, mod, schema, syncName, lastSync,
 utcOffset, qconf, conf, data from deck""")
         self.qconf = simplejson.loads(self.qconf)
         self.conf = simplejson.loads(self.conf)
@@ -137,9 +136,14 @@ qconf=?, conf=?, data=?""",
         self.db.rollback()
 
     def modSchema(self):
+        if not self.schemaDirty():
+            # next sync will be full
+            self.emptyTrash()
         self.schema = intTime()
-        # next sync will be full, so we can forget old gravestones
+
-        anki.graves.forgetAll(self.db)
+    def schemaDirty(self):
+        "True if schema changed since last sync, or syncing off."
+        return self.schema > self.lastSync
 
     # unsorted
     ##########################################################################
@@ -152,6 +156,13 @@ qconf=?, conf=?, data=?""",
     def getCard(self, id):
         return anki.cards.Card(self, id)
 
+    def getFact(self, id):
+        return anki.facts.Fact(self, id=id)
+
+    def getTemplate(self, id):
+        return anki.models.Template(self, self.deck.db.first(
+            "select * from templates where id = ?", id))
+
         # if card:
         #     return card
         # if sched.name == "main":
@@ -448,7 +459,7 @@ due > :now and due < :now""", now=time.time())
     def addFact(self, fact):
         "Add a fact to the deck. Return number of new cards."
         # check we have card models available
-        cms = self.availableCardModels(fact)
+        cms = self.findTemplates(fact)
         if not cms:
             return None
         # set pos
@@ -458,6 +469,8 @@ due > :now and due < :now""", now=time.time())
         isRandom = self.qconf['newCardOrder'] == NEW_CARDS_RANDOM
         if isRandom:
             due = random.randrange(0, 10000)
+        # flush the fact so we get its id
+        fact.flush(cache=False)
         for template in cms:
             print "fixme:specify group on fact add"
             group = self.groupForTemplate(template)
@@ -482,8 +495,8 @@ due > :now and due < :now""", now=time.time())
         id = self.conf['currentGroupId']
         return self.db.query(anki.groups.GroupConf).get(id).load()
 
-    def availableCardModels(self, fact, checkActive=True):
+    def findTemplates(self, fact, checkActive=True):
-        "List of active card models that aren't empty for FACT."
+        "Return active, non-empty templates."
         ok = []
         for template in fact.model.templates:
             if template.active or not checkActive:
@@ -505,7 +518,7 @@ due > :now and due < :now""", now=time.time())
 
     def addCards(self, fact, tids):
         ids = []
-        for template in self.availableCardModels(fact, False):
+        for template in self.findTemplates(fact, False):
             if template.id not in tids:
                 continue
             if self.db.scalar("""
@@ -538,40 +551,26 @@ where fid = :fid and tid = :cmid""",
         return self.db.scalar("select count(id) from cards where fid = :id",
                              id=fid)
 
-    def deleteFact(self, fid):
+    def _deleteFacts(self, ids):
-        "Delete a fact. Removes any associated cards. Don't flush."
+        "Bulk delete facts by ID. Don't call this directly."
-        # remove any remaining cards
-        self.db.execute("insert into cardsDeleted select id, :time "
-                         "from cards where fid = :fid",
-                         time=time.time(), fid=fid)
-        self.db.execute(
-            "delete from cards where fid = :id", id=fid)
-        # and then the fact
-        self.deleteFacts([fid])
-
-    def deleteFacts(self, ids):
-        "Bulk delete facts by ID; don't touch cards."
         if not ids:
             return
-        now = time.time()
         strids = ids2str(ids)
         self.db.execute("delete from facts where id in %s" % strids)
         self.db.execute("delete from fdata where fid in %s" % strids)
-        anki.graves.registerMany(self.db, anki.graves.FACT, ids)
 
-    def deleteDanglingFacts(self):
+    def _deleteDanglingFacts(self):
-        "Delete any facts without cards. Return deleted ids."
+        "Delete any facts without cards. Don't call this directly."
         ids = self.db.list("""
-select facts.id from facts
+select id from facts where id not in (select distinct fid from cards)""")
-where facts.id not in (select distinct fid from cards)""")
+        self._deleteFacts(ids)
-        self.deleteFacts(ids)
         return ids
 
     def previewFact(self, oldFact, cms=None):
         "Duplicate fact and generate cards for preview. Don't add to deck."
         # check we have card models available
         if cms is None:
-            cms = self.availableCardModels(oldFact, checkActive=True)
+            cms = self.findTemplates(oldFact, checkActive=True)
         if not cms:
             return []
         fact = self.cloneFact(oldFact)
@@ -596,30 +595,42 @@ where facts.id not in (select distinct fid from cards)""")
     ##########################################################################
 
     def cardCount(self):
-        return self.db.scalar("select count() from cards")
+        all = self.db.scalar("select count() from cards")
+        trash = self.db.scalar("select count() from cards where queue = -4")
+        return all - trash
 
     def deleteCard(self, id):
-        "Delete a card given its id. Delete any unused facts. Don't flush."
+        "Delete a card given its id. Delete any unused facts."
         self.deleteCards([id])
 
     def deleteCards(self, ids):
         "Bulk delete cards by ID."
         if not ids:
             return
-        now = time.time()
+        sids = ids2str(ids)
-        strids = ids2str(ids)
         self.startProgress()
-        # grab fact ids
+        if self.schemaDirty():
-        fids = self.db.list("select fid from cards where id in %s"
+            # immediate delete?
-                                 % strids)
+            self.db.execute("delete from cards where id in %s" % sids)
-        # drop from cards
+            # remove any dangling facts
-        self.db.execute("delete from cards where id in %s" % strids)
+            self._deleteDanglingFacts()
-        # note deleted
+        else:
-        anki.graves.registerMany(self.db, anki.graves.CARD, ids)
+            # trash
-        # remove any dangling facts
+            sfids = ids2str(
-        self.deleteDanglingFacts()
+                self.db.list("select fid from cards where id in "+sids))
+            self.db.execute("delete from revlog where cid in "+sids)
+            self.db.execute("update cards set crt = 0 where id in "+sids)
+            self.db.execute("update facts set crt = 0 where id in "+sfids)
+            self.db.execute("delete from fdata where fid in "+sfids)
         self.finishProgress()
 
+    def emptyTrash(self):
+        self.db.executescript("""
+delete from facts where id in (select fid from cards where queue = -4);
+delete from fdata where fid in (select fid from cards where queue = -4);
+delete from revlog where cid in (select id from cards where queue = -4);
+delete from cards where queue = -4;""")
+
     # Models
     ##########################################################################
 
@@ -639,6 +650,7 @@ where facts.id not in (select distinct fid from cards)""")
 
     def deleteModel(self, mid):
         "Delete MODEL, and all its cards/facts."
+        # do a direct delete
         self.modSchema()
         # delete facts/cards
         self.deleteCards(self.db.list("""
@@ -648,7 +660,6 @@ select id from cards where fid in (select id from facts where mid = ?)""",
         self.db.execute("delete from models where id = ?", mid)
         self.db.execute("delete from templates where mid = ?", mid)
         self.db.execute("delete from fields where mid = ?", mid)
-        anki.graves.registerOne(self.db, anki.graves.MODEL, mid)
         # GUI should ensure last model is not deleted
         if self.conf['currentModelId'] == mid:
             self.conf['currentModelId'] = self.db.scalar(
@@ -732,16 +743,15 @@ and fmid = :id""" % sfids, id=old.id)
             # new
             for field in newModel.fields:
                 if field not in seen:
-                    d = [{'id': genID(),
+                    d = [{'fid': f,
-                          'fid': f,
                           'fmid': field.id,
                           'ord': field.ord}
                          for f in fids]
                     self.db.executemany('''
 insert into fdata
-(id, fid, fmid, ord, value)
+(fid, fmid, ord, value)
 values
-(:id, :fid, :fmid, :ord, "")''', d)
+(:fid, :fmid, :ord, "")''', d)
             # fact modtime
             self.db.execute("""
 update facts set
@@ -895,7 +905,7 @@ where tid in %s""" % strids, now=time.time())
     ##########################################################################
 
     def updateCache(self, ids, type="card"):
-        "Update cache after cards, facts or models changed."
+        "Update cache after facts or models changed."
         # gather metadata
         if type == "card":
             where = "and c.id in " + ids2str(ids)
@@ -911,10 +921,6 @@ where tid in %s""" % strids, now=time.time())
         # generate q/a
         pend = [self.formatQA(cids[n], facts[fids[n]], meta[cids[n]])
                 for n in range(len(cids))]
-        # update q/a
-        self.db.executemany(
-            "update cards set q = :q, a = :a, mod = %d where id = :id" %
-            intTime(), pend)
         for p in pend:
             self.media.registerText(p['q'])
             self.media.registerText(p['a'])
@@ -927,17 +933,21 @@ where tid in %s""" % strids, now=time.time())
         "Returns hash of id, question, answer."
         d = {'id': cardId}
         fields = {}
+        tags = None
         for (k, v) in fact.items():
+            if k == None:
+                tags = v[1]
+                continue
             fields["text:"+k] = stripHTML(v[1])
             if v[1]:
                 fields[k] = '<span class="fm%s">%s</span>' % (
                     hexifyID(v[0]), v[1])
             else:
                 fields[k] = u""
-        fields['Tags'] = meta[3]
+        fields['Tags'] = tags
-        fields['Model'] = meta[4]
+        fields['Model'] = meta[3]
-        fields['Template'] = meta[5]
+        fields['Template'] = meta[4]
-        fields['Group'] = meta[6]
+        fields['Group'] = meta[5]
         # render q & a
         for (type, format) in (("q", meta[1]), ("a", meta[2])):
             if filters:
@@ -950,12 +960,12 @@ where tid in %s""" % strids, now=time.time())
 
     def _cacheMeta(self, where=""):
         "Return cids, fids, and cid -> data hash."
-        # data is [fid, qfmt, afmt, tags, model, template, group]
+        # data is [fid, qfmt, afmt, model, template, group]
         meta = {}
         cids = []
         fids = []
         for r in self.db.execute("""
-select c.id, f.id, t.qfmt, t.afmt, f.tags, m.name, t.name, g.name
+select c.id, f.id, t.qfmt, t.afmt, m.name, t.name, g.name
 from cards c, facts f, models m, templates t, groups g where
 c.fid == f.id and f.mid == m.id and
 c.tid = t.id and c.gid = g.id
@@ -970,9 +980,8 @@ c.tid = t.id and c.gid = g.id
         facts = {}
         for id, fields in groupby(self.db.all("""
 select fdata.fid, fields.name, fields.id, fdata.val
-from fdata, fields where fdata.fid in %s and
+from fdata left outer join fields on fdata.fmid = fields.id
-fdata.fmid = fields.id
+where fdata.fid in %s order by fdata.fid""" % ids2str(ids)), itemgetter(0)):
-order by fdata.fid""" % ids2str(ids)), itemgetter(0)):
             facts[id] = dict([(f[1], f[2:]) for f in fields])
         return facts
 
@@ -992,7 +1001,7 @@ order by fdata.fid""" % ids2str(ids)), itemgetter(0)):
         r = []
         for (fid, map) in facts.items():
             for (fmid, val) in map.values():
-                if fmid not in confs:
+                if fmid and fmid not in confs:
                     confs[fmid] = simplejson.loads(self.db.scalar(
                         "select conf from fields where id = ?",
                         fmid))
@@ -1046,15 +1055,15 @@ insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(),
         self.registerTags(newTags)
         # find facts missing the tags
         if add:
-            l = "tags not "
+            l = "val not "
             fn = addTags
         else:
-            l = "tags "
+            l = "val "
             fn = deleteTags
         lim = " or ".join(
             [l+"like :_%d" % c for c, t in enumerate(newTags)])
         res = self.db.all(
-            "select id, tags from facts where " + lim,
+            "select fid, val from fdata where ord = -1 and " + lim,
             **dict([("_%d" % x, '%% %s %%' % y) for x, y in enumerate(newTags)]))
         # update tags
         fids = []
@@ -1062,8 +1071,10 @@ insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(),
             fids.append(row[0])
             return {'id': row[0], 't': fn(tags, row[1])}
         self.db.executemany("""
-update facts set tags = :t, mod = %d
+update fdata set val = :t
-where id = :id""" % intTime(), [fix(row) for row in res])
+where fid = :id""", [fix(row) for row in res])
+        self.db.execute("update facts set mod = ? where id in " +
+                        ids2str(fids), intTime())
         # update q/a cache
         self.updateCache(fids, type="fact")
         self.finishProgress()
@@ -1235,41 +1246,6 @@ where id = :id""" % intTime(), [fix(row) for row in res])
     # DB maintenance
     ##########################################################################
 
-    def recoverCards(self, ids):
-        "Put cards with damaged facts into new facts."
-        # create a new model in case the user has mod a previous one
-        from anki.stdmodels import RecoveryModel
-        m = RecoveryModel()
-        last = self.currentModel
-        self.addModel(m)
-        def repl(s):
-            # strip field model text
-            return re.sub("<span class=\"fm.*?>(.*?)</span>", "\\1", s)
-        # add new facts, pointing old card at new fact
-        for (id, q, a) in self.db.all("""
-select id, question, answer from cards
-where id in %s""" % ids2str(ids)):
-            f = self.newFact()
-            f['Question'] = repl(q)
-            f['Answer'] = repl(a)
-            try:
-                f.tags = self.db.scalar("""
-select group_concat(name, " ") from tags t, cardTags ct
-where cardId = :cid and ct.tagId = t.id""", cid=id) or u""
-                if f.tags:
-                    f.tags = " " + f.tags + " "
-            except:
-                raise Exception("Your sqlite is too old.")
-            cards = self.addFact(f)
-            # delete the freshly created card and point old card to this fact
-            self.db.execute("delete from cards where id = :id",
-                             id=f.cards[0].id)
-            self.db.execute("""
-update cards set fid = :fid, tid = :cmid, ord = 0
-where id = :id""", fid=f.id, cmid=m.templates[0].id, id=id)
-        # restore old model
-        self.currentModel = last
-
     def fixIntegrity(self, quick=False):
         "Fix possible problems and rebuild caches."
         self.save()

anki/facts.py

@@ -4,7 +4,7 @@
 
 import time
 from anki.errors import AnkiError
-from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime, \
+from anki.utils import stripHTMLMedia, fieldChecksum, intTime, \
     addTags, deleteTags, parseTags
 
 class Fact(object):
@@ -16,10 +16,11 @@ class Fact(object):
             self.id = id
             self.load()
         else:
-            self.id = genID()
+            self.id = None
             self.model = model
             self.mid = model.id
-            self.mod = intTime()
+            self.crt = intTime()
+            self.mod = self.crt
             self.tags = ""
             self.cache = ""
             self._fields = [""] * len(self.model.fields)
@@ -27,22 +28,24 @@ class Fact(object):
 
     def load(self):
         (self.mid,
-         self.mod,
+         self.crt,
-         self.pos,
+         self.mod) = self.deck.db.first("""
-         self.tags) = self.deck.db.first("""
+select mid, crt, mod from facts where id = ?""", self.id)
-select mid, mod, pos, tags from facts where id = ?""", self.id)
         self._fields = self.deck.db.list("""
-select value from fdata where fid = ? order by ordinal""", self.id)
+select val from fdata where fid = ? and fmid order by ord""", self.id)
+        self.tags = self.deck.db.scalar("""
+select val from fdata where fid = ? and ord = -1""", self.id)
         self.model = self.deck.getModel(self.mid)
 
-    def flush(self):
+    def flush(self, cache=True):
         self.mod = intTime()
         # facts table
         self.cache = stripHTMLMedia(u" ".join(self._fields))
-        self.deck.db.execute("""
+        res = self.deck.db.execute("""
-insert or replace into facts values (?, ?, ?, ?, ?, ?)""",
+insert or replace into facts values (?, ?, ?, ?, ?)""",
-                             self.id, self.mid, self.mod,
+                             self.id, self.mid, self.crt,
-                             self.pos, self.tags, self.cache)
+                             self.mod, self.cache)
+        self.id = res.lastrowid
         # fdata table
         self.deck.db.execute("delete from fdata where fid = ?", self.id)
         d = []
@@ -50,6 +53,7 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?)""",
             val = self._fields[ord]
             d.append(dict(fid=self.id, fmid=fmid, ord=ord,
                           val=val))
+        d.append(dict(fid=self.id, fmid=0, ord=-1, val=self.tags))
         self.deck.db.executemany("""
 insert into fdata values (:fid, :fmid, :ord, :val, '')""", d)
         # media and caches
@@ -106,9 +110,14 @@ insert into fdata values (:fid, :fmid, :ord, :val, '')""", d)
             return True
         val = self[name]
         csum = fieldChecksum(val)
+        print "in check, ", self.id
+        if self.id:
+            lim = "and fid != :fid"
+        else:
+            lim = ""
         return not self.deck.db.scalar(
-            "select 1 from fdata where csum = ? and fid != ? and val = ?",
+            "select 1 from fdata where csum = :c %s and val = :v" % lim,
-            csum, self.id, val)
+            c=csum, v=val, fid=self.id)
 
     def fieldComplete(self, name, text=None):
         (fmid, ord, conf) = self._fmap[name]

anki/find.py

@@ -400,7 +400,8 @@ def _findCards(deck, query):
                 tquery += "select id from facts except "
             if token == "none":
                 tquery += """
-select cards.id from cards, facts where facts.tags = '' and cards.fid = facts.id """
+select id from cards where fid in (select fid from fdata where ord = -1 and
+val = ''"""
             else:
                 token = token.replace("*", "%")
                 if not token.startswith("%"):
@@ -409,7 +410,7 @@ select cards.id from cards, facts where facts.tags = '' and cards.fid = facts.id
                     token += " %"
                 args["_tag_%d" % c] = token
                 tquery += """
-select id from facts where tags like :_tag_%d""" % c
+select fid from fdata where ord = -1 and val like :_tag_%d""" % c
         elif type == SEARCH_TYPE:
             if qquery:
                 if isNeg:

anki/graves.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright: Damien Elmes <anki@ichi2.net>
-# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
-
-# FIXME:
-# - check if we have to int(time)
-# - port all the code referencing the old tables
-
-import time
-from anki.utils import intTime
-
-FACT = 0
-CARD = 1
-MODEL = 2
-MEDIA = 3
-GROUP = 4
-GROUPCONFIG = 5
-
-def registerOne(db, type, id):
-    db.execute("insert into graves values (:t, :id, :ty)",
-               t=intTime(), id=id, ty=type)
-
-def registerMany(db, type, ids):
-    db.executemany("insert into graves values (:t, :id, :ty)",
-                   [{'t':intTime(), 'id':x, 'ty':type} for x in ids])
-
-def forgetAll(db):
-    db.execute("delete from graves")

anki/groups.py

@@ -27,7 +27,7 @@ defaultConf = {
 class GroupConfig(object):
     def __init__(self, name):
         self.name = name
-        self.id = genID()
+        self.id = None
         self.config = defaultConf
 
     def load(self):

anki/importing/__init__.py

@@ -15,7 +15,7 @@ import time
 #from anki.cards import cardsTable
 #from anki.facts import factsTable, fieldsTable
 from anki.lang import _
-from anki.utils import genID, canonifyTags, fieldChecksum
+from anki.utils import canonifyTags, fieldChecksum
 from anki.utils import canonifyTags, ids2str
 from anki.errors import *
 #from anki.deck import NEW_CARDS_RANDOM

anki/latex.py

@@ -3,7 +3,7 @@
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
 import re, tempfile, os, sys, shutil, cgi, subprocess
-from anki.utils import genID, checksum, call
+from anki.utils import checksum, call
 from anki.hooks import addHook
 from htmlentitydefs import entitydefs
 from anki.lang import _

anki/media.py

@@ -3,7 +3,7 @@
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
 import os, shutil, re, urllib2, time, tempfile, unicodedata, urllib
-from anki.utils import checksum, genID, intTime
+from anki.utils import checksum, intTime
 from anki.lang import _
 
 class MediaRegistry(object):
@@ -176,10 +176,14 @@ If a file with the same name exists, return a unique name."""
             if isinstance(s, unicode):
                 return unicodedata.normalize('NFD', s)
             return s
-        for (question, answer) in self.deck.db.all(
+        # generate q/a and look through all references
-            "select q, a from cards"):
+        (cids, fids, meta) = self.deck._cacheMeta()
-            for txt in (question, answer):
+        facts = self.deck._cacheFacts(fids)
-                for f in self.mediaFiles(txt):
+        pend = [self.deck.formatQA(cids[n], facts[fids[n]], meta[cids[n]])
+                for n in range(len(cids))]
+        for p in pend:
+            for type in ("q", "a"):
+                for f in self.mediaFiles(p[type]):
                     normrefs[norm(f)] = True
                     self.registerFile(f)
         # find unused media

anki/models.py

@@ -8,14 +8,9 @@ template or field, you should call model.flush(), rather than trying to save
 the subobject directly.
 """
 
-import time, re, simplejson, copy as copyMod
+import simplejson
-from anki.utils import genID, canonifyTags, intTime
+from anki.utils import intTime
-from anki.fonts import toPlatformFont
-from anki.utils import parseTags, hexifyID, checksum, stripHTML, intTime
 from anki.lang import _
-from anki.hooks import runFilter
-from anki.template import render
-from copy import copy
 
 # Models
 ##########################################################################
@@ -104,15 +99,23 @@ insert or replace into models values (?, ?, ?, ?)""",
     def copy(self):
         "Copy, flush and return."
         new = Model(self.deck, self.id)
-        new.id = genID()
+        new.id = None
         new.name += _(" copy")
-        for f in new.fields:
+        # get new id
-            f.id = genID()
+        f = new.fields; new.fields = []
-            f.mid = new.id
+        t = new.templates; new.templates = []
-        for t in new.templates:
-            t.id = genID()
-            t.mid = new.id
         new.flush()
+        # then put back
+        new.fields = f
+        new.templates = t
+        for f in new.fields:
+            f.id = None
+            f.mid = new.id
+            f._flush()
+        for t in new.templates:
+            t.id = None
+            t.mid = new.id
+            t._flush()
         return new
 
 # Field model object
@@ -175,7 +178,7 @@ class Template(object):
         if data:
             self.initFromData(data)
         else:
-            self.id = genID()
+            self.id = None
             self.active = True
             self.conf = defaultTemplateConf.copy()
 

anki/sched.py

@@ -495,7 +495,7 @@ and queue between 1 and 2""",
         # cutoff must not be more than 24 hours in the future
         cutoff = min(time.time() + 86400, cutoff)
         self.dayCutoff = cutoff
-        self.today = int(cutoff/86400 - self.deck.created/86400)
+        self.today = int(cutoff/86400 - self.deck.crt/86400)
 
     def checkDay(self):
         # check if the day has rolled over

anki/storage.py

@@ -49,10 +49,10 @@ def _addSchema(db, setDeckConf=True):
     db.executescript("""
 create table if not exists deck (
     id              integer primary key,
-    created         integer not null,
+    crt             integer not null,
     mod             integer not null,
+    ver             integer not null,
     schema          integer not null,
-    version         integer not null,
     syncName        text not null,
     lastSync        integer not null,
     utcOffset       integer not null,
@@ -66,28 +66,27 @@ create table if not exists cards (
     fid             integer not null,
     tid             integer not null,
     gid             integer not null,
-    mod             integer not null,
-    q               text not null,
-    a               text not null,
     ord             integer not null,
+    crt             integer not null,
+    mod             integer not null,
     type            integer not null,
     queue           integer not null,
     due             integer not null,
-    interval        integer not null,
+    ivl             integer not null,
     factor          integer not null,
     reps            integer not null,
     streak          integer not null,
     lapses          integer not null,
     grade           integer not null,
-    cycles          integer not null
+    cycles          integer not null,
+    data            text not null
 );
 
 create table if not exists facts (
     id              integer primary key,
     mid             integer not null,
+    crt             integer not null,
     mod             integer not null,
-    pos             integer not null,
-    tags            text not null,
     cache           text not null
 );
 
@@ -112,7 +111,7 @@ create table if not exists templates (
     mid             integer not null,
     ord             integer not null,
     name            text not null,
-    active          integer not null,
+    actv            integer not null,
     qfmt            text not null,
     afmt            text not null,
     conf            text not null
@@ -126,12 +125,6 @@ create table if not exists fdata (
     csum            text not null
 );
 
-create table if not exists graves (
-    delTime         integer not null,
-    objectId        integer not null,
-    type            integer not null
-);
-
 create table if not exists gconf (
     id              integer primary key,
     mod             integer not null,
@@ -140,7 +133,7 @@ create table if not exists gconf (
 );
 
 create table if not exists groups (
-    id              integer primary key autoincrement,
+    id              integer primary key,
     mod             integer not null,
     name            text not null,
     gcid            integer not null
@@ -157,10 +150,10 @@ create table if not exists revlog (
     cid             integer not null,
     ease            integer not null,
     rep             integer not null,
+    int             integer not null,
     lastInt         integer not null,
-    interval        integer not null,
     factor          integer not null,
-    userTime        integer not null,
+    taken           integer not null,
     flags           integer not null
 );
 
@@ -175,6 +168,7 @@ values(1,%(t)s,%(t)s,%(t)s,%(v)s,'',0,-2,'', '', '');
 """ % ({'t': intTime(), 'v':CURRENT_VERSION}))
     import anki.deck
     import anki.groups
+    # create a default group/configuration, which should not be removed
     db.execute(
         "insert or ignore into gconf values (1, ?, ?, ?)""",
         intTime(), _("Default Config"),
@@ -194,15 +188,15 @@ def _updateIndices(db):
 -- sync summaries
 create index if not exists ix_cards_mod on cards (mod);
 create index if not exists ix_facts_mod on facts (mod);
--- card spacing
+-- card spacing, etc
 create index if not exists ix_cards_fid on cards (fid);
 -- fact data
 create index if not exists ix_fdata_fid on fdata (fid);
 create index if not exists ix_fdata_csum on fdata (csum);
+-- revlog by card
+create index if not exists ix_revlog_cid on revlog (cid);
 -- media
 create index if not exists ix_media_csum on media (csum);
--- deletion tracking
-create index if not exists ix_graves_delTime on graves (delTime);
 """)
 
 # 2.0 schema migration
@@ -210,19 +204,19 @@ create index if not exists ix_graves_delTime on graves (delTime);
 # we don't have access to the progress handler at this point, so the GUI code
 # will need to set up a progress handling window before opening a deck.
 
-def _moveTable(db, table):
+def _moveTable(db, table, insExtra=""):
     sql = db.scalar(
         "select sql from sqlite_master where name = '%s'" % table)
     sql = sql.replace("TABLE "+table, "temporary table %s2" % table)
     db.execute(sql)
-    db.execute("insert into %s2 select * from %s" % (table, table))
+    db.execute("insert into %s2 select * from %s%s" % (table, table, insExtra))
     db.execute("drop table "+table)
     _addSchema(db, False)
 
 def _upgradeSchema(db):
     "Alter tables prior to ORM initialization."
     try:
-        ver = db.scalar("select version from deck")
+        ver = db.scalar("select ver from deck")
     except:
         ver = db.scalar("select version from decks")
     # latest 1.2 is 65
@@ -233,11 +227,18 @@ def _upgradeSchema(db):
 
     # cards
     ###########
-    _moveTable(db, "cards")
+    # move into temp table
+    _moveTable(db, "cards", " order by created")
+    # use the new order to rewrite card ids
+    for (old, new) in db.all("select id, rowid from cards2"):
+        db.execute(
+            "update reviewHistory set cardId = ? where cardId = ?", new, old)
+    # move back, preserving new ids
     db.execute("""
-insert into cards select id, factId, cardModelId, 1, cast(modified as int),
+insert into cards select rowid, factId, cardModelId, 1, ordinal,
-question, answer, ordinal, relativeDelay, type, due, cast(interval as int),
+cast(created as int), cast(modified as int), relativeDelay, type, due,
-cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""")
+cast(interval as int), cast(factor*1000 as int), reps, successive, noCount,
+0, 0, "" from cards2 order by created""")
     db.execute("drop table cards2")
 
     # tags
@@ -245,6 +246,11 @@ cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""")
     _moveTable(db, "tags")
     db.execute("insert or ignore into tags select id, ?, tag from tags2",
                intTime())
+    db.execute("drop table tags2")
+    db.execute("drop table cardTags")
+
+    # facts
+    ###########
     # tags should have a leading and trailing space if not empty, and not
     # use commas
     db.execute("""
@@ -253,23 +259,26 @@ when trim(tags) == "" then ""
 else " " || replace(replace(trim(tags), ",", " "), "  ", " ") || " "
 end)
 """)
-    db.execute("drop table tags2")
+    # we store them as fields now
-    db.execute("drop table cardTags")
+    db.execute("insert into fields select null, id, 0, -1, tags from facts")
-
+    # put facts in a temporary table, sorted by created
-    # facts
-    ###########
     db.execute("""
 create table facts2
-(id, modelId, modified, tags, cache)""")
+(id, modelId, created, modified, cache)""")
-    # use the rowid to give them an integer order
     db.execute("""
-insert into facts2 select id, modelId, modified, tags, spaceUntil from
+insert into facts2 select id, modelId, created, modified, spaceUntil
-facts order by created""")
+from facts order by created""")
+    # use the new order to rewrite fact ids
+    for (old, new) in db.all("select id, rowid from facts2"):
+        db.execute("update fields set factId = ? where factId = ?",
+                        new, old)
+        db.execute("update cards set fid = ? where fid = ?", new, old)
+    # and put the facts into the new table
     db.execute("drop table facts")
     _addSchema(db, False)
     db.execute("""
-insert or ignore into facts select id, modelId, rowid,
+insert or ignore into facts select rowid, modelId,
-cast(modified as int), tags, cache from facts2""")
+cast(created as int), cast(modified as int), cache from facts2""")
     db.execute("drop table facts2")
 
     # media
@@ -283,15 +292,15 @@ originalPath from media2""")
     # fields -> fdata
     ###########
     db.execute("""
-insert or ignore into fdata select factId, fieldModelId, ordinal, value, ''
+insert into fdata select factId, fieldModelId, ordinal, value, ''
-from fields""")
+from fields order by factId, ordinal""")
     db.execute("drop table fields")
 
     # models
     ###########
     _moveTable(db, "models")
     db.execute("""
-insert or ignore into models select id, cast(modified as int),
+insert into models select id, cast(modified as int),
 name, "{}" from models2""")
     db.execute("drop table models2")
 
@@ -349,7 +358,7 @@ utcOffset, "", "", "" from decks""", t=intTime())
     dkeys = ("hexCache", "cssCache")
     for (k, v) in db.execute("select * from deckVars").fetchall():
         if k in dkeys:
-            data[k] = v
+            pass
         else:
             conf[k] = v
     db.execute("update deck set qconf = :l, conf = :c, data = :d",
@@ -412,7 +421,7 @@ allowEmptyAnswer, typeAnswer from cardModels"""):
     # clean up
     db.execute("drop table cardModels")
 
-def _rewriteIds(deck):
+def _rewriteModelIds(deck):
     # rewrite model/template/field ids
     models = deck.allModels()
     deck.db.execute("delete from models")
@@ -441,7 +450,7 @@ def _rewriteIds(deck):
 def _postSchemaUpgrade(deck):
     "Handle the rest of the upgrade to 2.0."
     import anki.deck
-    _rewriteIds(deck)
+    _rewriteModelIds(deck)
     # remove old views
     for v in ("failedCards", "revCardsOld", "revCardsNew",
               "revCardsDue", "revCardsRandom", "acqCardsRandom",
@@ -472,22 +481,23 @@ def _postSchemaUpgrade(deck):
         deck.db.execute("drop table if exists %sDeleted" % t)
     # rewrite due times for new cards
     deck.db.execute("""
-update cards set due = (select pos from facts where fid = facts.id) where type=2""")
+update cards set due = fid where type=2""")
     # convert due cards into day-based due
     deck.db.execute("""
 update cards set due = cast(
 (case when due < :stamp then 0 else 1 end) +
 ((due-:stamp)/86400) as int)+:today where type
 between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
-    # update factPos
+    # track ids
-    deck.conf['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1
+    #deck.conf['nextFact'] = deck.db.scalar("select max(id) from facts")+1
+    #deck.conf['nextCard'] = deck.db.scalar("select max(id) from cards")+1
     deck.save()
 
     # optimize and finish
     deck.updateDynamicIndices()
     deck.db.execute("vacuum")
     deck.db.execute("analyze")
-    deck.db.execute("update deck set version = ?", CURRENT_VERSION)
+    deck.db.execute("update deck set ver = ?", CURRENT_VERSION)
     deck.save()
 
 # Post-init upgrade

anki/sync.py

@@ -10,7 +10,7 @@ from anki.errors import *
 #from anki.models import Model, Field, Template
 #from anki.facts import Fact
 #from anki.cards import Card
-from anki.utils import ids2str, hexifyID, checksum
+from anki.utils import ids2str, checksum
 #from anki.media import mediaFiles
 from anki.lang import _
 from hooks import runHook

anki/utils.py

@@ -197,28 +197,6 @@ def entsToTxt(html):
 # IDs
 ##############################################################################
 
-def genID(static=[]):
-    "Generate a random, unique 64bit ID."
-    # 23 bits of randomness, 41 bits of current time
-    # random rather than a counter to ensure efficient btree
-    t = long(time.time()*1000)
-    if not static:
-        static.extend([t, {}])
-    else:
-        if static[0] != t:
-            static[0] = t
-            static[1] = {}
-    while 1:
-        rand = random.getrandbits(23)
-        if rand not in static[1]:
-            static[1][rand] = True
-            break
-    x = rand << 41 | t
-    # turn into a signed long
-    if x >= 9223372036854775808L:
-        x -= 18446744073709551616L
-    return x
-
 def hexifyID(id):
     if id < 0:
         id += 18446744073709551616L
@@ -231,11 +209,7 @@ def dehexifyID(id):
     return id
 
 def ids2str(ids):
-    """Given a list of integers, return a string '(int1,int2,.)'
+    """Given a list of integers, return a string '(int1,int2,...)'."""
-
-The caller is responsible for ensuring only integers are provided.
-This is safe if you use sqlite primary key columns, which are guaranteed
-to be integers."""
     return "(%s)" % ",".join([str(i) for i in ids])
 
 # Tags

tests/test_deck.py

@@ -58,7 +58,7 @@ def test_factAddDelete():
     assert n == 2
     # check q/a generation
     c0 = f.cards()[0]
-    assert re.sub("</?.+?>", "", c0.q) == u"one"
+    assert re.sub("</?.+?>", "", c0.q()) == u"one"
     # it should not be a duplicate
     for p in f.problems():
         assert not p

tests/test_models.py

@@ -90,8 +90,6 @@ def test_modelChange():
     assert deck.modelUseCount(m2) == 1
     assert deck.cardCount() == 3
     assert deck.factCount() == 2
-    (q, a) = deck.db.first("""
+    c = deck.getCard(deck.db.scalar("select id from cards where fid = ?", f.id))
-select q, a from cards where fid = :id""",
+    assert stripHTML(c.q()) == u"e"
-                          id=f.id)
+    assert stripHTML(c.a()) == u"r"
-    assert stripHTML(q) == u"e"
-    assert stripHTML(a) == u"r"