convert groups to json; use timestamp ids for all but default

Rather than use a combination of id lookups on the groups table and a group
configuration cache in the scheduler, I've moved the groups and group config
into json objects on the deck table. This results in a net saving of code and
saves one or more DB lookups on each card answer, in exchange for a small
increase in deck load/save work.

I did a quick survey of AnkiWeb, and the vast majority of decks use less than
100 tags, and it's safe to assume groups will follow a similar pattern.

All groups and group configs except the default one will use integer
timestamps now, to simplify merging when syncing and importing.

defaultGroup() has been removed in favour of keeping the models up to date
(not yet done).

anki/cards.py

@@ -104,12 +104,10 @@ lapses=?, grade=?, cycles=?, edue=? where id = ?""",
 
     def _getQA(self, reload=False):
         if not self._qa or reload:
-            gname = self.deck.db.scalar(
-                "select name from groups where id = ?", self.gid)
             f = self.fact(); m = self.model()
             data = [self.id, f.id, m.id, self.gid, self.ord, f.stringTags(),
                     f.joinedFields()]
-            self._qa = self.deck._renderQA(self.model(), gname, data)
+            self._qa = self.deck._renderQA(self.model(), data)
         return self._qa
 
     def _withClass(self, txt, extra):

anki/cram.py

@@ -24,7 +24,6 @@ class CramScheduler(Scheduler):
 
     def reset(self):
         self._updateCutoff()
-        self._resetConf()
         self._resetLrnCount()
         self._resetLrn()
         self._resetNew()

anki/deck.py

@@ -34,11 +34,7 @@ defaultQconf = {
 
 # other options
 defaultConf = {
-    'currentModelId': 1,
-    'currentGroupId': 1,
     'nextPos': 1,
-    'nextGid': 2,
-    'nextGcid': 2,
     'mediaURL': "",
     'fontFamilies': [
         [u'ＭＳ 明朝',u'ヒラギノ明朝 Pro W3',u'Kochi Mincho', u'東風明朝']
@@ -89,11 +85,15 @@ class _Deck(object):
          self.lastSync,
          self.qconf,
          self.conf,
+         self.groups,
+         self.gconf,
          self.data) = self.db.first("""
 select crt, mod, scm, dty, syncName, lastSync,
-qconf, conf, data from deck""")
+qconf, conf, groups, gconf, data from deck""")
         self.qconf = simplejson.loads(self.qconf)
         self.conf = simplejson.loads(self.conf)
+        self.groups = simplejson.loads(self.groups)
+        self.gconf = simplejson.loads(self.gconf)
         self.data = simplejson.loads(self.data)
 
     def flush(self, mod=None):
@@ -278,9 +278,9 @@ qconf=?, conf=?, data=?""",
                 # [cid, fid, mid, gid, ord, tags, flds]
                 data = [1, 1, model.id, 1, template['ord'],
                         "", fact.joinedFields()]
-                now = self._renderQA(model, "", data)
+                now = self._renderQA(model, data)
                 data[6] = "\x1f".join([""]*len(fact.fields))
-                empty = self._renderQA(model, "", data)
+                empty = self._renderQA(model, data)
                 if now['q'] == empty['q']:
                     continue
                 if not template['emptyAns']:
@@ -334,7 +334,7 @@ qconf=?, conf=?, data=?""",
         card = anki.cards.Card(self)
         card.fid = fact.id
         card.ord = template['ord']
-        card.gid = self.defaultGroup(template['gid'] or fact.gid)
+        card.gid = template['gid'] or fact.gid
         card.due = due
         if flush:
             card.flush()
@@ -446,12 +446,10 @@ select id from cards where fid in (select id from facts where mid = ?)""",
         else:
             raise Exception()
         mods = self.models()
-        groups = dict(self.db.all("select id, name from groups"))
+        return [self._renderQA(mods[row[2]], row)
-        return [self._renderQA(mods[row[2]], groups[row[3]], row)
                 for row in self._qaData(where)]
 
-    # fixme: don't need gid or data
+    def _renderQA(self, model, data):
-    def _renderQA(self, model, gname, data):
         "Returns hash of id, question, answer."
         # data is [cid, fid, mid, gid, ord, tags, flds]
         # unpack fields and create dict
@@ -466,7 +464,7 @@ select id from cards where fid in (select id from facts where mid = ?)""",
                 fields[name] = ""
         fields['Tags'] = data[5]
         fields['Model'] = model.name
-        fields['Group'] = gname
+        fields['Group'] = self.groupName(data[3])
         template = model.templates[data[4]]
         fields['Template'] = template['name']
         # render q & a
@@ -480,10 +478,10 @@ select id from cards where fid in (select id from facts where mid = ?)""",
                 else:
                     name = "ca:"
                 format = format.replace("cloze:", name)
-            fields = runFilter("mungeFields", fields, model, gname, data, self)
+            fields = runFilter("mungeFields", fields, model, data, self)
             html = anki.template.render(format, fields)
             d[type] = runFilter(
-                "mungeQA", html, type, fields, model, gname, data, self)
+                "mungeQA", html, type, fields, model, data, self)
         return d
 
     def _qaData(self, where=""):
@@ -552,51 +550,41 @@ update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res
 
     # Groups
     ##########################################################################
+    # the id keys are strings because that's the way they're stored in json,
+    # but the anki code passes around integers
 
-    def groups(self):
+    def groupID(self, name, create=True):
-        "A list of all group names."
+        "Add a group with NAME. Reuse group if already exists. Return id."
-        return self.db.list("select name from groups order by name")
+        for id, g in self.groups.items():
+            if g['name'].lower() == name.lower():
+                return int(id)
+        if not create:
+            return None
+        g = dict(name=name, conf=1, mod=intTime())
+        while 1:
+            id = intTime(1000)
+            if str(id) in self.groups:
+                continue
+            self.groups[str(id)] = g
+            return id
 
-    def groupName(self, id):
+    def groupName(self, gid):
-        return self.db.scalar("select name from groups where id = ?", id)
+        return self.groups[str(gid)]['name']
 
-    def groupId(self, name):
+    def groupConf(self, gid):
-        "Return the id for NAME, creating if necessary."
+        return self.gconf[str(self.groups[str(gid)]['conf'])]
-        id = self.db.scalar("select id from groups where name = ?", name)
-        if not id:
-            id = self.db.execute(
-                "insert into groups values (?,?,?,?, ?)",
-                self.nextID("gid"), intTime(), name, 1,
-                simplejson.dumps(anki.groups.defaultData)).lastrowid
-        return id
-
-    def defaultGroup(self, id):
-        if id == 1:
-            return 1
-        return self.db.scalar("select id from groups where id = ?", id) or 1
 
     def delGroup(self, gid):
         self.modSchema()
         self.db.execute("update cards set gid = 1 where gid = ?", gid)
         self.db.execute("update facts set gid = 1 where gid = ?", gid)
         self.db.execute("delete from groups where id = ?", gid)
+        print "fixme: loop through models and update stale gid references"
 
     def setGroup(self, cids, gid):
         self.db.execute(
             "update cards set gid = ? where id in "+ids2str(cids), gid)
 
-    # Group configuration
-    ##########################################################################
-
-    def groupConfs(self):
-        "Return [name, id]."
-        return self.db.all("select name, id from gconf order by name")
-
-    def groupConf(self, gcid):
-        return simplejson.loads(
-            self.db.scalar(
-                "select conf from gconf where id = ?", gcid))
-
     # Tag-based selective study
     ##########################################################################
 

anki/facts.py

@@ -19,7 +19,7 @@ class Fact(object):
         else:
             self.id = timestampID(deck.db, "facts")
             self._model = model
-            self.gid = deck.defaultGroup(model.conf['gid'])
+            self.gid = model.conf['gid']
             self.mid = model.id
             self.tags = []
             self.fields = [""] * len(self._model.fields)

anki/find.py

@@ -121,7 +121,7 @@ order by %s""" % (lim, sort)
             elif type == SEARCH_MODEL:
                 self._findModel(token, isNeg, c)
             elif type == SEARCH_GROUP:
-                self._findGroup(token, isNeg, c)
+                self._findGroup(token, isNeg)
             else:
                 self._findText(token, isNeg, c)
 
@@ -189,12 +189,10 @@ order by %s""" % (lim, sort)
                 extra, c))
         self.lims['args']['_mod_%d'%c] = val
 
-    def _findGroup(self, val, isNeg, c):
+    def _findGroup(self, val, isNeg):
-        extra = "not" if isNeg else ""
+        extra = "!" if isNeg else ""
-        self.lims['card'].append(
+        id = self.deck.groupID(val, create=False) or 0
-            "c.gid %s in (select id from groups where name like :_grp_%d)" % (
+        self.lims['card'].append("c.gid %s= %d" % (extra, id))
-                extra, c))
-        self.lims['args']['_grp_%d'%c] = val
 
     def _findTemplate(self, val, isNeg):
         lims = []

anki/latex.py

@@ -30,7 +30,7 @@ def stripLatex(text):
         text = text.replace(match.group(), "")
     return text
 
-def mungeQA(html, type, fields, model, gname, data, deck):
+def mungeQA(html, type, fields, model, data, deck):
     "Convert TEXT with embedded latex tags to image links."
     for match in regexps['standard'].finditer(html):
         html = html.replace(match.group(), _imgLink(deck, match.group(1), model))

anki/sched.py

@@ -35,7 +35,6 @@ class Scheduler(object):
             return c
 
     def reset(self):
-        self._resetConf()
         self._resetCounts()
         self._resetLrn()
         self._resetRev()
@@ -151,18 +150,23 @@ sum(case when queue = 2 and due <= ? then 1 else 0 end),
 sum(case when queue = 0 then 1 else 0 end)
 from cards group by gid""", self.today):
             gids[gid] = [all, rev, new]
-        return [[name, gid]+gids.get(gid, [0, 0, 0]) for (gid, name) in
+        return [[grp['name'], int(gid)]+gids.get(int(gid), [0, 0, 0])
-                self.deck.db.execute(
+                for (gid, grp) in self._orderedGroups()]
-                    "select id, name from groups order by name")]
+
+    def _orderedGroups(self):
+        grps = self.deck.groups.items()
+        def key(grp):
+            return grp[1]['name']
+        grps.sort(key=key)
+        return grps
 
     def groupCountTree(self):
         return self._groupChildren(self.groupCounts())
 
     def groupTree(self):
         "Like the count tree without the counts. Faster."
-        return self._groupChildren([[name, gid, 0, 0, 0] for (gid, name) in
+        return self._groupChildren([[grp['name'], int(gid), 0, 0, 0]
-                self.deck.db.execute(
+                                    for (gid, grp) in self._orderedGroups()])
-                    "select id, name from groups order by name")])
 
     def _groupChildren(self, grps):
         tree = []
@@ -590,16 +594,8 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % (
     # Tools
     ##########################################################################
 
-    def _resetConf(self):
-        "Update group conf cache."
-        self.groupConfs = dict(self.deck.db.all("select id, gcid from groups"))
-        self.confCache = {}
-
     def _cardConf(self, card):
-        id = self.groupConfs[card.gid]
+        return self.deck.groupConf(card.gid)
-        if id not in self.confCache:
-            self.confCache[id] = self.deck.groupConf(id)
-        return self.confCache[id]
 
     def _groupLimit(self):
         l = self.deck.qconf['groups']

anki/storage.py

@@ -67,6 +67,8 @@ create table if not exists deck (
     lastSync        integer not null,
     qconf           text not null,
     conf            text not null,
+    groups          text not null,
+    gconf           text not null,
     data            text not null
 );
 
@@ -116,21 +118,6 @@ create table if not exists models (
     css             text not null
 );
 
-create table if not exists groups (
-    id              integer primary key,
-    mod             integer not null,
-    name            text not null,
-    gcid            integer not null,
-    data            text not null
-);
-
-create table if not exists gconf (
-    id              integer primary key,
-    mod             integer not null,
-    name            text not null,
-    conf            text not null
-);
-
 create table if not exists graves (
     time            integer not null,
     oid             integer not null,
@@ -155,23 +142,18 @@ create table if not exists tags (
 );
 
 insert or ignore into deck
-values(1,0,0,0,%(v)s,0,'',0,'', '', '');
+values(1,0,0,0,%(v)s,0,'',0,'','','','','');
 """ % ({'v':CURRENT_VERSION}))
     import anki.deck
     import anki.groups
-    # create a default group/configuration, which should not be removed
-    db.execute(
-        "insert or ignore into gconf values (1, ?, ?, ?)""",
-        intTime(), _("Default"),
-        simplejson.dumps(anki.groups.defaultConf))
-    db.execute(
-        "insert or ignore into groups values (1, ?, ?, 1, ?)",
-        intTime(), _("Default"), simplejson.dumps(
-            anki.groups.defaultData))
     if setDeckConf:
-        db.execute("update deck set qconf = ?, conf = ?, data = ?",
+        db.execute("""
+update deck set qconf = ?, conf = ?, groups = ?, gconf = ?, data = ?""",
                    simplejson.dumps(anki.deck.defaultQconf),
                    simplejson.dumps(anki.deck.defaultConf),
+                   simplejson.dumps({'1': {'name': _("Default"), 'conf': 1,
+                                       'mod': intTime()}}),
+                   simplejson.dumps({'1': anki.groups.defaultConf}),
                    "{}")
 
 def _updateIndices(db):
@@ -384,7 +366,7 @@ def _migrateDeckTbl(db):
     db.execute("""
 insert or replace into deck select id, cast(created as int), :t,
 :t, 99, 0, ifnull(syncName, ""), cast(lastSync as int),
-"", "", "" from decks""", t=intTime())
+"", "", "", "", "" from decks""", t=intTime())
     # update selective study
     qconf = anki.deck.defaultQconf.copy()
     # delete old selective study settings, which we can't auto-upgrade easily
@@ -415,10 +397,13 @@ insert or replace into deck select id, cast(created as int), :t,
             pass
         else:
             conf[k] = v
-    db.execute("update deck set qconf = :l, conf = :c, data = :d",
+    import anki.groups
+    db.execute("update deck set qconf = :l,conf = :c,data = :d,groups=:g,gconf=:gc",
                l=simplejson.dumps(qconf),
                c=simplejson.dumps(conf),
-               d=simplejson.dumps(data))
+               d=simplejson.dumps(data),
+               g=simplejson.dumps({'1': {'name': _("Default"), 'conf': 1}}),
+               gc=simplejson.dumps({'1': anki.groups.defaultConf}))
     # clean up
     db.execute("drop table decks")
     db.execute("drop table deckVars")

tests/test_deck.py

@@ -133,18 +133,15 @@ def test_upgrade():
 def test_groups():
     deck = getEmptyDeck()
     # we start with a standard group
-    assert len(deck.groups()) == 1
+    assert len(deck.groups) == 1
     # it should have an id of 1
-    assert deck.groupId(deck.groups()[0]) == 1
+    assert deck.groups['1']
     # create a new group
-    assert deck.groupId("new group") == 2
+    ts = deck.groupID("new group")
-    assert len(deck.groups()) == 2
+    assert ts
+    assert len(deck.groups) == 2
     # should get the same id
-    assert deck.groupId("new group") == 2
+    assert deck.groupID("new group") == ts
-    # deleting a group should not recycle ids
-    deck.delGroup(2)
-    assert len(deck.groups()) == 1
-    assert deck.groupId("another group") == 3
     # by default, everything should be shown
     assert not deck.qconf['groups']
 

tests/test_sched.py

@@ -582,7 +582,7 @@ def test_ordcycle():
 def test_counts():
     d = getEmptyDeck()
     # add a second group
-    assert d.groupId("new group") == 2
+    assert d.groupID("new group")
     # for each card type
     for type in range(3):
         # and each of the groups
@@ -681,7 +681,7 @@ def test_groupCounts():
     # and one that's a child
     f = d.newFact()
     f['Front'] = u"two"
-    f.gid = d.groupId("Default::1")
+    default1 = f.gid = d.groupID("Default::1")
     d.addFact(f)
     # make it a review card
     c = f.cards()[0]
@@ -691,21 +691,21 @@ def test_groupCounts():
     # add one more with a new group
     f = d.newFact()
     f['Front'] = u"two"
-    f.gid = d.groupId("foo::bar")
+    foobar = f.gid = d.groupID("foo::bar")
     d.addFact(f)
     # and one that's a sibling
     f = d.newFact()
     f['Front'] = u"three"
-    f.gid = d.groupId("foo::baz")
+    foobaz = f.gid = d.groupID("foo::baz")
     d.addFact(f)
     d.reset()
     assert d.sched.counts() == (3, 0, 1)
-    assert len(d.groups()) == 4
+    assert len(d.groups) == 4
     cnts = d.sched.groupCounts()
     assert cnts[0] == ["Default", 1, 1, 0, 1]
-    assert cnts[1] == ["Default::1", 2, 1, 1, 0]
+    assert cnts[1] == ["Default::1", default1, 1, 1, 0]
-    assert cnts[2] == ["foo::bar", 3, 1, 0, 1]
+    assert cnts[2] == ["foo::bar", foobar, 1, 0, 1]
-    assert cnts[3] == ["foo::baz", 4, 1, 0, 1]
+    assert cnts[3] == ["foo::baz", foobaz, 1, 0, 1]
     tree = d.sched.groupCountTree()
     assert tree[0][0] == "Default"
     # sum of child and parent
@@ -715,7 +715,7 @@ def test_groupCounts():
     assert tree[0][4] == 1
     # child count is just review
     assert tree[0][5][0][0] == "1"
-    assert tree[0][5][0][1] == 2
+    assert tree[0][5][0][1] == default1
     assert tree[0][5][0][2] == 1
     assert tree[0][5][0][3] == 1
     assert tree[0][5][0][4] == 0