From 7afe6a9a7dcbfb9e689259474996e26f279ee6bf Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 27 Aug 2011 17:13:04 +0900 Subject: [PATCH] convert groups to json; use timestamp ids for all but default Rather than use a combination of id lookups on the groups table and a group configuration cache in the scheduler, I've moved the groups and group config into json objects on the deck table. This results in a net saving of code and saves one or more DB lookups on each card answer, in exchange for a small increase in deck load/save work. I did a quick survey of AnkiWeb, and the vast majority of decks use less than 100 tags, and it's safe to assume groups will follow a similar pattern. All groups and group configs except the default one will use integer timestamps now, to simplify merging when syncing and importing. defaultGroup() has been removed in favour of keeping the models up to date (not yet done). --- anki/cards.py | 4 +-- anki/cram.py | 1 - anki/deck.py | 80 +++++++++++++++++++-------------------------- anki/facts.py | 2 +- anki/find.py | 12 +++---- anki/latex.py | 2 +- anki/sched.py | 28 +++++++--------- anki/storage.py | 43 ++++++++---------------- tests/test_deck.py | 15 ++++----- tests/test_sched.py | 18 +++++----- 10 files changed, 83 insertions(+), 122 deletions(-) diff --git a/anki/cards.py b/anki/cards.py index 9ec9a2c98..a534fe687 100644 --- a/anki/cards.py +++ b/anki/cards.py @@ -104,12 +104,10 @@ lapses=?, grade=?, cycles=?, edue=? where id = ?""", def _getQA(self, reload=False): if not self._qa or reload: - gname = self.deck.db.scalar( - "select name from groups where id = ?", self.gid) f = self.fact(); m = self.model() data = [self.id, f.id, m.id, self.gid, self.ord, f.stringTags(), f.joinedFields()] - self._qa = self.deck._renderQA(self.model(), gname, data) + self._qa = self.deck._renderQA(self.model(), data) return self._qa def _withClass(self, txt, extra): diff --git a/anki/cram.py b/anki/cram.py index 6babb147a..048877e69 100644 --- a/anki/cram.py +++ b/anki/cram.py @@ -24,7 +24,6 @@ class CramScheduler(Scheduler): def reset(self): self._updateCutoff() - self._resetConf() self._resetLrnCount() self._resetLrn() self._resetNew() diff --git a/anki/deck.py b/anki/deck.py index 3e95a6873..f379fa94b 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -34,11 +34,7 @@ defaultQconf = { # other options defaultConf = { - 'currentModelId': 1, - 'currentGroupId': 1, 'nextPos': 1, - 'nextGid': 2, - 'nextGcid': 2, 'mediaURL': "", 'fontFamilies': [ [u'ＭＳ 明朝',u'ヒラギノ明朝 Pro W3',u'Kochi Mincho', u'東風明朝'] @@ -89,11 +85,15 @@ class _Deck(object): self.lastSync, self.qconf, self.conf, + self.groups, + self.gconf, self.data) = self.db.first(""" select crt, mod, scm, dty, syncName, lastSync, -qconf, conf, data from deck""") +qconf, conf, groups, gconf, data from deck""") self.qconf = simplejson.loads(self.qconf) self.conf = simplejson.loads(self.conf) + self.groups = simplejson.loads(self.groups) + self.gconf = simplejson.loads(self.gconf) self.data = simplejson.loads(self.data) def flush(self, mod=None): @@ -278,9 +278,9 @@ qconf=?, conf=?, data=?""", # [cid, fid, mid, gid, ord, tags, flds] data = [1, 1, model.id, 1, template['ord'], "", fact.joinedFields()] - now = self._renderQA(model, "", data) + now = self._renderQA(model, data) data[6] = "\x1f".join([""]*len(fact.fields)) - empty = self._renderQA(model, "", data) + empty = self._renderQA(model, data) if now['q'] == empty['q']: continue if not template['emptyAns']: @@ -334,7 +334,7 @@ qconf=?, conf=?, data=?""", card = anki.cards.Card(self) card.fid = fact.id card.ord = template['ord'] - card.gid = self.defaultGroup(template['gid'] or fact.gid) + card.gid = template['gid'] or fact.gid card.due = due if flush: card.flush() @@ -446,12 +446,10 @@ select id from cards where fid in (select id from facts where mid = ?)""", else: raise Exception() mods = self.models() - groups = dict(self.db.all("select id, name from groups")) - return [self._renderQA(mods[row[2]], groups[row[3]], row) + return [self._renderQA(mods[row[2]], row) for row in self._qaData(where)] - # fixme: don't need gid or data - def _renderQA(self, model, gname, data): + def _renderQA(self, model, data): "Returns hash of id, question, answer." # data is [cid, fid, mid, gid, ord, tags, flds] # unpack fields and create dict @@ -466,7 +464,7 @@ select id from cards where fid in (select id from facts where mid = ?)""", fields[name] = "" fields['Tags'] = data[5] fields['Model'] = model.name - fields['Group'] = gname + fields['Group'] = self.groupName(data[3]) template = model.templates[data[4]] fields['Template'] = template['name'] # render q & a @@ -480,10 +478,10 @@ select id from cards where fid in (select id from facts where mid = ?)""", else: name = "ca:" format = format.replace("cloze:", name) - fields = runFilter("mungeFields", fields, model, gname, data, self) + fields = runFilter("mungeFields", fields, model, data, self) html = anki.template.render(format, fields) d[type] = runFilter( - "mungeQA", html, type, fields, model, gname, data, self) + "mungeQA", html, type, fields, model, data, self) return d def _qaData(self, where=""): @@ -552,51 +550,41 @@ update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res # Groups ########################################################################## + # the id keys are strings because that's the way they're stored in json, + # but the anki code passes around integers - def groups(self): - "A list of all group names." - return self.db.list("select name from groups order by name") + def groupID(self, name, create=True): + "Add a group with NAME. Reuse group if already exists. Return id." + for id, g in self.groups.items(): + if g['name'].lower() == name.lower(): + return int(id) + if not create: + return None + g = dict(name=name, conf=1, mod=intTime()) + while 1: + id = intTime(1000) + if str(id) in self.groups: + continue + self.groups[str(id)] = g + return id - def groupName(self, id): - return self.db.scalar("select name from groups where id = ?", id) + def groupName(self, gid): + return self.groups[str(gid)]['name'] - def groupId(self, name): - "Return the id for NAME, creating if necessary." - id = self.db.scalar("select id from groups where name = ?", name) - if not id: - id = self.db.execute( - "insert into groups values (?,?,?,?, ?)", - self.nextID("gid"), intTime(), name, 1, - simplejson.dumps(anki.groups.defaultData)).lastrowid - return id - - def defaultGroup(self, id): - if id == 1: - return 1 - return self.db.scalar("select id from groups where id = ?", id) or 1 + def groupConf(self, gid): + return self.gconf[str(self.groups[str(gid)]['conf'])] def delGroup(self, gid): self.modSchema() self.db.execute("update cards set gid = 1 where gid = ?", gid) self.db.execute("update facts set gid = 1 where gid = ?", gid) self.db.execute("delete from groups where id = ?", gid) + print "fixme: loop through models and update stale gid references" def setGroup(self, cids, gid): self.db.execute( "update cards set gid = ? where id in "+ids2str(cids), gid) - # Group configuration - ########################################################################## - - def groupConfs(self): - "Return [name, id]." - return self.db.all("select name, id from gconf order by name") - - def groupConf(self, gcid): - return simplejson.loads( - self.db.scalar( - "select conf from gconf where id = ?", gcid)) - # Tag-based selective study ########################################################################## diff --git a/anki/facts.py b/anki/facts.py index 1d41004b7..45898a9e5 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -19,7 +19,7 @@ class Fact(object): else: self.id = timestampID(deck.db, "facts") self._model = model - self.gid = deck.defaultGroup(model.conf['gid']) + self.gid = model.conf['gid'] self.mid = model.id self.tags = [] self.fields = [""] * len(self._model.fields) diff --git a/anki/find.py b/anki/find.py index cfb074a42..5959111a6 100644 --- a/anki/find.py +++ b/anki/find.py @@ -121,7 +121,7 @@ order by %s""" % (lim, sort) elif type == SEARCH_MODEL: self._findModel(token, isNeg, c) elif type == SEARCH_GROUP: - self._findGroup(token, isNeg, c) + self._findGroup(token, isNeg) else: self._findText(token, isNeg, c) @@ -189,12 +189,10 @@ order by %s""" % (lim, sort) extra, c)) self.lims['args']['_mod_%d'%c] = val - def _findGroup(self, val, isNeg, c): - extra = "not" if isNeg else "" - self.lims['card'].append( - "c.gid %s in (select id from groups where name like :_grp_%d)" % ( - extra, c)) - self.lims['args']['_grp_%d'%c] = val + def _findGroup(self, val, isNeg): + extra = "!" if isNeg else "" + id = self.deck.groupID(val, create=False) or 0 + self.lims['card'].append("c.gid %s= %d" % (extra, id)) def _findTemplate(self, val, isNeg): lims = [] diff --git a/anki/latex.py b/anki/latex.py index e45544bfd..c31a11457 100644 --- a/anki/latex.py +++ b/anki/latex.py @@ -30,7 +30,7 @@ def stripLatex(text): text = text.replace(match.group(), "") return text -def mungeQA(html, type, fields, model, gname, data, deck): +def mungeQA(html, type, fields, model, data, deck): "Convert TEXT with embedded latex tags to image links." for match in regexps['standard'].finditer(html): html = html.replace(match.group(), _imgLink(deck, match.group(1), model)) diff --git a/anki/sched.py b/anki/sched.py index f8f2cf13a..3af84e5fd 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -35,7 +35,6 @@ class Scheduler(object): return c def reset(self): - self._resetConf() self._resetCounts() self._resetLrn() self._resetRev() @@ -151,18 +150,23 @@ sum(case when queue = 2 and due <= ? then 1 else 0 end), sum(case when queue = 0 then 1 else 0 end) from cards group by gid""", self.today): gids[gid] = [all, rev, new] - return [[name, gid]+gids.get(gid, [0, 0, 0]) for (gid, name) in - self.deck.db.execute( - "select id, name from groups order by name")] + return [[grp['name'], int(gid)]+gids.get(int(gid), [0, 0, 0]) + for (gid, grp) in self._orderedGroups()] + + def _orderedGroups(self): + grps = self.deck.groups.items() + def key(grp): + return grp[1]['name'] + grps.sort(key=key) + return grps def groupCountTree(self): return self._groupChildren(self.groupCounts()) def groupTree(self): "Like the count tree without the counts. Faster." - return self._groupChildren([[name, gid, 0, 0, 0] for (gid, name) in - self.deck.db.execute( - "select id, name from groups order by name")]) + return self._groupChildren([[grp['name'], int(gid), 0, 0, 0] + for (gid, grp) in self._orderedGroups()]) def _groupChildren(self, grps): tree = [] @@ -590,16 +594,8 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % ( # Tools ########################################################################## - def _resetConf(self): - "Update group conf cache." - self.groupConfs = dict(self.deck.db.all("select id, gcid from groups")) - self.confCache = {} - def _cardConf(self, card): - id = self.groupConfs[card.gid] - if id not in self.confCache: - self.confCache[id] = self.deck.groupConf(id) - return self.confCache[id] + return self.deck.groupConf(card.gid) def _groupLimit(self): l = self.deck.qconf['groups'] diff --git a/anki/storage.py b/anki/storage.py index abc657f27..312a4f1af 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -67,6 +67,8 @@ create table if not exists deck ( lastSync integer not null, qconf text not null, conf text not null, + groups text not null, + gconf text not null, data text not null ); @@ -116,21 +118,6 @@ create table if not exists models ( css text not null ); -create table if not exists groups ( - id integer primary key, - mod integer not null, - name text not null, - gcid integer not null, - data text not null -); - -create table if not exists gconf ( - id integer primary key, - mod integer not null, - name text not null, - conf text not null -); - create table if not exists graves ( time integer not null, oid integer not null, @@ -155,23 +142,18 @@ create table if not exists tags ( ); insert or ignore into deck -values(1,0,0,0,%(v)s,0,'',0,'', '', ''); +values(1,0,0,0,%(v)s,0,'',0,'','','','',''); """ % ({'v':CURRENT_VERSION})) import anki.deck import anki.groups - # create a default group/configuration, which should not be removed - db.execute( - "insert or ignore into gconf values (1, ?, ?, ?)""", - intTime(), _("Default"), - simplejson.dumps(anki.groups.defaultConf)) - db.execute( - "insert or ignore into groups values (1, ?, ?, 1, ?)", - intTime(), _("Default"), simplejson.dumps( - anki.groups.defaultData)) if setDeckConf: - db.execute("update deck set qconf = ?, conf = ?, data = ?", + db.execute(""" +update deck set qconf = ?, conf = ?, groups = ?, gconf = ?, data = ?""", simplejson.dumps(anki.deck.defaultQconf), simplejson.dumps(anki.deck.defaultConf), + simplejson.dumps({'1': {'name': _("Default"), 'conf': 1, + 'mod': intTime()}}), + simplejson.dumps({'1': anki.groups.defaultConf}), "{}") def _updateIndices(db): @@ -384,7 +366,7 @@ def _migrateDeckTbl(db): db.execute(""" insert or replace into deck select id, cast(created as int), :t, :t, 99, 0, ifnull(syncName, ""), cast(lastSync as int), -"", "", "" from decks""", t=intTime()) +"", "", "", "", "" from decks""", t=intTime()) # update selective study qconf = anki.deck.defaultQconf.copy() # delete old selective study settings, which we can't auto-upgrade easily @@ -415,10 +397,13 @@ insert or replace into deck select id, cast(created as int), :t, pass else: conf[k] = v - db.execute("update deck set qconf = :l, conf = :c, data = :d", + import anki.groups + db.execute("update deck set qconf = :l,conf = :c,data = :d,groups=:g,gconf=:gc", l=simplejson.dumps(qconf), c=simplejson.dumps(conf), - d=simplejson.dumps(data)) + d=simplejson.dumps(data), + g=simplejson.dumps({'1': {'name': _("Default"), 'conf': 1}}), + gc=simplejson.dumps({'1': anki.groups.defaultConf})) # clean up db.execute("drop table decks") db.execute("drop table deckVars") diff --git a/tests/test_deck.py b/tests/test_deck.py index 5cfdbd235..1f429fba1 100644 --- a/tests/test_deck.py +++ b/tests/test_deck.py @@ -133,18 +133,15 @@ def test_upgrade(): def test_groups(): deck = getEmptyDeck() # we start with a standard group - assert len(deck.groups()) == 1 + assert len(deck.groups) == 1 # it should have an id of 1 - assert deck.groupId(deck.groups()[0]) == 1 + assert deck.groups['1'] # create a new group - assert deck.groupId("new group") == 2 - assert len(deck.groups()) == 2 + ts = deck.groupID("new group") + assert ts + assert len(deck.groups) == 2 # should get the same id - assert deck.groupId("new group") == 2 - # deleting a group should not recycle ids - deck.delGroup(2) - assert len(deck.groups()) == 1 - assert deck.groupId("another group") == 3 + assert deck.groupID("new group") == ts # by default, everything should be shown assert not deck.qconf['groups'] diff --git a/tests/test_sched.py b/tests/test_sched.py index 14ff614e3..720b01984 100644 --- a/tests/test_sched.py +++ b/tests/test_sched.py @@ -582,7 +582,7 @@ def test_ordcycle(): def test_counts(): d = getEmptyDeck() # add a second group - assert d.groupId("new group") == 2 + assert d.groupID("new group") # for each card type for type in range(3): # and each of the groups @@ -681,7 +681,7 @@ def test_groupCounts(): # and one that's a child f = d.newFact() f['Front'] = u"two" - f.gid = d.groupId("Default::1") + default1 = f.gid = d.groupID("Default::1") d.addFact(f) # make it a review card c = f.cards()[0] @@ -691,21 +691,21 @@ def test_groupCounts(): # add one more with a new group f = d.newFact() f['Front'] = u"two" - f.gid = d.groupId("foo::bar") + foobar = f.gid = d.groupID("foo::bar") d.addFact(f) # and one that's a sibling f = d.newFact() f['Front'] = u"three" - f.gid = d.groupId("foo::baz") + foobaz = f.gid = d.groupID("foo::baz") d.addFact(f) d.reset() assert d.sched.counts() == (3, 0, 1) - assert len(d.groups()) == 4 + assert len(d.groups) == 4 cnts = d.sched.groupCounts() assert cnts[0] == ["Default", 1, 1, 0, 1] - assert cnts[1] == ["Default::1", 2, 1, 1, 0] - assert cnts[2] == ["foo::bar", 3, 1, 0, 1] - assert cnts[3] == ["foo::baz", 4, 1, 0, 1] + assert cnts[1] == ["Default::1", default1, 1, 1, 0] + assert cnts[2] == ["foo::bar", foobar, 1, 0, 1] + assert cnts[3] == ["foo::baz", foobaz, 1, 0, 1] tree = d.sched.groupCountTree() assert tree[0][0] == "Default" # sum of child and parent @@ -715,7 +715,7 @@ def test_groupCounts(): assert tree[0][4] == 1 # child count is just review assert tree[0][5][0][0] == "1" - assert tree[0][5][0][1] == 2 + assert tree[0][5][0][1] == default1 assert tree[0][5][0][2] == 1 assert tree[0][5][0][3] == 1 assert tree[0][5][0][4] == 0