diff --git a/anki/groups.py b/anki/groups.py index ae0f33d3a..a290c934b 100644 --- a/anki/groups.py +++ b/anki/groups.py @@ -2,9 +2,10 @@ # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import simplejson +import simplejson, copy from anki.utils import intTime, ids2str from anki.consts import * +from anki.lang import _ # fixmes: # - make sure lists like new[delays] are not being shared by multiple groups @@ -19,14 +20,19 @@ from anki.consts import * # references, and treat any invalid gids as the default group. # - deletions of group config force a full sync -# configuration only available to top level groups -defaultTopConf = { - 'newPerDay': 20, +# these are a cache of the current day's reviews. they may be wrong after a +# sync merge if someone reviewed from two locations +defaultGroup = { 'newToday': [0, 0], # currentDay, count 'revToday': [0, 0], 'lrnToday': [0, 0], - 'timeToday': [0, 0], # currentDay, time in ms - 'newTodayOrder': NEW_TODAY_ORD, + 'timeToday': [0, 0], # time in ms + 'conf': 1, +} + +# configuration only available to top level groups +defaultTopConf = { + 'revLim': 100, 'newSpread': NEW_CARDS_DISTRIBUTE, 'collapseTime': 1200, 'repLim': 0, @@ -40,6 +46,8 @@ defaultConf = { 'delays': [1, 10], 'ints': [1, 7, 4], 'initialFactor': 2500, + 'order': NEW_TODAY_ORD, + 'perDay': 20, }, 'lapse': { 'delays': [1, 10], @@ -110,8 +118,9 @@ class GroupManager(object): # not top level; ensure all parents exist g = {} self._ensureParents(name) + for (k,v) in defaultGroup.items(): + g[k] = v g['name'] = name - g['conf'] = 1 while 1: id = intTime(1000) if str(id) not in self.groups: @@ -119,7 +128,7 @@ class GroupManager(object): g['id'] = id self.groups[str(id)] = g self.save(g) - self.maybeAddToActive(g) + self.maybeAddToActive() return int(id) def rem(self, gid, cardsToo=False): @@ -142,9 +151,19 @@ class GroupManager(object): "A list of all groups." return self.groups.values() - def allConf(self): - "A list of all group config." - return self.gconf.values() + def get(self, gid, default=True): + id = str(gid) + if id in self.groups: + return self.groups[id] + elif default: + return self.groups['1'] + + def update(self, g): + "Add or update an existing group. Used for syncing and merging." + self.groups[str(g['id'])] = g + self.maybeAddToActive() + # mark registry changed, but don't bump mod time + self.save() def _ensureParents(self, name): path = name.split("::") @@ -156,42 +175,61 @@ class GroupManager(object): s += "::" + p self.id(s) + # Group configurations + ############################################################# + + def allConf(self): + "A list of all group config." + return self.gconf.values() + + def conf(self, gid): + return self.gconf[str(self.groups[str(gid)]['conf'])] + + def updateConf(self, g): + self.gconf[str(g['id'])] = g + self.save() + + def confId(self, name): + "Create a new configuration and return id." + c = copy.deepcopy(defaultConf) + while 1: + id = intTime(1000) + if str(id) not in self.gconf: + break + c['id'] = id + self.gconf[str(id)] = c + self.save(c) + return id + + def remConf(self, id): + "Remove a configuration and update all groups using it." + self.deck.modSchema() + del self.gconf[str(id)] + for g in self.all(): + if g['conf'] == id: + g['conf'] = 1 + self.save(g) + + def setConf(self, grp, id): + grp['conf'] = id + self.save(grp) + # Group utils ############################################################# def name(self, gid): return self.get(gid)['name'] - def conf(self, gid): - return self.gconf[str(self.groups[str(gid)]['conf'])] - - def get(self, gid, default=True): - id = str(gid) - if id in self.groups: - return self.groups[id] - elif default: - return self.groups['1'] - def setGroup(self, cids, gid): self.deck.db.execute( "update cards set gid=?,usn=?,mod=? where id in "+ ids2str(cids), gid, self.deck.usn(), intTime()) - def update(self, g): - "Add or update an existing group. Used for syncing and merging." - self.groups[str(g['id'])] = g - self.maybeAddToActive(g) - # mark registry changed, but don't bump mod time - self.save() - def maybeAddToActive(self, g): + def maybeAddToActive(self): # since order is important, we can't just append to the end self.select(self.selected()) - def updateConf(self, g): - self.gconf[str(g['id'])] = g - self.save() - def sendHome(self, cids): self.deck.db.execute(""" update cards set gid=(select gid from facts f where f.id=fid), @@ -205,9 +243,8 @@ usn=?,mod=? where id in %s""" % ids2str(cids), ############################################################# def top(self): - "The current top level group as an object, and marks as modified." + "The current top level group as an object." g = self.get(self.deck.conf['topGroup']) - self.save(g) return g def active(self): @@ -222,23 +259,23 @@ usn=?,mod=? where id in %s""" % ids2str(cids), "Select a new branch." # save the top level group name = self.groups[str(gid)]['name'] - self.deck.conf['topGroup'] = self.topFor(name) + self.deck.conf['topGroup'] = self._topFor(name) # current group self.deck.conf['curGroup'] = gid # and active groups (current + all children) - actv = [gid] + actv = [] for g in self.all(): if g['name'].startswith(name + "::"): - actv.append(g['id']) - self.deck.conf['activeGroups'] = actv + actv.append((g['name'], g['id'])) + actv.sort() + self.deck.conf['activeGroups'] = [gid] + [a[1] for a in actv] - def topFor(self, name): + def parents(self, gid): + "All parents of gid." + path = self.get(gid)['name'].split("::") + return [self.get(x) for x in path[:-1]] + + def _topFor(self, name): "The top level gid for NAME." path = name.split("::") return self.id(path[0]) - - def underSelected(self, name): - "True if name is under the selected group." - # if nothing is selected, always true - s = self.selected() - return name.startswith(self.get(s)['name']) diff --git a/anki/models.py b/anki/models.py index 15f774224..3fcbb4b70 100644 --- a/anki/models.py +++ b/anki/models.py @@ -99,7 +99,9 @@ class ModelManager(object): return self.models.values()[0] def setCurrent(self, m): - self.deck.groups.top()['curModel'] = m['id'] + t = self.deck.groups.top() + t['curModel'] = m['id'] + self.deck.groups.save(t) def get(self, id): "Get model with ID, or None." diff --git a/anki/sched.py b/anki/sched.py index 387b52b60..4a32c5fd1 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -20,8 +20,9 @@ class Scheduler(object): name = "std" def __init__(self, deck): self.deck = deck - self.queueLimit = 200 + self.queueLimit = 50 self.reportLimit = 1000 + # fixme: replace reps with group based counts self.reps = 0 self._updateCutoff() @@ -35,7 +36,7 @@ class Scheduler(object): return c def reset(self): - self._resetCounts() + self._updateCutoff() self._resetLrn() self._resetRev() self._resetNew() @@ -50,20 +51,23 @@ class Scheduler(object): # put it in the learn queue card.queue = 1 card.type = 1 - self._updateStats('new') + self._updateStats(card, 'new') if card.queue == 1: self._answerLrnCard(card, ease) if not wasNew: - self._updateStats('lrn') + self._updateStats(card, 'lrn') elif card.queue == 2: self._answerRevCard(card, ease) - self._updateStats('rev') + self._updateStats(card, 'rev') else: raise Exception("Invalid queue") - self._updateStats('time', card.timeTaken()) + self._updateStats(card, 'time', card.timeTaken()) card.mod = intTime() card.usn = self.deck.usn() card.flushSched() + # if nothing more to study, rebuild queue + if self.counts() == (0,0,0): + self.reset() def counts(self): "Does not include fetched but unanswered." @@ -101,24 +105,19 @@ order by due""" % self._groupLimit(), self.deck.db.execute( "update cards set queue = type where queue between -3 and -2") - # Counts + # Rev/lrn/time daily stats ########################################################################## - def _resetCounts(self): - self._updateCutoff() - self._resetLrnCount() - self._resetRevCount() - self._resetNewCount() - self._updateStatsDay("time") - - def _updateStatsDay(self, type): - l = self.deck.groups.top() - if l[type+'Today'][0] != self.today: - # it's a new day; reset counts - l[type+'Today'] = [self.today, 0] - - def _updateStats(self, type, cnt=1): - self.deck.groups.top()[type+'Today'][1] += cnt + def _updateStats(self, card, type, cnt=1): + key = type+"Today" + for g in ([self.deck.groups.get(card.gid)] + + self.deck.groups.parents(card.gid)): + # ensure we're on the correct day + if g[key][0] != self.today: + g[key] = [self.today, 0] + # add + g[key][1] += cnt + self.deck.groups.save(g) # Group counts ########################################################################## @@ -128,22 +127,8 @@ order by due""" % self._groupLimit(), # find groups with 1 or more due cards gids = {} for g in self.deck.groups.all(): - hasDue = self.deck.db.scalar(""" -select 1 from cards where gid = ? and -((queue = 2 and due <= ?) or (queue = 1 and due < ?)) limit 1""", - g['id'], self.today, intTime()) - top = self.deck.groups.get( - self.deck.groups.topFor(g['name'])) - if top['newToday'][0] != self.today: - # it's a new day; reset counts - top['newToday'] = [self.today, 0] - hasNew = max(0, top['newPerDay'] - top['newToday'][1]) - if hasNew: - # if the limit hasn't run out, check to see if there are - # actually cards - hasNew = self.deck.db.scalar( - "select 1 from cards where queue = 0 and gid = ? limit 1", - g['id']) + hasDue = self._groupHasLrn(g['id']) or self._groupHasRev(g['id']) + hasNew = self._groupHasNew(g['id']) gids[g['id']] = [hasDue or 0, hasNew or 0] return [[grp['name'], int(gid)]+gids.get(int(gid)) for (gid, grp) in self._orderedGroups()] @@ -222,43 +207,74 @@ select 1 from cards where gid = ? and ########################################################################## def _resetNewCount(self): - self._updateStatsDay("new") - l = self.deck.groups.top() - lim = min(self.reportLimit, l['newPerDay'] - l['newToday'][1]) - if lim <= 0: - self.newCount = 0 - else: - self.newCount = self.deck.db.scalar(""" -select count() from (select id from cards where -gid in %s and queue = 0 limit %d)""" % (self._groupLimit(), lim)) + self.newCount = 0 + pcounts = {} + # for each of the active groups + for gid in self.deck.groups.active(): + # get the individual group's limit + lim = self._groupNewLimitSingle(self.deck.groups.get(gid)) + if not lim: + continue + # check the parents + parents = self.deck.groups.parents(gid) + for p in parents: + # add if missing + if p['id'] not in pcounts: + pcounts[p['id']] = self._groupNewLimitSingle(p) + # take minimum of child and parent + lim = min(pcounts[p['id']], lim) + # see how many cards we actually have + cnt = self.deck.db.scalar(""" +select count() from (select 1 from cards where +gid = ? and queue = 0 limit ?)""", gid, lim) + # if non-zero, decrement from parent counts + for p in parents: + pcounts[p['id']] -= cnt + # we may also be a parent + pcounts[gid] = lim - cnt + # and add to running total + self.newCount += cnt def _resetNew(self): - lim = min(self.queueLimit, self.newCount) - self.newQueue = self.deck.db.all(""" -select id, due from cards where -gid in %s and queue = 0 limit %d""" % (self._groupLimit(), - lim)) - self.newQueue.reverse() + self._resetNewCount() + self.newGids = self.deck.groups.active() + self._newQueue = [] self._updateNewCardRatio() + def _fillNew(self): + if self._newQueue: + return True + if not self.newCount: + return False + while self.newGids: + gid = self.newGids[0] + lim = min(self.queueLimit, self._groupNewLimit(gid)) + if lim: + # fill the queue with the current gid + self._newQueue = self.deck.db.all(""" +select id, due from cards where gid = ? and queue = 0 limit ?""", gid, lim) + if self._newQueue: + self._newQueue.reverse() + return True + # nothing left in the group; move to next + self.newGids.pop(0) + def _getNewCard(self): - # We rely on sqlite to return the cards in id order. This may not - # correspond to the 'ord' order. The alternative would be to do - # something like due = fid*100+ord, but then we have no efficient way - # of spacing siblings as we'd need to fetch the fid as well. - if self.newQueue: - (id, due) = self.newQueue.pop() - # move any siblings to the end? - if self.deck.groups.top()['newTodayOrder'] == NEW_TODAY_ORD: - n = len(self.newQueue) - while self.newQueue and self.newQueue[-1][1] == due: - self.newQueue.insert(0, self.newQueue.pop()) - n -= 1 - if not n: - # we only have one fact in the queue; stop rotating - break - self.newCount -= 1 - return id + if not self._fillNew(): + return + (id, due) = self._newQueue.pop() + # move any siblings to the end? + conf = self.deck.groups.conf(self.newGids[0]) + if conf['new']['order'] == NEW_TODAY_ORD: + n = len(self._newQueue) + while self._newQueue and self._newQueue[-1][1] == due: + self._newQueue.insert(0, self._newQueue.pop()) + n -= 1 + if not n: + # we only have one fact in the queue; stop rotating + break + self.newCount -= 1 + return id def _updateNewCardRatio(self): if self.deck.groups.top()['newSpread'] == NEW_CARDS_DISTRIBUTE: @@ -282,30 +298,65 @@ gid in %s and queue = 0 limit %d""" % (self._groupLimit(), elif self.newCardModulus: return self.reps and self.reps % self.newCardModulus == 0 + def _groupHasNew(self, gid): + if not self._groupNewLimit(gid): + return False + return self.deck.db.scalar( + "select 1 from cards where gid = ? and queue = 0 limit 1", gid) + + def _groupNewLimit(self, gid): + sel = self.deck.groups.get(gid) + lim = -1 + # for the group and each of its parents + for g in [sel] + self.deck.groups.parents(gid): + rem = self._groupNewLimitSingle(g) + if lim == -1: + lim = rem + else: + lim = min(rem, lim) + return lim + + def _groupNewLimitSingle(self, g): + # update day if necessary + if g['newToday'][0] != self.today: + g['newToday'] = [self.today, 0] + c = self.deck.groups.conf(g['id']) + return max(0, c['new']['perDay'] - g['newToday'][1]) + # Learning queue ########################################################################## def _resetLrnCount(self): - self._updateStatsDay("lrn") self.lrnCount = self.deck.db.scalar(""" -select count() from (select id from cards where +select count() from (select 1 from cards where gid in %s and queue = 1 and due < ? limit %d)""" % ( self._groupLimit(), self.reportLimit), intTime() + self.deck.groups.top()['collapseTime']) def _resetLrn(self): - self.lrnQueue = self.deck.db.all(""" + self._resetLrnCount() + self._lrnQueue = [] + + def _fillLrn(self): + if not self.lrnCount: + return False + if self._lrnQueue: + return True + self._lrnQueue = self.deck.db.all(""" select due, id from cards where gid in %s and queue = 1 and due < :lim limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff) + # as it arrives sorted by gid first, we need to sort it + self._lrnQueue.sort() + return self._lrnQueue def _getLrnCard(self, collapse=False): - if self.lrnQueue: + if self._fillLrn(): cutoff = time.time() if collapse: cutoff += self.deck.groups.top()['collapseTime'] - if self.lrnQueue[0][0] < cutoff: - id = heappop(self.lrnQueue)[1] + if self._lrnQueue[0][0] < cutoff: + id = heappop(self._lrnQueue)[1] self.lrnCount -= 1 return id @@ -334,7 +385,7 @@ limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff) # not collapsed; add some randomness delay *= random.uniform(1, 1.25) card.due = int(time.time() + delay) - heappush(self.lrnQueue, (card.due, card.id)) + heappush(self._lrnQueue, (card.due, card.id)) # if it's due within the cutoff, increment count if delay <= self.deck.groups.top()['collapseTime']: self.lrnCount += 1 @@ -410,11 +461,22 @@ where queue = 1 and type = 2 %s """ % (intTime(), self.deck.usn(), extra)) + def _groupHasLrn(self, gid): + return self.deck.db.scalar( + "select 1 from cards where gid = ? and queue = 1 " + "and due < ? limit 1", + gid, intTime() + self.deck.groups.top()['collapseTime']) + # Reviews ########################################################################## + def _groupHasRev(self, gid): + return self.deck.db.scalar( + "select 1 from cards where gid = ? and queue = 2 " + "and due <= ? limit 1", + gid, self.today) + def _resetRevCount(self): - self._updateStatsDay("rev") self.revCount = self.deck.db.scalar(""" select count() from (select id from cards where gid in %s and queue = 2 and due <= :lim limit %d)""" % ( @@ -422,25 +484,29 @@ gid in %s and queue = 2 and due <= :lim limit %d)""" % ( lim=self.today) def _resetRev(self): - self.revQueue = self.deck.db.list(""" + self._resetRevCount() + self._revQueue = [] + + def _fillRev(self): + if not self.revCount: + return False + if self._revQueue: + return True + self._revQueue = self.deck.db.list(""" select id from cards where gid in %s and queue = 2 and due <= :lim %s limit %d""" % ( self._groupLimit(), self._revOrder(), self.queueLimit), lim=self.today) - r = random.Random() - r.seed(self.today) - r.shuffle(self.revQueue) + if not self.deck.conf['revOrder']: + r = random.Random() + r.seed(self.today) + r.shuffle(self._revQueue) + return True def _getRevCard(self): - if self._haveRevCards(): + if self._fillRev(): self.revCount -= 1 - return self.revQueue.pop() - - def _haveRevCards(self): - if self.revCount: - if not self.revQueue: - self._resetRev() - return self.revQueue + return self._revQueue.pop() def _revOrder(self): if self.deck.conf['revOrder']: @@ -472,7 +538,7 @@ gid in %s and queue = 2 and due <= :lim %s limit %d""" % ( self.lrnCount += 1 # leech? if not self._checkLeech(card, conf) and conf['relearn']: - heappush(self.lrnQueue, (card.due, card.id)) + heappush(self._lrnQueue, (card.due, card.id)) def _nextLapseIvl(self, card, conf): return int(card.ivl*conf['mult']) + 1 @@ -643,6 +709,8 @@ gid in %s and queue = 2 and due <= :lim %s limit %d""" % ( def newTomorrow(self): "Number of new cards tomorrow." + print "fixme: rethink newTomorrow() etc" + return 1 lim = self.deck.groups.top()['newPerDay'] return self.deck.db.scalar( "select count() from (select id from cards where " diff --git a/anki/storage.py b/anki/storage.py index ab9661f73..ff395351c 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -135,18 +135,26 @@ insert or ignore into deck values(1,0,0,0,%(v)s,0,0,0,'','{}','','','{}'); """ % ({'v':CURRENT_VERSION})) import anki.deck - import anki.groups if setDeckConf: - g = anki.groups.defaultTopConf.copy() - g['id'] = 1 - g['name'] = _("Default") - g['conf'] = 1 - g['mod'] = intTime() - gc = anki.groups.defaultConf.copy() - gc['id'] = 1 - db.execute(""" + _addDeckVars(db, *_getDeckVars(db)) + +def _getDeckVars(db): + import anki.groups + g = anki.groups.defaultGroup.copy() + for k,v in anki.groups.defaultTopConf.items(): + g[k] = v + g['id'] = 1 + g['name'] = _("Default") + g['conf'] = 1 + g['mod'] = intTime() + gc = anki.groups.defaultConf.copy() + gc['id'] = 1 + return g, gc, anki.deck.defaultConf.copy() + +def _addDeckVars(db, g, gc, c): + db.execute(""" update deck set conf = ?, groups = ?, gconf = ?""", - simplejson.dumps(anki.deck.defaultConf), + simplejson.dumps(c), simplejson.dumps({'1': g}), simplejson.dumps({'1': gc})) @@ -356,14 +364,7 @@ insert or replace into deck select id, cast(created as int), :t, :t, 99, 0, 0, cast(lastSync as int), "", "", "", "", "" from decks""", t=intTime()) # prepare a group to store the old deck options - import anki.groups - g = anki.groups.defaultTopConf.copy() - g['id'] = 1 - g['name'] = _("Default") - g['conf'] = 1 - g['mod'] = intTime() - # and deck conf - conf = anki.deck.defaultConf.copy() + g, gc, conf = _getDeckVars(db) # delete old selective study settings, which we can't auto-upgrade easily keys = ("newActive", "newInactive", "revActive", "revInactive") for k in keys: @@ -373,7 +374,6 @@ insert or replace into deck select id, cast(created as int), :t, g['newPerDay'] = db.scalar("select newCardsPerDay from decks") g['repLim'] = db.scalar("select sessionRepLimit from decks") g['timeLim'] = db.scalar("select sessionTimeLimit from decks") - # this needs to be placed in the model later on conf['oldNewOrder'] = db.scalar("select newCardOrder from decks") # no reverse option anymore @@ -385,10 +385,7 @@ insert or replace into deck select id, cast(created as int), :t, pass else: conf[k] = v - db.execute("update deck set conf=:c,groups=:g,gconf=:gc", - c=simplejson.dumps(conf), - g=simplejson.dumps({'1': g}), - gc=simplejson.dumps({'1': anki.groups.defaultConf})) + _addDeckVars(db, g, gc, conf) # clean up db.execute("drop table decks") db.execute("drop table deckVars") diff --git a/tests/test_sched.py b/tests/test_sched.py index 0b1dd68c5..6e46f124f 100644 --- a/tests/test_sched.py +++ b/tests/test_sched.py @@ -48,6 +48,36 @@ def test_new(): assert(stripHTML(c.q()) == qs[n]) d.sched.answerCard(c, 2) +def test_newLimits(): + d = getEmptyDeck() + # add some facts + g2 = d.groups.id("Default::foo") + for i in range(30): + f = d.newFact() + f['Front'] = str(i) + if i > 4: + f.gid = g2 + d.addFact(f) + # give the child group a different configuration + c2 = d.groups.confId("new conf") + d.groups.setConf(d.groups.get(g2), c2) + d.reset() + # both confs have defaulted to a limit of 20 + assert d.sched.newCount == 20 + # first card we get comes from parent + c = d.sched.getCard() + assert c.gid == 1 + # limit the parent to 10 cards, meaning we get 10 in total + conf1 = d.groups.conf(1) + conf1['new']['perDay'] = 10 + d.reset() + assert d.sched.newCount == 10 + # if we limit child to 4, we should get 9 + conf2 = d.groups.conf(g2) + conf2['new']['perDay'] = 4 + d.reset() + assert d.sched.newCount == 9 + def test_newOrder(): d = getEmptyDeck() m = d.models.current() @@ -69,7 +99,7 @@ def test_newOrder(): d.conf['newPerDay'] = 100 d.reset() # cards should be sorted by id - assert d.sched.newQueue == list(reversed(sorted(d.sched.newQueue))) + assert d.sched._newQueue == list(reversed(sorted(d.sched._newQueue))) def test_newBoxes(): d = getEmptyDeck() @@ -725,6 +755,30 @@ def test_groupCounts(): d.sched.groupCounts() d.sched.groupCountTree() +def test_groupFlow(): + d = getEmptyDeck() + # add a fact with default group + f = d.newFact() + f['Front'] = u"one" + d.addFact(f) + # and one that's a child + f = d.newFact() + f['Front'] = u"two" + default1 = f.gid = d.groups.id("Default::2") + d.addFact(f) + # and another that's higher up + f = d.newFact() + f['Front'] = u"three" + default1 = f.gid = d.groups.id("Default::1") + d.addFact(f) + # should get top level one first, then ::1, then ::2 + d.reset() + assert d.sched.counts() == (3,0,0) + for i in "one", "three", "two": + c = d.sched.getCard() + assert c.fact()['Front'] == i + d.sched.answerCard(c, 2) + def test_reorder(): d = getEmptyDeck() # add a fact with default group