reworked handling of spaced cards; add cms argument to previewFact()

- obsolete spaceUntil - it serves no useful purpose
- the old per-model spacing variables are obsolete, as the new approach
  requires uniform spacing across all models for new cards
- introduce a new per-deck variable: newSpacing
- don't fill new queue if we've done today's cards
- still need to check cramming / review early

newSpacing is a time in seconds to delay introduction of sibling new cards.
It can be applied as many times as necessary as there is no harm in new cards
being delayed repeatedly. Because the default queue length is 200 and it can
take quite some time for the spaced cards to be placed in the queue again, we
use a separate array to track spaced new cards provided the configured delay
is less than 20 minutes. At times under 20 minutes this number is not a
guaranteed minimum spacing - if the new card queue is empty the spaced cards
will be flushed before checking the new queue again, as otherwise we end up
trying to fill on every repetition. The due counts no longer decrease by more
than one if the spacing is less than the due cutoff, since that confused some
users.

Review cards are now placed at the end of the current review queue, and will
never be rescheduled to a different day. The old approach had a number of
problems:

- the more card models you had, the more likely a card would be spaced
  multiple times, resulting in you forgetting the card before you get a chance
  to review it
- spacing was applied even if the due card was already late
- repeatedly failing one card over a period of days or weeks would also stave
  the other cards of attention

anki/cards.py

@@ -58,7 +58,7 @@ cardsTable = Table(
     # data to the above
     Column('yesCount', Integer, nullable=False, default=0),
     Column('noCount', Integer, nullable=False, default=0),
-    # caching
+    # obsolete
     Column('spaceUntil', Float, nullable=False, default=0),
     # relativeDelay is reused as type without scheduling (ie, it remains 0-2
     # even if card is suspended, etc)
@@ -258,7 +258,7 @@ noCount=:noCount,
 spaceUntil = :spaceUntil,
 isDue = 0,
 type = :type,
-combinedDue = max(:spaceUntil, :due),
+combinedDue = :combinedDue,
 relativeDelay = 0,
 priority = :priority
 where id=:id""", self.__dict__)

anki/deck.py

@@ -161,7 +161,7 @@ class Deck(object):
         self.lastSessionStart = 0
         self.queueLimit = 200
         # if most recent deck var not defined, make sure defaults are set
-        if not self.s.scalar("select 1 from deckVars where key = 'revInactive'"):
+        if not self.s.scalar("select 1 from deckVars where key = 'newSpacing'"):
             self.setVarDefault("suspendLeeches", True)
             self.setVarDefault("leechFails", 16)
             self.setVarDefault("perDay", True)
@@ -169,6 +169,7 @@ class Deck(object):
             self.setVarDefault("revActive", "")
             self.setVarDefault("newInactive", self.suspended)
             self.setVarDefault("revInactive", self.suspended)
+            self.setVarDefault("newSpacing", 60)
         self.updateCutoff()
         self.setupStandardScheduler()
 
@@ -262,6 +263,7 @@ class Deck(object):
             "select count(*) from cards c where type = 2 "
             "and combinedDue < :lim"), lim=self.dueCutoff)
         self.updateNewCountToday()
+        self.spacedCards = []
 
     def _updateNewCountToday(self):
         self.newCountToday = max(min(
@@ -289,7 +291,7 @@ limit %d""" % (self.revOrder(), self.queueLimit)), lim=self.dueCutoff)
             self.revQueue.reverse()
 
     def _fillNewQueue(self):
-        if self.newCount and not self.newQueue:
+        if self.newCountToday and not self.newQueue and not self.spacedCards:
             self.newQueue = self.s.all(
                 self.cardLimit(
                 "newActive", "newInactive", """
@@ -298,37 +300,54 @@ type = 2 and combinedDue < :lim order by %s
 limit %d""" % (self.newOrder(), self.queueLimit)), lim=self.dueCutoff)
             self.newQueue.reverse()
 
-    def queueNotEmpty(self, queue, fillFunc):
+    def queueNotEmpty(self, queue, fillFunc, new=False):
         while True:
-            self.removeSpaced(queue)
+            self.removeSpaced(queue, new)
             if queue:
                 return True
             fillFunc()
             if not queue:
                 return False
 
-    def removeSpaced(self, queue):
+    def removeSpaced(self, queue, new=False):
+        popped = []
+        delay = None
         while queue:
             fid = queue[-1][1]
             if fid in self.spacedFacts:
-                if time.time() > self.spacedFacts[fid]:
-                    # no longer spaced; clean up
-                    del self.spacedFacts[fid]
-                else:
                 # still spaced
-                    queue.pop()
+                id = queue.pop()[0]
+                # assuming 10 cards/minute, track id if likely to expire
+                # before queue refilled
+                if new and self.newSpacing < self.queueLimit * 6:
+                    popped.append(id)
+                    delay = self.spacedFacts[fid]
             else:
+                if popped:
+                    self.spacedCards.append((delay, popped))
                 return
 
     def revNoSpaced(self):
         return self.queueNotEmpty(self.revQueue, self.fillRevQueue)
 
     def newNoSpaced(self):
-        return self.queueNotEmpty(self.newQueue, self.fillNewQueue)
+        return self.queueNotEmpty(self.newQueue, self.fillNewQueue, True)
 
     def _requeueCard(self, card, oldSuc):
+        newType = None
         try:
             if card.reps == 1:
+                if self.newFromCache:
+                    # fetched from spaced cache
+                    newType = 2
+                    cards = self.spacedCards.pop(0)[1]
+                    # reschedule the siblings
+                    if len(cards) > 1:
+                        self.spacedCards.append(
+                            (time.time() + self.newSpacing, cards[1:]))
+                else:
+                    # fetched from normal queue
+                    newType = 1
                     self.newQueue.pop()
             elif oldSuc == 0:
                 self.failedQueue.pop()
@@ -341,10 +360,11 @@ produce the problem.
 
 Counts %d %d %d
 Queue %d %d %d
-Card info: %d %d %d""" % (self.failedSoonCount, self.revCount, self.newCountToday,
+Card info: %d %d %d
+New type: %s""" % (self.failedSoonCount, self.revCount, self.newCountToday,
                           len(self.failedQueue), len(self.revQueue),
                           len(self.newQueue),
-                          card.reps, card.successive, oldSuc))
+                          card.reps, card.successive, oldSuc, `newType`))
 
     def revOrder(self):
         return ("priority desc, interval desc",
@@ -426,6 +446,9 @@ where type >= 0
             self.newCardModulus = 0
         # recache css
         self.rebuildCSS()
+        # spacing for delayed cards - not to be confused with newCardSpacing
+        # above
+        self.newSpacing = self.getFloat('newSpacing')
 
     def checkDailyStats(self):
         # check if the day has rolled over
@@ -630,13 +653,13 @@ limit %s""" % (self.cramOrder, self.queueLimit)))
                 return self.failedQueue[-1][0]
         # distribute new cards?
         if self.newNoSpaced() and self.timeForNewCard():
-            return self.newQueue[-1][0]
+            return self.getNewCard()
         # card due for review?
         if self.revNoSpaced():
             return self.revQueue[-1][0]
         # new cards left?
         if self.newCountToday:
-            return self.newQueue[-1][0]
+            return self.getNewCard()
         # display failed cards early/last
         if self.showFailedLast() and self.failedQueue:
             return self.failedQueue[-1][0]
@@ -673,6 +696,16 @@ limit %s""" % (self.cramOrder, self.queueLimit)))
         else:
             return False
 
+    def getNewCard(self):
+        if (not self.newQueue or
+            self.spacedCards and
+            self.spacedCards[0][0] < time.time()):
+            cards = self.spacedCards[0][1]
+            self.newFromCache = True
+            return cards[0]
+        self.newFromCache = False
+        return self.newQueue[-1][0]
+
     def showFailedLast(self):
         return self.collapseTime or not self.delay0
 
@@ -733,6 +766,7 @@ limit %s""" % (self.cramOrder, self.queueLimit)))
                     "finishedMsg": fin}
 
     def _getCardTables(self):
+        raise "needs to account for spaced new"
         t = time.time()
         c = self.getCard()
         sel = """
@@ -772,15 +806,15 @@ where id in """
             # only update if card was not new
             card.lastDue = card.due
         card.due = self.nextDue(card, ease, oldState)
+        card.combinedDue = card.due
         card.isDue = 0
         card.lastFactor = card.factor
-        card.spaceUntil = 0;
+        card.spaceUntil = 0
         if lastDelay >= 0:
             # don't update factor if learning ahead
             self.updateFactor(card, ease)
         # spacing
-        space = self.spaceUntilTime(card)
+        self.spaceCards(card)
-        self.spaceCards(card, space)
         # adjust counts for current card
         if ease == 1:
             if card.due < self.failedCutoff:
@@ -821,49 +855,30 @@ where id in """
         runHook("cardAnswered", card.id, isLeech)
         self.setUndoEnd(undoName)
 
-    def spaceUntilTime(self, card):
+    def _spaceCards(self, card):
-        (minSpacing, spaceFactor) = self.s.first("""
+        # update new counts
-select models.initialSpacing, models.spacing from
+        new = time.time() + self.newSpacing
-facts, models where facts.modelId = models.id and facts.id = :id""", id=card.factId)
+        if new > self.dueCutoff:
-        minOfOtherCards = self.s.scalar("""
+            self.newCount -= self.s.scalar("""
-select min(interval) from cards
+select count() from cards
-where factId = :fid and id != :id""", fid=card.factId, id=card.id) or 0
+where factId = :fid and id != :cid
-        if minOfOtherCards:
+and combinedDue < :cut and type = 2
-            space = min(minOfOtherCards, card.interval)
+""", cid=card.id, fid=card.factId, cut=self.dueCutoff, new=new)
-        else:
+        # space cards
-            space = 0
-        space = space * spaceFactor * 86400.0
-        space = max(minSpacing, space)
-        if space:
-            space += time.time()
-        return space
-
-    def _spaceCards(self, card, space):
-        if not space:
-            return
-        # adjust counts
-        for (type, count) in self.s.all("""
-select type, count(type) from cards
-where factId = :fid and
-combinedDue < :now and id != :cid
-group by type""", fid=card.factId, cid=card.id, now=self.dueCutoff):
-            if type == 0:
-                self.failedSoonCount -= count
-            elif type == 1:
-                self.revCount -= count
-            elif type == 2:
-                self.newCount -= count
-        # space other cards
         self.s.statement("""
 update cards set
-spaceUntil = :space,
+combinedDue = (case
-combinedDue = max(:space, due),
+when type = 1 then :cut - 1
+when type = 2 then :new
+end),
 modified = :now, isDue = 0
-where id != :id and factId = :factId""",
+where id != :id and factId = :factId
-                         id=card.id, space=space, now=time.time(),
+and combinedDue < :cut
-                         factId=card.factId)
+and type between 1 and 2""",
+                         id=card.id, now=time.time(), factId=card.factId,
+                         cut=self.dueCutoff, new=new)
         # update local cache of seen facts
-        self.spacedFacts[card.factId] = space
+        self.spacedFacts[card.factId] = time.time() + self.newSpacing
 
     def isLeech(self, card):
         no = card.noCount
@@ -1016,7 +1031,7 @@ where id in %s""" % ids2str(ids), now=time.time(), new=0)
         self.s.statements("""
 update cards
 set due = :rand + ordinal,
-combinedDue = max(:rand + ordinal, spaceUntil),
+combinedDue = :rand + ordinal,
 modified = :now
 where factId = :fid
 and relativeDelay = 2""", data)
@@ -1026,7 +1041,7 @@ and relativeDelay = 2""", data)
         self.s.statement("""
 update cards set
 due = created,
-combinedDue = max(spaceUntil, created),
+combinedDue = created,
 modified = :now
 where relativeDelay = 2""", now=time.time())
 
@@ -1533,9 +1548,10 @@ where facts.id not in (select distinct factId from cards)""")
         self.deleteFacts(ids)
         return ids
 
-    def previewFact(self, oldFact):
+    def previewFact(self, oldFact, cms=None):
         "Duplicate fact and generate cards for preview. Don't add to deck."
         # check we have card models available
+        if cms is None:
             cms = self.availableCardModels(oldFact, checkActive=True)
         if not cms:
             return []
@@ -2887,13 +2903,16 @@ select id from facts where spaceUntil like :_ff_%d escape '\\'""" % c
     # Meta vars
     ##########################################################################
 
-    def getInt(self, key):
+    def getInt(self, key, type=int):
         ret = self.s.scalar("select value from deckVars where key = :k",
                             k=key)
         if ret is not None:
-            ret = int(ret)
+            ret = type(ret)
         return ret
 
+    def getFloat(self, key):
+        return self.getInt(key, float)
+
     def getBool(self, key):
         ret = self.s.scalar("select value from deckVars where key = :k",
                             k=key)

anki/models.py

@@ -172,8 +172,8 @@ modelsTable = Table(
     Column('name', UnicodeText, nullable=False),
     Column('description', UnicodeText, nullable=False, default=u""), # obsolete
     Column('features', UnicodeText, nullable=False, default=u""), # used as mediaURL
-    Column('spacing', Float, nullable=False, default=0.1),
+    Column('spacing', Float, nullable=False, default=0.1), # obsolete
-    Column('initialSpacing', Float, nullable=False, default=60),
+    Column('initialSpacing', Float, nullable=False, default=60), # obsolete
     Column('source', Integer, nullable=False, default=0))
 
 class Model(object):