more type handling updates; don't munge counts on sync

In various parts of the code we need to get all cards of a given category
(new, failed, etc) regardless of whether they're suspended, buried, etc. So we
store the true type in the obsolete relativeDelay column and add in index for
it, because it's cheaper than putting indices on reps & successive.

anki/cards.py

@@ -60,7 +60,9 @@ cardsTable = Table(
     Column('noCount', Integer, nullable=False, default=0),
     # caching
     Column('spaceUntil', Float, nullable=False, default=0),
-    Column('relativeDelay', Float, nullable=False, default=0), # obsolete
+    # relativeDelay is reused as type without scheduling (ie, it remains 0-2
+    # even if card is suspended, etc)
+    Column('relativeDelay', Float, nullable=False, default=0),
     Column('isDue', Boolean, nullable=False, default=0), # obsolete
     Column('type', Integer, nullable=False, default=2),
     Column('combinedDue', Integer, nullable=False, default=0))
@@ -73,6 +75,7 @@ class Card(object):
         self.id = genID()
         # new cards start as new & due
         self.type = 2
+        self.relativeDelay = self.type
         self.timerStarted = False
         self.timerStopped = False
         self.modified = time.time()

anki/deck.py

@@ -68,7 +68,7 @@ SEARCH_FIELD = 6
 SEARCH_FIELD_EXISTS = 7
 SEARCH_QA = 8
 SEARCH_PHRASE_WB = 9
-DECK_VERSION = 49
+DECK_VERSION = 50
 
 deckVarsTable = Table(
     'deckVars', metadata,
@@ -357,14 +357,12 @@ Card info: %d %d %d""" % (self.failedSoonCount, self.revCount, self.newCountToda
         else:
             where = " where " + lim
         self.s.statement("""
-update cards
+update cards set
-set type = (case
+type = (case
-when successive = 0 and reps != 0
+when successive then 1 when reps then 0 else 2 end),
-then 0 -- failed
+relativeDelay = (case
-when successive != 0 and reps != 0
+when successive then 1 when reps then 0 else 2 end)
-then 1 -- review
+""")
-else 2 -- new
-end)""" + where)
         # old-style suspended cards
         self.s.statement(
             "update cards set type = type - 3 where priority = 0 and type >= 0")
@@ -788,6 +786,7 @@ where id in """
         anki.cards.Card.updateStats(card, ease, oldState)
         # update type & ensure past cutoff
         card.type = self.cardType(card)
+        card.relativeDelay = card.type
         if ease != 1:
             card.due = max(card.due, self.dueCutoff+1)
         # allow custom schedulers to munge the card
@@ -981,7 +980,7 @@ factor = 2.5, reps = 0, successive = 0, averageTime = 0, reviewTime = 0,
 youngEase0 = 0, youngEase1 = 0, youngEase2 = 0, youngEase3 = 0,
 youngEase4 = 0, matureEase0 = 0, matureEase1 = 0, matureEase2 = 0,
 matureEase3 = 0,matureEase4 = 0, yesCount = 0, noCount = 0,
-spaceUntil = 0, type = 2,
+spaceUntil = 0, type = 2, relativeDelay = 2,
 combinedDue = created, modified = :now, due = created
 where id in %s""" % ids2str(ids), now=time.time(), new=0)
         if self.newCardOrder == NEW_CARDS_RANDOM:
@@ -1006,7 +1005,7 @@ set due = :rand + ordinal,
 combinedDue = max(:rand + ordinal, spaceUntil),
 modified = :now
 where factId = :fid
-and type = 2""", data)
+and relativeDelay = 2""", data)
 
     def orderNewCards(self):
         "Set 'due' to card creation time."
@@ -1015,7 +1014,7 @@ update cards set
 due = created,
 combinedDue = max(spaceUntil, created),
 modified = :now
-where type = 2""", now=time.time())
+where relativeDelay = 2""", now=time.time())
 
     def rescheduleCards(self, ids, min, max):
         "Reset cards and schedule with new interval in days (min, max)."
@@ -1038,7 +1037,8 @@ reps = 1,
 successive = 1,
 yesCount = 1,
 firstAnswered = :t,
-type = 1
+type = 1,
+relativeDelay = 1
 where id = :id""", vals)
         self.flushMod()
 
@@ -1048,8 +1048,9 @@ where id = :id""", vals)
     def nextDueMsg(self):
         next = self.earliestTime()
         if next:
-            newCount = self.s.scalar(
+            # all new cards except suspended
-                "select count() from cards where type = 2")
+            newCount = self.s.scalar("""
+select count() from cards where relativeDelay = 2 and type != -1""")
             newCardsTomorrow = min(newCount, self.newCardsPerDay)
             cards = self.cardsDueBy(time.time() + 86400)
             msg = _('''\
@@ -1216,15 +1217,14 @@ group by cardTags.cardId""" % limit)
     # Suspending
     ##########################################################################
 
-    # when older clients are upgraded, we can move the code which touches
+    # when older clients are upgraded, we can remove the code which touches
     # priorities & isDue
 
     def suspendCards(self, ids):
         self.startProgress()
         self.s.statement("""
 update cards
-set type = (case
+set type = relativeDelay - 3,
-when successive then -2 when reps then -3 else -1 end),
 priority = -3, modified = :t, isDue=0
 where type >= 0 and id in %s""" % ids2str(ids), t=time.time())
         self.flushMod()
@@ -1234,7 +1234,7 @@ where type >= 0 and id in %s""" % ids2str(ids), t=time.time())
     def unsuspendCards(self, ids):
         self.startProgress()
         self.s.statement("""
-update cards set type = type + 3, priority=0, modified=:t
+update cards set type = relativeDelay, priority=0, modified=:t
 where type < 0 and id in %s""" %
             ids2str(ids), t=time.time())
         self.updatePriorities(ids)
@@ -1279,7 +1279,7 @@ select count(id) from cards where type < 0""")
         "Number of spaced new cards."
         return self.s.scalar("""
 select count(cards.id) from cards where
-type = 2 and combinedDue > :now
+relativeDelay = 2 and combinedDue > :now
 and due < :now""", now=time.time())
 
     def isEmpty(self):
@@ -1298,7 +1298,7 @@ and due < :now""", now=time.time())
     def newCountAll(self):
         "All new cards, including spaced."
         return self.s.scalar(
-            "select count(id) from cards where type = 2")
+            "select count(id) from cards where relativeDelay = 2")
 
     # Card predicates
     ##########################################################################
@@ -3565,9 +3565,7 @@ update cards set type = type - 3 where type between 0 and 2 and priority = -3"""
         if ids:
             deck.updatePriorities(ids)
             deck.s.statement(
-                "update cards set type = type - 3 where type between 3 and 5")
+                "update cards set type = relativeDelay where type > 2")
-            deck.s.statement(
-                "update cards set type = type - 6 where type between 6 and 8")
             deck.s.commit()
         # determine starting factor for new cards
         deck.averageFactor = (deck.s.scalar(
@@ -3620,11 +3618,15 @@ update cards set type = type - 3 where type between 0 and 2 and priority = -3"""
         deck.s.statement("""
 create index if not exists ix_cards_typeCombined on cards
 (type, combinedDue)""")
-        # failed cards, review early
+        # scheduler-agnostic type
+        deck.s.statement("""
+create index if not exists ix_cards_relativeDelay on cards
+(relativeDelay)""")
+        # failed cards, review early - obsolete
         deck.s.statement("""
 create index if not exists ix_cards_duePriority on cards
 (type, isDue, combinedDue, priority)""")
-        # check due
+        # check due - obsolete
         deck.s.statement("""
 create index if not exists ix_cards_priorityDue on cards
 (type, isDue, priority, combinedDue)""")
@@ -4119,6 +4121,13 @@ nextFactor, reps, thinkingTime, yesCount, noCount from reviewHistory""")
             deck.rebuildTypes()
             deck.version = 49
             deck.s.commit()
+        if deck.version < 50:
+            # more new type handling
+            deck.rebuildTypes()
+            # add an index for relativeDelay (type cache)
+            DeckStorage._addIndices(deck)
+            deck.version = 50
+            deck.s.commit()
         # executing a pragma here is very slow on large decks, so we store
         # our own record
         if not deck.getInt("pageSize") == 4096:

anki/exporting.py

@@ -129,6 +129,7 @@ yesCount = 0,
 noCount = 0,
 spaceUntil = 0,
 type = 2,
+relativeDelay = 2,
 combinedDue = created,
 modified = :now
 """, now=time.time())

anki/graphs.py

@@ -79,10 +79,10 @@ class DeckGraphs(object):
             t = time.time()
             young = self.deck.s.all("""
 select interval, combinedDue
-from cards where type in (0, 1) and interval <= 21""")
+from cards where relativeDelay between 0 and 1 and interval <= 21""")
             mature = self.deck.s.all("""
 select interval, combinedDue
-from cards where type = 1 and interval > 21""")
+from cards where relativeDelay = 1 and interval > 21""")
 
             for (src, dest) in [(young, daysYoung),
                                 (mature, daysMature)]:

anki/importing/__init__.py

@@ -191,8 +191,8 @@ where factId in (%s)""" % ",".join([str(s) for s in factIds]))
                     'cardModelId': cm.id,
                     'ordinal': cm.ordinal,
                     'question': u"",
-                    'answer': u"",
+                    'answer': u""
-                    'type': 2},cards[m]) for m in range(len(cards))]
+                    },cards[m]) for m in range(len(cards))]
                 self.deck.s.execute(cardsTable.insert(),
                                     data)
         self.deck.updateProgress()
@@ -212,6 +212,14 @@ where factId in (%s)""" % ",".join([str(s) for s in factIds]))
         data['tags'] = u""
         self.cardIds.append(data['id'])
         data['combinedDue'] = data['due']
+        if data.get('successive', 0):
+            t = 1
+        elif data.get('reps', 0):
+            t = 0
+        else:
+            t = 2
+        data['type'] = t
+        data['relativeDelay'] = t
         return data
 
     def stripInvalid(self, cards):

anki/importing/mnemosyne10.py

@@ -61,8 +61,6 @@ class Mnemosyne10Importer(Importer):
             card.reps = card.yesCount + card.noCount
             if item.cat.name != u"<default>":
                 card.tags = item.cat.name.replace(" ", "_")
-            if card.reps:
-                card.type = 1
             cards.append(card)
         return cards
 

anki/stats.py

@@ -539,7 +539,7 @@ class DeckStats(object):
         return (self.deck.s.scalar("""
 select count(id) from cards
 where combinedDue < :cutoff
-and priority > 0 and type in (0,1)""", cutoff=cutoff) or 0) / float(period)
+and priority > 0 and relativeDelay in (0,1)""", cutoff=cutoff) or 0) / float(period)
 
     def getPastWorkloadPeriod(self, period):
         cutoff = time.time() - 86400 * period

anki/sync.py

@@ -491,9 +491,6 @@ where factId in %s""" % factIds))
             'spaceUntil': f[5] or "",
             'lastCardId': f[6]
             } for f in facts]
-        self.deck.factCount += (len(facts) - self.deck.s.scalar(
-            "select count(*) from facts where id in %s" %
-            ids2str([f[0] for f in facts])))
         self.deck.s.execute("""
 insert or replace into facts
 (id, modelId, created, modified, tags, spaceUntil, lastCardId)
@@ -534,12 +531,21 @@ priority, interval, lastInterval, due, lastDue, factor,
 firstAnswered, reps, successive, averageTime, reviewTime, youngEase0,
 youngEase1, youngEase2, youngEase3, youngEase4, matureEase0,
 matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount,
-question, answer, lastFactor, spaceUntil, type, combinedDue
+question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay
 from cards where id in %s""" % ids2str(ids)))
 
     def updateCards(self, cards):
         if not cards:
             return
+        # FIXME: older clients won't send this, so this is temp compat code
+        def getType(row):
+            if len(row) > 37:
+                return row[37]
+            if row[15]:
+                return 1
+            elif row[14]:
+                return 0
+            return 2
         dlist = [{'id': c[0],
                   'factId': c[1],
                   'cardModelId': c[2],
@@ -576,10 +582,8 @@ from cards where id in %s""" % ids2str(ids)))
                   'spaceUntil': c[33],
                   'type': c[34],
                   'combinedDue': c[35],
+                  'rd': getType(c)
                   } for c in cards]
-        self.deck.cardCount += (len(cards) - self.deck.s.scalar(
-            "select count(*) from cards where id in %s" %
-            ids2str([c[0] for c in cards])))
         self.deck.s.execute("""
 insert or replace into cards
 (id, factId, cardModelId, created, modified, tags, ordinal,
@@ -596,7 +600,7 @@ values
 :youngEase1, :youngEase2, :youngEase3, :youngEase4, :matureEase0,
 :matureEase1, :matureEase2, :matureEase3, :matureEase4, :yesCount,
 :noCount, :question, :answer, :lastFactor, :spaceUntil,
-:type, :combinedDue, 0, 0)""", dlist)
+:type, :combinedDue, :rd, 0)""", dlist)
         self.deck.s.statement(
             "delete from cardsDeleted where cardId in %s" %
             ids2str([c[0] for c in cards]))
@@ -841,9 +845,6 @@ where media.id in %s""" % sids, now=time.time())
         t = time.time()
         dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2],
                   'ordinal': c[3], 'created': c[4], 't': t} for c in cards]
-        self.deck.cardCount += (len(cards) - self.deck.s.scalar(
-            "select count(*) from cards where id in %s" %
-            ids2str([c[0] for c in cards])))
         # add any missing cards
         self.deck.s.statements("""
 insert or ignore into cards