improve dynamic indices, implement new queue

anki/consts.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Copyright: Damien Elmes <anki@ichi2.net>
+# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
+
+MATURE_THRESHOLD = 21
+
+# whether new cards should be mixed with reviews, or shown first or last
+NEW_CARDS_DISTRIBUTE = 0
+NEW_CARDS_LAST = 1
+NEW_CARDS_FIRST = 2
+
+# new card insertion order
+NEW_CARDS_RANDOM = 0
+NEW_CARDS_DUE = 1
+
+# sort order for day's new cards
+NEW_TODAY_ORDINAL = 0
+NEW_TODAY_FACT = 1
+NEW_TODAY_DUE = 2
+
+# review card sort order
+REV_CARDS_OLD_FIRST = 0
+REV_CARDS_NEW_FIRST = 1
+REV_CARDS_RANDOM = 2
+
+# searching
+SEARCH_TAG = 0
+SEARCH_TYPE = 1
+SEARCH_PHRASE = 2
+SEARCH_FID = 3
+SEARCH_CARD = 4
+SEARCH_DISTINCT = 5
+SEARCH_FIELD = 6
+SEARCH_FIELD_EXISTS = 7
+SEARCH_QA = 8
+SEARCH_PHRASE_WB = 9

anki/deck.py

@@ -23,40 +23,24 @@ from anki.media import updateMediaCount, mediaFiles, \
      rebuildMediaDir
 from anki.upgrade import upgradeSchema, updateIndices, upgradeDeck, DECK_VERSION
 from anki.sched import Scheduler
+from anki.consts import *
 import anki.latex # sets up hook
 
 # ensure all the DB metadata in other files is loaded before proceeding
 import anki.models, anki.facts, anki.cards, anki.media
 
-# rest
+# Selective study and new card limits. These vars are necessary to determine
-MATURE_THRESHOLD = 21
+# counts even on a minimum deck load, and thus are separate from the rest of
-NEW_CARDS_DISTRIBUTE = 0
+# the config.
-NEW_CARDS_LAST = 1
-NEW_CARDS_FIRST = 2
-NEW_CARDS_RANDOM = 0
-NEW_CARDS_OLD_FIRST = 1
-NEW_CARDS_NEW_FIRST = 2
-REV_CARDS_OLD_FIRST = 0
-REV_CARDS_NEW_FIRST = 1
-REV_CARDS_DUE_FIRST = 2
-REV_CARDS_RANDOM = 3
-SEARCH_TAG = 0
-SEARCH_TYPE = 1
-SEARCH_PHRASE = 2
-SEARCH_FID = 3
-SEARCH_CARD = 4
-SEARCH_DISTINCT = 5
-SEARCH_FIELD = 6
-SEARCH_FIELD_EXISTS = 7
-SEARCH_QA = 8
-SEARCH_PHRASE_WB = 9
-
-# selective study
 defaultLim = {
     'newActive': u"",
     'newInactive': u"",
     'revActive': u"",
     'revInactive': u"",
+    'newPerDay': 20,
+    # currentDay, count
+    'newToday': [0, 0],
+    'newTodayOrder': NEW_TODAY_ORDINAL,
 }
 
 # scheduling and other options
@@ -64,8 +48,7 @@ defaultConf = {
     'utcOffset': -2,
     'newCardOrder': 1,
     'newCardSpacing': NEW_CARDS_DISTRIBUTE,
-    'newCardsPerDay': 20,
+    'revCardOrder': REV_CARDS_OLD_FIRST,
-    'revCardOrder': 0,
     'collapseTime': 600,
     'sessionRepLimit': 0,
     'sessionTimeLimit': 600,
@@ -120,8 +103,11 @@ class Deck(object):
         self.sessionStartReps = 0
         self.sessionStartTime = 0
         self.lastSessionStart = 0
+        # counter for reps since deck open
+        self.reps = 0
         self.sched = Scheduler(self)
 
+
     def modifiedSinceSave(self):
         return self.modified > self.lastLoaded
 
@@ -161,7 +147,7 @@ interval=0, due=created, factor=2.5, reps=0, successive=0, lapses=0, flags=0"""
             sql2 += "  where cardId in "+sids
         self.db.statement(sql, now=time.time())
         self.db.statement(sql2)
-        if self.newCardOrder == NEW_CARDS_RANDOM:
+        if self.config['newCardOrder'] == NEW_CARDS_RANDOM:
             # we need to re-randomize now
             self.randomizeNewCards(ids)
         self.flushMod()
@@ -452,7 +438,7 @@ due > :now and due < :now""", now=time.time())
         self.db.save(fact)
         # update field cache
         self.flushMod()
-        isRandom = self.newCardOrder == NEW_CARDS_RANDOM
+        isRandom = self.config['newCardOrder'] == NEW_CARDS_RANDOM
         if isRandom:
             due = random.uniform(0, time.time())
         t = time.time()
@@ -2167,9 +2153,9 @@ Return new path, relative to media dir."""
 
     def flushConfig(self):
         print "make flushConfig() more intelligent"
-        deck._config = simplejson.dumps(deck.config)
+        self._config = unicode(simplejson.dumps(self.config))
-        deck._limits = simplejson.dumps(deck.limits)
+        self._limits = unicode(simplejson.dumps(self.limits))
-        deck._data = simplejson.dumps(deck.data)
+        self._data = unicode(simplejson.dumps(self.data))
 
     def close(self):
         if self.db:
@@ -2655,48 +2641,25 @@ seq > :s and seq <= :e order by seq desc""", s=start, e=end)
     ##########################################################################
 
     def updateDynamicIndices(self):
-        print "fix dynamicIndices()"
+        # determine required columns
-        return
-        indices = {
-            'intervalDesc':
-            '(queue, interval desc, factId, due)',
-            'intervalAsc':
-            '(queue, interval, factId, due)',
-            'randomOrder':
-            '(queue, factId, ordinal, due)',
-            'dueAsc':
-            '(queue, position, factId, due)',
-            'dueDesc':
-            '(queue, position desc, factId, due)',
-            }
-        # determine required
         required = []
-        if self.revCardOrder == REV_CARDS_OLD_FIRST:
+        if self.limits['newTodayOrder'] == NEW_TODAY_ORDINAL:
-            required.append("intervalDesc")
+            required.append("ordinal")
-        if self.revCardOrder == REV_CARDS_NEW_FIRST:
+        elif self.limits['newTodayOrder'] == NEW_TODAY_FACT:
-            required.append("intervalAsc")
+            required.append("factId")
-        if self.revCardOrder == REV_CARDS_RANDOM:
+        if self.config['revCardOrder'] in (REV_CARDS_OLD_FIRST, REV_CARDS_NEW_FIRST):
-            required.append("randomOrder")
+            required.append("interval")
-        if (self.revCardOrder == REV_CARDS_DUE_FIRST or
+        cols = ["queue", "due"] + required
-            self.newCardOrder == NEW_CARDS_OLD_FIRST or
+        # update if changed
-            self.newCardOrder == NEW_CARDS_RANDOM):
+        if self.db.scalar(
-            required.append("dueAsc")
+            "select 1 from sqlite_master where name = 'ix_cards_multi'"):
-        if (self.newCardOrder == NEW_CARDS_NEW_FIRST):
+            rows = self.db.all("pragma index_info('ix_cards_multi')")
-            required.append("dueDesc")
+        else:
-        # add/delete
+            rows = None
-        analyze = False
+        if not (rows and cols == [r[2] for r in rows]):
-        for (k, v) in indices.items():
+            self.db.statement("drop index if exists ix_cards_multi")
-            n = "ix_cards_%s" % k
+            self.db.statement("create index ix_cards_multi on cards (%s)" %
-            if k in required:
+                              ", ".join(cols))
-                if not self.db.scalar(
-                    "select 1 from sqlite_master where name = :n", n=n):
-                    self.db.statement(
-                        "create index %s on cards %s" %
-                        (n, v))
-                    analyze = True
-            else:
-                self.db.statement("drop index if exists %s" % n)
-        if analyze:
             self.db.statement("analyze")
 
 mapper(Deck, deckTable, properties={
@@ -2723,9 +2686,8 @@ sourcesTable = Table(
 
 def newCardOrderLabels():
     return {
-        0: _("Show new cards in random order"),
+        0: _("Add new cards in random order"),
-        1: _("Show new cards in order added"),
+        1: _("Add new cards to end of queue"),
-        2: _("Show new cards in reverse order added"),
         }
 
 def newCardSchedulingLabels():

anki/importing/__init__.py

@@ -18,7 +18,7 @@ from anki.lang import _
 from anki.utils import genID, canonifyTags, fieldChecksum
 from anki.utils import canonifyTags, ids2str
 from anki.errors import *
-from anki.deck import NEW_CARDS_RANDOM
+#from anki.deck import NEW_CARDS_RANDOM
 
 # Base importer
 ##########################################################################

anki/importing/anki10.py

@@ -7,7 +7,7 @@ from anki.importing import Importer
 from anki.sync import SyncClient, SyncServer, copyLocalMedia
 from anki.lang import _
 from anki.utils import ids2str
-from anki.deck import NEW_CARDS_RANDOM
+#from anki.deck import NEW_CARDS_RANDOM
 import time
 
 class Anki10Importer(Importer):

anki/sched.py

@@ -3,12 +3,14 @@
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
 import time, datetime, simplejson
+from operator import itemgetter
 from heapq import *
 from anki.db import *
 from anki.cards import Card
 from anki.utils import parseTags, ids2str
 from anki.tags import tagIds
 from anki.lang import _
+from anki.consts import *
 
 # the standard Anki scheduler
 class Scheduler(object):
@@ -45,7 +47,13 @@ class Scheduler(object):
         if card.queue == 0:
             self.answerLearnCard(card, ease)
         elif card.queue == 1:
+            self.revCount -= 1
             self.answerRevCard(card, ease)
+        elif card.queue == 2:
+            # put it in the learn queue
+            card.queue = 0
+            self.newCount -= 1
+            self.answerLearnCard(card, ease)
         else:
             raise Exception("Invalid queue")
         card.toDB(self.db)
@@ -80,6 +88,62 @@ class Scheduler(object):
         # collapse or finish
         return self.getLearnCard(collapse=True)
 
+    # New cards
+    ##########################################################################
+
+    # need to keep track of reps for timebox and new card introduction
+
+    def resetNew(self):
+        l = self.deck.limits
+        if l['newToday'][0] != self.today:
+            # it's a new day; reset counts
+            l['newToday'] = [self.today, 0]
+        lim = min(self.queueLimit, l['newPerDay'] - l['newToday'][1])
+        if lim <= 0:
+            self.newQueue = []
+            self.newCount = 0
+        else:
+            self.newQueue = self.db.all(
+                self.cardLimit(
+                    "newActive", "newInactive", """
+select id, %s from cards c where
+queue = 2 order by due limit %d""" % (self.newOrder(), lim)))
+            self.newQueue.sort(key=itemgetter(1), reverse=True)
+            self.newCount = len(self.newQueue)
+        self.updateNewCardRatio()
+
+    def getNewCard(self):
+        if self.newQueue:
+            return self.newQueue.pop()[0]
+
+    def newOrder(self):
+        return ("ordinal",
+                "factId",
+                "due",
+            )[self.deck.limits['newTodayOrder']]
+
+    def updateNewCardRatio(self):
+        if self.deck.config['newCardSpacing'] == NEW_CARDS_DISTRIBUTE:
+            if self.newCount:
+                self.newCardModulus = (
+                    (self.newCount + self.revCount) / self.newCount)
+                # if there are cards to review, ensure modulo >= 2
+                if self.revCount:
+                    self.newCardModulus = max(2, self.newCardModulus)
+                return
+        self.newCardModulus = 0
+
+    def timeForNewCard(self):
+        "True if it's time to display a new card when distributing."
+        if not self.newCount:
+            return False
+        if self.deck.config['newCardSpacing'] == NEW_CARDS_LAST:
+            return False
+        elif self.deck.config['newCardSpacing'] == NEW_CARDS_FIRST:
+            return True
+        elif self.newCardModulus:
+            return self.deck.reps and self.deck.reps % self.newCardModulus == 0
+
     # Learning queue
     ##########################################################################
 
@@ -407,86 +471,6 @@ and queue between 1 and 2""",
         self.reset()
         self.refreshSession()
 
-    # New cards
-    ##########################################################################
-
-#         # day counts
-#         (self.repsToday, self.newSeenToday) = self.db.first("""
-# select count(), sum(case when rep = 1 then 1 else 0 end) from revlog
-# where time > :t""", t=self.dayCutoff-86400)
-#         self.newSeenToday = self.newSeenToday or 0
-#         print "newSeenToday in answer(), reset called twice"
-#         print "newSeenToday needs to account for drill mode too."
-
-    # when do we do this?
-    #self.updateNewCountToday()
-
-    def resetNew(self):
-#        self.updateNewCardRatio()
-        pass
-
-    def rebuildNewCount(self):
-        self.newAvail = self.db.scalar(
-            self.cardLimit(
-            "newActive", "newInactive",
-            "select count(*) from cards c where queue = 2 "
-            "and due < :lim"), lim=self.dayCutoff)
-        self.updateNewCountToday()
-
-    def updateNewCountToday(self):
-        self.newCount = max(min(
-            self.newAvail, self.newCardsPerDay -
-            self.newSeenToday), 0)
-
-    def fillNewQueue(self):
-        if self.newCount and not self.newQueue:
-            self.newQueue = self.db.all(
-                self.cardLimit(
-                "newActive", "newInactive", """
-select c.id, factId from cards c where
-queue = 2 and due < :lim order by %s
-limit %d""" % (self.newOrder(), self.queueLimit)), lim=self.dayCutoff)
-            self.newQueue.reverse()
-
-    def updateNewCardRatio(self):
-        if self.newCardSpacing == NEW_CARDS_DISTRIBUTE:
-            if self.newCount:
-                self.newCardModulus = (
-                    (self.newCount + self.revCount) / self.newCount)
-                # if there are cards to review, ensure modulo >= 2
-                if self.revCount:
-                    self.newCardModulus = max(2, self.newCardModulus)
-            else:
-                self.newCardModulus = 0
-        else:
-            self.newCardModulus = 0
-
-    def timeForNewCard(self):
-        "True if it's time to display a new card when distributing."
-        # FIXME
-        return False
-
-        if not self.newCount:
-            return False
-        if self.newCardSpacing == NEW_CARDS_LAST:
-            return False
-        if self.newCardSpacing == NEW_CARDS_FIRST:
-            return True
-        if self.newCardModulus:
-            return self.repsToday % self.newCardModulus == 0
-        else:
-            return False
-
-    def getNewCard(self):
-        # FIXME
-        return None
-        #return self.newQueue[-1][0]
-
-    def newOrder(self):
-        return ("due",
-                "due",
-                "due desc")[self.newCardOrder]
-
     # Tools
     ##########################################################################
 
@@ -541,7 +525,7 @@ limit %d""" % (self.newOrder(), self.queueLimit)), lim=self.dayCutoff)
         # cutoff must not be more than 24 hours in the future
         cutoff = min(time.time() + 86400, cutoff)
         self.dayCutoff = cutoff
-        self.dayCount = int(cutoff/86400 - self.deck.created/86400)
+        self.today = int(cutoff/86400 - self.deck.created/86400)
 
     def checkDay(self):
         # check if the day has rolled over

anki/upgrade.py

@@ -99,13 +99,19 @@ ifnull(syncName, ""), lastSync, utcOffset, "", "", "" from decks""")
         lim[k] = s.execute("select value from deckVars where key=:k",
                            {'k':k}).scalar()
         s.execute("delete from deckVars where key=:k", {'k':k})
+    lim['newPerDay'] = s.execute(
+        "select newCardsPerDay from decks").scalar()
     # fetch remaining settings from decks table
     conf = deck.defaultConf.copy()
     data = {}
-    keys = ("newCardOrder", "newCardSpacing", "newCardsPerDay",
+    keys = ("newCardOrder", "newCardSpacing", "revCardOrder",
-            "revCardOrder", "sessionRepLimit", "sessionTimeLimit")
+            "sessionRepLimit", "sessionTimeLimit")
     for k in keys:
         conf[k] = s.execute("select %s from decks" % k).scalar()
+    # random and due options merged
+    conf['revCardOrder'] = min(2, conf['revCardOrder'])
+    # no reverse option anymore
+    conf['newCardOrder'] = min(1, conf['newCardOrder'])
     # add any deck vars and save
     dkeys = ("hexCache", "cssCache")
     for (k, v) in s.execute("select * from deckVars").fetchall():
@@ -123,14 +129,6 @@ ifnull(syncName, ""), lastSync, utcOffset, "", "", "" from decks""")
 
 def updateIndices(db):
     "Add indices to the DB."
-    # due counts, failed card queue
-    db.execute("""
-create index if not exists ix_cards_queueDue on cards
-(queue, due, factId)""")
-    # counting cards of a given type
-    db.execute("""
-create index if not exists ix_cards_type on cards
-(type)""")
     # sync summaries
     db.execute("""
 create index if not exists ix_cards_modified on cards
@@ -177,7 +175,6 @@ def upgradeDeck(deck):
                   "dueAsc", "dueDesc"):
             deck.db.statement("drop index if exists ix_cards_%s2" % d)
             deck.db.statement("drop index if exists ix_cards_%s" % d)
-        deck.updateDynamicIndices()
         # remove old views
         for v in ("failedCards", "revCardsOld", "revCardsNew",
                   "revCardsDue", "revCardsRandom", "acqCardsRandom",
@@ -209,9 +206,17 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
         deck.db.statement("drop index if exists ix_fields_fieldModelId")
         # update schema time
         deck.db.statement("update deck set schemaMod = :t", t=time.time())
+        # remove queueDue as it's become dynamic, and type index
+        deck.db.statement("drop index if exists ix_cards_queueDue")
+        deck.db.statement("drop index if exists ix_cards_type")
 
         # finally, update indices & optimize
         updateIndices(deck.db)
+        # setup limits & config for dynamicIndices()
+        deck.limits = simplejson.loads(deck._limits)
+        deck.config = simplejson.loads(deck._config)
+
+        deck.updateDynamicIndices()
         deck.db.execute("vacuum")
         deck.db.execute("analyze")
         deck.version = 100

tests/test_deck.py

@@ -311,5 +311,6 @@ def test_findCards():
 def test_upgrade():
     src = os.path.expanduser("~/Scratch/upgrade.anki")
     (fd, dst) = tempfile.mkstemp(suffix=".anki")
+    print "upgrade to", dst
     shutil.copy(src, dst)
     deck = Deck(dst)

tests/test_sched.py

@@ -15,6 +15,26 @@ def test_basics():
     d = getEmptyDeck()
     assert not d.getCard()
 
+def test_new():
+    d = getEmptyDeck()
+    assert d.sched.newCount == 0
+    # add a fact
+    f = d.newFact()
+    f['Front'] = u"one"; f['Back'] = u"two"
+    f = d.addFact(f)
+    d.db.flush()
+    d.reset()
+    assert d.sched.newCount == 1
+    # fetch it
+    c = d.getCard()
+    assert c
+    assert c.queue == 2
+    assert c.type == 2
+    # if we answer it, it should become a learn card
+    d.answerCard(c, 1)
+    assert c.queue == 0
+    assert c.type == 2
+
 def test_learn():
     d = getEmptyDeck()
     # add a fact