experimental queue building

Still a work in progress, and hidden behind a feature flag.

Cargo.lock

@@ -62,6 +62,7 @@ dependencies = [
  "flate2",
  "fluent",
  "fluent-syntax",
+ "fnv",
  "futures",
  "hex",
  "htmlescape",

pylib/anki/collection.py

@@ -139,7 +139,7 @@ class Collection:
         if ver == 1:
             self.sched = V1Scheduler(self)
         elif ver == 2:
-            if os.getenv("TEST_SCHEDULER"):
+            if self.is_2021_test_scheduler_enabled():
                 self.sched = V2TestScheduler(self)  # type: ignore
             else:
                 self.sched = V2Scheduler(self)
@@ -149,6 +149,14 @@ class Collection:
         self.clearUndo()
         self._loadScheduler()
 
+    def is_2021_test_scheduler_enabled(self) -> bool:
+        return self.get_config_bool(Config.Bool.SCHED_2021)
+
+    def set_2021_test_scheduler_enabled(self, enabled: bool) -> None:
+        if self.is_2021_test_scheduler_enabled() != enabled:
+            self.set_config_bool(Config.Bool.SCHED_2021, enabled)
+            self._loadScheduler()
+
     # DB-related
     ##########################################################################
 
@@ -774,6 +782,9 @@ table.review-log {{ {revlog_style} }}
             c.nid,
         )
         # and finally, update daily counts
+        if self.sched.is_2021:
+            self._backend.requeue_undone_card(c.id)
+        else:
             n = c.queue
             if c.queue in (QUEUE_TYPE_DAY_LEARN_RELEARN, QUEUE_TYPE_PREVIEW):
                 n = QUEUE_TYPE_LRN

pylib/anki/scheduler.py

@@ -8,458 +8,113 @@ used by Anki.
 
 from __future__ import annotations
 
-import pprint
-import random
-import time
 from heapq import *
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
+from typing import Any, List, Optional, Sequence, Tuple, Union
 
 import anki  # pylint: disable=unused-import
 import anki._backend.backend_pb2 as _pb
 from anki import hooks
 from anki.cards import Card
 from anki.consts import *
-from anki.decks import Deck, DeckConfig, DeckManager, DeckTreeNode, QueueConfig
+from anki.decks import DeckConfig, DeckTreeNode, QueueConfig
 from anki.notes import Note
 from anki.types import assert_exhaustive
 from anki.utils import from_json_bytes, ids2str, intTime
 
+QueuedCards = _pb.GetQueuedCardsOut.QueuedCards
 CongratsInfo = _pb.CongratsInfoOut
-CountsForDeckToday = _pb.CountsForDeckTodayOut
 SchedTimingToday = _pb.SchedTimingTodayOut
 UnburyCurrentDeck = _pb.UnburyCardsInCurrentDeckIn
 BuryOrSuspend = _pb.BuryOrSuspendCardsIn
 
+# fixme: reviewer.cardQueue/editCurrent/undo handling/retaining current card
+# fixme: .reps
+
 
 class Scheduler:
-    _burySiblingsOnAnswer = True
+    is_2021 = True
 
     def __init__(self, col: anki.collection.Collection) -> None:
         self.col = col.weakref()
-        self.queueLimit = 50
+        # fixme: only used by the timeboxing code, and was double-incremented
-        self.reportLimit = 1000
+        # for ages - just move to gui?
-        self.dynReportLimit = 99999
         self.reps = 0
-        self.today: Optional[int] = None
-        self._haveQueues = False
-        self._lrnCutoff = 0
-        self._updateCutoff()
 
-    # Daily cutoff
+    # Timing
     ##########################################################################
 
-    def _updateCutoff(self) -> None:
+    def timing_today(self) -> SchedTimingToday:
-        timing = self._timing_today()
-        self.today = timing.days_elapsed
-        self.dayCutoff = timing.next_day_at
-
-    def _checkDay(self) -> None:
-        # check if the day has rolled over
-        if time.time() > self.dayCutoff:
-            self.reset()
-
-    def _timing_today(self) -> SchedTimingToday:
         return self.col._backend.sched_timing_today()
 
+    @property
+    def today(self) -> int:
+        return self.timing_today().days_elapsed
+
+    @property
+    def dayCutoff(self) -> int:
+        return self.timing_today().next_day_at
+
     # Fetching the next card
     ##########################################################################
 
     def reset(self) -> None:
-        self.col.decks.update_active()
+        self.col._backend.clear_card_queues()
-        self._updateCutoff()
-        self._reset_counts()
-        self._resetLrn()
-        self._resetRev()
-        self._resetNew()
-        self._haveQueues = True
 
-    def _reset_counts(self) -> None:
+    def get_queued_cards(
-        tree = self.deck_due_tree(self.col.decks.selected())
+        self,
-        node = self.col.decks.find_deck_in_tree(tree, int(self.col.conf["curDeck"]))
+        *,
-        if not node:
+        fetch_limit: int = 1,
-            # current deck points to a missing deck
+        intraday_learning_only: bool = False,
-            self.newCount = 0
+    ) -> Union[QueuedCards, CongratsInfo]:
-            self.revCount = 0
+        info = self.col._backend.get_queued_cards(
-            self._immediate_learn_count = 0
+            fetch_limit=fetch_limit, intraday_learning_only=intraday_learning_only
+        )
+        kind = info.WhichOneof("value")
+        if kind == "queued_cards":
+            return info.queued_cards
+        elif kind == "congrats_info":
+            return info.congrats_info
         else:
-            self.newCount = node.new_count
+            assert_exhaustive(kind)
-            self.revCount = node.review_count
+            assert False
-            self._immediate_learn_count = node.learn_count
 
     def getCard(self) -> Optional[Card]:
-        """Pop the next card from the queue. None if finished."""
+        """Fetch the next card from the queue. None if finished."""
-        self._checkDay()
+        response = self.get_queued_cards()
-        if not self._haveQueues:
+        if isinstance(response, QueuedCards):
-            self.reset()
+            backend_card = response.cards[0].card
-        card = self._getCard()
+            card = Card(self.col)
-        if card:
+            card._load_from_backend_card(backend_card)
-            self.col.log(card)
-            if not self._burySiblingsOnAnswer:
-                self._burySiblings(card)
-            self.reps += 1
             card.startTimer()
             return card
-        return None
-
-    def _getCard(self) -> Optional[Card]:
-        """Return the next due card, or None."""
-        # learning card due?
-        c = self._getLrnCard()
-        if c:
-            return c
-
-        # new first, or time for one?
-        if self._timeForNewCard():
-            c = self._getNewCard()
-            if c:
-                return c
-
-        # day learning first and card due?
-        dayLearnFirst = self.col.conf.get("dayLearnFirst", False)
-        if dayLearnFirst:
-            c = self._getLrnDayCard()
-            if c:
-                return c
-
-        # card due for review?
-        c = self._getRevCard()
-        if c:
-            return c
-
-        # day learning card due?
-        if not dayLearnFirst:
-            c = self._getLrnDayCard()
-            if c:
-                return c
-
-        # new cards left?
-        c = self._getNewCard()
-        if c:
-            return c
-
-        # collapse or finish
-        return self._getLrnCard(collapse=True)
-
-    # Fetching new cards
-    ##########################################################################
-
-    def _resetNew(self) -> None:
-        self._newDids = self.col.decks.active()[:]
-        self._newQueue: List[int] = []
-        self._updateNewCardRatio()
-
-    def _fillNew(self, recursing: bool = False) -> bool:
-        if self._newQueue:
-            return True
-        if not self.newCount:
-            return False
-        while self._newDids:
-            did = self._newDids[0]
-            lim = min(self.queueLimit, self._deckNewLimit(did))
-            if lim:
-                # fill the queue with the current did
-                self._newQueue = self.col.db.list(
-                    f"""
-                select id from cards where did = ? and queue = {QUEUE_TYPE_NEW} order by due,ord limit ?""",
-                    did,
-                    lim,
-                )
-                if self._newQueue:
-                    self._newQueue.reverse()
-                    return True
-            # nothing left in the deck; move to next
-            self._newDids.pop(0)
-
-        # if we didn't get a card but the count is non-zero,
-        # we need to check again for any cards that were
-        # removed from the queue but not buried
-        if recursing:
-            print("bug: fillNew()")
-            return False
-        self._reset_counts()
-        self._resetNew()
-        return self._fillNew(recursing=True)
-
-    def _getNewCard(self) -> Optional[Card]:
-        if self._fillNew():
-            self.newCount -= 1
-            return self.col.getCard(self._newQueue.pop())
-        return None
-
-    def _updateNewCardRatio(self) -> None:
-        if self.col.conf["newSpread"] == NEW_CARDS_DISTRIBUTE:
-            if self.newCount:
-                self.newCardModulus = (self.newCount + self.revCount) // self.newCount
-                # if there are cards to review, ensure modulo >= 2
-                if self.revCount:
-                    self.newCardModulus = max(2, self.newCardModulus)
-                return
-        self.newCardModulus = 0
-
-    def _timeForNewCard(self) -> Optional[bool]:
-        "True if it's time to display a new card when distributing."
-        if not self.newCount:
-            return False
-        if self.col.conf["newSpread"] == NEW_CARDS_LAST:
-            return False
-        elif self.col.conf["newSpread"] == NEW_CARDS_FIRST:
-            return True
-        elif self.newCardModulus:
-            return self.reps != 0 and self.reps % self.newCardModulus == 0
         else:
-            # shouldn't reach
             return None
 
-    def _deckNewLimit(
+    def _is_finished(self) -> bool:
-        self, did: int, fn: Optional[Callable[[Deck], int]] = None
+        "Don't use this, it is a stop-gap until this code is refactored."
-    ) -> int:
+        info = self.get_queued_cards()
-        if not fn:
+        return isinstance(info, CongratsInfo)
-            fn = self._deckNewLimitSingle
+
-        sel = self.col.decks.get(did)
+    def counts(self, card: Optional[Card] = None) -> Tuple[int, int, int]:
-        lim = -1
+        info = self.get_queued_cards()
-        # for the deck and each of its parents
+        if isinstance(info, CongratsInfo):
-        for g in [sel] + self.col.decks.parents(did):
+            counts = [0, 0, 0]
-            rem = fn(g)
-            if lim == -1:
-                lim = rem
         else:
-                lim = min(rem, lim)
+            counts = [info.new_count, info.learning_count, info.review_count]
-        return lim
 
-    def _newForDeck(self, did: int, lim: int) -> int:
+        return tuple(counts)  # type: ignore
-        "New count for a single deck."
-        if not lim:
-            return 0
-        lim = min(lim, self.reportLimit)
-        return self.col.db.scalar(
-            f"""
-select count() from
-(select 1 from cards where did = ? and queue = {QUEUE_TYPE_NEW} limit ?)""",
-            did,
-            lim,
-        )
 
-    def _deckNewLimitSingle(self, g: DeckConfig) -> int:
+    @property
-        "Limit for deck without parent limits."
+    def newCount(self) -> int:
-        if g["dyn"]:
+        return self.counts()[0]
-            return self.dynReportLimit
-        c = self.col.decks.confForDid(g["id"])
-        limit = max(0, c["new"]["perDay"] - self.counts_for_deck_today(g["id"]).new)
-        return hooks.scheduler_new_limit_for_single_deck(limit, g)
 
-    def totalNewForCurrentDeck(self) -> int:
+    @property
-        return self.col.db.scalar(
+    def lrnCount(self) -> int:
-            f"""
+        return self.counts()[1]
-select count() from cards where id in (
-select id from cards where did in %s and queue = {QUEUE_TYPE_NEW} limit ?)"""
-            % self._deckLimit(),
-            self.reportLimit,
-        )
 
-    # Fetching learning cards
+    @property
-    ##########################################################################
+    def reviewCount(self) -> int:
-
+        return self.counts()[2]
-    # scan for any newly due learning cards every minute
-    def _updateLrnCutoff(self, force: bool) -> bool:
-        nextCutoff = intTime() + self.col.conf["collapseTime"]
-        if nextCutoff - self._lrnCutoff > 60 or force:
-            self._lrnCutoff = nextCutoff
-            return True
-        return False
-
-    def _maybeResetLrn(self, force: bool) -> None:
-        if self._updateLrnCutoff(force):
-            self._resetLrn()
-
-    def _resetLrnCount(self) -> None:
-        # sub-day
-        self.lrnCount = (
-            self.col.db.scalar(
-                f"""
-select count() from cards where did in %s and queue = {QUEUE_TYPE_LRN}
-and due < ?"""
-                % (self._deckLimit()),
-                self._lrnCutoff,
-            )
-            or 0
-        )
-        # day
-        self.lrnCount += self.col.db.scalar(
-            f"""
-select count() from cards where did in %s and queue = {QUEUE_TYPE_DAY_LEARN_RELEARN}
-and due <= ?"""
-            % (self._deckLimit()),
-            self.today,
-        )
-        # previews
-        self.lrnCount += self.col.db.scalar(
-            f"""
-select count() from cards where did in %s and queue = {QUEUE_TYPE_PREVIEW}
-"""
-            % (self._deckLimit())
-        )
-
-    def _resetLrn(self) -> None:
-        self._updateLrnCutoff(force=True)
-        self._resetLrnCount()
-        self._lrnQueue: List[Tuple[int, int]] = []
-        self._lrnDayQueue: List[int] = []
-        self._lrnDids = self.col.decks.active()[:]
-
-    # sub-day learning
-    def _fillLrn(self) -> Union[bool, List[Any]]:
-        if not self.lrnCount:
-            return False
-        if self._lrnQueue:
-            return True
-        cutoff = intTime() + self.col.conf["collapseTime"]
-        self._lrnQueue = self.col.db.all(  # type: ignore
-            f"""
-select due, id from cards where
-did in %s and queue in ({QUEUE_TYPE_LRN},{QUEUE_TYPE_PREVIEW}) and due < ?
-limit %d"""
-            % (self._deckLimit(), self.reportLimit),
-            cutoff,
-        )
-        for i in range(len(self._lrnQueue)):
-            self._lrnQueue[i] = (self._lrnQueue[i][0], self._lrnQueue[i][1])
-        # as it arrives sorted by did first, we need to sort it
-        self._lrnQueue.sort()
-        return self._lrnQueue
-
-    def _getLrnCard(self, collapse: bool = False) -> Optional[Card]:
-        self._maybeResetLrn(force=collapse and self.lrnCount == 0)
-        if self._fillLrn():
-            cutoff = time.time()
-            if collapse:
-                cutoff += self.col.conf["collapseTime"]
-            if self._lrnQueue[0][0] < cutoff:
-                id = heappop(self._lrnQueue)[1]
-                card = self.col.getCard(id)
-                self.lrnCount -= 1
-                return card
-        return None
-
-    # daily learning
-    def _fillLrnDay(self) -> Optional[bool]:
-        if not self.lrnCount:
-            return False
-        if self._lrnDayQueue:
-            return True
-        while self._lrnDids:
-            did = self._lrnDids[0]
-            # fill the queue with the current did
-            self._lrnDayQueue = self.col.db.list(
-                f"""
-select id from cards where
-did = ? and queue = {QUEUE_TYPE_DAY_LEARN_RELEARN} and due <= ? limit ?""",
-                did,
-                self.today,
-                self.queueLimit,
-            )
-            if self._lrnDayQueue:
-                # order
-                r = random.Random()
-                r.seed(self.today)
-                r.shuffle(self._lrnDayQueue)
-                # is the current did empty?
-                if len(self._lrnDayQueue) < self.queueLimit:
-                    self._lrnDids.pop(0)
-                return True
-            # nothing left in the deck; move to next
-            self._lrnDids.pop(0)
-        # shouldn't reach here
-        return False
-
-    def _getLrnDayCard(self) -> Optional[Card]:
-        if self._fillLrnDay():
-            self.lrnCount -= 1
-            return self.col.getCard(self._lrnDayQueue.pop())
-        return None
-
-    # Fetching reviews
-    ##########################################################################
-
-    def _currentRevLimit(self) -> int:
-        d = self.col.decks.get(self.col.decks.selected(), default=False)
-        return self._deckRevLimitSingle(d)
-
-    def _deckRevLimitSingle(
-        self, d: Dict[str, Any], parentLimit: Optional[int] = None
-    ) -> int:
-        # invalid deck selected?
-        if not d:
-            return 0
-
-        if d["dyn"]:
-            return self.dynReportLimit
-
-        c = self.col.decks.confForDid(d["id"])
-        lim = max(0, c["rev"]["perDay"] - self.counts_for_deck_today(d["id"]).review)
-
-        if parentLimit is not None:
-            lim = min(parentLimit, lim)
-        elif "::" in d["name"]:
-            for parent in self.col.decks.parents(d["id"]):
-                # pass in dummy parentLimit so we don't do parent lookup again
-                lim = min(lim, self._deckRevLimitSingle(parent, parentLimit=lim))
-        return hooks.scheduler_review_limit_for_single_deck(lim, d)
-
-    def _revForDeck(
-        self, did: int, lim: int, childMap: DeckManager.childMapNode
-    ) -> Any:
-        dids = [did] + self.col.decks.childDids(did, childMap)
-        lim = min(lim, self.reportLimit)
-        return self.col.db.scalar(
-            f"""
-select count() from
-(select 1 from cards where did in %s and queue = {QUEUE_TYPE_REV}
-and due <= ? limit ?)"""
-            % ids2str(dids),
-            self.today,
-            lim,
-        )
-
-    def _resetRev(self) -> None:
-        self._revQueue: List[int] = []
-
-    def _fillRev(self, recursing: bool = False) -> bool:
-        "True if a review card can be fetched."
-        if self._revQueue:
-            return True
-        if not self.revCount:
-            return False
-
-        lim = min(self.queueLimit, self._currentRevLimit())
-        if lim:
-            self._revQueue = self.col.db.list(
-                f"""
-select id from cards where
-did in %s and queue = {QUEUE_TYPE_REV} and due <= ?
-order by due, random()
-limit ?"""
-                % self._deckLimit(),
-                self.today,
-                lim,
-            )
-
-            if self._revQueue:
-                # preserve order
-                self._revQueue.reverse()
-                return True
-
-        if recursing:
-            print("bug: fillRev2()")
-            return False
-        self._reset_counts()
-        self._resetRev()
-        return self._fillRev(recursing=True)
-
-    def _getRevCard(self) -> Optional[Card]:
-        if self._fillRev():
-            self.revCount -= 1
-            return self.col.getCard(self._revQueue.pop())
-        return None
 
     # Answering a card
     ##########################################################################
@@ -470,13 +125,11 @@ limit ?"""
 
         self.col.markReview(card)
 
-        if self._burySiblingsOnAnswer:
-            self._burySiblings(card)
-
         new_state = self._answerCard(card, ease)
 
-        if not self._handle_leech(card, new_state):
+        self._handle_leech(card, new_state)
-            self._maybe_requeue_card(card)
+
+        self.reps += 1
 
     def _answerCard(self, card: Card, ease: int) -> _pb.SchedulingState:
         states = self.col._backend.get_next_card_states(card.id)
@@ -523,45 +176,6 @@ limit ?"""
         else:
             return False
 
-    def _maybe_requeue_card(self, card: Card) -> None:
-        # preview cards
-        if card.queue == QUEUE_TYPE_PREVIEW:
-            # adjust the count immediately, and rely on the once a minute
-            # checks to requeue it
-            self.lrnCount += 1
-            return
-
-        # learning cards
-        if not card.queue == QUEUE_TYPE_LRN:
-            return
-        if card.due >= (intTime() + self.col.conf["collapseTime"]):
-            return
-
-        # card is due within collapse time, so we'll want to add it
-        # back to the learning queue
-        self.lrnCount += 1
-
-        # if the queue is not empty and there's nothing else to do, make
-        # sure we don't put it at the head of the queue and end up showing
-        # it twice in a row
-        if self._lrnQueue and not self.revCount and not self.newCount:
-            smallestDue = self._lrnQueue[0][0]
-            card.due = max(card.due, smallestDue + 1)
-
-        heappush(self._lrnQueue, (card.due, card.id))
-
-    def _cardConf(self, card: Card) -> DeckConfig:
-        return self.col.decks.confForDid(card.did)
-
-    def _home_config(self, card: Card) -> DeckConfig:
-        return self.col.decks.confForDid(card.odid or card.did)
-
-    def _deckLimit(self) -> str:
-        return ids2str(self.col.decks.active())
-
-    def counts_for_deck_today(self, deck_id: int) -> CountsForDeckToday:
-        return self.col._backend.counts_for_deck_today(deck_id)
-
     # Next times
     ##########################################################################
     # fixme: move these into tests_schedv2 in the future
@@ -618,52 +232,9 @@ limit ?"""
 
         return self._interval_for_state(new_state)
 
-    # Sibling spacing
-    ##########################################################################
-
-    def _burySiblings(self, card: Card) -> None:
-        toBury: List[int] = []
-        conf = self._home_config(card)
-        bury_new = conf["new"].get("bury", True)
-        bury_rev = conf["rev"].get("bury", True)
-        # loop through and remove from queues
-        for cid, queue in self.col.db.execute(
-            f"""
-select id, queue from cards where nid=? and id!=?
-and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""",
-            card.nid,
-            card.id,
-            self.today,
-        ):
-            if queue == QUEUE_TYPE_REV:
-                queue_obj = self._revQueue
-                if bury_rev:
-                    toBury.append(cid)
-            else:
-                queue_obj = self._newQueue
-                if bury_new:
-                    toBury.append(cid)
-
-            # even if burying disabled, we still discard to give same-day spacing
-            try:
-                queue_obj.remove(cid)
-            except ValueError:
-                pass
-        # then bury
-        if toBury:
-            self.bury_cards(toBury, manual=False)
-
     # Review-related UI helpers
     ##########################################################################
 
-    def counts(self, card: Optional[Card] = None) -> Tuple[int, int, int]:
-        counts = [self.newCount, self.lrnCount, self.revCount]
-        if card:
-            idx = self.countIdx(card)
-            counts[idx] += 1
-        new, lrn, rev = counts
-        return (new, lrn, rev)
-
     def countIdx(self, card: Card) -> int:
         if card.queue in (QUEUE_TYPE_DAY_LEARN_RELEARN, QUEUE_TYPE_PREVIEW):
             return QUEUE_TYPE_LRN
@@ -708,18 +279,14 @@ and (queue={QUEUE_TYPE_NEW} or (queue={QUEUE_TYPE_REV} and due<=?))""",
         did = self.col.decks.current()["id"]
         self.col._backend.extend_limits(deck_id=did, new_delta=new, review_delta=rev)
 
-    def _is_finished(self) -> bool:
+    # fixme: used by custom study
-        "Don't use this, it is a stop-gap until this code is refactored."
-        return not any((self.newCount, self.revCount, self._immediate_learn_count))
-
     def totalRevForCurrentDeck(self) -> int:
         return self.col.db.scalar(
             f"""
 select count() from cards where id in (
-select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? limit ?)"""
+select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? limit 9999)"""
             % self._deckLimit(),
             self.today,
-            self.reportLimit,
         )
 
     # Filtered deck handling
@@ -832,11 +399,6 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l
 
     ##########################################################################
 
-    def __repr__(self) -> str:
-        d = dict(self.__dict__)
-        del d["col"]
-        return f"{super().__repr__()} {pprint.pformat(d, width=300)}"
-
     # unit tests
     def _fuzzIvlRange(self, ivl: int) -> Tuple[int, int]:
         return (ivl, ivl)
@@ -844,6 +406,11 @@ select id from cards where did in %s and queue = {QUEUE_TYPE_REV} and due <= ? l
     # Legacy aliases and helpers
     ##########################################################################
 
+    # fixme: only used by totalRevForCurrentDeck and old deck stats
+    def _deckLimit(self) -> str:
+        self.col.decks.update_active()
+        return ids2str(self.col.decks.active())
+
     def reschedCards(
         self, card_ids: List[int], min_interval: int, max_interval: int
     ) -> None:
@@ -943,6 +510,12 @@ due = (case when odue>0 then odue else due end), odue = 0, odid = 0, usn = ? whe
         )
         return from_json_bytes(self.col._backend.deck_tree_legacy())[5]
 
+    def _cardConf(self, card: Card) -> DeckConfig:
+        return self.col.decks.confForDid(card.did)
+
+    def _home_config(self, card: Card) -> DeckConfig:
+        return self.col.decks.confForDid(card.odid or card.did)
+
     def _newConf(self, card: Card) -> QueueConfig:
         return self._home_config(card)["new"]
 

pylib/anki/schedv2.py

@@ -39,6 +39,7 @@ class Scheduler:
     haveCustomStudy = True
     _burySiblingsOnAnswer = True
     revCount: int
+    is_2021 = False
 
     def __init__(self, col: anki.collection.Collection) -> None:
         self.col = col.weakref()
@@ -102,7 +103,6 @@ class Scheduler:
             self.col.log(card)
             if not self._burySiblingsOnAnswer:
                 self._burySiblings(card)
-            self.reps += 1
             card.startTimer()
             return card
         return None

pylib/tests/test_sched2021.py

@@ -0,0 +1 @@
+test_schedv2.py

pylib/tests/test_schedv2.py

@@ -2,6 +2,9 @@
 
 import copy
 import time
+from typing import Tuple
+
+import pytest
 
 from anki import hooks
 from anki.consts import *
@@ -10,10 +13,19 @@ from anki.schedv2 import UnburyCurrentDeck
 from anki.utils import intTime
 from tests.shared import getEmptyCol as getEmptyColOrig
 
+# This file is used to exercise both the legacy Python 2.1 scheduler,
+# and the experimental new one in Rust. Most tests run on both, but a few
+# tests have been implemented separately where the behaviour differs.
+is_2021 = "2021" in __file__
+new_sched_only = pytest.mark.skipif(not is_2021, reason="2021 only")
+old_sched_only = pytest.mark.skipif(is_2021, reason="old only")
+
 
 def getEmptyCol():
     col = getEmptyColOrig()
     col.upgrade_to_v2_scheduler()
+    if is_2021:
+        col.set_2021_test_scheduler_enabled(True)
     return col
 
 
@@ -183,6 +195,7 @@ def test_learn():
     c.type = CARD_TYPE_NEW
     c.queue = QUEUE_TYPE_LRN
     c.flush()
+    col.sched.reset()
     col.sched.answerCard(c, 4)
     assert c.type == CARD_TYPE_REV
     assert c.queue == QUEUE_TYPE_REV
@@ -274,6 +287,9 @@ def test_learn_day():
     note = col.newNote()
     note["Front"] = "one"
     col.addNote(note)
+    note = col.newNote()
+    note["Front"] = "two"
+    col.addNote(note)
     col.sched.reset()
     c = col.sched.getCard()
     conf = col.sched._cardConf(c)
@@ -283,11 +299,14 @@ def test_learn_day():
     col.sched.answerCard(c, 3)
     # two reps to graduate, 1 more today
     assert c.left % 1000 == 3
-    assert col.sched.counts() == (0, 1, 0)
+    assert col.sched.counts() == (1, 1, 0)
-    c = col.sched.getCard()
+    c.load()
     ni = col.sched.nextIvl
     assert ni(c, 3) == 86400
-    # answering it will place it in queue 3
+    # answer the other dummy card
+    col.sched.answerCard(col.sched.getCard(), 4)
+    # answering the first one will place it in queue 3
+    c = col.sched.getCard()
     col.sched.answerCard(c, 3)
     assert c.due == col.sched.today + 1
     assert c.queue == QUEUE_TYPE_DAY_LEARN_RELEARN
@@ -296,6 +315,10 @@ def test_learn_day():
     c.due -= 1
     c.flush()
     col.reset()
+    if is_2021:
+        # it appears in the review queue
+        assert col.sched.counts() == (0, 0, 1)
+    else:
         assert col.sched.counts() == (0, 1, 0)
     c = col.sched.getCard()
     # nextIvl should work
@@ -408,7 +431,7 @@ def test_reviews():
     assert "leech" in c.note().tags
 
 
-def test_review_limits():
+def review_limits_setup() -> Tuple[anki.collection.Collection, Dict]:
     col = getEmptyCol()
 
     parent = col.decks.get(col.decks.id("parent"))
@@ -442,6 +465,13 @@ def test_review_limits():
         c.due = 0
         c.flush()
 
+    return col, child
+
+
+@old_sched_only
+def test_review_limits():
+    col, child = review_limits_setup()
+
     tree = col.sched.deck_due_tree().children
     # (('parent', 1514457677462, 5, 0, 0, (('child', 1514457677463, 5, 0, 0, ()),)))
     assert tree[0].review_count == 5  # parent
@@ -462,6 +492,29 @@ def test_review_limits():
     assert tree[0].children[0].review_count == 9  # child
 
 
+@new_sched_only
+def test_review_limits_new():
+    col, child = review_limits_setup()
+
+    tree = col.sched.deck_due_tree().children
+    assert tree[0].review_count == 5  # parent
+    assert tree[0].children[0].review_count == 5  # child capped by parent
+
+    # child .counts() are bound by parents
+    col.decks.select(child["id"])
+    col.sched.reset()
+    assert col.sched.counts() == (0, 0, 5)
+
+    # answering a card in the child should decrement both child and parent count
+    c = col.sched.getCard()
+    col.sched.answerCard(c, 3)
+    assert col.sched.counts() == (0, 0, 4)
+
+    tree = col.sched.deck_due_tree().children
+    assert tree[0].review_count == 4  # parent
+    assert tree[0].children[0].review_count == 4  # child
+
+
 def test_button_spacing():
     col = getEmptyCol()
     note = col.newNote()
@@ -851,13 +904,20 @@ def test_ordcycle():
     note["Back"] = "1"
     col.addNote(note)
     assert col.cardCount() == 3
+
+    conf = col.decks.get_config(1)
+    conf["new"]["bury"] = False
+    col.decks.save(conf)
     col.reset()
+
     # ordinals should arrive in order
-    assert col.sched.getCard().ord == 0
+    for i in range(3):
-    assert col.sched.getCard().ord == 1
+        c = col.sched.getCard()
-    assert col.sched.getCard().ord == 2
+        assert c.ord == i
+        col.sched.answerCard(c, 4)
 
 
+@old_sched_only
 def test_counts_idx():
     col = getEmptyCol()
     note = col.newNote()
@@ -882,57 +942,87 @@ def test_counts_idx():
     assert col.sched.counts() == (0, 1, 0)
 
 
+@new_sched_only
+def test_counts_idx_new():
+    col = getEmptyCol()
+    note = col.newNote()
+    note["Front"] = "one"
+    note["Back"] = "two"
+    col.addNote(note)
+    note = col.newNote()
+    note["Front"] = "two"
+    note["Back"] = "two"
+    col.addNote(note)
+    col.reset()
+    assert col.sched.counts() == (2, 0, 0)
+    c = col.sched.getCard()
+    # getCard does not decrement counts
+    assert col.sched.counts() == (2, 0, 0)
+    assert col.sched.countIdx(c) == 0
+    # answer to move to learn queue
+    col.sched.answerCard(c, 1)
+    assert col.sched.counts() == (1, 1, 0)
+    assert col.sched.countIdx(c) == 1
+    # fetching next will not decrement the count
+    c = col.sched.getCard()
+    assert col.sched.counts() == (1, 1, 0)
+    assert col.sched.countIdx(c) == 0
+
+
 def test_repCounts():
     col = getEmptyCol()
     note = col.newNote()
     note["Front"] = "one"
     col.addNote(note)
-    col.reset()
-    # lrnReps should be accurate on pass/fail
-    assert col.sched.counts() == (1, 0, 0)
-    col.sched.answerCard(col.sched.getCard(), 1)
-    assert col.sched.counts() == (0, 1, 0)
-    col.sched.answerCard(col.sched.getCard(), 1)
-    assert col.sched.counts() == (0, 1, 0)
-    col.sched.answerCard(col.sched.getCard(), 3)
-    assert col.sched.counts() == (0, 1, 0)
-    col.sched.answerCard(col.sched.getCard(), 1)
-    assert col.sched.counts() == (0, 1, 0)
-    col.sched.answerCard(col.sched.getCard(), 3)
-    assert col.sched.counts() == (0, 1, 0)
-    col.sched.answerCard(col.sched.getCard(), 3)
-    assert col.sched.counts() == (0, 0, 0)
     note = col.newNote()
     note["Front"] = "two"
     col.addNote(note)
     col.reset()
-    # initial pass should be correct too
+    # lrnReps should be accurate on pass/fail
-    col.sched.answerCard(col.sched.getCard(), 3)
+    assert col.sched.counts() == (2, 0, 0)
-    assert col.sched.counts() == (0, 1, 0)
     col.sched.answerCard(col.sched.getCard(), 1)
+    assert col.sched.counts() == (1, 1, 0)
+    col.sched.answerCard(col.sched.getCard(), 1)
+    assert col.sched.counts() == (0, 2, 0)
+    col.sched.answerCard(col.sched.getCard(), 3)
+    assert col.sched.counts() == (0, 2, 0)
+    col.sched.answerCard(col.sched.getCard(), 1)
+    assert col.sched.counts() == (0, 2, 0)
+    col.sched.answerCard(col.sched.getCard(), 3)
     assert col.sched.counts() == (0, 1, 0)
     col.sched.answerCard(col.sched.getCard(), 4)
     assert col.sched.counts() == (0, 0, 0)
-    # immediate graduate should work
     note = col.newNote()
     note["Front"] = "three"
     col.addNote(note)
+    note = col.newNote()
+    note["Front"] = "four"
+    col.addNote(note)
     col.reset()
+    # initial pass and immediate graduate should be correct too
+    assert col.sched.counts() == (2, 0, 0)
+    col.sched.answerCard(col.sched.getCard(), 3)
+    assert col.sched.counts() == (1, 1, 0)
+    col.sched.answerCard(col.sched.getCard(), 4)
+    assert col.sched.counts() == (0, 1, 0)
     col.sched.answerCard(col.sched.getCard(), 4)
     assert col.sched.counts() == (0, 0, 0)
     # and failing a review should too
     note = col.newNote()
-    note["Front"] = "three"
+    note["Front"] = "five"
     col.addNote(note)
     c = note.cards()[0]
     c.type = CARD_TYPE_REV
     c.queue = QUEUE_TYPE_REV
     c.due = col.sched.today
     c.flush()
+    note = col.newNote()
+    note["Front"] = "six"
+    col.addNote(note)
     col.reset()
-    assert col.sched.counts() == (0, 0, 1)
+    assert col.sched.counts() == (1, 0, 1)
     col.sched.answerCard(col.sched.getCard(), 1)
-    assert col.sched.counts() == (0, 1, 0)
+    assert col.sched.counts() == (1, 1, 0)
 
 
 def test_timing():
@@ -968,12 +1058,25 @@ def test_collapse():
     note = col.newNote()
     note["Front"] = "one"
     col.addNote(note)
+    # and another, so we don't get the same twice in a row
+    note = col.newNote()
+    note["Front"] = "two"
+    col.addNote(note)
     col.reset()
-    # test collapsing
+    # first note
     c = col.sched.getCard()
     col.sched.answerCard(c, 1)
-    c = col.sched.getCard()
+    # second note
-    col.sched.answerCard(c, 4)
+    c2 = col.sched.getCard()
+    assert c2.nid != c.nid
+    col.sched.answerCard(c2, 1)
+    # first should become available again, despite it being due in the future
+    c3 = col.sched.getCard()
+    assert c3.due > intTime()
+    col.sched.answerCard(c3, 4)
+    # answer other
+    c4 = col.sched.getCard()
+    col.sched.answerCard(c4, 4)
     assert not col.sched.getCard()
 
 
@@ -1049,9 +1152,16 @@ def test_deckFlow():
     note["Front"] = "three"
     default1 = note.model()["did"] = col.decks.id("Default::1")
     col.addNote(note)
-    # should get top level one first, then ::1, then ::2
     col.reset()
     assert col.sched.counts() == (3, 0, 0)
+    if is_2021:
+        # cards arrive in position order by default
+        for i in "one", "two", "three":
+            c = col.sched.getCard()
+            assert c.note()["Front"] == i
+            col.sched.answerCard(c, 3)
+    else:
+        # should get top level one first, then ::1, then ::2
         for i in "one", "three", "two":
             c = col.sched.getCard()
             assert c.note()["Front"] == i
@@ -1120,13 +1230,13 @@ def test_resched():
     note["Front"] = "one"
     col.addNote(note)
     c = note.cards()[0]
-    col.sched.reschedCards([c.id], 0, 0)
+    col.sched.set_due_date([c.id], "0")
     c.load()
     assert c.due == col.sched.today
     assert c.ivl == 1
     assert c.queue == QUEUE_TYPE_REV and c.type == CARD_TYPE_REV
     # make it due tomorrow
-    col.sched.reschedCards([c.id], 1, 1)
+    col.sched.set_due_date([c.id], "1")
     c.load()
     assert c.due == col.sched.today + 1
     assert c.ivl == 1

pylib/tests/test_undo.py

@@ -50,31 +50,29 @@ def test_review():
     note = col.newNote()
     note["Front"] = "one"
     col.addNote(note)
+    note = col.newNote()
+    note["Front"] = "two"
+    col.addNote(note)
     col.reset()
     assert not col.undoName()
     # answer
-    assert col.sched.counts() == (1, 0, 0)
+    assert col.sched.counts() == (2, 0, 0)
     c = col.sched.getCard()
     assert c.queue == QUEUE_TYPE_NEW
     col.sched.answerCard(c, 3)
     assert c.left % 1000 == 1
-    assert col.sched.counts() == (0, 1, 0)
+    assert col.sched.counts() == (1, 1, 0)
     assert c.queue == QUEUE_TYPE_LRN
     # undo
     assert col.undoName()
     col.undo()
     col.reset()
-    assert col.sched.counts() == (1, 0, 0)
+    assert col.sched.counts() == (2, 0, 0)
     c.load()
     assert c.queue == QUEUE_TYPE_NEW
     assert c.left % 1000 != 1
     assert not col.undoName()
     # we should be able to undo multiple answers too
-    note = col.newNote()
-    note["Front"] = "two"
-    col.addNote(note)
-    col.reset()
-    assert col.sched.counts() == (2, 0, 0)
     c = col.sched.getCard()
     col.sched.answerCard(c, 3)
     c = col.sched.getCard()

rslib/BUILD.bazel

@@ -84,6 +84,7 @@ rust_library(
         "//rslib/cargo:failure",
         "//rslib/cargo:flate2",
         "//rslib/cargo:fluent",
+        "//rslib/cargo:fnv",
         "//rslib/cargo:futures",
         "//rslib/cargo:hex",
         "//rslib/cargo:htmlescape",

rslib/Cargo.toml

@@ -81,3 +81,4 @@ async-trait = "0.1.42"
 proc-macro-nested = "=0.1.6"
 ammonia = "3.1.0"
 pulldown-cmark = "0.8.0"
+fnv = "1.0.7"

rslib/backend.proto

@@ -120,6 +120,9 @@ service BackendService {
   rpc StateIsLeech(SchedulingState) returns (Bool);
   rpc AnswerCard(AnswerCardIn) returns (Empty);
   rpc UpgradeScheduler(Empty) returns (Empty);
+  rpc GetQueuedCards(GetQueuedCardsIn) returns (GetQueuedCardsOut);
+  rpc ClearCardQueues(Empty) returns (Empty);
+  rpc RequeueUndoneCard(CardID) returns (Empty);
 
   // stats
 
@@ -252,7 +255,17 @@ message DeckConfigInner {
     NEW_CARD_ORDER_DUE = 0;
     NEW_CARD_ORDER_RANDOM = 1;
   }
-
+  enum ReviewCardOrder {
+    REVIEW_CARD_ORDER_SHUFFLED_BY_DAY = 0;
+    REVIEW_CARD_ORDER_SHUFFLED = 1;
+    REVIEW_CARD_ORDER_INTERVALS_ASCENDING = 2;
+    REVIEW_CARD_ORDER_INTERVALS_DESCENDING = 3;
+  }
+  enum ReviewMix {
+    REVIEW_MIX_MIX_WITH_REVIEWS = 0;
+    REVIEW_MIX_AFTER_REVIEWS = 1;
+    REVIEW_MIX_BEFORE_REVIEWS = 2;
+  }
   enum LeechAction {
     LEECH_ACTION_SUSPEND = 0;
     LEECH_ACTION_TAG_ONLY = 1;
@@ -265,6 +278,7 @@ message DeckConfigInner {
 
   uint32 new_per_day = 9;
   uint32 reviews_per_day = 10;
+  uint32 new_per_day_minimum = 29;
 
   float initial_ease = 11;
   float easy_multiplier = 12;
@@ -279,6 +293,10 @@ message DeckConfigInner {
   uint32 graduating_interval_easy = 19;
 
   NewCardOrder new_card_order = 20;
+  ReviewCardOrder review_order = 32;
+
+  ReviewMix new_mix = 30;
+  ReviewMix interday_learning_mix = 31;
 
   LeechAction leech_action = 21;
   uint32 leech_threshold = 22;
@@ -1243,6 +1261,7 @@ message Config {
       COLLAPSE_TODAY = 6;
       COLLAPSE_CARD_STATE = 7;
       COLLAPSE_FLAGS = 8;
+      SCHED_2021 = 9;
     }
     Key key = 1;
   }
@@ -1341,3 +1360,34 @@ message AnswerCardIn {
   int64 answered_at_millis = 5;
   uint32 milliseconds_taken = 6;
 }
+
+message GetQueuedCardsIn {
+  uint32 fetch_limit = 1;
+  bool intraday_learning_only = 2;
+}
+
+message GetQueuedCardsOut {
+  enum Queue {
+    New = 0;
+    Learning = 1;
+    Review = 2;
+  }
+
+  message QueuedCard {
+    Card card = 1;
+    Queue queue = 5;
+    NextCardStates next_states = 6;
+  }
+
+  message QueuedCards {
+    repeated QueuedCard cards = 1;
+    uint32 new_count = 2;
+    uint32 learning_count = 3;
+    uint32 review_count = 4;
+  }
+
+  oneof value {
+    QueuedCards queued_cards = 1;
+    CongratsInfoOut congrats_info = 2;
+  }
+}

rslib/cargo/BUILD.bazel

@@ -129,6 +129,15 @@ alias(
     ],
 )
 
+alias(
+    name = "fnv",
+    actual = "@raze__fnv__1_0_7//:fnv",
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+)
+
 alias(
     name = "futures",
     actual = "@raze__futures__0_3_12//:futures",

rslib/src/backend/mod.rs

@@ -698,6 +698,25 @@ impl BackendService for Backend {
             .map(Into::into)
     }
 
+    fn get_queued_cards(
+        &self,
+        input: pb::GetQueuedCardsIn,
+    ) -> BackendResult<pb::GetQueuedCardsOut> {
+        self.with_col(|col| col.get_queued_cards(input.fetch_limit, input.intraday_learning_only))
+    }
+
+    fn clear_card_queues(&self, _input: pb::Empty) -> BackendResult<pb::Empty> {
+        self.with_col(|col| {
+            col.clear_queues();
+            Ok(().into())
+        })
+    }
+
+    fn requeue_undone_card(&self, input: pb::CardId) -> BackendResult<pb::Empty> {
+        self.with_col(|col| col.requeue_undone_card(input.into()))
+            .map(Into::into)
+    }
+
     // statistics
     //-----------------------------------------------
 
@@ -2059,8 +2078,8 @@ fn pbcard_to_native(c: pb::Card) -> Result<Card> {
     })
 }
 
-impl From<crate::scheduler::cutoff::SchedTimingToday> for pb::SchedTimingTodayOut {
+impl From<crate::scheduler::timing::SchedTimingToday> for pb::SchedTimingTodayOut {
-    fn from(t: crate::scheduler::cutoff::SchedTimingToday) -> pb::SchedTimingTodayOut {
+    fn from(t: crate::scheduler::timing::SchedTimingToday) -> pb::SchedTimingTodayOut {
         pb::SchedTimingTodayOut {
             days_elapsed: t.days_elapsed,
             next_day_at: t.next_day_at,

rslib/src/card.rs

@@ -118,6 +118,10 @@ impl Card {
     pub fn ease_factor(&self) -> f32 {
         (self.ease_factor as f32) / 1000.0
     }
+
+    pub fn is_intraday_learning(&self) -> bool {
+        matches!(self.queue, CardQueue::Learn | CardQueue::PreviewRepeat)
+    }
 }
 #[derive(Debug)]
 pub(crate) struct UpdateCardUndo(Card);

rslib/src/collection.rs

@@ -1,7 +1,6 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::err::Result;
 use crate::i18n::I18n;
 use crate::log::Logger;
 use crate::types::Usn;
@@ -11,6 +10,7 @@ use crate::{
     storage::SqliteStorage,
     undo::UndoManager,
 };
+use crate::{err::Result, scheduler::queue::CardQueues};
 use std::{collections::HashMap, path::PathBuf, sync::Arc};
 
 pub fn open_collection<P: Into<PathBuf>>(
@@ -63,6 +63,7 @@ pub struct CollectionState {
     pub(crate) undo: UndoManager,
     pub(crate) notetype_cache: HashMap<NoteTypeID, Arc<NoteType>>,
     pub(crate) deck_cache: HashMap<DeckID, Arc<Deck>>,
+    pub(crate) card_queues: Option<CardQueues>,
 }
 
 pub struct Collection {

rslib/src/config.rs

@@ -5,8 +5,8 @@ use crate::{
     backend_proto as pb, collection::Collection, decks::DeckID, err::Result, notetype::NoteTypeID,
     timestamp::TimestampSecs,
 };
-use pb::config::bool::Key as BoolKey;
+pub use pb::config::bool::Key as BoolKey;
-use pb::config::string::Key as StringKey;
+pub use pb::config::string::Key as StringKey;
 use serde::{de::DeserializeOwned, Serialize};
 use serde_aux::field_attributes::deserialize_bool_from_anything;
 use serde_derive::Deserialize;
@@ -63,6 +63,7 @@ pub(crate) enum ConfigKey {
     NormalizeNoteText,
     PreviewBothSides,
     Rollover,
+    Sched2021,
     SchedulerVersion,
     SetDueBrowser,
     SetDueReviewer,
@@ -104,6 +105,7 @@ impl From<ConfigKey> for &'static str {
             ConfigKey::NormalizeNoteText => "normalize_note_text",
             ConfigKey::PreviewBothSides => "previewBothSides",
             ConfigKey::Rollover => "rollover",
+            ConfigKey::Sched2021 => "sched2021",
             ConfigKey::SchedulerVersion => "schedVer",
             ConfigKey::SetDueBrowser => "setDueBrowser",
             ConfigKey::SetDueReviewer => "setDueReviewer",
@@ -126,6 +128,7 @@ impl From<BoolKey> for ConfigKey {
             BoolKey::CollapseTags => ConfigKey::CollapseTags,
             BoolKey::CollapseToday => ConfigKey::CollapseToday,
             BoolKey::PreviewBothSides => ConfigKey::PreviewBothSides,
+            BoolKey::Sched2021 => ConfigKey::Sched2021,
         }
     }
 }
@@ -365,7 +368,12 @@ impl Collection {
 
     #[allow(clippy::match_single_binding)]
     pub(crate) fn get_bool(&self, config: pb::config::Bool) -> bool {
-        match config.key() {
+        self.get_bool_key(config.key())
+    }
+
+    #[allow(clippy::match_single_binding)]
+    pub(crate) fn get_bool_key(&self, key: BoolKey) -> bool {
+        match key {
             // all options default to false at the moment
             other => self.get_config_default(ConfigKey::from(other)),
         }
@@ -421,12 +429,19 @@ impl Default for SortKind {
     }
 }
 
+// 2021 scheduler moves this into deck config
 pub(crate) enum NewReviewMix {
     Mix = 0,
     ReviewsFirst = 1,
     NewFirst = 2,
 }
 
+impl Default for NewReviewMix {
+    fn default() -> Self {
+        NewReviewMix::Mix
+    }
+}
+
 #[derive(PartialEq, Serialize_repr, Deserialize_repr, Clone, Copy)]
 #[repr(u8)]
 pub(crate) enum Weekday {

rslib/src/deckconf/mod.rs

@@ -11,7 +11,7 @@ use crate::{
 };
 
 pub use crate::backend_proto::{
-    deck_config_inner::{LeechAction, NewCardOrder},
+    deck_config_inner::{LeechAction, NewCardOrder, ReviewCardOrder, ReviewMix},
     DeckConfigInner,
 };
 pub use schema11::{DeckConfSchema11, NewCardOrderSchema11};
@@ -41,14 +41,9 @@ impl Default for DeckConf {
             inner: DeckConfigInner {
                 learn_steps: vec![1.0, 10.0],
                 relearn_steps: vec![10.0],
-                disable_autoplay: false,
-                cap_answer_time_to_secs: 60,
-                visible_timer_secs: 0,
-                skip_question_when_replaying_answer: false,
                 new_per_day: 20,
                 reviews_per_day: 200,
-                bury_new: false,
+                new_per_day_minimum: 0,
-                bury_reviews: false,
                 initial_ease: 2.5,
                 easy_multiplier: 1.3,
                 hard_multiplier: 1.2,
@@ -59,8 +54,17 @@ impl Default for DeckConf {
                 graduating_interval_good: 1,
                 graduating_interval_easy: 4,
                 new_card_order: NewCardOrder::Due as i32,
+                review_order: ReviewCardOrder::ShuffledByDay as i32,
+                new_mix: ReviewMix::MixWithReviews as i32,
+                interday_learning_mix: ReviewMix::MixWithReviews as i32,
                 leech_action: LeechAction::TagOnly as i32,
                 leech_threshold: 8,
+                disable_autoplay: false,
+                cap_answer_time_to_secs: 60,
+                visible_timer_secs: 0,
+                skip_question_when_replaying_answer: false,
+                bury_new: false,
+                bury_reviews: false,
                 other: vec![],
             },
         }

rslib/src/deckconf/schema11.rs

@@ -32,6 +32,18 @@ pub struct DeckConfSchema11 {
     pub(crate) lapse: LapseConfSchema11,
     #[serde(rename = "dyn", default, deserialize_with = "default_on_invalid")]
     dynamic: bool,
+
+    // 2021 scheduler options: these were not in schema 11, but we need to persist them
+    // so the settings are not lost on upgrade/downgrade
+    #[serde(default)]
+    new_mix: i32,
+    #[serde(default)]
+    new_per_day_minimum: u32,
+    #[serde(default)]
+    interday_learning_mix: i32,
+    #[serde(default)]
+    review_order: i32,
+
     #[serde(flatten)]
     other: HashMap<String, Value>,
 }
@@ -191,6 +203,10 @@ impl Default for DeckConfSchema11 {
             rev: Default::default(),
             lapse: Default::default(),
             other: Default::default(),
+            new_mix: 0,
+            new_per_day_minimum: 0,
+            interday_learning_mix: 0,
+            review_order: 0,
         }
     }
 }
@@ -229,14 +245,9 @@ impl From<DeckConfSchema11> for DeckConf {
             inner: DeckConfigInner {
                 learn_steps: c.new.delays,
                 relearn_steps: c.lapse.delays,
-                disable_autoplay: !c.autoplay,
-                cap_answer_time_to_secs: c.max_taken.max(0) as u32,
-                visible_timer_secs: c.timer as u32,
-                skip_question_when_replaying_answer: !c.replayq,
                 new_per_day: c.new.per_day,
                 reviews_per_day: c.rev.per_day,
-                bury_new: c.new.bury,
+                new_per_day_minimum: c.new_per_day_minimum,
-                bury_reviews: c.rev.bury,
                 initial_ease: (c.new.initial_factor as f32) / 1000.0,
                 easy_multiplier: c.rev.ease4,
                 hard_multiplier: c.rev.hard_factor,
@@ -250,8 +261,17 @@ impl From<DeckConfSchema11> for DeckConf {
                     NewCardOrderSchema11::Random => NewCardOrder::Random,
                     NewCardOrderSchema11::Due => NewCardOrder::Due,
                 } as i32,
+                review_order: c.review_order,
+                new_mix: c.new_mix,
+                interday_learning_mix: c.interday_learning_mix,
                 leech_action: c.lapse.leech_action as i32,
                 leech_threshold: c.lapse.leech_fails,
+                disable_autoplay: !c.autoplay,
+                cap_answer_time_to_secs: c.max_taken.max(0) as u32,
+                visible_timer_secs: c.timer as u32,
+                skip_question_when_replaying_answer: !c.replayq,
+                bury_new: c.new.bury,
+                bury_reviews: c.rev.bury,
                 other: other_bytes,
             },
         }
@@ -332,6 +352,10 @@ impl From<DeckConf> for DeckConfSchema11 {
                 other: lapse_other,
             },
             other: top_other,
+            new_mix: i.new_mix,
+            new_per_day_minimum: i.new_per_day_minimum,
+            interday_learning_mix: i.interday_learning_mix,
+            review_order: i.review_order,
         }
     }
 }

rslib/src/decks/tree.rs

@@ -5,7 +5,7 @@ use super::{Deck, DeckKind, DueCounts};
 use crate::{
     backend_proto::DeckTreeNode,
     collection::Collection,
-    config::SchedulerVersion,
+    config::{BoolKey, SchedulerVersion},
     deckconf::{DeckConf, DeckConfID},
     decks::DeckID,
     err::Result,
@@ -123,12 +123,11 @@ fn apply_limits(
     node.review_count = (node.review_count + child_rev_total).min(remaining_rev);
 }
 
-/// Apply parent new limits to children, and add child counts to parents.
+/// Apply parent new limits to children, and add child counts to parents. Unlike
-/// Unlike v1, reviews are not capped by their parents, and we return the
+/// v1 and the 2021 scheduler, reviews are not capped by their parents, and we
-/// uncapped review amount to add to the parent. This is a bit of a hack, and
+/// return the uncapped review amount to add to the parent.
-/// just tides us over until the v2 queue building code can be reworked.
 /// Counts are (new, review).
-fn apply_limits_v2(
+fn apply_limits_v2_old(
     node: &mut DeckTreeNode,
     today: u32,
     decks: &HashMap<DeckID, Deck>,
@@ -148,7 +147,7 @@ fn apply_limits_v2(
     let mut child_rev_total = 0;
     for child in &mut node.children {
         child_rev_total +=
-            apply_limits_v2(child, today, decks, dconf, (remaining_new, remaining_rev));
+            apply_limits_v2_old(child, today, decks, dconf, (remaining_new, remaining_rev));
         child_new_total += child.new_count;
         // no limit on learning cards
         node.learn_count += child.learn_count;
@@ -283,8 +282,10 @@ impl Collection {
             let counts = self.due_counts(days_elapsed, learn_cutoff, limit)?;
             let dconf = self.storage.get_deck_config_map()?;
             add_counts(&mut tree, &counts);
-            if self.scheduler_version() == SchedulerVersion::V2 {
+            if self.scheduler_version() == SchedulerVersion::V2
-                apply_limits_v2(
+                && !self.get_bool_key(BoolKey::Sched2021)
+            {
+                apply_limits_v2_old(
                     &mut tree,
                     days_elapsed,
                     &decks_map,

rslib/src/preferences.rs

@@ -8,7 +8,7 @@ use crate::{
     },
     collection::Collection,
     err::Result,
-    scheduler::cutoff::local_minutes_west_for_stamp,
+    scheduler::timing::local_minutes_west_for_stamp,
 };
 
 impl Collection {

rslib/src/scheduler/answering/mod.rs

@@ -20,11 +20,11 @@ use crate::{
 use revlog::RevlogEntryPartial;
 
 use super::{
-    cutoff::SchedTimingToday,
     states::{
         steps::LearningSteps, CardState, FilteredState, NextCardStates, NormalState, StateContext,
     },
     timespan::answer_button_time_collapsible,
+    timing::SchedTimingToday,
 };
 
 #[derive(Copy, Clone)]
@@ -239,6 +239,7 @@ impl Collection {
             self.add_partial_revlog(revlog_partial, usn, &answer)?;
         }
         self.update_deck_stats_from_answer(usn, &answer, &updater)?;
+        let timing = updater.timing;
 
         let mut card = updater.into_card();
         self.update_card(&mut card, &original, usn)?;
@@ -246,6 +247,8 @@ impl Collection {
             self.add_leech_tag(card.note_id)?;
         }
 
+        self.update_queues_after_answering_card(&card, timing)?;
+
         Ok(())
     }
 

rslib/src/scheduler/bury_and_suspend.rs

@@ -10,7 +10,7 @@ use crate::{
     search::SortMode,
 };
 
-use super::cutoff::SchedTimingToday;
+use super::timing::SchedTimingToday;
 use pb::unbury_cards_in_current_deck_in::Mode as UnburyDeckMode;
 
 impl Card {

rslib/src/scheduler/mod.rs

@@ -6,20 +6,21 @@ use crate::{collection::Collection, config::SchedulerVersion, err::Result, prelu
 pub mod answering;
 pub mod bury_and_suspend;
 pub(crate) mod congrats;
-pub mod cutoff;
 mod learning;
 pub mod new;
+pub(crate) mod queue;
 mod reviews;
 pub mod states;
 pub mod timespan;
+pub mod timing;
 mod upgrade;
 
 use chrono::FixedOffset;
-use cutoff::{
+pub use reviews::parse_due_date_str;
+use timing::{
     sched_timing_today, v1_creation_date_adjusted_to_hour, v1_rollover_from_creation_stamp,
     SchedTimingToday,
 };
-pub use reviews::parse_due_date_str;
 
 impl Collection {
     pub fn timing_today(&self) -> Result<SchedTimingToday> {

rslib/src/scheduler/queue/builder/gathering.rs

@@ -0,0 +1,174 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{super::limits::RemainingLimits, DueCard, NewCard, QueueBuilder};
+use crate::{card::CardQueue, prelude::*};
+
+impl QueueBuilder {
+    /// Assumes cards will arrive sorted in (queue, due) order, so learning
+    /// cards come first, and reviews come before day-learning and preview cards.
+    pub(in super::super) fn add_due_card(
+        &mut self,
+        limit: &mut RemainingLimits,
+        queue: CardQueue,
+        card: DueCard,
+    ) -> bool {
+        let should_add = self.should_add_review_card(card.note_id);
+
+        match queue {
+            CardQueue::Learn | CardQueue::PreviewRepeat => self.learning.push(card),
+            CardQueue::DayLearn => {
+                self.day_learning.push(card);
+            }
+            CardQueue::Review => {
+                if should_add {
+                    self.review.push(card);
+                    limit.review -= 1;
+                }
+            }
+            CardQueue::New
+            | CardQueue::Suspended
+            | CardQueue::SchedBuried
+            | CardQueue::UserBuried => {
+                unreachable!()
+            }
+        }
+
+        limit.review != 0
+    }
+
+    pub(in super::super) fn add_new_card(
+        &mut self,
+        limit: &mut RemainingLimits,
+        card: NewCard,
+    ) -> bool {
+        let already_seen = self.have_seen_note_id(card.note_id);
+        if !already_seen {
+            self.new.push(card);
+            limit.new -= 1;
+            return limit.new != 0;
+        }
+
+        // Cards will be arriving in (due, card_id) order, with all
+        // siblings sharing the same due number by default. In the
+        // common case, card ids will match template order, and nothing
+        // special is required. But if some cards have been generated
+        // after the initial note creation, they will have higher card
+        // ids, and the siblings will thus arrive in the wrong order.
+        // Sorting by ordinal in the DB layer is fairly costly, as it
+        // doesn't allow us to exit early when the daily limits have
+        // been met, so we want to enforce ordering as we add instead.
+        let previous_card_was_sibling_with_higher_ordinal = self
+            .new
+            .last()
+            .map(|previous| previous.note_id == card.note_id && previous.extra > card.extra)
+            .unwrap_or(false);
+
+        if previous_card_was_sibling_with_higher_ordinal {
+            if self.bury_new {
+                // When burying is enabled, we replace the existing sibling
+                // with the lower ordinal one.
+                *self.new.last_mut().unwrap() = card;
+            } else {
+                // When burying disabled, we'll want to add this card as well, but
+                // not at the end of the list.
+                let target_idx = self
+                    .new
+                    .iter()
+                    .enumerate()
+                    .rev()
+                    .filter_map(|(idx, queued_card)| {
+                        if queued_card.note_id != card.note_id || queued_card.extra < card.extra {
+                            Some(idx + 1)
+                        } else {
+                            None
+                        }
+                    })
+                    .next()
+                    .unwrap_or(0);
+                self.new.insert(target_idx, card);
+                limit.new -= 1;
+            }
+        } else {
+            // card has arrived in expected order - add if burying disabled
+            if !self.bury_new {
+                self.new.push(card);
+                limit.new -= 1;
+            }
+        }
+
+        limit.new != 0
+    }
+
+    fn should_add_review_card(&mut self, note_id: NoteID) -> bool {
+        !self.have_seen_note_id(note_id) || !self.bury_reviews
+    }
+
+    /// Mark note seen, and return true if seen before.
+    fn have_seen_note_id(&mut self, note_id: NoteID) -> bool {
+        !self.seen_note_ids.insert(note_id)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn new_siblings() {
+        let mut builder = QueueBuilder::default();
+        builder.bury_new = true;
+        let mut limits = RemainingLimits {
+            review: 0,
+            new: 100,
+        };
+
+        let cards = vec![
+            NewCard {
+                id: CardID(1),
+                note_id: NoteID(1),
+                extra: 0,
+                ..Default::default()
+            },
+            NewCard {
+                id: CardID(2),
+                note_id: NoteID(2),
+                extra: 1,
+                ..Default::default()
+            },
+            NewCard {
+                id: CardID(3),
+                note_id: NoteID(2),
+                extra: 2,
+                ..Default::default()
+            },
+            NewCard {
+                id: CardID(4),
+                note_id: NoteID(2),
+                extra: 0,
+                ..Default::default()
+            },
+        ];
+
+        for card in &cards {
+            builder.add_new_card(&mut limits, card.clone());
+        }
+
+        assert_eq!(builder.new[0].id, CardID(1));
+        assert_eq!(builder.new[1].id, CardID(4));
+        assert_eq!(builder.new.len(), 2);
+
+        // with burying disabled, we should get all siblings in order
+        builder.bury_new = false;
+        builder.new.truncate(0);
+
+        for card in &cards {
+            builder.add_new_card(&mut limits, card.clone());
+        }
+
+        assert_eq!(builder.new[0].id, CardID(1));
+        assert_eq!(builder.new[1].id, CardID(4));
+        assert_eq!(builder.new[2].id, CardID(2));
+        assert_eq!(builder.new[3].id, CardID(3));
+    }
+}

rslib/src/scheduler/queue/builder/intersperser.rs

@@ -0,0 +1,123 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/// Adapter to evenly mix two iterators of varying lengths into one.
+pub(crate) struct Intersperser<I, I2>
+where
+    I: Iterator + ExactSizeIterator,
+{
+    one: I,
+    two: I2,
+    one_idx: usize,
+    two_idx: usize,
+    one_len: usize,
+    two_len: usize,
+    ratio: f32,
+}
+
+impl<I, I2> Intersperser<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+    pub fn new(one: I, two: I2) -> Self {
+        let one_len = one.len();
+        let two_len = two.len();
+        let ratio = one_len as f32 / two_len as f32;
+        Intersperser {
+            one,
+            two,
+            one_idx: 0,
+            two_idx: 0,
+            one_len,
+            two_len,
+            ratio,
+        }
+    }
+
+    fn one_idx(&self) -> Option<usize> {
+        if self.one_idx == self.one_len {
+            None
+        } else {
+            Some(self.one_idx)
+        }
+    }
+
+    fn two_idx(&self) -> Option<usize> {
+        if self.two_idx == self.two_len {
+            None
+        } else {
+            Some(self.two_idx)
+        }
+    }
+
+    fn next_one(&mut self) -> Option<I::Item> {
+        self.one_idx += 1;
+        self.one.next()
+    }
+
+    fn next_two(&mut self) -> Option<I::Item> {
+        self.two_idx += 1;
+        self.two.next()
+    }
+}
+
+impl<I, I2> Iterator for Intersperser<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+    type Item = I::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match (self.one_idx(), self.two_idx()) {
+            (Some(idx1), Some(idx2)) => {
+                let relative_idx2 = idx2 as f32 * self.ratio;
+                if relative_idx2 < idx1 as f32 {
+                    self.next_two()
+                } else {
+                    self.next_one()
+                }
+            }
+            (Some(_), None) => self.next_one(),
+            (None, Some(_)) => self.next_two(),
+            (None, None) => None,
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let remaining = (self.one_len + self.two_len) - (self.one_idx + self.two_idx);
+        (remaining, Some(remaining))
+    }
+}
+
+impl<I, I2> ExactSizeIterator for Intersperser<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+}
+
+#[cfg(test)]
+mod test {
+    use super::Intersperser;
+
+    fn intersperse(a: &[u32], b: &[u32]) -> Vec<u32> {
+        Intersperser::new(a.iter().cloned(), b.iter().cloned()).collect()
+    }
+
+    #[test]
+    fn interspersing() {
+        let a = &[1, 2, 3];
+        let b = &[11, 22, 33];
+        assert_eq!(&intersperse(a, b), &[1, 11, 2, 22, 3, 33]);
+
+        let b = &[11, 22];
+        assert_eq!(&intersperse(a, b), &[1, 11, 2, 22, 3]);
+
+        // when both lists have the same relative position, we add from
+        // list 1 even if list 2 has more elements
+        let b = &[11, 22, 33, 44, 55, 66];
+        assert_eq!(&intersperse(a, b), &[1, 11, 22, 2, 33, 44, 3, 55, 66]);
+    }
+}

rslib/src/scheduler/queue/builder/mod.rs

@@ -0,0 +1,224 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+mod gathering;
+pub(crate) mod intersperser;
+pub(crate) mod sized_chain;
+mod sorting;
+
+use std::{
+    cmp::Reverse,
+    collections::{BinaryHeap, HashSet, VecDeque},
+};
+
+use super::{
+    limits::{remaining_limits_capped_to_parents, RemainingLimits},
+    CardQueues, LearningQueueEntry, QueueEntry, QueueEntryKind,
+};
+use crate::deckconf::{NewCardOrder, ReviewCardOrder, ReviewMix};
+use crate::prelude::*;
+use {intersperser::Intersperser, sized_chain::SizedChain};
+
+/// Temporary holder for review cards that will be built into a queue.
+#[derive(Debug, Default, Clone)]
+pub(crate) struct DueCard {
+    pub id: CardID,
+    pub note_id: NoteID,
+    pub mtime: TimestampSecs,
+    pub due: i32,
+    /// Used to store interval, and for shuffling
+    pub extra: u64,
+}
+
+/// Temporary holder for new cards that will be built into a queue.
+#[derive(Debug, Default, Clone)]
+pub(crate) struct NewCard {
+    pub id: CardID,
+    pub note_id: NoteID,
+    pub mtime: TimestampSecs,
+    pub due: i32,
+    /// Used to store template_idx, and for shuffling
+    pub extra: u64,
+}
+
+impl From<DueCard> for QueueEntry {
+    fn from(c: DueCard) -> Self {
+        QueueEntry {
+            id: c.id,
+            mtime: c.mtime,
+            kind: QueueEntryKind::Review,
+        }
+    }
+}
+
+impl From<NewCard> for QueueEntry {
+    fn from(c: NewCard) -> Self {
+        QueueEntry {
+            id: c.id,
+            mtime: c.mtime,
+            kind: QueueEntryKind::New,
+        }
+    }
+}
+
+impl From<DueCard> for LearningQueueEntry {
+    fn from(c: DueCard) -> Self {
+        LearningQueueEntry {
+            due: TimestampSecs(c.due as i64),
+            id: c.id,
+            mtime: c.mtime,
+        }
+    }
+}
+
+#[derive(Default)]
+pub(super) struct QueueBuilder {
+    pub(super) new: Vec<NewCard>,
+    pub(super) review: Vec<DueCard>,
+    pub(super) learning: Vec<DueCard>,
+    pub(super) day_learning: Vec<DueCard>,
+    pub(super) seen_note_ids: HashSet<NoteID>,
+    pub(super) new_order: NewCardOrder,
+    pub(super) review_order: ReviewCardOrder,
+    pub(super) day_learn_mix: ReviewMix,
+    pub(super) new_review_mix: ReviewMix,
+    pub(super) bury_new: bool,
+    pub(super) bury_reviews: bool,
+}
+
+impl QueueBuilder {
+    pub(super) fn build(
+        mut self,
+        top_deck_limits: RemainingLimits,
+        learn_ahead_secs: u32,
+        selected_deck: DeckID,
+        current_day: u32,
+    ) -> CardQueues {
+        self.sort_new();
+        self.sort_reviews();
+
+        // split and sort learning
+        let learn_ahead_secs = learn_ahead_secs as i64;
+        let (due_learning, later_learning) = split_learning(self.learning, learn_ahead_secs);
+        let learn_count = due_learning.len();
+
+        // merge day learning in, and cap to parent review count
+        let main_iter = merge_day_learning(self.review, self.day_learning, self.day_learn_mix);
+        let main_iter = main_iter.take(top_deck_limits.review as usize);
+        let review_count = main_iter.len();
+
+        // cap to parent new count, note down the new count, then merge new in
+        self.new.truncate(top_deck_limits.new as usize);
+        let new_count = self.new.len();
+        let main_iter = merge_new(main_iter, self.new, self.new_review_mix);
+
+        CardQueues {
+            new_count,
+            review_count,
+            learn_count,
+            main: main_iter.collect(),
+            due_learning,
+            later_learning,
+            learn_ahead_secs,
+            selected_deck,
+            current_day,
+        }
+    }
+}
+
+fn merge_day_learning(
+    reviews: Vec<DueCard>,
+    day_learning: Vec<DueCard>,
+    mode: ReviewMix,
+) -> Box<dyn ExactSizeIterator<Item = QueueEntry>> {
+    let day_learning_iter = day_learning.into_iter().map(Into::into);
+    let reviews_iter = reviews.into_iter().map(Into::into);
+
+    match mode {
+        ReviewMix::AfterReviews => Box::new(SizedChain::new(reviews_iter, day_learning_iter)),
+        ReviewMix::BeforeReviews => Box::new(SizedChain::new(day_learning_iter, reviews_iter)),
+        ReviewMix::MixWithReviews => Box::new(Intersperser::new(reviews_iter, day_learning_iter)),
+    }
+}
+
+fn merge_new(
+    review_iter: impl ExactSizeIterator<Item = QueueEntry> + 'static,
+    new: Vec<NewCard>,
+    mode: ReviewMix,
+) -> Box<dyn ExactSizeIterator<Item = QueueEntry>> {
+    let new_iter = new.into_iter().map(Into::into);
+
+    match mode {
+        ReviewMix::BeforeReviews => Box::new(SizedChain::new(new_iter, review_iter)),
+        ReviewMix::AfterReviews => Box::new(SizedChain::new(review_iter, new_iter)),
+        ReviewMix::MixWithReviews => Box::new(Intersperser::new(review_iter, new_iter)),
+    }
+}
+
+/// Split the learning queue into cards due within limit, and cards due later
+/// today. Learning does not need to be sorted in advance, as the sorting is
+/// done as the heaps/dequeues are built.
+fn split_learning(
+    learning: Vec<DueCard>,
+    learn_ahead_secs: i64,
+) -> (
+    VecDeque<LearningQueueEntry>,
+    BinaryHeap<Reverse<LearningQueueEntry>>,
+) {
+    let cutoff = TimestampSecs(TimestampSecs::now().0 + learn_ahead_secs);
+
+    // split learning into now and later
+    let (mut now, later): (Vec<_>, Vec<_>) = learning
+        .into_iter()
+        .map(LearningQueueEntry::from)
+        .partition(|c| c.due <= cutoff);
+
+    // sort due items in ascending order, as we pop the deque from the front
+    now.sort_unstable_by(|a, b| a.due.cmp(&b.due));
+    // partition() requires both outputs to be the same, so we need to create the deque
+    // separately
+    let now = VecDeque::from(now);
+
+    // build the binary min heap
+    let later: BinaryHeap<_> = later.into_iter().map(Reverse).collect();
+
+    (now, later)
+}
+
+impl Collection {
+    pub(crate) fn build_queues(&mut self, deck_id: DeckID) -> Result<CardQueues> {
+        let now = TimestampSecs::now();
+        let timing = self.timing_for_timestamp(now)?;
+        let (decks, parent_count) = self.storage.deck_with_parents_and_children(deck_id)?;
+        let config = self.storage.get_deck_config_map()?;
+        let limits = remaining_limits_capped_to_parents(&decks, &config);
+        let selected_deck_limits = limits[parent_count];
+
+        let mut queues = QueueBuilder::default();
+
+        for (deck, mut limit) in decks.iter().zip(limits).skip(parent_count) {
+            if limit.review > 0 {
+                self.storage.for_each_due_card_in_deck(
+                    timing.days_elapsed,
+                    timing.next_day_at,
+                    deck.id,
+                    |queue, card| queues.add_due_card(&mut limit, queue, card),
+                )?;
+            }
+            if limit.new > 0 {
+                self.storage.for_each_new_card_in_deck(deck.id, |card| {
+                    queues.add_new_card(&mut limit, card)
+                })?;
+            }
+        }
+
+        let queues = queues.build(
+            selected_deck_limits,
+            self.learn_ahead_secs(),
+            deck_id,
+            timing.days_elapsed,
+        );
+
+        Ok(queues)
+    }
+}

rslib/src/scheduler/queue/builder/sized_chain.rs

@@ -0,0 +1,80 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/// The standard Rust chain does not implement ExactSizeIterator, and we need
+/// to keep track of size so we can intersperse.
+pub(crate) struct SizedChain<I, I2> {
+    one: I,
+    two: I2,
+    one_idx: usize,
+    two_idx: usize,
+    one_len: usize,
+    two_len: usize,
+}
+
+impl<I, I2> SizedChain<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+    pub fn new(one: I, two: I2) -> Self {
+        let one_len = one.len();
+        let two_len = two.len();
+        SizedChain {
+            one,
+            two,
+            one_idx: 0,
+            two_idx: 0,
+            one_len,
+            two_len,
+        }
+    }
+}
+
+impl<I, I2> Iterator for SizedChain<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+    type Item = I::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.one_idx < self.one_len {
+            self.one_idx += 1;
+            self.one.next()
+        } else if self.two_idx < self.two_len {
+            self.two_idx += 1;
+            self.two.next()
+        } else {
+            None
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let remaining = (self.one_len + self.two_len) - (self.one_idx + self.two_idx);
+        (remaining, Some(remaining))
+    }
+}
+
+impl<I, I2> ExactSizeIterator for SizedChain<I, I2>
+where
+    I: ExactSizeIterator,
+    I2: ExactSizeIterator<Item = I::Item>,
+{
+}
+
+#[cfg(test)]
+mod test {
+    use super::SizedChain;
+
+    fn chain(a: &[u32], b: &[u32]) -> Vec<u32> {
+        SizedChain::new(a.iter().cloned(), b.iter().cloned()).collect()
+    }
+
+    #[test]
+    fn sized_chain() {
+        let a = &[1, 2, 3];
+        let b = &[11, 22, 33];
+        assert_eq!(&chain(a, b), &[1, 2, 3, 11, 22, 33]);
+    }
+}

rslib/src/scheduler/queue/builder/sorting.rs

@@ -0,0 +1,71 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{DueCard, NewCard, NewCardOrder, QueueBuilder, ReviewCardOrder};
+use fnv::FnvHasher;
+use std::{cmp::Ordering, hash::Hasher};
+
+impl QueueBuilder {
+    pub(super) fn sort_new(&mut self) {
+        match self.new_order {
+            NewCardOrder::Random => {
+                self.new.iter_mut().for_each(NewCard::hash_id_and_mtime);
+                self.new.sort_unstable_by(|a, b| a.extra.cmp(&b.extra));
+            }
+            NewCardOrder::Due => {
+                self.new.sort_unstable_by(|a, b| a.due.cmp(&b.due));
+            }
+        }
+    }
+
+    pub(super) fn sort_reviews(&mut self) {
+        self.review.iter_mut().for_each(DueCard::hash_id_and_mtime);
+        self.day_learning
+            .iter_mut()
+            .for_each(DueCard::hash_id_and_mtime);
+
+        match self.review_order {
+            ReviewCardOrder::ShuffledByDay => {
+                self.review.sort_unstable_by(shuffle_by_day);
+                self.day_learning.sort_unstable_by(shuffle_by_day);
+            }
+            ReviewCardOrder::Shuffled => {
+                self.review.sort_unstable_by(|a, b| a.extra.cmp(&b.extra));
+                self.day_learning
+                    .sort_unstable_by(|a, b| a.extra.cmp(&b.extra));
+            }
+            ReviewCardOrder::IntervalsAscending => {
+                // fixme: implement; may require separate field if we want
+                // to shuffle cards that share an interval
+            }
+            ReviewCardOrder::IntervalsDescending => {
+                // fixme: implement; may require separate field if we want
+                // to shuffle cards that share an interval
+            }
+        }
+    }
+}
+
+// We sort based on a hash so that if the queue is rebuilt, remaining
+// cards come back in the same order.
+impl DueCard {
+    fn hash_id_and_mtime(&mut self) {
+        let mut hasher = FnvHasher::default();
+        hasher.write_i64(self.id.0);
+        hasher.write_i64(self.mtime.0);
+        self.extra = hasher.finish();
+    }
+}
+
+impl NewCard {
+    fn hash_id_and_mtime(&mut self) {
+        let mut hasher = FnvHasher::default();
+        hasher.write_i64(self.id.0);
+        hasher.write_i64(self.mtime.0);
+        self.extra = hasher.finish();
+    }
+}
+
+fn shuffle_by_day(a: &DueCard, b: &DueCard) -> Ordering {
+    (a.due, a.extra).cmp(&(b.due, b.extra))
+}

rslib/src/scheduler/queue/learning.rs

@@ -0,0 +1,133 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use std::{
+    cmp::{Ordering, Reverse},
+    collections::VecDeque,
+};
+
+use super::{CardQueues, LearningQueueEntry};
+use crate::{prelude::*, scheduler::timing::SchedTimingToday};
+
+impl CardQueues {
+    /// Check for any newly due cards, and then return the first, if any,
+    /// that is due before now.
+    pub(super) fn next_learning_entry_due_before_now(
+        &mut self,
+        now: TimestampSecs,
+    ) -> Option<LearningQueueEntry> {
+        let learn_ahead_cutoff = now.adding_secs(self.learn_ahead_secs);
+        self.check_for_newly_due_learning_cards(learn_ahead_cutoff);
+        self.next_learning_entry_learning_ahead()
+            .filter(|c| c.due <= now)
+    }
+
+    /// Check for due learning cards up to the learn ahead limit.
+    /// Does not check for newly due cards, as that is already done by
+    /// next_learning_entry_due_before_now()
+    pub(super) fn next_learning_entry_learning_ahead(&self) -> Option<LearningQueueEntry> {
+        self.due_learning.front().copied()
+    }
+
+    pub(super) fn pop_learning_entry(&mut self, id: CardID) -> Option<LearningQueueEntry> {
+        if let Some(top) = self.due_learning.front() {
+            if top.id == id {
+                self.learn_count -= 1;
+                return self.due_learning.pop_front();
+            }
+        }
+
+        // fixme: remove this in the future
+        // the current python unit tests answer learning cards before they're due,
+        // so for now we also check the head of the later_due queue
+        if let Some(top) = self.later_learning.peek() {
+            if top.0.id == id {
+                // self.learn_count -= 1;
+                return self.later_learning.pop().map(|c| c.0);
+            }
+        }
+
+        None
+    }
+
+    /// Given the just-answered `card`, place it back in the learning queues if it's still
+    /// due today. Avoid placing it in a position where it would be shown again immediately.
+    pub(super) fn maybe_requeue_learning_card(&mut self, card: &Card, timing: SchedTimingToday) {
+        if !card.is_intraday_learning() {
+            return;
+        }
+
+        let learn_ahead_limit = timing.now.adding_secs(self.learn_ahead_secs);
+
+        if card.due < learn_ahead_limit.0 as i32 {
+            let mut entry = LearningQueueEntry {
+                due: TimestampSecs(card.due as i64),
+                id: card.id,
+                mtime: card.mtime,
+            };
+
+            if self.learning_collapsed() {
+                if let Some(next) = self.due_learning.front() {
+                    if next.due >= entry.due {
+                        // the earliest due card is due later than this one; make this one
+                        // due after that one
+                        entry.due = next.due.adding_secs(1);
+                    }
+                    self.push_due_learning_card(entry);
+                } else {
+                    // nothing else waiting to review; make this due in a minute
+                    entry.due = learn_ahead_limit.adding_secs(60);
+                    self.later_learning.push(Reverse(entry));
+                }
+            } else {
+                // not collapsed; can add normally
+                self.push_due_learning_card(entry);
+            }
+        } else if card.due < timing.next_day_at as i32 {
+            self.later_learning.push(Reverse(LearningQueueEntry {
+                due: TimestampSecs(card.due as i64),
+                id: card.id,
+                mtime: card.mtime,
+            }));
+        };
+    }
+
+    fn learning_collapsed(&self) -> bool {
+        self.main.is_empty()
+    }
+
+    /// Adds card, maintaining correct sort order, and increments learning count.
+    pub(super) fn push_due_learning_card(&mut self, entry: LearningQueueEntry) {
+        self.learn_count += 1;
+        let target_idx =
+            binary_search_by(&self.due_learning, |e| e.due.cmp(&entry.due)).unwrap_or_else(|e| e);
+        self.due_learning.insert(target_idx, entry);
+    }
+
+    fn check_for_newly_due_learning_cards(&mut self, cutoff: TimestampSecs) {
+        while let Some(earliest) = self.later_learning.peek() {
+            if earliest.0.due > cutoff {
+                break;
+            }
+            let entry = self.later_learning.pop().unwrap().0;
+            self.push_due_learning_card(entry);
+        }
+    }
+}
+
+/// Adapted from the Rust stdlib VecDeque implementation; we can drop this when the following
+/// lands: https://github.com/rust-lang/rust/issues/78021
+fn binary_search_by<'a, F, T>(deque: &'a VecDeque<T>, mut f: F) -> Result<usize, usize>
+where
+    F: FnMut(&'a T) -> Ordering,
+{
+    let (front, back) = deque.as_slices();
+
+    match back.first().map(|elem| f(elem)) {
+        Some(Ordering::Less) | Some(Ordering::Equal) => back
+            .binary_search_by(f)
+            .map(|idx| idx + front.len())
+            .map_err(|idx| idx + front.len()),
+        _ => front.binary_search_by(f),
+    }
+}

rslib/src/scheduler/queue/limits.rs

@@ -0,0 +1,216 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{Deck, DeckKind};
+use crate::deckconf::{DeckConf, DeckConfID};
+use std::collections::HashMap;
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct RemainingLimits {
+    pub review: u32,
+    pub new: u32,
+}
+
+impl RemainingLimits {
+    pub(crate) fn new(deck: &Deck, config: Option<&DeckConf>) -> Self {
+        if let Some(config) = config {
+            RemainingLimits {
+                review: ((config.inner.reviews_per_day as i32) - deck.common.review_studied).max(0)
+                    as u32,
+                new: ((config.inner.new_per_day as i32) - deck.common.new_studied).max(0) as u32,
+            }
+        } else {
+            RemainingLimits {
+                review: std::u32::MAX,
+                new: std::u32::MAX,
+            }
+        }
+    }
+
+    fn limit_to_parent(&mut self, parent: RemainingLimits) {
+        self.review = self.review.min(parent.review);
+        self.new = self.new.min(parent.new);
+    }
+}
+
+pub(super) fn remaining_limits_capped_to_parents(
+    decks: &[Deck],
+    config: &HashMap<DeckConfID, DeckConf>,
+) -> Vec<RemainingLimits> {
+    let mut limits = get_remaining_limits(decks, config);
+    cap_limits_to_parents(decks.iter().map(|d| d.name.as_str()), &mut limits);
+    limits
+}
+
+/// Return the remaining limits for each of the provided decks, in
+/// the provided deck order.
+fn get_remaining_limits(
+    decks: &[Deck],
+    config: &HashMap<DeckConfID, DeckConf>,
+) -> Vec<RemainingLimits> {
+    decks
+        .iter()
+        .map(move |deck| {
+            // get deck config if not filtered
+            let config = if let DeckKind::Normal(normal) = &deck.kind {
+                config.get(&DeckConfID(normal.config_id))
+            } else {
+                None
+            };
+            RemainingLimits::new(deck, config)
+        })
+        .collect()
+}
+
+/// Given a sorted list of deck names and their current limits,
+/// cap child limits to their parents.
+fn cap_limits_to_parents<'a>(
+    names: impl IntoIterator<Item = &'a str>,
+    limits: &'a mut Vec<RemainingLimits>,
+) {
+    let mut parent_limits = vec![];
+    let mut last_limit = None;
+    let mut last_level = 0;
+
+    names
+        .into_iter()
+        .zip(limits.iter_mut())
+        .for_each(|(name, limits)| {
+            let level = name.matches('\x1f').count() + 1;
+            if last_limit.is_none() {
+                // top-level deck
+                last_limit = Some(*limits);
+                last_level = level;
+            } else {
+                // add/remove parent limits if descending/ascending
+                let mut target = level;
+                while target != last_level {
+                    if target < last_level {
+                        // current deck is at higher level than previous
+                        parent_limits.pop();
+                        target += 1;
+                    } else {
+                        // current deck is at a lower level than previous. this
+                        // will push the same remaining counts multiple times if
+                        // the deck tree is missing a parent
+                        parent_limits.push(last_limit.unwrap());
+                        target -= 1;
+                    }
+                }
+
+                // apply current parent limit
+                limits.limit_to_parent(*parent_limits.last().unwrap());
+                last_level = level;
+                last_limit = Some(*limits);
+            }
+        })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn limits() {
+        let limits_map = vec![
+            (
+                "A",
+                RemainingLimits {
+                    review: 100,
+                    new: 20,
+                },
+            ),
+            (
+                "A\x1fB",
+                RemainingLimits {
+                    review: 50,
+                    new: 30,
+                },
+            ),
+            (
+                "A\x1fC",
+                RemainingLimits {
+                    review: 10,
+                    new: 10,
+                },
+            ),
+            ("A\x1fC\x1fD", RemainingLimits { review: 5, new: 30 }),
+            (
+                "A\x1fE",
+                RemainingLimits {
+                    review: 200,
+                    new: 100,
+                },
+            ),
+        ];
+        let (names, mut limits): (Vec<_>, Vec<_>) = limits_map.into_iter().unzip();
+        cap_limits_to_parents(names.into_iter(), &mut limits);
+        assert_eq!(
+            &limits,
+            &[
+                RemainingLimits {
+                    review: 100,
+                    new: 20
+                },
+                RemainingLimits {
+                    review: 50,
+                    new: 20
+                },
+                RemainingLimits {
+                    review: 10,
+                    new: 10
+                },
+                RemainingLimits { review: 5, new: 10 },
+                RemainingLimits {
+                    review: 100,
+                    new: 20
+                }
+            ]
+        );
+
+        // missing parents should not break it
+        let limits_map = vec![
+            (
+                "A",
+                RemainingLimits {
+                    review: 100,
+                    new: 20,
+                },
+            ),
+            (
+                "A\x1fB\x1fC\x1fD",
+                RemainingLimits {
+                    review: 50,
+                    new: 30,
+                },
+            ),
+            (
+                "A\x1fC",
+                RemainingLimits {
+                    review: 100,
+                    new: 100,
+                },
+            ),
+        ];
+
+        let (names, mut limits): (Vec<_>, Vec<_>) = limits_map.into_iter().unzip();
+        cap_limits_to_parents(names.into_iter(), &mut limits);
+        assert_eq!(
+            &limits,
+            &[
+                RemainingLimits {
+                    review: 100,
+                    new: 20
+                },
+                RemainingLimits {
+                    review: 50,
+                    new: 20
+                },
+                RemainingLimits {
+                    review: 100,
+                    new: 20
+                },
+            ]
+        );
+    }
+}

rslib/src/scheduler/queue/main.rs

@@ -0,0 +1,37 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{CardQueues, QueueEntry, QueueEntryKind};
+use crate::prelude::*;
+
+impl CardQueues {
+    pub(super) fn next_main_entry(&self) -> Option<QueueEntry> {
+        self.main.front().copied()
+    }
+
+    pub(super) fn pop_main_entry(&mut self, id: CardID) -> Option<QueueEntry> {
+        if let Some(last) = self.main.front() {
+            if last.id == id {
+                match last.kind {
+                    QueueEntryKind::New => self.new_count -= 1,
+                    QueueEntryKind::Review => self.review_count -= 1,
+                    QueueEntryKind::Learning => unreachable!(),
+                }
+                return self.main.pop_front();
+            }
+        }
+
+        None
+    }
+
+    /// Add an undone card back to the 'front' of the list, and update
+    /// the counts.
+    pub(super) fn push_main_entry(&mut self, entry: QueueEntry) {
+        match entry.kind {
+            QueueEntryKind::New => self.new_count += 1,
+            QueueEntryKind::Review => self.review_count += 1,
+            QueueEntryKind::Learning => unreachable!(),
+        }
+        self.main.push_front(entry);
+    }
+}

rslib/src/scheduler/queue/mod.rs

@@ -0,0 +1,252 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+mod builder;
+mod learning;
+mod limits;
+mod main;
+
+use std::{
+    cmp::Reverse,
+    collections::{BinaryHeap, VecDeque},
+};
+
+use crate::{backend_proto as pb, card::CardQueue, prelude::*, timestamp::TimestampSecs};
+pub(crate) use builder::{DueCard, NewCard};
+
+use super::timing::SchedTimingToday;
+
+#[derive(Debug)]
+pub(crate) struct CardQueues {
+    new_count: usize,
+    review_count: usize,
+    learn_count: usize,
+
+    main: VecDeque<QueueEntry>,
+    due_learning: VecDeque<LearningQueueEntry>,
+    later_learning: BinaryHeap<Reverse<LearningQueueEntry>>,
+
+    selected_deck: DeckID,
+    current_day: u32,
+    learn_ahead_secs: i64,
+}
+
+#[derive(Debug)]
+pub(crate) struct Counts {
+    new: usize,
+    learning: usize,
+    review: usize,
+}
+
+impl CardQueues {
+    /// Get the next due card, if there is one.
+    fn next_entry(&mut self, now: TimestampSecs) -> Option<QueueEntry> {
+        self.next_learning_entry_due_before_now(now)
+            .map(Into::into)
+            .or_else(|| self.next_main_entry())
+            .or_else(|| self.next_learning_entry_learning_ahead().map(Into::into))
+    }
+
+    /// Remove the provided card from the top of the learning or main queues.
+    /// If it was not at the top, return an error.
+    fn pop_answered(&mut self, id: CardID) -> Result<()> {
+        if self.pop_main_entry(id).is_none() && self.pop_learning_entry(id).is_none() {
+            Err(AnkiError::invalid_input("not at top of queue"))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn counts(&self) -> Counts {
+        Counts {
+            new: self.new_count,
+            learning: self.learn_count,
+            review: self.review_count,
+        }
+    }
+
+    fn is_stale(&self, deck: DeckID, current_day: u32) -> bool {
+        self.selected_deck != deck || self.current_day != current_day
+    }
+
+    fn update_after_answering_card(&mut self, card: &Card, timing: SchedTimingToday) -> Result<()> {
+        self.pop_answered(card.id)?;
+        self.maybe_requeue_learning_card(card, timing);
+        Ok(())
+    }
+
+    /// Add a just-undone card back to the appropriate queue, updating counts.
+    pub(crate) fn push_undone_card(&mut self, card: &Card) {
+        if card.is_intraday_learning() {
+            self.push_due_learning_card(LearningQueueEntry {
+                due: TimestampSecs(card.due as i64),
+                id: card.id,
+                mtime: card.mtime,
+            })
+        } else {
+            self.push_main_entry(card.into())
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct QueueEntry {
+    id: CardID,
+    mtime: TimestampSecs,
+    kind: QueueEntryKind,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) enum QueueEntryKind {
+    New,
+    /// Includes day-learning cards
+    Review,
+    Learning,
+}
+
+impl PartialOrd for QueueEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        self.id.partial_cmp(&other.id)
+    }
+}
+
+impl From<&Card> for QueueEntry {
+    fn from(card: &Card) -> Self {
+        let kind = match card.queue {
+            CardQueue::Learn | CardQueue::PreviewRepeat => QueueEntryKind::Learning,
+            CardQueue::New => QueueEntryKind::New,
+            CardQueue::Review | CardQueue::DayLearn => QueueEntryKind::Review,
+            CardQueue::Suspended | CardQueue::SchedBuried | CardQueue::UserBuried => {
+                unreachable!()
+            }
+        };
+        QueueEntry {
+            id: card.id,
+            mtime: card.mtime,
+            kind,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)]
+struct LearningQueueEntry {
+    // due comes first, so the derived ordering sorts by due
+    due: TimestampSecs,
+    id: CardID,
+    mtime: TimestampSecs,
+}
+
+impl From<LearningQueueEntry> for QueueEntry {
+    fn from(e: LearningQueueEntry) -> Self {
+        Self {
+            id: e.id,
+            mtime: e.mtime,
+            kind: QueueEntryKind::Learning,
+        }
+    }
+}
+
+impl Collection {
+    pub(crate) fn get_queued_cards(
+        &mut self,
+        fetch_limit: u32,
+        intraday_learning_only: bool,
+    ) -> Result<pb::GetQueuedCardsOut> {
+        if let Some(next_cards) = self.next_cards(fetch_limit, intraday_learning_only)? {
+            Ok(pb::GetQueuedCardsOut {
+                value: Some(pb::get_queued_cards_out::Value::QueuedCards(next_cards)),
+            })
+        } else {
+            Ok(pb::GetQueuedCardsOut {
+                value: Some(pb::get_queued_cards_out::Value::CongratsInfo(
+                    self.congrats_info()?,
+                )),
+            })
+        }
+    }
+
+    pub(crate) fn clear_queues(&mut self) {
+        self.state.card_queues = None;
+    }
+
+    /// FIXME: remove this once undoing is moved into backend
+    pub(crate) fn requeue_undone_card(&mut self, card_id: CardID) -> Result<()> {
+        let card = self.storage.get_card(card_id)?.ok_or(AnkiError::NotFound)?;
+        self.get_queues()?.push_undone_card(&card);
+        Ok(())
+    }
+
+    pub(crate) fn update_queues_after_answering_card(
+        &mut self,
+        card: &Card,
+        timing: SchedTimingToday,
+    ) -> Result<()> {
+        if let Some(queues) = &mut self.state.card_queues {
+            queues.update_after_answering_card(card, timing)
+        } else {
+            // we currenly allow the queues to be empty for unit tests
+            Ok(())
+        }
+    }
+
+    fn get_queues(&mut self) -> Result<&mut CardQueues> {
+        let timing = self.timing_today()?;
+        let deck = self.get_current_deck_id();
+        let need_rebuild = self
+            .state
+            .card_queues
+            .as_ref()
+            .map(|q| q.is_stale(deck, timing.days_elapsed))
+            .unwrap_or(true);
+        if need_rebuild {
+            self.state.card_queues = Some(self.build_queues(deck)?);
+        }
+
+        Ok(self.state.card_queues.as_mut().unwrap())
+    }
+
+    fn next_cards(
+        &mut self,
+        _fetch_limit: u32,
+        _intraday_learning_only: bool,
+    ) -> Result<Option<pb::get_queued_cards_out::QueuedCards>> {
+        let queues = self.get_queues()?;
+        let mut cards = vec![];
+        if let Some(entry) = queues.next_entry(TimestampSecs::now()) {
+            let card = self
+                .storage
+                .get_card(entry.id)?
+                .ok_or(AnkiError::NotFound)?;
+            if card.mtime != entry.mtime {
+                return Err(AnkiError::invalid_input(
+                    "bug: card modified without updating queue",
+                ));
+            }
+
+            // fixme: pass in card instead of id
+            let next_states = self.get_next_card_states(card.id)?;
+
+            cards.push(pb::get_queued_cards_out::QueuedCard {
+                card: Some(card.into()),
+                next_states: Some(next_states.into()),
+                queue: match entry.kind {
+                    QueueEntryKind::New => 0,
+                    QueueEntryKind::Learning => 1,
+                    QueueEntryKind::Review => 2,
+                },
+            });
+        }
+
+        if cards.is_empty() {
+            Ok(None)
+        } else {
+            let counts = self.get_queues()?.counts();
+            Ok(Some(pb::get_queued_cards_out::QueuedCards {
+                cards,
+                new_count: counts.new as u32,
+                learning_count: counts.learning as u32,
+                review_count: counts.review as u32,
+            }))
+        }
+    }
+}

rslib/src/scheduler/timing.rs

@@ -6,6 +6,7 @@ use chrono::{Date, Duration, FixedOffset, Local, TimeZone, Timelike};
 
 #[derive(Debug, PartialEq, Clone, Copy)]
 pub struct SchedTimingToday {
+    pub now: TimestampSecs,
     /// The number of days that have passed since the collection was created.
     pub days_elapsed: u32,
     /// Timestamp of the next day rollover.
@@ -43,6 +44,7 @@ pub fn sched_timing_today_v2_new(
     let days_elapsed = days_elapsed(created_date, today, rollover_passed);
 
     SchedTimingToday {
+        now: current_secs,
         days_elapsed,
         next_day_at,
     }
@@ -119,6 +121,7 @@ fn sched_timing_today_v1(crt: TimestampSecs, now: TimestampSecs) -> SchedTimingT
     let days_elapsed = (now.0 - crt.0) / 86_400;
     let next_day_at = crt.0 + (days_elapsed + 1) * 86_400;
     SchedTimingToday {
+        now,
         days_elapsed: days_elapsed as u32,
         next_day_at,
     }
@@ -147,6 +150,7 @@ fn sched_timing_today_v2_legacy(
     }
 
     SchedTimingToday {
+        now,
         days_elapsed: days_elapsed as u32,
         next_day_at,
     }
@@ -351,6 +355,7 @@ mod test {
         assert_eq!(
             sched_timing_today_v1(TimestampSecs(1575226800), now),
             SchedTimingToday {
+                now,
                 days_elapsed: 107,
                 next_day_at: 1584558000
             }
@@ -359,6 +364,7 @@ mod test {
         assert_eq!(
             sched_timing_today_v2_legacy(TimestampSecs(1533564000), 0, now, aest_offset()),
             SchedTimingToday {
+                now,
                 days_elapsed: 589,
                 next_day_at: 1584540000
             }
@@ -367,6 +373,7 @@ mod test {
         assert_eq!(
             sched_timing_today_v2_legacy(TimestampSecs(1524038400), 4, now, aest_offset()),
             SchedTimingToday {
+                now,
                 days_elapsed: 700,
                 next_day_at: 1584554400
             }

rslib/src/scheduler/upgrade.rs

@@ -10,7 +10,7 @@ use crate::{
     search::SortMode,
 };
 
-use super::cutoff::local_minutes_west_for_stamp;
+use super::timing::local_minutes_west_for_stamp;
 
 struct V1FilteredDeckInfo {
     /// True if the filtered deck had rescheduling enabled.

rslib/src/storage/card/due_cards.sql

@@ -0,0 +1,18 @@
+SELECT queue,
+  id,
+  nid,
+  due,
+  cast(ivl AS integer),
+  cast(mod AS integer)
+FROM cards
+WHERE did = ?1
+  AND (
+    (
+      queue IN (2, 3)
+      AND due <= ?2
+    )
+    OR (
+      queue IN (1, 4)
+      AND due <= ?3
+    )
+  )

rslib/src/storage/card/mod.rs

@@ -7,7 +7,10 @@ use crate::{
     decks::{Deck, DeckID, DeckKind},
     err::Result,
     notes::NoteID,
-    scheduler::congrats::CongratsInfo,
+    scheduler::{
+        congrats::CongratsInfo,
+        queue::{DueCard, NewCard},
+    },
     timestamp::{TimestampMillis, TimestampSecs},
     types::Usn,
 };
@@ -159,6 +162,67 @@ impl super::SqliteStorage {
         Ok(())
     }
 
+    /// Call func() for each due card, stopping when it returns false
+    /// or no more cards found.
+    pub(crate) fn for_each_due_card_in_deck<F>(
+        &self,
+        day_cutoff: u32,
+        learn_cutoff: i64,
+        deck: DeckID,
+        mut func: F,
+    ) -> Result<()>
+    where
+        F: FnMut(CardQueue, DueCard) -> bool,
+    {
+        let mut stmt = self.db.prepare_cached(include_str!("due_cards.sql"))?;
+        let mut rows = stmt.query(params![
+            // with many subdecks, avoiding named params shaves off a few milliseconds
+            deck,
+            day_cutoff,
+            learn_cutoff
+        ])?;
+        while let Some(row) = rows.next()? {
+            let queue: CardQueue = row.get(0)?;
+            if !func(
+                queue,
+                DueCard {
+                    id: row.get(1)?,
+                    note_id: row.get(2)?,
+                    due: row.get(3).ok().unwrap_or_default(),
+                    extra: row.get::<_, u32>(4)? as u64,
+                    mtime: row.get(5)?,
+                },
+            ) {
+                break;
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Call func() for each new card, stopping when it returns false
+    /// or no more cards found. Cards will arrive in (deck_id, due) order.
+    pub(crate) fn for_each_new_card_in_deck<F>(&self, deck: DeckID, mut func: F) -> Result<()>
+    where
+        F: FnMut(NewCard) -> bool,
+    {
+        let mut stmt = self.db.prepare_cached(include_str!("new_cards.sql"))?;
+        let mut rows = stmt.query(params![deck])?;
+        while let Some(row) = rows.next()? {
+            if !func(NewCard {
+                id: row.get(0)?,
+                note_id: row.get(1)?,
+                due: row.get(2)?,
+                extra: row.get::<_, u32>(3)? as u64,
+                mtime: row.get(4)?,
+            }) {
+                break;
+            }
+        }
+
+        Ok(())
+    }
+
     /// Fix some invalid card properties, and return number of changed cards.
     pub(crate) fn fix_card_properties(
         &self,

rslib/src/storage/card/new_cards.sql

@@ -0,0 +1,8 @@
+SELECT id,
+  nid,
+  due,
+  ord,
+  cast(mod AS integer)
+FROM cards
+WHERE did = ?
+  AND queue = 0

rslib/src/storage/deck/mod.rs

@@ -162,6 +162,38 @@ impl SqliteStorage {
             .collect()
     }
 
+    /// Return the provided deck with its parents and children in an ordered list, and
+    /// the number of parent decks that need to be skipped to get to the chosen deck.
+    pub(crate) fn deck_with_parents_and_children(
+        &self,
+        deck_id: DeckID,
+    ) -> Result<(Vec<Deck>, usize)> {
+        let deck = self.get_deck(deck_id)?.ok_or(AnkiError::NotFound)?;
+        let mut parents = self.parent_decks(&deck)?;
+        parents.reverse();
+        let parent_count = parents.len();
+
+        let prefix_start = format!("{}\x1f", deck.name);
+        let prefix_end = format!("{}\x20", deck.name);
+        parents.push(deck);
+
+        let decks = parents
+            .into_iter()
+            .map(Result::Ok)
+            .chain(
+                self.db
+                    .prepare_cached(concat!(
+                        include_str!("get_deck.sql"),
+                        " where name > ? and name < ?"
+                    ))?
+                    .query_and_then(&[prefix_start, prefix_end], row_to_deck)?,
+            )
+            .collect::<Result<_>>()?;
+
+        Ok((decks, parent_count))
+    }
+
+    /// Return the parents of `child`, with the most immediate parent coming first.
     pub(crate) fn parent_decks(&self, child: &Deck) -> Result<Vec<Deck>> {
         let mut decks: Vec<Deck> = vec![];
         while let Some(parent_name) =

rslib/src/storage/sqlite.rs

@@ -5,7 +5,7 @@ use crate::config::schema11_config_as_string;
 use crate::err::Result;
 use crate::err::{AnkiError, DBErrorKind};
 use crate::timestamp::{TimestampMillis, TimestampSecs};
-use crate::{i18n::I18n, scheduler::cutoff::v1_creation_date, text::without_combining};
+use crate::{i18n::I18n, scheduler::timing::v1_creation_date, text::without_combining};
 use regex::Regex;
 use rusqlite::{functions::FunctionFlags, params, Connection, NO_PARAMS};
 use std::cmp::Ordering;

rslib/src/timestamp.rs

@@ -35,6 +35,10 @@ impl TimestampSecs {
     pub fn datetime(self, utc_offset: FixedOffset) -> DateTime<FixedOffset> {
         utc_offset.timestamp(self.0, 0)
     }
+
+    pub fn adding_secs(self, secs: i64) -> Self {
+        TimestampSecs(self.0 + secs)
+    }
 }
 
 impl TimestampMillis {