favour integers, change due representation, fact&card ordering, more

- removed 'created' column from various tables. We don't care when things like
  models are created, and card creation time didn't reflect the actual time a
  card was created
- facts were previously ordered by their creation date. The code would
  manually set the creation time for subsequent facts on import by 0.0001
  seconds, and then card due times were set by adding the fact time to the
  ordinal number*0.000001. This was prone to error, and the number of zeros used
  was actually different in different parts of the code. Instead of this, we
  replace it with a 'pos' column on facts, which increments for each new fact.
- importing should add new facts with a higher pos, but concurrent updates in
  a synced deck can have multiple facts with the same pos

- due times are completely different now, and depend on the card type
- new cards have due=fact.pos or random(0, 10000)
- reviews have due set to an integer representing days since deck
  creation/download
- cards in the learn queue use an integer timestamp in seconds

- many columns like modified, lastSync, factor, interval, etc have been converted to
  integer columns. They are cheaper to store (large decks can save 10s of
  megabytes) and faster to search for.

- cards have their group assigned on fact creation. In the future we'll add a
  per-template option for a default group.

- switch to due/random order for the review queue on upgrade. Users can still
  switch to the old behaviour if they want, but many people don't care what
  it's set to, and due is considerably faster, which may result in a better
  user experience

anki/cards.py

@@ -6,7 +6,7 @@ import time, sys, math, random
 from anki.db import *
 from anki.models import CardModel, Model, FieldModel, formatQA
 from anki.facts import Fact, factsTable, Field
-from anki.utils import parseTags, findTag, stripHTML, genID, hexifyID
+from anki.utils import parseTags, findTag, stripHTML, genID, hexifyID, intTime
 from anki.media import updateMediaCount, mediaFiles
 
 MAX_TIMER = 60
@@ -21,13 +21,18 @@ MAX_TIMER = 60
 # Ordinal: card template # for fact
 # Flags: unused; reserved for future use
 
+# Due is used differently for different queues.
+# - new queue: fact.pos
+# - rev queue: integer day
+# - lrn queue: integer timestamp
+
 cardsTable = Table(
     'cards', metadata,
     Column('id', Integer, primary_key=True),
     Column('factId', Integer, ForeignKey("facts.id"), nullable=False),
     Column('groupId', Integer, nullable=False, default=1),
     Column('cardModelId', Integer, ForeignKey("cardModels.id"), nullable=False),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
     # general
     Column('question', UnicodeText, nullable=False, default=u""),
     Column('answer', UnicodeText, nullable=False, default=u""),
@@ -36,10 +41,10 @@ cardsTable = Table(
     # shared scheduling
     Column('type', Integer, nullable=False, default=2),
     Column('queue', Integer, nullable=False, default=2),
-    Column('due', Float, nullable=False),
+    Column('due', Integer, nullable=False),
     # sm2
-    Column('interval', Float, nullable=False, default=0),
+    Column('interval', Integer, nullable=False, default=0),
-    Column('factor', Float, nullable=False, default=2.5),
+    Column('factor', Integer, nullable=False),
     Column('reps', Integer, nullable=False, default=0),
     Column('streak', Integer, nullable=False, default=0),
     Column('lapses', Integer, nullable=False, default=0),
@@ -50,27 +55,27 @@ cardsTable = Table(
 
 class Card(object):
 
-    # FIXME: this needs tidying up
+    # called one of three ways:
-    def __init__(self, fact=None, cardModel=None, due=None):
+    # - with no args, followed by .fromDB()
-        self.id = genID()
+    # - with all args, when adding cards to db
-        self.modified = time.time()
+    def __init__(self, fact=None, cardModel=None, group=None):
-        if due:
+        # timer
-            self.due = due
+        self.timerStarted = None
-        else:
-            self.due = self.modified
         if fact:
+            self.id = genID()
+            self.modified = intTime()
+            self.due = fact.pos
             self.fact = fact
             self.modelId = fact.modelId
-        if cardModel:
             self.cardModel = cardModel
+            self.groupId = group.id
+            self.factor = group.config['initialFactor']
             # for non-orm use
             self.cardModelId = cardModel.id
             self.ordinal = cardModel.ordinal
-        # timer
-        self.timerStarted = None
 
     def setModified(self):
-        self.modified = time.time()
+        self.modified = intTime()
 
     def startTimer(self):
         self.timerStarted = time.time()

anki/deck.py

@@ -10,7 +10,7 @@ from anki.lang import _, ngettext
 from anki.errors import DeckAccessError
 from anki.stdmodels import BasicModel
 from anki.utils import parseTags, tidyHTML, genID, ids2str, hexifyID, \
-     canonifyTags, joinTags, addTags, checksum, fieldChecksum
+     canonifyTags, joinTags, addTags, checksum, fieldChecksum, intTime
 from anki.revlog import logReview
 from anki.models import Model, CardModel, formatQA
 from anki.fonts import toPlatformFont
@@ -48,6 +48,8 @@ defaultConf = {
     'sessionRepLimit': 0,
     'sessionTimeLimit': 600,
     'currentModelId': None,
+    'currentGroupId': 1,
+    'nextFactPos': 1,
     'mediaURL': "",
     'latexPre': """\
 \\documentclass[12pt]{article}
@@ -66,9 +68,9 @@ defaultConf = {
 deckTable = Table(
     'deck', metadata,
     Column('id', Integer, nullable=False, primary_key=True),
-    Column('created', Float, nullable=False, default=time.time),
+    Column('created', Integer, nullable=False, default=intTime),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
-    Column('schemaMod', Float, nullable=False, default=0),
+    Column('schemaMod', Integer, nullable=False, default=intTime),
     Column('version', Integer, nullable=False, default=DECK_VERSION),
     Column('syncName', UnicodeText, nullable=False, default=u""),
     Column('lastSync', Integer, nullable=False, default=0),
@@ -412,7 +414,7 @@ due > :now and due < :now""", now=time.time())
         "Return a new fact with the current model."
         if model is None:
             model = self.currentModel
-        return anki.facts.Fact(model)
+        return anki.facts.Fact(model, self.getFactPos())
 
     def addFact(self, fact, reset=True):
         "Add a fact to the deck. Return list of new cards."
@@ -432,11 +434,10 @@ due > :now and due < :now""", now=time.time())
         self.flushMod()
         isRandom = self.qconf['newCardOrder'] == NEW_CARDS_RANDOM
         if isRandom:
-            due = random.uniform(0, time.time())
+            due = random.randrange(0, 10000)
-        t = time.time()
         for cardModel in cms:
-            created = fact.created + 0.00001*cardModel.ordinal
+            group = self.groupForTemplate(cardModel)
-            card = anki.cards.Card(fact, cardModel, created)
+            card = anki.cards.Card(fact, cardModel, group)
             if isRandom:
                 card.due = due
             self.flushMod()
@@ -449,6 +450,11 @@ due > :now and due < :now""", now=time.time())
             self.reset()
         return fact
 
+    def groupForTemplate(self, template):
+        print "add default group to template"
+        id = self.config['currentGroupId']
+        return self.db.query(anki.groups.GroupConfig).get(id).load()
+
     def availableCardModels(self, fact, checkActive=True):
         "List of active card models that aren't empty for FACT."
         models = []
@@ -2132,11 +2138,12 @@ Return new path, relative to media dir."""
     # DB helpers
     ##########################################################################
 
-    def save(self):
+    def save(self, config=True):
         "Commit any pending changes to disk."
         if self.lastLoaded == self.modified:
             return
         self.lastLoaded = self.modified
+        if config:
             self.flushConfig()
         self.db.commit()
 
@@ -2187,14 +2194,22 @@ Return new path, relative to media dir."""
             self.db = None
             self.s = None
 
-    def setModified(self, newTime=None):
+    def setModified(self):
         #import traceback; traceback.print_stack()
-        self.modified = newTime or time.time()
+        self.modified = intTime()
 
     def setSchemaModified(self):
-        self.schemaMod = time.time()
+        self.schemaMod = intTime()
         anki.graves.forgetAll(self.db)
 
+    def getFactPos(self):
+        "Return next fact position, incrementing it."
+        # note this is incremented even if facts are not added; gaps are not a bug
+        p = self.config['nextFactPos']
+        self.config['nextFactPos'] += 1
+        self.setModified()
+        return p
+
     def flushMod(self):
         "Mark modified and flush to DB."
         self.setModified()
@@ -2659,8 +2674,8 @@ sourcesTable = Table(
     'sources', metadata,
     Column('id', Integer, nullable=False, primary_key=True),
     Column('name', UnicodeText, nullable=False, default=""),
-    Column('created', Float, nullable=False, default=time.time),
+    Column('created', Integer, nullable=False, default=intTime),
-    Column('lastSync', Float, nullable=False, default=0),
+    Column('lastSync', Integer, nullable=False, default=0),
     # -1 = never check, 0 = always check, 1+ = number of seconds passed.
     # not currently exposed in the GUI
     Column('syncPeriod', Integer, nullable=False, default=0))
@@ -2783,11 +2798,11 @@ class DeckStorage(object):
         "Add a default group & config."
         s.execute("""
 insert into groupConfig values (1, :t, :name, :conf)""",
-                  t=time.time(), name=_("Default Config"),
+                  t=intTime(), name=_("Default Config"),
                   conf=simplejson.dumps(anki.groups.defaultConf))
         s.execute("""
 insert into groups values (1, :t, "Default", 1)""",
-                  t=time.time())
+                  t=intTime())
     _addConfig = staticmethod(_addConfig)
 
     def _addTables(s):

anki/facts.py

@@ -6,7 +6,7 @@ import time
 from anki.db import *
 from anki.errors import *
 from anki.models import Model, FieldModel, fieldModelsTable
-from anki.utils import genID, stripHTMLMedia, fieldChecksum
+from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime
 from anki.hooks import runHook
 
 # Fields in a fact
@@ -43,6 +43,8 @@ mapper(Field, fieldsTable, properties={
 # Facts: a set of fields and a model
 ##########################################################################
 
+# Pos: incrementing number defining add order. There may be duplicates if
+# content is added on two sync locations at once. Importing adds to end.
 # Cache: a HTML-stripped amalgam of the field contents, so we can perform
 # searches of marked up text in a reasonable time.
 
@@ -50,20 +52,21 @@ factsTable = Table(
     'facts', metadata,
     Column('id', Integer, primary_key=True),
     Column('modelId', Integer, ForeignKey("models.id"), nullable=False),
-    Column('created', Float, nullable=False, default=time.time),
+    Column('pos', Integer, nullable=False),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
     Column('tags', UnicodeText, nullable=False, default=u""),
     Column('cache', UnicodeText, nullable=False, default=u""))
 
 class Fact(object):
     "A single fact. Fields exposed as dict interface."
 
-    def __init__(self, model=None):
+    def __init__(self, model=None, pos=None):
         self.model = model
         self.id = genID()
         if model:
             for fm in model.fieldModels:
                 self.fields.append(Field(fm))
+            self.pos = pos
         self.new = True
 
     def isNew(self):
@@ -130,7 +133,7 @@ class Fact(object):
 
     def setModified(self, textChanged=False, deck=None, media=True):
         "Mark modified and update cards."
-        self.modified = time.time()
+        self.modified = intTime()
         if textChanged:
             if not deck:
                 # FIXME: compat code

anki/graves.py

@@ -8,6 +8,7 @@
 
 import time
 from anki.db import *
+from anki.utils import intTime
 
 FACT = 0
 CARD = 1
@@ -24,11 +25,11 @@ gravestonesTable = Table(
 
 def registerOne(db, type, id):
     db.statement("insert into gravestones values (:t, :id, :ty)",
-                 t=time.time(), id=id, ty=type)
+                 t=intTime(), id=id, ty=type)
 
 def registerMany(db, type, ids):
     db.statements("insert into gravestones values (:t, :id, :ty)",
-                 [{'t':time.time(), 'id':x, 'ty':type} for x in ids])
+                 [{'t':intTime(), 'id':x, 'ty':type} for x in ids])
 
 def forgetAll(db):
     db.statement("delete from gravestones")

anki/groups.py

@@ -3,12 +3,13 @@
 # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
 
 import simplejson, time
+from anki.utils import intTime
 from anki.db import *
 
 groupsTable = Table(
     'groups', metadata,
     Column('id', Integer, primary_key=True),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
     Column('name', UnicodeText, nullable=False),
     Column('confId', Integer, nullable=False))
 
@@ -34,7 +35,7 @@ defaultConf = {
 groupConfigTable = Table(
     'groupConfig', metadata,
     Column('id', Integer, primary_key=True),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
     Column('name', UnicodeText, nullable=False),
     Column('config', UnicodeText, nullable=False,
            default=unicode(simplejson.dumps(defaultConf))))
@@ -45,9 +46,13 @@ class GroupConfig(object):
         self.id = genID()
         self.config = defaultConf
 
+    def load(self):
+        self.config = simplejson.loads(self._config)
+        return self
+
     def save(self):
         self._config = simplejson.dumps(self.config)
-        self.modified = time.time()
+        self.modified = intTime()
 
 mapper(GroupConfig, groupConfigTable, properties={
     '_config': groupConfigTable.c.config,

anki/media.py

@@ -4,7 +4,7 @@
 
 import os, shutil, re, urllib2, time, tempfile, unicodedata, urllib
 from anki.db import *
-from anki.utils import checksum, genID
+from anki.utils import checksum, genID, intTime
 from anki.lang import _
 
 # other code depends on this order, so don't reorder
@@ -19,7 +19,7 @@ mediaTable = Table(
     Column('id', Integer, primary_key=True, nullable=False),
     Column('filename', UnicodeText, nullable=False, unique=True),
     Column('refcnt', Integer, nullable=False),
-    Column('modified', Float, nullable=False),
+    Column('modified', Integer, nullable=False),
     Column('chksum', UnicodeText, nullable=False, default=u""))
 
 # File handling
@@ -71,7 +71,7 @@ def updateMediaCount(deck, file, count=1):
         "select 1 from media where filename = :file", file=file):
         deck.db.statement(
             "update media set refcnt = refcnt + :c, modified = :t where filename = :file",
-            file=file, c=count, t=time.time())
+            file=file, c=count, t=intTime())
     elif count > 0:
         try:
             sum = unicode(
@@ -81,7 +81,7 @@ def updateMediaCount(deck, file, count=1):
         deck.db.statement("""
 insert into media (id, filename, refcnt, modified, chksum)
 values (:id, :file, :c, :mod, :sum)""",
-                         id=genID(), file=file, c=count, mod=time.time(),
+                         id=genID(), file=file, c=count, mod=intTime(),
                          sum=sum)
 
 def removeUnusedMedia(deck):
@@ -173,12 +173,12 @@ def rebuildMediaDir(deck, delete=False, dirty=True):
         path = os.path.join(mdir, file)
         if not os.path.exists(path):
             if md5:
-                update.append({'f':file, 'sum':u"", 'c':time.time()})
+                update.append({'f':file, 'sum':u"", 'c':intTime()})
         else:
             sum = unicode(
                 checksum(open(os.path.join(mdir, file), "rb").read()))
             if md5 != sum:
-                update.append({'f':file, 'sum':sum, 'c':time.time()})
+                update.append({'f':file, 'sum':sum, 'c':intTime()})
     if update:
         deck.db.statements("""
 update media set chksum = :sum, modified = :c where filename = :f""",

anki/models.py

@@ -5,9 +5,9 @@
 import time, re, simplejson
 from sqlalchemy.ext.orderinglist import ordering_list
 from anki.db import *
-from anki.utils import genID, canonifyTags
+from anki.utils import genID, canonifyTags, intTime
 from anki.fonts import toPlatformFont
-from anki.utils import parseTags, hexifyID, checksum, stripHTML
+from anki.utils import parseTags, hexifyID, checksum, stripHTML, intTime
 from anki.lang import _
 from anki.hooks import runFilter
 from anki.template import render
@@ -156,7 +156,7 @@ def formatQA(cid, mid, fact, tags, cm, deck):
 modelsTable = Table(
     'models', metadata,
     Column('id', Integer, primary_key=True),
-    Column('modified', Float, nullable=False, default=time.time),
+    Column('modified', Integer, nullable=False, default=intTime),
     Column('name', UnicodeText, nullable=False),
     # currently unused
     Column('config', UnicodeText, nullable=False, default=u"")
@@ -169,7 +169,7 @@ class Model(object):
         self.id = genID()
 
     def setModified(self):
-        self.modified = time.time()
+        self.modified = intTime()
 
     def addFieldModel(self, field):
         "Add a field model. Don't call this directly."

anki/revlog.py

@@ -18,9 +18,9 @@ revlogTable = Table(
     Column('cardId', Integer, nullable=False),
     Column('ease', Integer, nullable=False),
     Column('rep', Integer, nullable=False),
-    Column('lastInterval', Float, nullable=False),
+    Column('lastInterval', Integer, nullable=False),
-    Column('interval', Float, nullable=False),
+    Column('interval', Integer, nullable=False),
-    Column('factor', Float, nullable=False),
+    Column('factor', Integer, nullable=False),
     Column('userTime', Integer, nullable=False),
     Column('flags', Integer, nullable=False, default=0))
 

anki/sched.py

@@ -22,7 +22,7 @@ class Scheduler(object):
         self.learnLimit = 1000
         self.updateCutoff()
 
-    def getCard(self, orm=True):
+    def getCard(self):
         "Pop the next card from the queue. None if finished."
         self.checkDay()
         id = self.getCardId()

anki/upgrade.py

@@ -8,6 +8,7 @@ import time, simplejson
 from anki.db import *
 from anki.lang import _
 from anki.media import rebuildMediaDir
+from anki.utils import intTime
 
 def moveTable(s, table):
     sql = s.scalar(
@@ -36,9 +37,9 @@ def upgradeSchema(engine, s):
         import cards
         metadata.create_all(engine, tables=[cards.cardsTable])
         s.execute("""
-insert into cards select id, factId, 1, cardModelId, modified, question,
+insert into cards select id, factId, 1, cardModelId, cast(modified as int),
-answer, ordinal, 0, relativeDelay, type, due, interval, factor, reps,
+question, answer, ordinal, 0, relativeDelay, type, due, cast(interval as int),
-successive, noCount, 0, 0 from cards2""")
+cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""")
         s.execute("drop table cards2")
         # tags
         ###########
@@ -53,12 +54,19 @@ insert or ignore into cardTags select cardId, tagId, src from cardTags2""")
         s.execute("drop table cardTags2")
         # facts
         ###########
-        moveTable(s, "facts")
+        s.execute("""
+create table facts2
+(id, modelId, modified, tags, cache)""")
+        # use the rowid to give them an integer order
+        s.execute("""
+insert into facts2 select id, modelId, modified, tags, spaceUntil from
+facts order by created""")
+        s.execute("drop table facts")
         import facts
         metadata.create_all(engine, tables=[facts.factsTable])
         s.execute("""
-insert or ignore into facts select id, modelId, created, modified, tags,
+insert or ignore into facts select id, modelId, rowid,
-spaceUntil from facts2""")
+cast(modified as int), tags, cache from facts2""")
         s.execute("drop table facts2")
         # media
         ###########
@@ -66,7 +74,7 @@ spaceUntil from facts2""")
         import media
         metadata.create_all(engine, tables=[media.mediaTable])
         s.execute("""
-insert or ignore into media select id, filename, size, created,
+insert or ignore into media select id, filename, size, cast(created as int),
 originalPath from media2""")
         s.execute("drop table media2")
         # deck
@@ -78,7 +86,7 @@ originalPath from media2""")
         import models
         metadata.create_all(engine, tables=[models.modelsTable])
         s.execute("""
-insert or ignore into models select id, modified, name, "" from models2""")
+insert or ignore into models select id, cast(modified as int), name, "" from models2""")
         s.execute("drop table models2")
 
     return ver
@@ -87,16 +95,17 @@ def migrateDeck(s, engine):
     import deck
     metadata.create_all(engine, tables=[deck.deckTable])
     s.execute("""
-insert into deck select id, created, modified, 0, 99,
+insert into deck select id, cast(created as int), cast(modified as int),
-ifnull(syncName, ""), lastSync, utcOffset, "", "", "" from decks""")
+0, 99, ifnull(syncName, ""), cast(lastSync as int),
+utcOffset, "", "", "" from decks""")
     # update selective study
     qconf = deck.defaultQconf.copy()
     # delete old selective study settings, which we can't auto-upgrade easily
     keys = ("newActive", "newInactive", "revActive", "revInactive")
     for k in keys:
         s.execute("delete from deckVars where key=:k", {'k':k})
-    # copy other settings
+    # copy other settings, ignoring deck order as there's a new default
-    keys = ("newCardOrder", "newCardSpacing", "revCardOrder")
+    keys = ("newCardOrder", "newCardSpacing")
     for k in keys:
         qconf[k] = s.execute("select %s from decks" % k).scalar()
     qconf['newPerDay'] = s.execute(
@@ -184,8 +193,10 @@ def upgradeDeck(deck):
         # migrate revlog data to new table
         deck.db.statement("""
 insert or ignore into revlog select
-cast(time*1000 as int), cardId, ease, reps, lastInterval, nextInterval, nextFactor,
+cast(time*1000 as int), cardId, ease, reps,
-cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
+cast(lastInterval as int), cast(nextInterval as int),
+cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
+0 from reviewHistory""")
         deck.db.statement("drop table reviewHistory")
         # convert old ease0 into ease1
         deck.db.statement("update revlog set ease = 1 where ease = 0")
@@ -198,7 +209,7 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
         # don't need an index on fieldModelId
         deck.db.statement("drop index if exists ix_fields_fieldModelId")
         # update schema time
-        deck.db.statement("update deck set schemaMod = :t", t=time.time())
+        deck.db.statement("update deck set schemaMod = :t", t=intTime())
         # remove queueDue as it's become dynamic, and type index
         deck.db.statement("drop index if exists ix_cards_queueDue")
         deck.db.statement("drop index if exists ix_cards_type")
@@ -207,9 +218,24 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
             deck.db.statement("drop table if exists %sDeleted" % t)
         # finally, update indices & optimize
         updateIndices(deck.db)
+        # rewrite due times for new cards
+        deck.db.statement("""
+update cards set due = (select pos from facts where factId = facts.id) where type=2""")
+        # convert due cards into day-based due
+        deck.db.statement("""
+update cards set due = cast(
+(case when due < :stamp then 0 else 1 end) +
+((due-:stamp)/86400) as int)+:today where type
+between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today)
+        print "today", deck.sched.today
+        print "cut", deck.sched.dayCutoff
         # setup qconf & config for dynamicIndices()
         deck.qconf = simplejson.loads(deck._qconf)
         deck.config = simplejson.loads(deck._config)
+        deck.data = simplejson.loads(deck._data)
+        # update factPos
+        deck.config['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1
+        deck.flushConfig()
         # add default config
         import deck as deckMod
         deckMod.DeckStorage._addConfig(deck.engine)

anki/utils.py

@@ -23,6 +23,9 @@ if sys.version_info[1] < 5:
 # Time handling
 ##############################################################################
 
+def intTime():
+    return int(time.time())
+
 timeTable = {
     "years": lambda n: ngettext("%s year", "%s years", n),
     "months": lambda n: ngettext("%s month", "%s months", n),