store revlog time in integer MS

using floats for the primary key causes sqlite to generate an extra index,
which is expensive for large histories

anki/graphs.py

@@ -114,10 +114,10 @@ from cards c where queue = 1 and interval > 21"""
         return self.deck.db.all("""
 select
 count() as combinedNewReps,
-date(time-:off, "unixepoch") as day,
+date(time/1000-:off, "unixepoch") as day,
 sum(case when lastInterval > 21 then 1 else 0 end) as matureReps,
 count() - sum(case when rep = 1 then 1 else 0 end) as combinedYoungReps,
-sum(userTime) as reviewTime from revlog
+sum(userTime/1000) as reviewTime from revlog
 group by day order by day
 """, off=self.deck.utcOffset)
 
@@ -244,7 +244,7 @@ group by day order by day
         else:
             # firstAnswered
             res = self.deck.db.column0(
-                "select time from revlog where rep = 1")
+                "select time/1000 from revlog where rep = 1")
         for r in res:
             d = int((r - self.endOfDay) / 86400.0)
             days[d] = days.get(d, 0) + 1

anki/revlog.py

@@ -8,17 +8,20 @@ from anki.db import *
 # Flags: 0=standard review, 1=reschedule due to cram, drill, etc
 # Rep: Repetition number. The same number may appear twice if a card has been
 # manually rescheduled or answered on multiple sites before a sync.
+#
+# We store the times in integer milliseconds to avoid an extra index on the
+# primary key.
 
 revlogTable = Table(
     'revlog', metadata,
-    Column('time', Float, nullable=False, primary_key=True, default=time.time),
+    Column('time', Integer, nullable=False, primary_key=True),
     Column('cardId', Integer, nullable=False),
     Column('ease', Integer, nullable=False),
     Column('rep', Integer, nullable=False),
     Column('lastInterval', Float, nullable=False),
     Column('interval', Float, nullable=False),
     Column('factor', Float, nullable=False),
-    Column('userTime', Float, nullable=False),
+    Column('userTime', Integer, nullable=False),
     Column('flags', Integer, nullable=False, default=0))
 
 def logReview(db, card, ease, flags=0):
@@ -26,7 +29,7 @@ def logReview(db, card, ease, flags=0):
 insert into revlog values (
 :created, :cardId, :ease, :rep, :lastInterval, :interval, :factor,
 :userTime, :flags)""",
-                 created=time.time(), cardId=card.id, ease=ease, rep=card.reps,
+                 created=int(time.time()*1000), cardId=card.id, ease=ease, rep=card.reps,
                  lastInterval=card.lastInterval, interval=card.interval,
-                 factor=card.factor, userTime=card.userTime(),
+                 factor=card.factor, userTime=int(card.userTime()*1000),
                  flags=flags)

anki/stats.py

@@ -25,7 +25,7 @@ class CardStats(object):
         self.txt = "<table>"
         self.addLine(_("Added"), self.strTime(c.created))
         first = self.deck.db.scalar(
-            "select time from revlog where rep = 1 and cardId = :id", id=c.id)
+            "select time/1000 from revlog where rep = 1 and cardId = :id", id=c.id)
         if first:
             self.addLine(_("First Review"), self.strTime(first))
         self.addLine(_("Changed"), self.strTime(c.modified))
@@ -42,7 +42,7 @@ class CardStats(object):
             self.addLine(_("Reviews"), "%d/%d (s=%d)" % (
                 c.reps-c.lapses, c.reps, c.successive))
         (cnt, total) = self.deck.db.first(
-            "select count(), sum(userTime) from revlog where cardId = :id", id=c.id)
+            "select count(), sum(userTime)/1000 from revlog where cardId = :id", id=c.id)
         if cnt:
             self.addLine(_("Average Time"), fmt(total / float(cnt), point=2))
             self.addLine(_("Total Time"), fmt(total, point=2))
@@ -250,15 +250,15 @@ class DeckStats(object):
         x = today + 86400*start
         y = today + 86400*finish
         return self.deck.db.scalar("""
-select count(distinct(cast((time-:off)/86400 as integer))) from revlog
-where time >= :x and time <= :y""",x=x,y=y, off=self.deck.utcOffset)
+select count(distinct(cast((time/1000-:off)/86400 as integer))) from revlog
+where time >= :x*1000 and time <= :y*1000""",x=x,y=y, off=self.deck.utcOffset)
 
     def getRepsDone(self, start, finish):
         now = datetime.datetime.today()
         x = time.mktime((now + datetime.timedelta(start)).timetuple())
         y = time.mktime((now + datetime.timedelta(finish)).timetuple())
         return self.deck.db.scalar(
-            "select count() from revlog where time >= :x and time <= :y",
+            "select count() from revlog where time >= :x*1000 and time <= :y*1000",
             x=x, y=y)
 
     def getAverageInterval(self):
@@ -317,7 +317,7 @@ and queue != -1 and type between 0 and 1""", cutoff=cutoff) or 0) / float(period
         cutoff = time.time() - 86400 * period
         return (self.deck.db.scalar("""
 select count(*) from revlog
-where time > :cutoff""", cutoff=cutoff) or 0) / float(period)
+where time > :cutoff*1000""", cutoff=cutoff) or 0) / float(period)
 
     def getNewPeriod(self, period):
         cutoff = time.time() - 86400 * period
@@ -329,4 +329,4 @@ where created > :cutoff""", cutoff=cutoff) or 0)
         cutoff = time.time() - 86400 * period
         return (self.deck.db.scalar("""
 select count(*) from revlog
-where rep = 1 and time > :cutoff""", cutoff=cutoff) or 0)
+where rep = 1 and time > :cutoff*1000""", cutoff=cutoff) or 0)

anki/upgrade.py

@@ -146,9 +146,9 @@ def upgradeDeck(deck):
     if deck.version < 74:
         # migrate revlog data to new table
         deck.db.statement("""
-insert into revlog select
-time, cardId, ease, reps, lastInterval, nextInterval, nextFactor,
-min(thinkingTime, 60), 0 from reviewHistory""")
+insert or ignore into revlog select
+cast(time*1000 as int), cardId, ease, reps, lastInterval, nextInterval, nextFactor,
+cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
         deck.db.statement("drop table reviewHistory")
         # convert old ease0 into ease1
         deck.db.statement("update revlog set ease = 1 where ease = 0")