From 55f4b9b7d0a8cc43f5833d91333280f0e94f70a8 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 4 Mar 2011 01:55:59 +0900 Subject: [PATCH] favour integers, change due representation, fact&card ordering, more - removed 'created' column from various tables. We don't care when things like models are created, and card creation time didn't reflect the actual time a card was created - facts were previously ordered by their creation date. The code would manually set the creation time for subsequent facts on import by 0.0001 seconds, and then card due times were set by adding the fact time to the ordinal number*0.000001. This was prone to error, and the number of zeros used was actually different in different parts of the code. Instead of this, we replace it with a 'pos' column on facts, which increments for each new fact. - importing should add new facts with a higher pos, but concurrent updates in a synced deck can have multiple facts with the same pos - due times are completely different now, and depend on the card type - new cards have due=fact.pos or random(0, 10000) - reviews have due set to an integer representing days since deck creation/download - cards in the learn queue use an integer timestamp in seconds - many columns like modified, lastSync, factor, interval, etc have been converted to integer columns. They are cheaper to store (large decks can save 10s of megabytes) and faster to search for. - cards have their group assigned on fact creation. In the future we'll add a per-template option for a default group. - switch to due/random order for the review queue on upgrade. Users can still switch to the old behaviour if they want, but many people don't care what it's set to, and due is considerably faster, which may result in a better user experience --- anki/cards.py | 39 +++++++++++++++++++--------------- anki/deck.py | 51 ++++++++++++++++++++++++++++---------------- anki/facts.py | 13 +++++++----- anki/graves.py | 5 +++-- anki/groups.py | 11 +++++++--- anki/media.py | 12 +++++------ anki/models.py | 8 +++---- anki/revlog.py | 6 +++--- anki/sched.py | 2 +- anki/upgrade.py | 56 ++++++++++++++++++++++++++++++++++++------------- anki/utils.py | 3 +++ 11 files changed, 132 insertions(+), 74 deletions(-) diff --git a/anki/cards.py b/anki/cards.py index a0b45fa00..4b33bdd60 100644 --- a/anki/cards.py +++ b/anki/cards.py @@ -6,7 +6,7 @@ import time, sys, math, random from anki.db import * from anki.models import CardModel, Model, FieldModel, formatQA from anki.facts import Fact, factsTable, Field -from anki.utils import parseTags, findTag, stripHTML, genID, hexifyID +from anki.utils import parseTags, findTag, stripHTML, genID, hexifyID, intTime from anki.media import updateMediaCount, mediaFiles MAX_TIMER = 60 @@ -21,13 +21,18 @@ MAX_TIMER = 60 # Ordinal: card template # for fact # Flags: unused; reserved for future use +# Due is used differently for different queues. +# - new queue: fact.pos +# - rev queue: integer day +# - lrn queue: integer timestamp + cardsTable = Table( 'cards', metadata, Column('id', Integer, primary_key=True), Column('factId', Integer, ForeignKey("facts.id"), nullable=False), Column('groupId', Integer, nullable=False, default=1), Column('cardModelId', Integer, ForeignKey("cardModels.id"), nullable=False), - Column('modified', Float, nullable=False, default=time.time), + Column('modified', Integer, nullable=False, default=intTime), # general Column('question', UnicodeText, nullable=False, default=u""), Column('answer', UnicodeText, nullable=False, default=u""), @@ -36,10 +41,10 @@ cardsTable = Table( # shared scheduling Column('type', Integer, nullable=False, default=2), Column('queue', Integer, nullable=False, default=2), - Column('due', Float, nullable=False), + Column('due', Integer, nullable=False), # sm2 - Column('interval', Float, nullable=False, default=0), - Column('factor', Float, nullable=False, default=2.5), + Column('interval', Integer, nullable=False, default=0), + Column('factor', Integer, nullable=False), Column('reps', Integer, nullable=False, default=0), Column('streak', Integer, nullable=False, default=0), Column('lapses', Integer, nullable=False, default=0), @@ -50,27 +55,27 @@ cardsTable = Table( class Card(object): - # FIXME: this needs tidying up - def __init__(self, fact=None, cardModel=None, due=None): - self.id = genID() - self.modified = time.time() - if due: - self.due = due - else: - self.due = self.modified + # called one of three ways: + # - with no args, followed by .fromDB() + # - with all args, when adding cards to db + def __init__(self, fact=None, cardModel=None, group=None): + # timer + self.timerStarted = None if fact: + self.id = genID() + self.modified = intTime() + self.due = fact.pos self.fact = fact self.modelId = fact.modelId - if cardModel: self.cardModel = cardModel + self.groupId = group.id + self.factor = group.config['initialFactor'] # for non-orm use self.cardModelId = cardModel.id self.ordinal = cardModel.ordinal - # timer - self.timerStarted = None def setModified(self): - self.modified = time.time() + self.modified = intTime() def startTimer(self): self.timerStarted = time.time() diff --git a/anki/deck.py b/anki/deck.py index 416cdc863..4e859dac1 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -10,7 +10,7 @@ from anki.lang import _, ngettext from anki.errors import DeckAccessError from anki.stdmodels import BasicModel from anki.utils import parseTags, tidyHTML, genID, ids2str, hexifyID, \ - canonifyTags, joinTags, addTags, checksum, fieldChecksum + canonifyTags, joinTags, addTags, checksum, fieldChecksum, intTime from anki.revlog import logReview from anki.models import Model, CardModel, formatQA from anki.fonts import toPlatformFont @@ -48,6 +48,8 @@ defaultConf = { 'sessionRepLimit': 0, 'sessionTimeLimit': 600, 'currentModelId': None, + 'currentGroupId': 1, + 'nextFactPos': 1, 'mediaURL': "", 'latexPre': """\ \\documentclass[12pt]{article} @@ -66,9 +68,9 @@ defaultConf = { deckTable = Table( 'deck', metadata, Column('id', Integer, nullable=False, primary_key=True), - Column('created', Float, nullable=False, default=time.time), - Column('modified', Float, nullable=False, default=time.time), - Column('schemaMod', Float, nullable=False, default=0), + Column('created', Integer, nullable=False, default=intTime), + Column('modified', Integer, nullable=False, default=intTime), + Column('schemaMod', Integer, nullable=False, default=intTime), Column('version', Integer, nullable=False, default=DECK_VERSION), Column('syncName', UnicodeText, nullable=False, default=u""), Column('lastSync', Integer, nullable=False, default=0), @@ -412,7 +414,7 @@ due > :now and due < :now""", now=time.time()) "Return a new fact with the current model." if model is None: model = self.currentModel - return anki.facts.Fact(model) + return anki.facts.Fact(model, self.getFactPos()) def addFact(self, fact, reset=True): "Add a fact to the deck. Return list of new cards." @@ -432,11 +434,10 @@ due > :now and due < :now""", now=time.time()) self.flushMod() isRandom = self.qconf['newCardOrder'] == NEW_CARDS_RANDOM if isRandom: - due = random.uniform(0, time.time()) - t = time.time() + due = random.randrange(0, 10000) for cardModel in cms: - created = fact.created + 0.00001*cardModel.ordinal - card = anki.cards.Card(fact, cardModel, created) + group = self.groupForTemplate(cardModel) + card = anki.cards.Card(fact, cardModel, group) if isRandom: card.due = due self.flushMod() @@ -449,6 +450,11 @@ due > :now and due < :now""", now=time.time()) self.reset() return fact + def groupForTemplate(self, template): + print "add default group to template" + id = self.config['currentGroupId'] + return self.db.query(anki.groups.GroupConfig).get(id).load() + def availableCardModels(self, fact, checkActive=True): "List of active card models that aren't empty for FACT." models = [] @@ -2132,12 +2138,13 @@ Return new path, relative to media dir.""" # DB helpers ########################################################################## - def save(self): + def save(self, config=True): "Commit any pending changes to disk." if self.lastLoaded == self.modified: return self.lastLoaded = self.modified - self.flushConfig() + if config: + self.flushConfig() self.db.commit() def flushConfig(self): @@ -2187,14 +2194,22 @@ Return new path, relative to media dir.""" self.db = None self.s = None - def setModified(self, newTime=None): + def setModified(self): #import traceback; traceback.print_stack() - self.modified = newTime or time.time() + self.modified = intTime() def setSchemaModified(self): - self.schemaMod = time.time() + self.schemaMod = intTime() anki.graves.forgetAll(self.db) + def getFactPos(self): + "Return next fact position, incrementing it." + # note this is incremented even if facts are not added; gaps are not a bug + p = self.config['nextFactPos'] + self.config['nextFactPos'] += 1 + self.setModified() + return p + def flushMod(self): "Mark modified and flush to DB." self.setModified() @@ -2659,8 +2674,8 @@ sourcesTable = Table( 'sources', metadata, Column('id', Integer, nullable=False, primary_key=True), Column('name', UnicodeText, nullable=False, default=""), - Column('created', Float, nullable=False, default=time.time), - Column('lastSync', Float, nullable=False, default=0), + Column('created', Integer, nullable=False, default=intTime), + Column('lastSync', Integer, nullable=False, default=0), # -1 = never check, 0 = always check, 1+ = number of seconds passed. # not currently exposed in the GUI Column('syncPeriod', Integer, nullable=False, default=0)) @@ -2783,11 +2798,11 @@ class DeckStorage(object): "Add a default group & config." s.execute(""" insert into groupConfig values (1, :t, :name, :conf)""", - t=time.time(), name=_("Default Config"), + t=intTime(), name=_("Default Config"), conf=simplejson.dumps(anki.groups.defaultConf)) s.execute(""" insert into groups values (1, :t, "Default", 1)""", - t=time.time()) + t=intTime()) _addConfig = staticmethod(_addConfig) def _addTables(s): diff --git a/anki/facts.py b/anki/facts.py index 8c515cc21..008d24a96 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -6,7 +6,7 @@ import time from anki.db import * from anki.errors import * from anki.models import Model, FieldModel, fieldModelsTable -from anki.utils import genID, stripHTMLMedia, fieldChecksum +from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime from anki.hooks import runHook # Fields in a fact @@ -43,6 +43,8 @@ mapper(Field, fieldsTable, properties={ # Facts: a set of fields and a model ########################################################################## +# Pos: incrementing number defining add order. There may be duplicates if +# content is added on two sync locations at once. Importing adds to end. # Cache: a HTML-stripped amalgam of the field contents, so we can perform # searches of marked up text in a reasonable time. @@ -50,20 +52,21 @@ factsTable = Table( 'facts', metadata, Column('id', Integer, primary_key=True), Column('modelId', Integer, ForeignKey("models.id"), nullable=False), - Column('created', Float, nullable=False, default=time.time), - Column('modified', Float, nullable=False, default=time.time), + Column('pos', Integer, nullable=False), + Column('modified', Integer, nullable=False, default=intTime), Column('tags', UnicodeText, nullable=False, default=u""), Column('cache', UnicodeText, nullable=False, default=u"")) class Fact(object): "A single fact. Fields exposed as dict interface." - def __init__(self, model=None): + def __init__(self, model=None, pos=None): self.model = model self.id = genID() if model: for fm in model.fieldModels: self.fields.append(Field(fm)) + self.pos = pos self.new = True def isNew(self): @@ -130,7 +133,7 @@ class Fact(object): def setModified(self, textChanged=False, deck=None, media=True): "Mark modified and update cards." - self.modified = time.time() + self.modified = intTime() if textChanged: if not deck: # FIXME: compat code diff --git a/anki/graves.py b/anki/graves.py index 815a5d8dc..423ad3a5d 100644 --- a/anki/graves.py +++ b/anki/graves.py @@ -8,6 +8,7 @@ import time from anki.db import * +from anki.utils import intTime FACT = 0 CARD = 1 @@ -24,11 +25,11 @@ gravestonesTable = Table( def registerOne(db, type, id): db.statement("insert into gravestones values (:t, :id, :ty)", - t=time.time(), id=id, ty=type) + t=intTime(), id=id, ty=type) def registerMany(db, type, ids): db.statements("insert into gravestones values (:t, :id, :ty)", - [{'t':time.time(), 'id':x, 'ty':type} for x in ids]) + [{'t':intTime(), 'id':x, 'ty':type} for x in ids]) def forgetAll(db): db.statement("delete from gravestones") diff --git a/anki/groups.py b/anki/groups.py index 98910940d..1985c21b7 100644 --- a/anki/groups.py +++ b/anki/groups.py @@ -3,12 +3,13 @@ # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html import simplejson, time +from anki.utils import intTime from anki.db import * groupsTable = Table( 'groups', metadata, Column('id', Integer, primary_key=True), - Column('modified', Float, nullable=False, default=time.time), + Column('modified', Integer, nullable=False, default=intTime), Column('name', UnicodeText, nullable=False), Column('confId', Integer, nullable=False)) @@ -34,7 +35,7 @@ defaultConf = { groupConfigTable = Table( 'groupConfig', metadata, Column('id', Integer, primary_key=True), - Column('modified', Float, nullable=False, default=time.time), + Column('modified', Integer, nullable=False, default=intTime), Column('name', UnicodeText, nullable=False), Column('config', UnicodeText, nullable=False, default=unicode(simplejson.dumps(defaultConf)))) @@ -45,9 +46,13 @@ class GroupConfig(object): self.id = genID() self.config = defaultConf + def load(self): + self.config = simplejson.loads(self._config) + return self + def save(self): self._config = simplejson.dumps(self.config) - self.modified = time.time() + self.modified = intTime() mapper(GroupConfig, groupConfigTable, properties={ '_config': groupConfigTable.c.config, diff --git a/anki/media.py b/anki/media.py index 7023d4a69..83e9d4182 100644 --- a/anki/media.py +++ b/anki/media.py @@ -4,7 +4,7 @@ import os, shutil, re, urllib2, time, tempfile, unicodedata, urllib from anki.db import * -from anki.utils import checksum, genID +from anki.utils import checksum, genID, intTime from anki.lang import _ # other code depends on this order, so don't reorder @@ -19,7 +19,7 @@ mediaTable = Table( Column('id', Integer, primary_key=True, nullable=False), Column('filename', UnicodeText, nullable=False, unique=True), Column('refcnt', Integer, nullable=False), - Column('modified', Float, nullable=False), + Column('modified', Integer, nullable=False), Column('chksum', UnicodeText, nullable=False, default=u"")) # File handling @@ -71,7 +71,7 @@ def updateMediaCount(deck, file, count=1): "select 1 from media where filename = :file", file=file): deck.db.statement( "update media set refcnt = refcnt + :c, modified = :t where filename = :file", - file=file, c=count, t=time.time()) + file=file, c=count, t=intTime()) elif count > 0: try: sum = unicode( @@ -81,7 +81,7 @@ def updateMediaCount(deck, file, count=1): deck.db.statement(""" insert into media (id, filename, refcnt, modified, chksum) values (:id, :file, :c, :mod, :sum)""", - id=genID(), file=file, c=count, mod=time.time(), + id=genID(), file=file, c=count, mod=intTime(), sum=sum) def removeUnusedMedia(deck): @@ -173,12 +173,12 @@ def rebuildMediaDir(deck, delete=False, dirty=True): path = os.path.join(mdir, file) if not os.path.exists(path): if md5: - update.append({'f':file, 'sum':u"", 'c':time.time()}) + update.append({'f':file, 'sum':u"", 'c':intTime()}) else: sum = unicode( checksum(open(os.path.join(mdir, file), "rb").read())) if md5 != sum: - update.append({'f':file, 'sum':sum, 'c':time.time()}) + update.append({'f':file, 'sum':sum, 'c':intTime()}) if update: deck.db.statements(""" update media set chksum = :sum, modified = :c where filename = :f""", diff --git a/anki/models.py b/anki/models.py index fe6970430..00f7493ce 100644 --- a/anki/models.py +++ b/anki/models.py @@ -5,9 +5,9 @@ import time, re, simplejson from sqlalchemy.ext.orderinglist import ordering_list from anki.db import * -from anki.utils import genID, canonifyTags +from anki.utils import genID, canonifyTags, intTime from anki.fonts import toPlatformFont -from anki.utils import parseTags, hexifyID, checksum, stripHTML +from anki.utils import parseTags, hexifyID, checksum, stripHTML, intTime from anki.lang import _ from anki.hooks import runFilter from anki.template import render @@ -156,7 +156,7 @@ def formatQA(cid, mid, fact, tags, cm, deck): modelsTable = Table( 'models', metadata, Column('id', Integer, primary_key=True), - Column('modified', Float, nullable=False, default=time.time), + Column('modified', Integer, nullable=False, default=intTime), Column('name', UnicodeText, nullable=False), # currently unused Column('config', UnicodeText, nullable=False, default=u"") @@ -169,7 +169,7 @@ class Model(object): self.id = genID() def setModified(self): - self.modified = time.time() + self.modified = intTime() def addFieldModel(self, field): "Add a field model. Don't call this directly." diff --git a/anki/revlog.py b/anki/revlog.py index e4b0220b6..bf8d7cf81 100644 --- a/anki/revlog.py +++ b/anki/revlog.py @@ -18,9 +18,9 @@ revlogTable = Table( Column('cardId', Integer, nullable=False), Column('ease', Integer, nullable=False), Column('rep', Integer, nullable=False), - Column('lastInterval', Float, nullable=False), - Column('interval', Float, nullable=False), - Column('factor', Float, nullable=False), + Column('lastInterval', Integer, nullable=False), + Column('interval', Integer, nullable=False), + Column('factor', Integer, nullable=False), Column('userTime', Integer, nullable=False), Column('flags', Integer, nullable=False, default=0)) diff --git a/anki/sched.py b/anki/sched.py index aa23706e8..f048620a6 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -22,7 +22,7 @@ class Scheduler(object): self.learnLimit = 1000 self.updateCutoff() - def getCard(self, orm=True): + def getCard(self): "Pop the next card from the queue. None if finished." self.checkDay() id = self.getCardId() diff --git a/anki/upgrade.py b/anki/upgrade.py index d299c09cc..9a06c1a77 100644 --- a/anki/upgrade.py +++ b/anki/upgrade.py @@ -8,6 +8,7 @@ import time, simplejson from anki.db import * from anki.lang import _ from anki.media import rebuildMediaDir +from anki.utils import intTime def moveTable(s, table): sql = s.scalar( @@ -36,9 +37,9 @@ def upgradeSchema(engine, s): import cards metadata.create_all(engine, tables=[cards.cardsTable]) s.execute(""" -insert into cards select id, factId, 1, cardModelId, modified, question, -answer, ordinal, 0, relativeDelay, type, due, interval, factor, reps, -successive, noCount, 0, 0 from cards2""") +insert into cards select id, factId, 1, cardModelId, cast(modified as int), +question, answer, ordinal, 0, relativeDelay, type, due, cast(interval as int), +cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") s.execute("drop table cards2") # tags ########### @@ -53,12 +54,19 @@ insert or ignore into cardTags select cardId, tagId, src from cardTags2""") s.execute("drop table cardTags2") # facts ########### - moveTable(s, "facts") + s.execute(""" +create table facts2 +(id, modelId, modified, tags, cache)""") + # use the rowid to give them an integer order + s.execute(""" +insert into facts2 select id, modelId, modified, tags, spaceUntil from +facts order by created""") + s.execute("drop table facts") import facts metadata.create_all(engine, tables=[facts.factsTable]) s.execute(""" -insert or ignore into facts select id, modelId, created, modified, tags, -spaceUntil from facts2""") +insert or ignore into facts select id, modelId, rowid, +cast(modified as int), tags, cache from facts2""") s.execute("drop table facts2") # media ########### @@ -66,7 +74,7 @@ spaceUntil from facts2""") import media metadata.create_all(engine, tables=[media.mediaTable]) s.execute(""" -insert or ignore into media select id, filename, size, created, +insert or ignore into media select id, filename, size, cast(created as int), originalPath from media2""") s.execute("drop table media2") # deck @@ -78,7 +86,7 @@ originalPath from media2""") import models metadata.create_all(engine, tables=[models.modelsTable]) s.execute(""" -insert or ignore into models select id, modified, name, "" from models2""") +insert or ignore into models select id, cast(modified as int), name, "" from models2""") s.execute("drop table models2") return ver @@ -87,16 +95,17 @@ def migrateDeck(s, engine): import deck metadata.create_all(engine, tables=[deck.deckTable]) s.execute(""" -insert into deck select id, created, modified, 0, 99, -ifnull(syncName, ""), lastSync, utcOffset, "", "", "" from decks""") +insert into deck select id, cast(created as int), cast(modified as int), +0, 99, ifnull(syncName, ""), cast(lastSync as int), +utcOffset, "", "", "" from decks""") # update selective study qconf = deck.defaultQconf.copy() # delete old selective study settings, which we can't auto-upgrade easily keys = ("newActive", "newInactive", "revActive", "revInactive") for k in keys: s.execute("delete from deckVars where key=:k", {'k':k}) - # copy other settings - keys = ("newCardOrder", "newCardSpacing", "revCardOrder") + # copy other settings, ignoring deck order as there's a new default + keys = ("newCardOrder", "newCardSpacing") for k in keys: qconf[k] = s.execute("select %s from decks" % k).scalar() qconf['newPerDay'] = s.execute( @@ -184,8 +193,10 @@ def upgradeDeck(deck): # migrate revlog data to new table deck.db.statement(""" insert or ignore into revlog select -cast(time*1000 as int), cardId, ease, reps, lastInterval, nextInterval, nextFactor, -cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""") +cast(time*1000 as int), cardId, ease, reps, +cast(lastInterval as int), cast(nextInterval as int), +cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), +0 from reviewHistory""") deck.db.statement("drop table reviewHistory") # convert old ease0 into ease1 deck.db.statement("update revlog set ease = 1 where ease = 0") @@ -198,7 +209,7 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""") # don't need an index on fieldModelId deck.db.statement("drop index if exists ix_fields_fieldModelId") # update schema time - deck.db.statement("update deck set schemaMod = :t", t=time.time()) + deck.db.statement("update deck set schemaMod = :t", t=intTime()) # remove queueDue as it's become dynamic, and type index deck.db.statement("drop index if exists ix_cards_queueDue") deck.db.statement("drop index if exists ix_cards_type") @@ -207,9 +218,24 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""") deck.db.statement("drop table if exists %sDeleted" % t) # finally, update indices & optimize updateIndices(deck.db) + # rewrite due times for new cards + deck.db.statement(""" +update cards set due = (select pos from facts where factId = facts.id) where type=2""") + # convert due cards into day-based due + deck.db.statement(""" +update cards set due = cast( +(case when due < :stamp then 0 else 1 end) + +((due-:stamp)/86400) as int)+:today where type +between 0 and 1""", stamp=deck.sched.dayCutoff, today=deck.sched.today) + print "today", deck.sched.today + print "cut", deck.sched.dayCutoff # setup qconf & config for dynamicIndices() deck.qconf = simplejson.loads(deck._qconf) deck.config = simplejson.loads(deck._config) + deck.data = simplejson.loads(deck._data) + # update factPos + deck.config['nextFactPos'] = deck.db.scalar("select max(pos) from facts")+1 + deck.flushConfig() # add default config import deck as deckMod deckMod.DeckStorage._addConfig(deck.engine) diff --git a/anki/utils.py b/anki/utils.py index f9e570543..f1e59c8d3 100644 --- a/anki/utils.py +++ b/anki/utils.py @@ -23,6 +23,9 @@ if sys.version_info[1] < 5: # Time handling ############################################################################## +def intTime(): + return int(time.time()) + timeTable = { "years": lambda n: ngettext("%s year", "%s years", n), "months": lambda n: ngettext("%s month", "%s months", n),