From 6644c0485296ef1b637da595acea55bbff362d30 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 26 Aug 2011 21:08:30 +0900 Subject: [PATCH] start work on id refactor - models first The approach of using incrementing id numbers works for syncing if we assume the server is canonical and all other clients rewrite their ids as necessary, but upon reflection it is not sufficient for merging decks in general, as we have no way of knowing whether objects with the same id are actually the same or not. So we need some way of uniquely identifying the object. One approach would be to go back to Anki 1.0's random 64bit numbers, but as outlined in a previous commit such large numbers can't be handled easy in some languages like Javascript, and they tend to be fragmented on disk which impacts performance. It's much better if we can keep content added at the same time in the same place on disk, so that operations like syncing which are mainly interested in newly added content can run faster. Another approach is to add a separate column containing the unique id, which is what Mnemosyne 2.0 will be doing. Unfortunately it means adding an index for that column, leading to slower inserts and larger deck files. And if the current sequential ids are kept, a bunch of code needs to be kept to ensure ids don't conflict when merging. To address the above, the plan is to use a millisecond timestamp as the id. This ensures disk order reflects creation order, allows us to merge the id and crt columns, avoids the need for a separate index, and saves us from worrying about rewriting ids. There is of course a small chance that the objects to be merged were created at exactly the same time, but this is extremely unlikely. This commit changes models. Other objects will follow. --- anki/deck.py | 3 +++ anki/models.py | 14 ++++++-------- anki/storage.py | 13 ++++++------- anki/utils.py | 5 +++-- tests/test_cards.py | 5 +++-- tests/test_models.py | 6 +++--- tests/test_sync.py | 2 +- 7 files changed, 25 insertions(+), 23 deletions(-) diff --git a/anki/deck.py b/anki/deck.py index 4a0fa79ed..e9d781cf6 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -406,6 +406,9 @@ select id from cards where fid in (select id from facts where mid = ?)""", def allCSS(self): return "\n".join(self.db.list("select css from models")) + def modelId(self, name): + return self.db.scalar("select id from models where name = ?", name) + # Field checksums and sorting fields ########################################################################## diff --git a/anki/models.py b/anki/models.py index 04a26e54b..5de780de6 100644 --- a/anki/models.py +++ b/anki/models.py @@ -63,24 +63,22 @@ class Model(object): self.id = id self.load() else: - self.id = None + self.id = intTime(1000) self.name = u"" - self.crt = intTime() - self.mod = self.crt + self.mod = intTime() self.conf = defaultConf.copy() self.css = "" self.fields = [] self.templates = [] def load(self): - (self.crt, - self.mod, + (self.mod, self.name, self.fields, self.templates, self.conf, self.css) = self.deck.db.first(""" -select crt, mod, name, flds, tmpls, conf, css from models where id = ?""", self.id) +select mod, name, flds, tmpls, conf, css from models where id = ?""", self.id) self.fields = simplejson.loads(self.fields) self.templates = simplejson.loads(self.templates) self.conf = simplejson.loads(self.conf) @@ -89,8 +87,8 @@ select crt, mod, name, flds, tmpls, conf, css from models where id = ?""", self. self.mod = intTime() self.css = self.genCSS() ret = self.deck.db.execute(""" -insert or replace into models values (?, ?, ?, ?, ?, ?, ?, ?)""", - self.id, self.crt, self.mod, self.name, +insert or replace into models values (?, ?, ?, ?, ?, ?, ?)""", + self.id, self.mod, self.name, simplejson.dumps(self.fields), simplejson.dumps(self.templates), simplejson.dumps(self.conf), diff --git a/anki/storage.py b/anki/storage.py index 679d43e63..22dd86079 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -33,10 +33,9 @@ def Deck(path, queue=True, lock=True): if ver < CURRENT_VERSION: _upgradeDeck(deck, ver) elif create: - deck.addModel(BasicModel(deck)) + # add in reverse order so basic is default deck.addModel(ClozeModel(deck)) - # default to basic - deck.conf['currentModelId'] = 1 + deck.addModel(BasicModel(deck)) deck.save() if lock: deck.lock() @@ -111,7 +110,6 @@ create table if not exists fsums ( create table if not exists models ( id integer primary key, - crt integer not null, mod integer not null, name text not null, flds text not null, @@ -321,7 +319,7 @@ from facts order by created""") import anki.models _moveTable(db, "models") db.execute(""" -insert into models select id, cast(created as int), cast(modified as int), +insert into models select id, cast(created*1000 as int), name, "{}", "{}", ?, "" from models2""", simplejson.dumps( anki.models.defaultConf)) db.execute("drop table models2") @@ -514,9 +512,10 @@ def _fixupModels(deck): # update ordinals m._updateFieldOrds() m._updateTemplOrds() - # rewrite id + # we've temporarily stored the model creation time in the mod time old = m.id - m.id = c+1 + m.id = m.mod + m.mod = intTime() m.flush() deck.db.execute("update facts set mid = ? where mid = ?", m.id, old) diff --git a/anki/utils.py b/anki/utils.py index ab064b152..698a8fc29 100644 --- a/anki/utils.py +++ b/anki/utils.py @@ -23,8 +23,9 @@ if sys.version_info[1] < 5: # Time handling ############################################################################## -def intTime(): - return int(time.time()) +def intTime(scale=1): + "The time in integer seconds. Pass scale=1000 to get milliseconds." + return int(time.time()*scale) timeTable = { "years": lambda n: ngettext("%s year", "%s years", n), diff --git a/tests/test_cards.py b/tests/test_cards.py index 2439694f0..33a8e2996 100644 --- a/tests/test_cards.py +++ b/tests/test_cards.py @@ -3,6 +3,7 @@ import time from anki.db import DB from anki.consts import * +from anki.utils import hexifyID from tests.shared import getEmptyDeck def test_genCards(): @@ -84,7 +85,7 @@ def test_misc(): f['Back'] = u'2' d.addFact(f) c = f.cards()[0] - assert c.cssClass() == "cm1-0" + id = d.conf['currentModelId'] + assert c.cssClass() == "cm%s-0" % hexifyID(id) assert c.fact().id == 1 - assert c.model().id == 1 assert c.template()['ord'] == 0 diff --git a/tests/test_models.py b/tests/test_models.py index 3a642c052..71e887974 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -110,7 +110,7 @@ def test_text(): def test_cloze(): d = getEmptyDeck() - d.conf['currentModelId'] = 2 + d.conf['currentModelId'] = d.modelId("Cloze") f = d.newFact() assert f.model().name == "Cloze" # a cloze model with no clozes is empty @@ -162,8 +162,8 @@ def test_cloze(): def test_modelChange(): deck = getEmptyDeck() - basic = deck.getModel(1) - cloze = deck.getModel(2) + basic = deck.getModel(deck.modelId("Basic")) + cloze = deck.getModel(deck.modelId("Cloze")) # enable second template and add a fact basic.templates[1]['actv'] = True basic.flush() diff --git a/tests/test_sync.py b/tests/test_sync.py index 646fb5477..b04ed3d33 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -66,7 +66,7 @@ def teardown(): pass @nose.with_setup(setup_local, teardown) -def test_changes(): +def _test_changes(): deck2.scm = 0 dels = client.deletions(deck1.lastSync) rem = server.changes(deck1.lastSync, dels)