diff --git a/anki/cards.py b/anki/cards.py index 2f558aade..8cd86bdc9 100644 --- a/anki/cards.py +++ b/anki/cards.py @@ -48,6 +48,7 @@ class Card(object): self.gid, self.ord, self.mod, + self.usn, self.type, self.queue, self.due, @@ -65,15 +66,17 @@ class Card(object): def flush(self): self.mod = intTime() + self.usn = self.deck.usn() self.deck.db.execute( """ insert or replace into cards values -(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", +(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", self.id, self.fid, self.gid, self.ord, self.mod, + self.usn, self.type, self.queue, self.due, @@ -88,11 +91,12 @@ insert or replace into cards values def flushSched(self): self.mod = intTime() + self.usn = self.deck.usn() self.deck.db.execute( """update cards set -mod=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?, +mod=?, usn=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?, lapses=?, grade=?, cycles=?, edue=? where id = ?""", - self.mod, self.type, self.queue, self.due, self.ivl, + self.mod, self.usn, self.type, self.queue, self.due, self.ivl, self.factor, self.reps, self.lapses, self.grade, self.cycles, self.edue, self.id) diff --git a/anki/deck.py b/anki/deck.py index b2c9842eb..273f1f3ae 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -74,13 +74,14 @@ class _Deck(object): self.mod, self.scm, self.dty, - self.lastSync, + self._usn, + self.ls, self.conf, models, groups, gconf, tags) = self.db.first(""" -select crt, mod, scm, dty, lastSync, +select crt, mod, scm, dty, usn, ls, conf, models, groups, gconf, tags from deck""") self.conf = simplejson.loads(self.conf) self.models.load(models) @@ -92,9 +93,9 @@ conf, models, groups, gconf, tags from deck""") self.mod = intTime() if mod is None else mod self.db.execute( """update deck set -crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""", +crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""", self.crt, self.mod, self.scm, self.dty, - self.lastSync, simplejson.dumps(self.conf)) + self._usn, self.ls, simplejson.dumps(self.conf)) self.models.flush() self.groups.flush() self.tags.flush() @@ -148,7 +149,7 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""", def schemaChanged(self): "True if schema changed since last sync." - return self.scm > self.lastSync + return self.scm > self.ls def setDirty(self): "Signal there are temp. suspended cards that need cleaning up on close." @@ -161,6 +162,7 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""", self.dty = False def rename(self, path): + raise "nyi" # close our DB connection self.close() # move to new path @@ -173,6 +175,9 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""", self.reopen() self.media.move(olddir) + def usn(self): + return self._usn + # Object creation helpers ########################################################################## @@ -370,6 +375,7 @@ select id from facts where id in %s and id not in (select fid from cards)""" % if csum: self.db.execute("delete from fsums where fid in "+sfids) self.db.executemany("insert into fsums values (?,?,?)", r) + # rely on calling code to bump usn+mod self.db.executemany("update facts set sfld = ? where id = ?", r2) # Q/A generation diff --git a/anki/facts.py b/anki/facts.py index 7d015e101..50ef835a2 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -29,10 +29,11 @@ class Fact(object): (self.mid, self.gid, self.mod, + self.usn, self.tags, self.fields, self.data) = self.deck.db.first(""" -select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id) +select mid, gid, mod, usn, tags, flds, data from facts where id = ?""", self.id) self.fields = splitFields(self.fields) self.tags = self.deck.tags.split(self.tags) self._model = self.deck.models.get(self.mid) @@ -40,13 +41,14 @@ select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id) def flush(self, mod=None): self.mod = mod if mod else intTime() + self.usn = self.deck.usn() sfld = stripHTML(self.fields[self.deck.models.sortIdx(self._model)]) tags = self.stringTags() res = self.deck.db.execute(""" -insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""", +insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", self.id, self.mid, self.gid, - self.mod, tags, self.joinedFields(), - sfld, self.data) + self.mod, self.usn, tags, + self.joinedFields(), sfld, self.data) self.id = res.lastrowid self.updateFieldChecksums() self.deck.tags.register(self.tags) diff --git a/anki/find.py b/anki/find.py index a1e86eece..873267f24 100644 --- a/anki/find.py +++ b/anki/find.py @@ -3,7 +3,8 @@ # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html import re -from anki.utils import ids2str, splitFields, joinFields, stripHTML +from anki.utils import ids2str, splitFields, joinFields, stripHTML, intTime + SEARCH_TAG = 0 SEARCH_TYPE = 1 @@ -401,11 +402,11 @@ def findReplace(deck, fids, src, dst, regex=False, field=None, fold=True): sflds[c] = repl(sflds[c]) flds = joinFields(sflds) if flds != origFlds: - d.append(dict(fid=fid, flds=flds)) + d.append(dict(fid=fid,flds=flds,u=deck.usn(),m=intTime())) if not d: return 0 # replace - deck.db.executemany("update facts set flds = :flds where id=:fid", d) + deck.db.executemany("update facts set flds=:flds,mod=:m,usn=:u where id=:fid", d) deck.updateFieldCache(fids) return len(d) diff --git a/anki/groups.py b/anki/groups.py index 6cc3f7c6a..2fe5892d8 100644 --- a/anki/groups.py +++ b/anki/groups.py @@ -57,6 +57,7 @@ defaultConf = { }, 'maxTaken': 60, 'mod': 0, + 'usn': 0, } class GroupManager(object): @@ -76,6 +77,7 @@ class GroupManager(object): "Can be called with either a group or a group configuration." if g: g['mod'] = intTime() + g['usn'] = self.deck.usn() self.changed = True def flush(self): @@ -114,8 +116,12 @@ class GroupManager(object): def rem(self, gid): self.deck.modSchema() - self.deck.db.execute("update cards set gid = 1 where gid = ?", gid) - self.deck.db.execute("update facts set gid = 1 where gid = ?", gid) + self.deck.db.execute( + "update cards set gid=1,usn=?,mod=? where gid = ?", + gid, self.deck.usn(), intTime()) + self.deck.db.execute( + "update facts set gid=1,usn=?,mod=? where gid = ?", + gid, self.deck.usn(), intTime()) self.deck.db.execute("delete from groups where id = ?", gid) print "fixme: loop through models and update stale gid references" @@ -157,7 +163,8 @@ class GroupManager(object): def setGroup(self, cids, gid): self.db.execute( - "update cards set gid = ? where id in "+ids2str(cids), gid) + "update cards set gid=?,usn=?,mod=? where id in "+ + ids2str(cids), gid, self.deck.usn(), intTime()) def update(self, g): "Add or update an existing group. Used for syncing and merging." diff --git a/anki/importing/__init__.py b/anki/importing/__init__.py index 07c7a3d26..fc601ca6e 100644 --- a/anki/importing/__init__.py +++ b/anki/importing/__init__.py @@ -19,9 +19,6 @@ from anki.utils import fieldChecksum, ids2str from anki.errors import * #from anki.deck import NEW_CARDS_RANDOM -# FIXME: when importing an anki file, if any revlog entries are less than the -# last sync time, we need to bump the deck schema - # Base importer ########################################################################## diff --git a/anki/models.py b/anki/models.py index 7d216ab0a..55e799269 100644 --- a/anki/models.py +++ b/anki/models.py @@ -28,6 +28,8 @@ defaultModel = { \\begin{document} """, 'latexPost': "\\end{document}", + 'mod': 0, + 'usn': 0, } defaultField = { @@ -75,6 +77,7 @@ class ModelManager(object): "Mark M modified if provided, and schedule registry flush." if m: m['mod'] = intTime() + m['usn'] = self.deck.usn() m['css'] = self._css(m) self.changed = True @@ -308,8 +311,10 @@ select id from cards where fid in (select id from facts where mid = ?)""", r = [] for (id, flds) in self.deck.db.execute( "select id, flds from facts where mid = ?", m['id']): - r.append((joinFields(fn(splitFields(flds))), id)) - self.deck.db.executemany("update facts set flds = ? where id = ?", r) + r.append((joinFields(fn(splitFields(flds))), + intTime(), self.deck.usn(), id)) + self.deck.db.executemany( + "update facts set flds=?,mod=?,usn=? where id = ?", r) # Templates ################################################## @@ -334,8 +339,9 @@ select c.id from cards c, facts f where c.fid=f.id and mid = ? and ord = ?""", self.deck.remCards(cids) # shift ordinals self.deck.db.execute(""" -update cards set ord = ord - 1 where fid in (select id from facts -where mid = ?) and ord > ?""", m['id'], ord) +update cards set ord = ord - 1, usn = ?, mod = ? + where fid in (select id from facts where mid = ?) and ord > ?""", + self.deck.usn(), intTime(), m['id'], ord) m['tmpls'].remove(template) self._updateTemplOrds(m) self.save(m) @@ -359,8 +365,9 @@ where mid = ?) and ord > ?""", m['id'], ord) # apply self.save(m) self.deck.db.execute(""" -update cards set ord = (case %s end) where fid in ( -select id from facts where mid = ?)""" % " ".join(map), m['id']) +update cards set ord = (case %s end),usn=?,mod=? where fid in ( +select id from facts where mid = ?)""" % " ".join(map), + self.deck.usn(), intTime(), m['id']) # Model changing ########################################################################## @@ -388,9 +395,10 @@ select id from facts where mid = ?)""" % " ".join(map), m['id']) for c in range(nfields): flds.append(newflds.get(c, "")) flds = joinFields(flds) - d.append(dict(fid=fid, flds=flds, mid=newModel['id'])) + d.append(dict(fid=fid, flds=flds, mid=newModel['id'], + m=intTime(),u=self.deck.usn())) self.deck.db.executemany( - "update facts set flds=:flds, mid=:mid where id = :fid", d) + "update facts set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :fid", d) self.deck.updateFieldCache(fids) def _changeCards(self, fids, newModel, map): @@ -399,9 +407,11 @@ select id from facts where mid = ?)""" % " ".join(map), m['id']) for (cid, ord) in self.deck.db.execute( "select id, ord from cards where fid in "+ids2str(fids)): if map[ord] is not None: - d.append(dict(cid=cid, new=map[ord])) + d.append(dict( + cid=cid,new=map[ord],u=self.deck.usn(),m=intTime())) else: deleted.append(cid) self.deck.db.executemany( - "update cards set ord=:new where id=:cid", d) + "update cards set ord=:new,usn=:u,mod=:m where id=:cid", + d) self.deck.remCards(deleted) diff --git a/anki/sched.py b/anki/sched.py index 17ccbe3d4..dd9e94fe0 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -62,6 +62,7 @@ class Scheduler(object): raise Exception("Invalid queue") self._updateStats('time', card.timeTaken()) card.mod = intTime() + card.usn = self.deck.usn() card.flushSched() def counts(self): @@ -397,10 +398,9 @@ limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff) ivl = card.ivl if leaving else -(self._delayForGrade(conf, card.grade)) def log(): self.deck.db.execute( - "insert into revlog values (?,?,?,?,?,?,?,?)", - int(time.time()*1000), card.id, ease, - ivl, lastIvl, - card.factor, card.timeTaken(), type) + "insert into revlog values (?,?,?,?,?,?,?,?,?)", + int(time.time()*1000), card.id, self.deck.usn(), ease, + ivl, lastIvl, card.factor, card.timeTaken(), type) try: log() except: @@ -415,10 +415,10 @@ limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff) extra = " and id in "+ids2str(ids) self.deck.db.execute(""" update cards set -due = edue, queue = 2, mod = %d +due = edue, queue = 2, mod = %d, usn = %d where queue = 1 and type = 2 %s -""" % (intTime(), extra)) +""" % (intTime(), self.deck.usn(), extra)) # Reviews ########################################################################## @@ -501,8 +501,8 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % ( def _logRev(self, card, ease): def log(): self.deck.db.execute( - "insert into revlog values (?,?,?,?,?,?,?,?)", - int(time.time()*1000), card.id, ease, + "insert into revlog values (?,?,?,?,?,?,?,?,?)", + int(time.time()*1000), card.id, self.deck.usn(), ease, card.ivl, card.lastIvl, card.factor, card.timeTaken(), 1) try: @@ -712,15 +712,15 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % ( "Suspend cards." self.removeFailed(ids) self.deck.db.execute( - "update cards set queue = -1, mod = ? where id in "+ - ids2str(ids), intTime()) + "update cards set queue=-1,mod=?,usn=? where id in "+ + ids2str(ids), intTime(), self.deck.usn()) def unsuspendCards(self, ids): "Unsuspend cards." self.deck.db.execute( - "update cards set queue = type, mod = ? " + "update cards set queue=type,mod=?,usn=? " "where queue = -1 and id in "+ ids2str(ids), - intTime()) + intTime(), self.deck.usn()) def buryFact(self, fid): "Bury all cards for fact until next session." @@ -772,8 +772,9 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % ( def forgetCards(self, ids): "Put cards at the end of the new queue." self.deck.db.execute( - "update cards set type=0, queue=0, ivl=0 where id in "+ids2str(ids)) + "update cards set type=0,queue=0,ivl=0 where id in "+ids2str(ids)) pmax = self.deck.db.scalar("select max(due) from cards where type=0") + # takes care of mod + usn self.sortCards(ids, start=pmax+1, shuffle=self.deck.models.randomNew()) def reschedCards(self, ids, imin, imax): @@ -816,15 +817,15 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % ( if low is not None: shiftby = high - low + 1 self.deck.db.execute(""" -update cards set mod=?, due=due+? where id not in %s -and due >= ?""" % scids, now, shiftby, low) +update cards set mod=?, usn=?, due=due+? where id not in %s +and due >= ?""" % scids, now, self.deck.usn(), shiftby, low) # reorder cards d = [] for id, fid in self.deck.db.execute( "select id, fid from cards where type = 0 and id in "+scids): - d.append(dict(now=now, due=due[fid], cid=id)) + d.append(dict(now=now, due=due[fid], usn=self.deck.usn(), cid=id)) self.deck.db.executemany( - "update cards set due = :due, mod = :now where id = :cid""", d) + "update cards set due=:due,mod=:now,usn=:usn where id = :cid""", d) # fixme: because it's a model property now, these should be done on a # per-model basis diff --git a/anki/storage.py b/anki/storage.py index f09cf9c2a..71d65b734 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -63,7 +63,8 @@ create table if not exists deck ( scm integer not null, ver integer not null, dty integer not null, - lastSync integer not null, + usn integer not null, + ls integer not null, conf text not null, models text not null, groups text not null, @@ -71,12 +72,30 @@ create table if not exists deck ( tags text not null ); +create table if not exists facts ( + id integer primary key, + mid integer not null, + gid integer not null, + mod integer not null, + usn integer not null, + tags text not null, + flds text not null, + sfld integer not null, + data text not null +); + +create table if not exists fsums ( + fid integer not null, + mid integer not null, + csum integer not null +); create table if not exists cards ( id integer primary key, fid integer not null, gid integer not null, ord integer not null, mod integer not null, + usn integer not null, type integer not null, queue integer not null, due integer not null, @@ -90,32 +109,10 @@ create table if not exists cards ( data text not null ); -create table if not exists facts ( - id integer primary key, - mid integer not null, - gid integer not null, - mod integer not null, - tags text not null, - flds text not null, - sfld integer not null, - data text not null -); - -create table if not exists fsums ( - fid integer not null, - mid integer not null, - csum integer not null -); - -create table if not exists graves ( - id integer not null, - oid integer not null, - type integer not null -); - create table if not exists revlog ( id integer primary key, cid integer not null, + usn integer not null, ease integer not null, ivl integer not null, lastIvl integer not null, @@ -124,8 +121,14 @@ create table if not exists revlog ( type integer not null ); +create table if not exists graves ( + usn integer not null, + oid integer not null, + type integer not null +); + insert or ignore into deck -values(1,0,0,0,%(v)s,0,0,'','{}','','','{}'); +values(1,0,0,0,%(v)s,0,0,0,'','{}','','','{}'); """ % ({'v':CURRENT_VERSION})) import anki.deck import anki.groups @@ -147,11 +150,13 @@ def _updateIndices(db): "Add indices to the DB." db.executescript(""" -- avoid loading entire facts table in for sync summary -create index if not exists ix_facts_mod on facts (mod); +create index if not exists ix_facts_usn on facts (usn); -- card spacing, etc create index if not exists ix_cards_fid on cards (fid); -- revlog by card create index if not exists ix_revlog_cid on revlog (cid); +-- revlog syncing +create index if not exists ix_revlog_usn on revlog (usn); -- field uniqueness check create index if not exists ix_fsums_fid on fsums (fid); create index if not exists ix_fsums_csum on fsums (csum); @@ -200,7 +205,7 @@ end) """) # pull facts into memory, so we can merge them with fields efficiently facts = db.all(""" -select id, modelId, 1, cast(created*1000 as int), cast(modified as int), tags +select id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags from facts order by created""") # build field hash fields = {} @@ -232,7 +237,7 @@ from facts order by created""") # and put the facts into the new table db.execute("drop table facts") _addSchema(db, False) - db.executemany("insert into facts values (?,?,?,?,?,?,'','')", data) + db.executemany("insert into facts values (?,?,?,?,?,?,?,'','')", data) db.execute("drop table fields") # cards @@ -244,7 +249,7 @@ from facts order by created""") cardidmap = {} for row in db.execute(""" select id, cast(created*1000 as int), factId, ordinal, -cast(modified as int), +cast(modified as int), 0, (case relativeDelay when 0 then 1 when 1 then 2 @@ -271,7 +276,7 @@ order by created"""): db.execute("drop table cards") _addSchema(db, False) db.executemany(""" -insert into cards values (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, "")""", +insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""", rows) # reviewHistory -> revlog @@ -280,7 +285,7 @@ insert into cards values (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, "")""", r = [] for row in db.execute(""" select -cast(time*1000 as int), cardId, ease, +cast(time*1000 as int), cardId, 0, ease, cast(nextInterval as int), cast(lastInterval as int), cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int), yesCount from reviewHistory"""): @@ -313,7 +318,7 @@ yesCount from reviewHistory"""): row[7] = 1 r.append(row) db.executemany( - "insert or ignore into revlog values (?,?,?,?,?,?,?,?)", r) + "insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r) db.execute("drop table reviewHistory") # deck @@ -342,7 +347,7 @@ def _migrateDeckTbl(db): db.execute("delete from deck") db.execute(""" insert or replace into deck select id, cast(created as int), :t, -:t, 99, 0, cast(lastSync as int), +:t, 99, 0, 0, cast(lastSync as int), "", "", "", "", "" from decks""", t=intTime()) # prepare a group to store the old deck options import anki.groups diff --git a/anki/sync.py b/anki/sync.py index e9b119a07..6b6f91337 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -58,14 +58,12 @@ class Syncer(object): def sync(self): "Returns 'noChanges', 'fullSync', or 'success'." # get local and remote modified, schema and sync times - self.lmod, lscm, lsyn = self.times() - self.rmod, rscm, rsyn = self.server.times() + self.lmod, lscm, lsyn, self.minUsn = self.times() + self.rmod, rscm, rsyn, self.maxUsn = self.server.times() if self.lmod == self.rmod: return "noChanges" elif lscm != rscm: return "fullSync" - # find last sync time minus 10 mins for clock drift - self.ls = self._lastSync(lsyn, rsyn) self.lnewer = self.lmod > self.rmod # get local changes and switch to full sync if there were too many self.status("getLocal") @@ -74,7 +72,8 @@ class Syncer(object): return "fullSync" # send them to the server, and get the server's changes self.status("getServer") - rchg = self.server.changes(ls=self.ls, lnewer=self.lnewer, changes=lchg) + rchg = self.server.changes(minUsn=self.minUsn, lnewer=self.lnewer, + changes=lchg) if rchg == "fullSync": return "fullSync" # otherwise, merge @@ -86,16 +85,14 @@ class Syncer(object): self.finish(mod) return "success" - def _lastSync(self, lsyn, rsyn): - return min(lsyn, rsyn) - 600 - def times(self): - return (self.deck.mod, self.deck.scm, self.deck.lastSync) + return (self.deck.mod, self.deck.scm, self.deck.ls, self.deck._usn) - def changes(self, ls=None, lnewer=None, changes=None): - if ls: + def changes(self, minUsn=None, lnewer=None, changes=None): + if minUsn is not None: # we're the server; save info - self.ls = ls + self.maxUsn = self.deck._usn + self.minUsn = minUsn self.lnewer = not lnewer self.rchg = changes try: @@ -110,7 +107,7 @@ class Syncer(object): # collection-level configuration from last modified side if self.lnewer: d['conf'] = self.getConf() - if ls: + if minUsn is not None: # we're the server, we can merge our side before returning self.merge(d, self.rchg) return d @@ -130,7 +127,8 @@ class Syncer(object): if not mod: # server side; we decide new mod time mod = intTime() - self.deck.lastSync = mod + self.deck.ls = mod + self.deck._usn = self.maxUsn + 1 self.deck.save(mod=mod) return mod @@ -138,7 +136,7 @@ class Syncer(object): ########################################################################## def getModels(self): - return [m for m in self.deck.models.all() if m['mod'] > self.ls] + return [m for m in self.deck.models.all() if m['usn'] >= self.minUsn] def mergeModels(self, rchg): # deletes result in schema mod, so we only have to worry about @@ -154,8 +152,8 @@ class Syncer(object): def getGroups(self): return [ - [g for g in self.deck.groups.all() if g['mod'] > self.ls], - [g for g in self.deck.groups.allConf() if g['mod'] > self.ls] + [g for g in self.deck.groups.all() if g['usn'] >= self.minUsn], + [g for g in self.deck.groups.allConf() if g['usn'] >= self.minUsn] ] def mergeGroups(self, rchg): @@ -175,7 +173,7 @@ class Syncer(object): ########################################################################## def getTags(self): - return self.deck.tags.allSince(self.ls) + return self.deck.tags.allSinceUSN(self.minUsn) def mergeTags(self, tags): self.deck.tags.register(tags) @@ -184,38 +182,40 @@ class Syncer(object): ########################################################################## def getRevlog(self): - r = self.deck.db.all("select * from revlog where id > ? limit ?", - self.ls*1000, self.MAX_REVLOG) + r = self.deck.db.all("select * from revlog where usn >= ? limit ?", + self.minUsn, self.MAX_REVLOG) if len(r) == self.MAX_REVLOG: raise SyncTooLarge return r def mergeRevlog(self, logs): + for l in logs: + l[2] = self.maxUsn self.deck.db.executemany( - "insert or ignore into revlog values (?,?,?,?,?,?,?,?)", + "insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", logs) # Facts ########################################################################## def getFacts(self): - f = self.deck.db.all("select * from facts where mod > ? limit ?", - self.ls, self.MAX_FACTS) + f = self.deck.db.all("select * from facts where usn >= ? limit ?", + self.minUsn, self.MAX_FACTS) if len(f) == self.MAX_FACTS: raise SyncTooLarge return [ f, self.deck.db.list( - "select oid from graves where id > ? and type = ?", - self.ls, REM_FACT) + "select oid from graves where usn >= ? and type = ?", + self.minUsn, REM_FACT) ] def mergeFacts(self, lchg, rchg): (toAdd, toRem) = self.findChanges( - lchg[0], lchg[1], rchg[0], rchg[1], 3) + lchg[0], lchg[1], rchg[0], rchg[1], 3, 4) # add missing self.deck.db.executemany( - "insert or replace into facts values (?,?,?,?,?,?,?,?)", + "insert or replace into facts values (?,?,?,?,?,?,?,?,?)", toAdd) # update fsums table - fixme: in future could skip sort cache self.deck.updateFieldCache([f[0] for f in toAdd]) @@ -226,26 +226,26 @@ class Syncer(object): ########################################################################## def getCards(self): - c = self.deck.db.all("select * from cards where mod > ? limit ?", - self.ls, self.MAX_CARDS) + c = self.deck.db.all("select * from cards where usn >= ? limit ?", + self.minUsn, self.MAX_CARDS) if len(c) == self.MAX_CARDS: raise SyncTooLarge return [ c, self.deck.db.list( - "select oid from graves where id > ? and type = ?", - self.ls, REM_CARD) + "select oid from graves where usn >= ? and type = ?", + self.minUsn, REM_CARD) ] def mergeCards(self, lchg, rchg): # cards with higher reps preserved, so that gid changes don't clobber # older reviews (toAdd, toRem) = self.findChanges( - lchg[0], lchg[1], rchg[0], rchg[1], 10) + lchg[0], lchg[1], rchg[0], rchg[1], 11, 5) # add missing self.deck.db.executemany( "insert or replace into cards values " - "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", toAdd) # remove remotely deleted self.deck.remCards(toRem) @@ -262,7 +262,7 @@ class Syncer(object): # Merging ########################################################################## - def findChanges(self, localAdds, localRems, remoteAdds, remoteRems, key): + def findChanges(self, localAdds, localRems, remoteAdds, remoteRems, key, usn): local = {} toAdd = [] toRem = [] @@ -280,6 +280,7 @@ class Syncer(object): # added on both sides if r[key] > l[key]: # remote newer; update + r[usn] = self.maxUsn toAdd.append(r) else: # local newer; remote will update @@ -289,15 +290,16 @@ class Syncer(object): pass else: # changed on server only + r[usn] = self.maxUsn toAdd.append(r) return toAdd, remoteRems class LocalServer(Syncer): # serialize/deserialize payload, so we don't end up sharing objects # between decks in testing - def changes(self, ls, lnewer, changes): + def changes(self, minUsn, lnewer, changes): l = simplejson.loads; d = simplejson.dumps - return l(d(Syncer.changes(self, ls, lnewer, l(d(changes))))) + return l(d(Syncer.changes(self, minUsn, lnewer, l(d(changes))))) # not yet ported class RemoteServer(Syncer): diff --git a/anki/tags.py b/anki/tags.py index 7b7dfa41d..d7f54bf69 100644 --- a/anki/tags.py +++ b/anki/tags.py @@ -39,7 +39,7 @@ class TagManager(object): # versions of the same tag if they ignore the qt autocomplete. for t in tags: if t not in self.tags: - self.tags[t] = intTime() + self.tags[t] = self.deck.usn() self.changed = True def all(self): @@ -57,8 +57,8 @@ class TagManager(object): self.register(set(self.split( " ".join(self.deck.db.list("select distinct tags from facts"+lim))))) - def allSince(self, mod): - return [k for k,v in self.tags.items() if v > mod] + def allSinceUSN(self, usn): + return [k for k,v in self.tags.items() if v >= usn] # Bulk addition/removal from facts ############################################################# @@ -88,9 +88,10 @@ class TagManager(object): fids = [] def fix(row): fids.append(row[0]) - return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()} + return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime(), + 'u':self.deck.usn()} self.deck.db.executemany( - "update facts set tags = :t, mod = :n where id = :id", + "update facts set tags=:t,mod=:n,usn=:u where id = :id", [fix(row) for row in res]) def bulkRem(self, ids, tags): @@ -171,5 +172,5 @@ class TagManager(object): def setGroupForTags(self, yes, no, gid): fids = self.selTagFids(yes, no) self.deck.db.execute( - "update cards set gid = ? where fid in "+ids2str(fids), - gid) + "update cards set gid=?,mod=?,usn=? where fid in "+ids2str(fids), + gid, intTime(), self.deck.usn()) diff --git a/tests/test_sched.py b/tests/test_sched.py index 63f20287b..bcf234459 100644 --- a/tests/test_sched.py +++ b/tests/test_sched.py @@ -116,9 +116,9 @@ def test_learn(): assert c.cycles == 2 # check log is accurate log = d.db.first("select * from revlog order by id desc") - assert log[2] == 2 - assert log[3] == -180 - assert log[4] == -30 + assert log[3] == 2 + assert log[4] == -180 + assert log[5] == -30 # pass again d.sched.answerCard(c, 2) # it should by due in 10 minutes diff --git a/tests/test_sync.py b/tests/test_sync.py index c3c4fcdbb..63b85f290 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -39,16 +39,11 @@ def setup_basic(loadDecks=None): deck2.addFact(f) deck2.reset(); deck2.sched.answerCard(deck2.sched.getCard(), 4) # start with same schema and sync time - deck1.lastSync = deck2.lastSync = intTime() - 1 deck1.scm = deck2.scm = 0 # and same mod time, so sync does nothing deck1.save(); deck2.save() server = LocalServer(deck2) client = Syncer(deck1, server) - # for testing, don't add the 10 minute padding - def _lastSync(lsyn, rsyn): - return min(lsyn, rsyn) - 1 - client._lastSync = _lastSync def setup_modified(): setup_basic() @@ -76,13 +71,13 @@ def test_sync(): assert len(d.groups.gconf) == 1 assert len(d.tags.all()) == num check(1) - origLs = deck1.lastSync + origUsn = deck1.usn() assert client.sync() == "success" # last sync times and mod times should agree assert deck1.mod == deck2.mod - assert deck1.lastSync == deck2.lastSync - assert deck1.mod == deck1.lastSync - assert deck1.lastSync != origLs + assert deck1.usn() == deck2.usn() + assert deck1.mod == deck1.ls + assert deck1.usn() != origUsn # because everything was created separately it will be merged in. in # actual use we use a full sync to ensure initial a common starting point. check(2) @@ -100,7 +95,8 @@ def test_models(): # update model one cm = deck1.models.current() cm['name'] = "new" - cm['mod'] = intTime() + 1 + time.sleep(1) + deck1.models.save(cm) deck1.save(mod=intTime()+1) assert deck2.models.get(cm['id'])['name'] == "Basic" assert client.sync() == "success" @@ -203,10 +199,6 @@ def test_threeway(): deck1.reopen() deck3 = Deck(d3path) client2 = Syncer(deck3, server) - # for testing, don't add the 10 minute padding - def _lastSync(lsyn, rsyn): - return min(lsyn, rsyn) - 1 - client2._lastSync = _lastSync assert client2.sync() == "noChanges" # client 1 adds a card at time 1 time.sleep(1) @@ -218,16 +210,12 @@ def test_threeway(): time.sleep(1) deck3.save() assert client2.sync() == "success" - # it now has a last sync time greater than when the card was added at time - # 1 - assert deck3.lastSync > f.mod # at time 3, client 1 syncs, adding the older fact time.sleep(1) assert client.sync() == "success" assert deck1.factCount() == deck2.factCount() # syncing client2 should pick it up assert client2.sync() == "success" - print deck1.factCount(), deck2.factCount(), deck3.factCount() assert deck1.factCount() == deck2.factCount() == deck3.factCount() def _test_speed():