mirror of
https://github.com/ankitects/anki.git
synced 2025-09-21 15:32:23 -04:00
add USNs
Decks now have an "update sequence number". All objects also have a USN, which is set to the deck USN each time they are modified. When syncing, each side sends any objects with a USN >= clientUSN. When objects are copied via sync, they have their USNs bumped to the current serverUSN. After a sync, the USN on both sides is set to serverUSN + 1. This solves the failing three way test, ensures we receive all changes regardless of clock drift, and as the revlog also has a USN now, ensures that old revlog entries are imported properly too. Objects retain a separate modification time, which is used for conflict resolution, deck subscriptions/importing, and info for the user. Note that if the clock is too far off, it will still cause confusion for users, as the due counts may be different depending on the time. For this reason it's probably a good idea to keep a limit on how far the clock can deviate. We still keep track of the last sync time, but only so we can determine if the schema has changed since the last sync. The media code needs to be updated to use USNs too.
This commit is contained in:
Damien Elmes
parent
b391202e47
commit
bc9f6e6a24
13 changed files with 169 additions and 145 deletions
|
|
@ -48,6 +48,7 @@ class Card(object):
|
|||
self.gid,
|
||||
self.ord,
|
||||
self.mod,
|
||||
self.usn,
|
||||
self.type,
|
||||
self.queue,
|
||||
self.due,
|
||||
|
|
@ -65,15 +66,17 @@ class Card(object):
|
|||
|
||||
def flush(self):
|
||||
self.mod = intTime()
|
||||
self.usn = self.deck.usn()
|
||||
self.deck.db.execute(
|
||||
"""
|
||||
insert or replace into cards values
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
self.id,
|
||||
self.fid,
|
||||
self.gid,
|
||||
self.ord,
|
||||
self.mod,
|
||||
self.usn,
|
||||
self.type,
|
||||
self.queue,
|
||||
self.due,
|
||||
|
|
@ -88,11 +91,12 @@ insert or replace into cards values
|
|||
|
||||
def flushSched(self):
|
||||
self.mod = intTime()
|
||||
self.usn = self.deck.usn()
|
||||
self.deck.db.execute(
|
||||
"""update cards set
|
||||
mod=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?,
|
||||
mod=?, usn=?, type=?, queue=?, due=?, ivl=?, factor=?, reps=?,
|
||||
lapses=?, grade=?, cycles=?, edue=? where id = ?""",
|
||||
self.mod, self.type, self.queue, self.due, self.ivl,
|
||||
self.mod, self.usn, self.type, self.queue, self.due, self.ivl,
|
||||
self.factor, self.reps, self.lapses,
|
||||
self.grade, self.cycles, self.edue, self.id)
|
||||
|
||||
|
|
|
|||
16
anki/deck.py
16
anki/deck.py
|
|
@ -74,13 +74,14 @@ class _Deck(object):
|
|||
self.mod,
|
||||
self.scm,
|
||||
self.dty,
|
||||
self.lastSync,
|
||||
self._usn,
|
||||
self.ls,
|
||||
self.conf,
|
||||
models,
|
||||
groups,
|
||||
gconf,
|
||||
tags) = self.db.first("""
|
||||
select crt, mod, scm, dty, lastSync,
|
||||
select crt, mod, scm, dty, usn, ls,
|
||||
conf, models, groups, gconf, tags from deck""")
|
||||
self.conf = simplejson.loads(self.conf)
|
||||
self.models.load(models)
|
||||
|
|
@ -92,9 +93,9 @@ conf, models, groups, gconf, tags from deck""")
|
|||
self.mod = intTime() if mod is None else mod
|
||||
self.db.execute(
|
||||
"""update deck set
|
||||
crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""",
|
||||
crt=?, mod=?, scm=?, dty=?, usn=?, ls=?, conf=?""",
|
||||
self.crt, self.mod, self.scm, self.dty,
|
||||
self.lastSync, simplejson.dumps(self.conf))
|
||||
self._usn, self.ls, simplejson.dumps(self.conf))
|
||||
self.models.flush()
|
||||
self.groups.flush()
|
||||
self.tags.flush()
|
||||
|
|
@ -148,7 +149,7 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""",
|
|||
|
||||
def schemaChanged(self):
|
||||
"True if schema changed since last sync."
|
||||
return self.scm > self.lastSync
|
||||
return self.scm > self.ls
|
||||
|
||||
def setDirty(self):
|
||||
"Signal there are temp. suspended cards that need cleaning up on close."
|
||||
|
|
@ -161,6 +162,7 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""",
|
|||
self.dty = False
|
||||
|
||||
def rename(self, path):
|
||||
raise "nyi"
|
||||
# close our DB connection
|
||||
self.close()
|
||||
# move to new path
|
||||
|
|
@ -173,6 +175,9 @@ crt=?, mod=?, scm=?, dty=?, lastSync=?, conf=?""",
|
|||
self.reopen()
|
||||
self.media.move(olddir)
|
||||
|
||||
def usn(self):
|
||||
return self._usn
|
||||
|
||||
# Object creation helpers
|
||||
##########################################################################
|
||||
|
||||
|
|
@ -370,6 +375,7 @@ select id from facts where id in %s and id not in (select fid from cards)""" %
|
|||
if csum:
|
||||
self.db.execute("delete from fsums where fid in "+sfids)
|
||||
self.db.executemany("insert into fsums values (?,?,?)", r)
|
||||
# rely on calling code to bump usn+mod
|
||||
self.db.executemany("update facts set sfld = ? where id = ?", r2)
|
||||
|
||||
# Q/A generation
|
||||
|
|
|
|||
|
|
@ -29,10 +29,11 @@ class Fact(object):
|
|||
(self.mid,
|
||||
self.gid,
|
||||
self.mod,
|
||||
self.usn,
|
||||
self.tags,
|
||||
self.fields,
|
||||
self.data) = self.deck.db.first("""
|
||||
select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id)
|
||||
select mid, gid, mod, usn, tags, flds, data from facts where id = ?""", self.id)
|
||||
self.fields = splitFields(self.fields)
|
||||
self.tags = self.deck.tags.split(self.tags)
|
||||
self._model = self.deck.models.get(self.mid)
|
||||
|
|
@ -40,13 +41,14 @@ select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id)
|
|||
|
||||
def flush(self, mod=None):
|
||||
self.mod = mod if mod else intTime()
|
||||
self.usn = self.deck.usn()
|
||||
sfld = stripHTML(self.fields[self.deck.models.sortIdx(self._model)])
|
||||
tags = self.stringTags()
|
||||
res = self.deck.db.execute("""
|
||||
insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
self.id, self.mid, self.gid,
|
||||
self.mod, tags, self.joinedFields(),
|
||||
sfld, self.data)
|
||||
self.mod, self.usn, tags,
|
||||
self.joinedFields(), sfld, self.data)
|
||||
self.id = res.lastrowid
|
||||
self.updateFieldChecksums()
|
||||
self.deck.tags.register(self.tags)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@
|
|||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
import re
|
||||
from anki.utils import ids2str, splitFields, joinFields, stripHTML
|
||||
from anki.utils import ids2str, splitFields, joinFields, stripHTML, intTime
|
||||
|
||||
|
||||
SEARCH_TAG = 0
|
||||
SEARCH_TYPE = 1
|
||||
|
|
@ -401,11 +402,11 @@ def findReplace(deck, fids, src, dst, regex=False, field=None, fold=True):
|
|||
sflds[c] = repl(sflds[c])
|
||||
flds = joinFields(sflds)
|
||||
if flds != origFlds:
|
||||
d.append(dict(fid=fid, flds=flds))
|
||||
d.append(dict(fid=fid,flds=flds,u=deck.usn(),m=intTime()))
|
||||
if not d:
|
||||
return 0
|
||||
# replace
|
||||
deck.db.executemany("update facts set flds = :flds where id=:fid", d)
|
||||
deck.db.executemany("update facts set flds=:flds,mod=:m,usn=:u where id=:fid", d)
|
||||
deck.updateFieldCache(fids)
|
||||
return len(d)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ defaultConf = {
|
|||
},
|
||||
'maxTaken': 60,
|
||||
'mod': 0,
|
||||
'usn': 0,
|
||||
}
|
||||
|
||||
class GroupManager(object):
|
||||
|
|
@ -76,6 +77,7 @@ class GroupManager(object):
|
|||
"Can be called with either a group or a group configuration."
|
||||
if g:
|
||||
g['mod'] = intTime()
|
||||
g['usn'] = self.deck.usn()
|
||||
self.changed = True
|
||||
|
||||
def flush(self):
|
||||
|
|
@ -114,8 +116,12 @@ class GroupManager(object):
|
|||
|
||||
def rem(self, gid):
|
||||
self.deck.modSchema()
|
||||
self.deck.db.execute("update cards set gid = 1 where gid = ?", gid)
|
||||
self.deck.db.execute("update facts set gid = 1 where gid = ?", gid)
|
||||
self.deck.db.execute(
|
||||
"update cards set gid=1,usn=?,mod=? where gid = ?",
|
||||
gid, self.deck.usn(), intTime())
|
||||
self.deck.db.execute(
|
||||
"update facts set gid=1,usn=?,mod=? where gid = ?",
|
||||
gid, self.deck.usn(), intTime())
|
||||
self.deck.db.execute("delete from groups where id = ?", gid)
|
||||
print "fixme: loop through models and update stale gid references"
|
||||
|
||||
|
|
@ -157,7 +163,8 @@ class GroupManager(object):
|
|||
|
||||
def setGroup(self, cids, gid):
|
||||
self.db.execute(
|
||||
"update cards set gid = ? where id in "+ids2str(cids), gid)
|
||||
"update cards set gid=?,usn=?,mod=? where id in "+
|
||||
ids2str(cids), gid, self.deck.usn(), intTime())
|
||||
|
||||
def update(self, g):
|
||||
"Add or update an existing group. Used for syncing and merging."
|
||||
|
|
|
|||
|
|
@ -19,9 +19,6 @@ from anki.utils import fieldChecksum, ids2str
|
|||
from anki.errors import *
|
||||
#from anki.deck import NEW_CARDS_RANDOM
|
||||
|
||||
# FIXME: when importing an anki file, if any revlog entries are less than the
|
||||
# last sync time, we need to bump the deck schema
|
||||
|
||||
# Base importer
|
||||
##########################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ defaultModel = {
|
|||
\\begin{document}
|
||||
""",
|
||||
'latexPost': "\\end{document}",
|
||||
'mod': 0,
|
||||
'usn': 0,
|
||||
}
|
||||
|
||||
defaultField = {
|
||||
|
|
@ -75,6 +77,7 @@ class ModelManager(object):
|
|||
"Mark M modified if provided, and schedule registry flush."
|
||||
if m:
|
||||
m['mod'] = intTime()
|
||||
m['usn'] = self.deck.usn()
|
||||
m['css'] = self._css(m)
|
||||
self.changed = True
|
||||
|
||||
|
|
@ -308,8 +311,10 @@ select id from cards where fid in (select id from facts where mid = ?)""",
|
|||
r = []
|
||||
for (id, flds) in self.deck.db.execute(
|
||||
"select id, flds from facts where mid = ?", m['id']):
|
||||
r.append((joinFields(fn(splitFields(flds))), id))
|
||||
self.deck.db.executemany("update facts set flds = ? where id = ?", r)
|
||||
r.append((joinFields(fn(splitFields(flds))),
|
||||
intTime(), self.deck.usn(), id))
|
||||
self.deck.db.executemany(
|
||||
"update facts set flds=?,mod=?,usn=? where id = ?", r)
|
||||
|
||||
# Templates
|
||||
##################################################
|
||||
|
|
@ -334,8 +339,9 @@ select c.id from cards c, facts f where c.fid=f.id and mid = ? and ord = ?""",
|
|||
self.deck.remCards(cids)
|
||||
# shift ordinals
|
||||
self.deck.db.execute("""
|
||||
update cards set ord = ord - 1 where fid in (select id from facts
|
||||
where mid = ?) and ord > ?""", m['id'], ord)
|
||||
update cards set ord = ord - 1, usn = ?, mod = ?
|
||||
where fid in (select id from facts where mid = ?) and ord > ?""",
|
||||
self.deck.usn(), intTime(), m['id'], ord)
|
||||
m['tmpls'].remove(template)
|
||||
self._updateTemplOrds(m)
|
||||
self.save(m)
|
||||
|
|
@ -359,8 +365,9 @@ where mid = ?) and ord > ?""", m['id'], ord)
|
|||
# apply
|
||||
self.save(m)
|
||||
self.deck.db.execute("""
|
||||
update cards set ord = (case %s end) where fid in (
|
||||
select id from facts where mid = ?)""" % " ".join(map), m['id'])
|
||||
update cards set ord = (case %s end),usn=?,mod=? where fid in (
|
||||
select id from facts where mid = ?)""" % " ".join(map),
|
||||
self.deck.usn(), intTime(), m['id'])
|
||||
|
||||
# Model changing
|
||||
##########################################################################
|
||||
|
|
@ -388,9 +395,10 @@ select id from facts where mid = ?)""" % " ".join(map), m['id'])
|
|||
for c in range(nfields):
|
||||
flds.append(newflds.get(c, ""))
|
||||
flds = joinFields(flds)
|
||||
d.append(dict(fid=fid, flds=flds, mid=newModel['id']))
|
||||
d.append(dict(fid=fid, flds=flds, mid=newModel['id'],
|
||||
m=intTime(),u=self.deck.usn()))
|
||||
self.deck.db.executemany(
|
||||
"update facts set flds=:flds, mid=:mid where id = :fid", d)
|
||||
"update facts set flds=:flds,mid=:mid,mod=:m,usn=:u where id = :fid", d)
|
||||
self.deck.updateFieldCache(fids)
|
||||
|
||||
def _changeCards(self, fids, newModel, map):
|
||||
|
|
@ -399,9 +407,11 @@ select id from facts where mid = ?)""" % " ".join(map), m['id'])
|
|||
for (cid, ord) in self.deck.db.execute(
|
||||
"select id, ord from cards where fid in "+ids2str(fids)):
|
||||
if map[ord] is not None:
|
||||
d.append(dict(cid=cid, new=map[ord]))
|
||||
d.append(dict(
|
||||
cid=cid,new=map[ord],u=self.deck.usn(),m=intTime()))
|
||||
else:
|
||||
deleted.append(cid)
|
||||
self.deck.db.executemany(
|
||||
"update cards set ord=:new where id=:cid", d)
|
||||
"update cards set ord=:new,usn=:u,mod=:m where id=:cid",
|
||||
d)
|
||||
self.deck.remCards(deleted)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ class Scheduler(object):
|
|||
raise Exception("Invalid queue")
|
||||
self._updateStats('time', card.timeTaken())
|
||||
card.mod = intTime()
|
||||
card.usn = self.deck.usn()
|
||||
card.flushSched()
|
||||
|
||||
def counts(self):
|
||||
|
|
@ -397,10 +398,9 @@ limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff)
|
|||
ivl = card.ivl if leaving else -(self._delayForGrade(conf, card.grade))
|
||||
def log():
|
||||
self.deck.db.execute(
|
||||
"insert into revlog values (?,?,?,?,?,?,?,?)",
|
||||
int(time.time()*1000), card.id, ease,
|
||||
ivl, lastIvl,
|
||||
card.factor, card.timeTaken(), type)
|
||||
"insert into revlog values (?,?,?,?,?,?,?,?,?)",
|
||||
int(time.time()*1000), card.id, self.deck.usn(), ease,
|
||||
ivl, lastIvl, card.factor, card.timeTaken(), type)
|
||||
try:
|
||||
log()
|
||||
except:
|
||||
|
|
@ -415,10 +415,10 @@ limit %d""" % (self._groupLimit(), self.reportLimit), lim=self.dayCutoff)
|
|||
extra = " and id in "+ids2str(ids)
|
||||
self.deck.db.execute("""
|
||||
update cards set
|
||||
due = edue, queue = 2, mod = %d
|
||||
due = edue, queue = 2, mod = %d, usn = %d
|
||||
where queue = 1 and type = 2
|
||||
%s
|
||||
""" % (intTime(), extra))
|
||||
""" % (intTime(), self.deck.usn(), extra))
|
||||
|
||||
# Reviews
|
||||
##########################################################################
|
||||
|
|
@ -501,8 +501,8 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % (
|
|||
def _logRev(self, card, ease):
|
||||
def log():
|
||||
self.deck.db.execute(
|
||||
"insert into revlog values (?,?,?,?,?,?,?,?)",
|
||||
int(time.time()*1000), card.id, ease,
|
||||
"insert into revlog values (?,?,?,?,?,?,?,?,?)",
|
||||
int(time.time()*1000), card.id, self.deck.usn(), ease,
|
||||
card.ivl, card.lastIvl, card.factor, card.timeTaken(),
|
||||
1)
|
||||
try:
|
||||
|
|
@ -712,15 +712,15 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % (
|
|||
"Suspend cards."
|
||||
self.removeFailed(ids)
|
||||
self.deck.db.execute(
|
||||
"update cards set queue = -1, mod = ? where id in "+
|
||||
ids2str(ids), intTime())
|
||||
"update cards set queue=-1,mod=?,usn=? where id in "+
|
||||
ids2str(ids), intTime(), self.deck.usn())
|
||||
|
||||
def unsuspendCards(self, ids):
|
||||
"Unsuspend cards."
|
||||
self.deck.db.execute(
|
||||
"update cards set queue = type, mod = ? "
|
||||
"update cards set queue=type,mod=?,usn=? "
|
||||
"where queue = -1 and id in "+ ids2str(ids),
|
||||
intTime())
|
||||
intTime(), self.deck.usn())
|
||||
|
||||
def buryFact(self, fid):
|
||||
"Bury all cards for fact until next session."
|
||||
|
|
@ -772,8 +772,9 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % (
|
|||
def forgetCards(self, ids):
|
||||
"Put cards at the end of the new queue."
|
||||
self.deck.db.execute(
|
||||
"update cards set type=0, queue=0, ivl=0 where id in "+ids2str(ids))
|
||||
"update cards set type=0,queue=0,ivl=0 where id in "+ids2str(ids))
|
||||
pmax = self.deck.db.scalar("select max(due) from cards where type=0")
|
||||
# takes care of mod + usn
|
||||
self.sortCards(ids, start=pmax+1, shuffle=self.deck.models.randomNew())
|
||||
|
||||
def reschedCards(self, ids, imin, imax):
|
||||
|
|
@ -816,15 +817,15 @@ queue = 2 %s and due <= :lim order by %s limit %d""" % (
|
|||
if low is not None:
|
||||
shiftby = high - low + 1
|
||||
self.deck.db.execute("""
|
||||
update cards set mod=?, due=due+? where id not in %s
|
||||
and due >= ?""" % scids, now, shiftby, low)
|
||||
update cards set mod=?, usn=?, due=due+? where id not in %s
|
||||
and due >= ?""" % scids, now, self.deck.usn(), shiftby, low)
|
||||
# reorder cards
|
||||
d = []
|
||||
for id, fid in self.deck.db.execute(
|
||||
"select id, fid from cards where type = 0 and id in "+scids):
|
||||
d.append(dict(now=now, due=due[fid], cid=id))
|
||||
d.append(dict(now=now, due=due[fid], usn=self.deck.usn(), cid=id))
|
||||
self.deck.db.executemany(
|
||||
"update cards set due = :due, mod = :now where id = :cid""", d)
|
||||
"update cards set due=:due,mod=:now,usn=:usn where id = :cid""", d)
|
||||
|
||||
# fixme: because it's a model property now, these should be done on a
|
||||
# per-model basis
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@ create table if not exists deck (
|
|||
scm integer not null,
|
||||
ver integer not null,
|
||||
dty integer not null,
|
||||
lastSync integer not null,
|
||||
usn integer not null,
|
||||
ls integer not null,
|
||||
conf text not null,
|
||||
models text not null,
|
||||
groups text not null,
|
||||
|
|
@ -71,12 +72,30 @@ create table if not exists deck (
|
|||
tags text not null
|
||||
);
|
||||
|
||||
create table if not exists facts (
|
||||
id integer primary key,
|
||||
mid integer not null,
|
||||
gid integer not null,
|
||||
mod integer not null,
|
||||
usn integer not null,
|
||||
tags text not null,
|
||||
flds text not null,
|
||||
sfld integer not null,
|
||||
data text not null
|
||||
);
|
||||
|
||||
create table if not exists fsums (
|
||||
fid integer not null,
|
||||
mid integer not null,
|
||||
csum integer not null
|
||||
);
|
||||
create table if not exists cards (
|
||||
id integer primary key,
|
||||
fid integer not null,
|
||||
gid integer not null,
|
||||
ord integer not null,
|
||||
mod integer not null,
|
||||
usn integer not null,
|
||||
type integer not null,
|
||||
queue integer not null,
|
||||
due integer not null,
|
||||
|
|
@ -90,32 +109,10 @@ create table if not exists cards (
|
|||
data text not null
|
||||
);
|
||||
|
||||
create table if not exists facts (
|
||||
id integer primary key,
|
||||
mid integer not null,
|
||||
gid integer not null,
|
||||
mod integer not null,
|
||||
tags text not null,
|
||||
flds text not null,
|
||||
sfld integer not null,
|
||||
data text not null
|
||||
);
|
||||
|
||||
create table if not exists fsums (
|
||||
fid integer not null,
|
||||
mid integer not null,
|
||||
csum integer not null
|
||||
);
|
||||
|
||||
create table if not exists graves (
|
||||
id integer not null,
|
||||
oid integer not null,
|
||||
type integer not null
|
||||
);
|
||||
|
||||
create table if not exists revlog (
|
||||
id integer primary key,
|
||||
cid integer not null,
|
||||
usn integer not null,
|
||||
ease integer not null,
|
||||
ivl integer not null,
|
||||
lastIvl integer not null,
|
||||
|
|
@ -124,8 +121,14 @@ create table if not exists revlog (
|
|||
type integer not null
|
||||
);
|
||||
|
||||
create table if not exists graves (
|
||||
usn integer not null,
|
||||
oid integer not null,
|
||||
type integer not null
|
||||
);
|
||||
|
||||
insert or ignore into deck
|
||||
values(1,0,0,0,%(v)s,0,0,'','{}','','','{}');
|
||||
values(1,0,0,0,%(v)s,0,0,0,'','{}','','','{}');
|
||||
""" % ({'v':CURRENT_VERSION}))
|
||||
import anki.deck
|
||||
import anki.groups
|
||||
|
|
@ -147,11 +150,13 @@ def _updateIndices(db):
|
|||
"Add indices to the DB."
|
||||
db.executescript("""
|
||||
-- avoid loading entire facts table in for sync summary
|
||||
create index if not exists ix_facts_mod on facts (mod);
|
||||
create index if not exists ix_facts_usn on facts (usn);
|
||||
-- card spacing, etc
|
||||
create index if not exists ix_cards_fid on cards (fid);
|
||||
-- revlog by card
|
||||
create index if not exists ix_revlog_cid on revlog (cid);
|
||||
-- revlog syncing
|
||||
create index if not exists ix_revlog_usn on revlog (usn);
|
||||
-- field uniqueness check
|
||||
create index if not exists ix_fsums_fid on fsums (fid);
|
||||
create index if not exists ix_fsums_csum on fsums (csum);
|
||||
|
|
@ -200,7 +205,7 @@ end)
|
|||
""")
|
||||
# pull facts into memory, so we can merge them with fields efficiently
|
||||
facts = db.all("""
|
||||
select id, modelId, 1, cast(created*1000 as int), cast(modified as int), tags
|
||||
select id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
|
||||
from facts order by created""")
|
||||
# build field hash
|
||||
fields = {}
|
||||
|
|
@ -232,7 +237,7 @@ from facts order by created""")
|
|||
# and put the facts into the new table
|
||||
db.execute("drop table facts")
|
||||
_addSchema(db, False)
|
||||
db.executemany("insert into facts values (?,?,?,?,?,?,'','')", data)
|
||||
db.executemany("insert into facts values (?,?,?,?,?,?,?,'','')", data)
|
||||
db.execute("drop table fields")
|
||||
|
||||
# cards
|
||||
|
|
@ -244,7 +249,7 @@ from facts order by created""")
|
|||
cardidmap = {}
|
||||
for row in db.execute("""
|
||||
select id, cast(created*1000 as int), factId, ordinal,
|
||||
cast(modified as int),
|
||||
cast(modified as int), 0,
|
||||
(case relativeDelay
|
||||
when 0 then 1
|
||||
when 1 then 2
|
||||
|
|
@ -271,7 +276,7 @@ order by created"""):
|
|||
db.execute("drop table cards")
|
||||
_addSchema(db, False)
|
||||
db.executemany("""
|
||||
insert into cards values (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, "")""",
|
||||
insert into cards values (?,?,1,?,?,?,?,?,?,?,?,?,?,0,0,0,"")""",
|
||||
rows)
|
||||
|
||||
# reviewHistory -> revlog
|
||||
|
|
@ -280,7 +285,7 @@ insert into cards values (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, 0, "")""",
|
|||
r = []
|
||||
for row in db.execute("""
|
||||
select
|
||||
cast(time*1000 as int), cardId, ease,
|
||||
cast(time*1000 as int), cardId, 0, ease,
|
||||
cast(nextInterval as int), cast(lastInterval as int),
|
||||
cast(nextFactor*1000 as int), cast(min(thinkingTime, 60)*1000 as int),
|
||||
yesCount from reviewHistory"""):
|
||||
|
|
@ -313,7 +318,7 @@ yesCount from reviewHistory"""):
|
|||
row[7] = 1
|
||||
r.append(row)
|
||||
db.executemany(
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?)", r)
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)", r)
|
||||
db.execute("drop table reviewHistory")
|
||||
|
||||
# deck
|
||||
|
|
@ -342,7 +347,7 @@ def _migrateDeckTbl(db):
|
|||
db.execute("delete from deck")
|
||||
db.execute("""
|
||||
insert or replace into deck select id, cast(created as int), :t,
|
||||
:t, 99, 0, cast(lastSync as int),
|
||||
:t, 99, 0, 0, cast(lastSync as int),
|
||||
"", "", "", "", "" from decks""", t=intTime())
|
||||
# prepare a group to store the old deck options
|
||||
import anki.groups
|
||||
|
|
|
|||
74
anki/sync.py
74
anki/sync.py
|
|
@ -58,14 +58,12 @@ class Syncer(object):
|
|||
def sync(self):
|
||||
"Returns 'noChanges', 'fullSync', or 'success'."
|
||||
# get local and remote modified, schema and sync times
|
||||
self.lmod, lscm, lsyn = self.times()
|
||||
self.rmod, rscm, rsyn = self.server.times()
|
||||
self.lmod, lscm, lsyn, self.minUsn = self.times()
|
||||
self.rmod, rscm, rsyn, self.maxUsn = self.server.times()
|
||||
if self.lmod == self.rmod:
|
||||
return "noChanges"
|
||||
elif lscm != rscm:
|
||||
return "fullSync"
|
||||
# find last sync time minus 10 mins for clock drift
|
||||
self.ls = self._lastSync(lsyn, rsyn)
|
||||
self.lnewer = self.lmod > self.rmod
|
||||
# get local changes and switch to full sync if there were too many
|
||||
self.status("getLocal")
|
||||
|
|
@ -74,7 +72,8 @@ class Syncer(object):
|
|||
return "fullSync"
|
||||
# send them to the server, and get the server's changes
|
||||
self.status("getServer")
|
||||
rchg = self.server.changes(ls=self.ls, lnewer=self.lnewer, changes=lchg)
|
||||
rchg = self.server.changes(minUsn=self.minUsn, lnewer=self.lnewer,
|
||||
changes=lchg)
|
||||
if rchg == "fullSync":
|
||||
return "fullSync"
|
||||
# otherwise, merge
|
||||
|
|
@ -86,16 +85,14 @@ class Syncer(object):
|
|||
self.finish(mod)
|
||||
return "success"
|
||||
|
||||
def _lastSync(self, lsyn, rsyn):
|
||||
return min(lsyn, rsyn) - 600
|
||||
|
||||
def times(self):
|
||||
return (self.deck.mod, self.deck.scm, self.deck.lastSync)
|
||||
return (self.deck.mod, self.deck.scm, self.deck.ls, self.deck._usn)
|
||||
|
||||
def changes(self, ls=None, lnewer=None, changes=None):
|
||||
if ls:
|
||||
def changes(self, minUsn=None, lnewer=None, changes=None):
|
||||
if minUsn is not None:
|
||||
# we're the server; save info
|
||||
self.ls = ls
|
||||
self.maxUsn = self.deck._usn
|
||||
self.minUsn = minUsn
|
||||
self.lnewer = not lnewer
|
||||
self.rchg = changes
|
||||
try:
|
||||
|
|
@ -110,7 +107,7 @@ class Syncer(object):
|
|||
# collection-level configuration from last modified side
|
||||
if self.lnewer:
|
||||
d['conf'] = self.getConf()
|
||||
if ls:
|
||||
if minUsn is not None:
|
||||
# we're the server, we can merge our side before returning
|
||||
self.merge(d, self.rchg)
|
||||
return d
|
||||
|
|
@ -130,7 +127,8 @@ class Syncer(object):
|
|||
if not mod:
|
||||
# server side; we decide new mod time
|
||||
mod = intTime()
|
||||
self.deck.lastSync = mod
|
||||
self.deck.ls = mod
|
||||
self.deck._usn = self.maxUsn + 1
|
||||
self.deck.save(mod=mod)
|
||||
return mod
|
||||
|
||||
|
|
@ -138,7 +136,7 @@ class Syncer(object):
|
|||
##########################################################################
|
||||
|
||||
def getModels(self):
|
||||
return [m for m in self.deck.models.all() if m['mod'] > self.ls]
|
||||
return [m for m in self.deck.models.all() if m['usn'] >= self.minUsn]
|
||||
|
||||
def mergeModels(self, rchg):
|
||||
# deletes result in schema mod, so we only have to worry about
|
||||
|
|
@ -154,8 +152,8 @@ class Syncer(object):
|
|||
|
||||
def getGroups(self):
|
||||
return [
|
||||
[g for g in self.deck.groups.all() if g['mod'] > self.ls],
|
||||
[g for g in self.deck.groups.allConf() if g['mod'] > self.ls]
|
||||
[g for g in self.deck.groups.all() if g['usn'] >= self.minUsn],
|
||||
[g for g in self.deck.groups.allConf() if g['usn'] >= self.minUsn]
|
||||
]
|
||||
|
||||
def mergeGroups(self, rchg):
|
||||
|
|
@ -175,7 +173,7 @@ class Syncer(object):
|
|||
##########################################################################
|
||||
|
||||
def getTags(self):
|
||||
return self.deck.tags.allSince(self.ls)
|
||||
return self.deck.tags.allSinceUSN(self.minUsn)
|
||||
|
||||
def mergeTags(self, tags):
|
||||
self.deck.tags.register(tags)
|
||||
|
|
@ -184,38 +182,40 @@ class Syncer(object):
|
|||
##########################################################################
|
||||
|
||||
def getRevlog(self):
|
||||
r = self.deck.db.all("select * from revlog where id > ? limit ?",
|
||||
self.ls*1000, self.MAX_REVLOG)
|
||||
r = self.deck.db.all("select * from revlog where usn >= ? limit ?",
|
||||
self.minUsn, self.MAX_REVLOG)
|
||||
if len(r) == self.MAX_REVLOG:
|
||||
raise SyncTooLarge
|
||||
return r
|
||||
|
||||
def mergeRevlog(self, logs):
|
||||
for l in logs:
|
||||
l[2] = self.maxUsn
|
||||
self.deck.db.executemany(
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?)",
|
||||
"insert or ignore into revlog values (?,?,?,?,?,?,?,?,?)",
|
||||
logs)
|
||||
|
||||
# Facts
|
||||
##########################################################################
|
||||
|
||||
def getFacts(self):
|
||||
f = self.deck.db.all("select * from facts where mod > ? limit ?",
|
||||
self.ls, self.MAX_FACTS)
|
||||
f = self.deck.db.all("select * from facts where usn >= ? limit ?",
|
||||
self.minUsn, self.MAX_FACTS)
|
||||
if len(f) == self.MAX_FACTS:
|
||||
raise SyncTooLarge
|
||||
return [
|
||||
f,
|
||||
self.deck.db.list(
|
||||
"select oid from graves where id > ? and type = ?",
|
||||
self.ls, REM_FACT)
|
||||
"select oid from graves where usn >= ? and type = ?",
|
||||
self.minUsn, REM_FACT)
|
||||
]
|
||||
|
||||
def mergeFacts(self, lchg, rchg):
|
||||
(toAdd, toRem) = self.findChanges(
|
||||
lchg[0], lchg[1], rchg[0], rchg[1], 3)
|
||||
lchg[0], lchg[1], rchg[0], rchg[1], 3, 4)
|
||||
# add missing
|
||||
self.deck.db.executemany(
|
||||
"insert or replace into facts values (?,?,?,?,?,?,?,?)",
|
||||
"insert or replace into facts values (?,?,?,?,?,?,?,?,?)",
|
||||
toAdd)
|
||||
# update fsums table - fixme: in future could skip sort cache
|
||||
self.deck.updateFieldCache([f[0] for f in toAdd])
|
||||
|
|
@ -226,26 +226,26 @@ class Syncer(object):
|
|||
##########################################################################
|
||||
|
||||
def getCards(self):
|
||||
c = self.deck.db.all("select * from cards where mod > ? limit ?",
|
||||
self.ls, self.MAX_CARDS)
|
||||
c = self.deck.db.all("select * from cards where usn >= ? limit ?",
|
||||
self.minUsn, self.MAX_CARDS)
|
||||
if len(c) == self.MAX_CARDS:
|
||||
raise SyncTooLarge
|
||||
return [
|
||||
c,
|
||||
self.deck.db.list(
|
||||
"select oid from graves where id > ? and type = ?",
|
||||
self.ls, REM_CARD)
|
||||
"select oid from graves where usn >= ? and type = ?",
|
||||
self.minUsn, REM_CARD)
|
||||
]
|
||||
|
||||
def mergeCards(self, lchg, rchg):
|
||||
# cards with higher reps preserved, so that gid changes don't clobber
|
||||
# older reviews
|
||||
(toAdd, toRem) = self.findChanges(
|
||||
lchg[0], lchg[1], rchg[0], rchg[1], 10)
|
||||
lchg[0], lchg[1], rchg[0], rchg[1], 11, 5)
|
||||
# add missing
|
||||
self.deck.db.executemany(
|
||||
"insert or replace into cards values "
|
||||
"(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
"(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
toAdd)
|
||||
# remove remotely deleted
|
||||
self.deck.remCards(toRem)
|
||||
|
|
@ -262,7 +262,7 @@ class Syncer(object):
|
|||
# Merging
|
||||
##########################################################################
|
||||
|
||||
def findChanges(self, localAdds, localRems, remoteAdds, remoteRems, key):
|
||||
def findChanges(self, localAdds, localRems, remoteAdds, remoteRems, key, usn):
|
||||
local = {}
|
||||
toAdd = []
|
||||
toRem = []
|
||||
|
|
@ -280,6 +280,7 @@ class Syncer(object):
|
|||
# added on both sides
|
||||
if r[key] > l[key]:
|
||||
# remote newer; update
|
||||
r[usn] = self.maxUsn
|
||||
toAdd.append(r)
|
||||
else:
|
||||
# local newer; remote will update
|
||||
|
|
@ -289,15 +290,16 @@ class Syncer(object):
|
|||
pass
|
||||
else:
|
||||
# changed on server only
|
||||
r[usn] = self.maxUsn
|
||||
toAdd.append(r)
|
||||
return toAdd, remoteRems
|
||||
|
||||
class LocalServer(Syncer):
|
||||
# serialize/deserialize payload, so we don't end up sharing objects
|
||||
# between decks in testing
|
||||
def changes(self, ls, lnewer, changes):
|
||||
def changes(self, minUsn, lnewer, changes):
|
||||
l = simplejson.loads; d = simplejson.dumps
|
||||
return l(d(Syncer.changes(self, ls, lnewer, l(d(changes)))))
|
||||
return l(d(Syncer.changes(self, minUsn, lnewer, l(d(changes)))))
|
||||
|
||||
# not yet ported
|
||||
class RemoteServer(Syncer):
|
||||
|
|
|
|||
15
anki/tags.py
15
anki/tags.py
|
|
@ -39,7 +39,7 @@ class TagManager(object):
|
|||
# versions of the same tag if they ignore the qt autocomplete.
|
||||
for t in tags:
|
||||
if t not in self.tags:
|
||||
self.tags[t] = intTime()
|
||||
self.tags[t] = self.deck.usn()
|
||||
self.changed = True
|
||||
|
||||
def all(self):
|
||||
|
|
@ -57,8 +57,8 @@ class TagManager(object):
|
|||
self.register(set(self.split(
|
||||
" ".join(self.deck.db.list("select distinct tags from facts"+lim)))))
|
||||
|
||||
def allSince(self, mod):
|
||||
return [k for k,v in self.tags.items() if v > mod]
|
||||
def allSinceUSN(self, usn):
|
||||
return [k for k,v in self.tags.items() if v >= usn]
|
||||
|
||||
# Bulk addition/removal from facts
|
||||
#############################################################
|
||||
|
|
@ -88,9 +88,10 @@ class TagManager(object):
|
|||
fids = []
|
||||
def fix(row):
|
||||
fids.append(row[0])
|
||||
return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()}
|
||||
return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime(),
|
||||
'u':self.deck.usn()}
|
||||
self.deck.db.executemany(
|
||||
"update facts set tags = :t, mod = :n where id = :id",
|
||||
"update facts set tags=:t,mod=:n,usn=:u where id = :id",
|
||||
[fix(row) for row in res])
|
||||
|
||||
def bulkRem(self, ids, tags):
|
||||
|
|
@ -171,5 +172,5 @@ class TagManager(object):
|
|||
def setGroupForTags(self, yes, no, gid):
|
||||
fids = self.selTagFids(yes, no)
|
||||
self.deck.db.execute(
|
||||
"update cards set gid = ? where fid in "+ids2str(fids),
|
||||
gid)
|
||||
"update cards set gid=?,mod=?,usn=? where fid in "+ids2str(fids),
|
||||
gid, intTime(), self.deck.usn())
|
||||
|
|
|
|||
|
|
@ -116,9 +116,9 @@ def test_learn():
|
|||
assert c.cycles == 2
|
||||
# check log is accurate
|
||||
log = d.db.first("select * from revlog order by id desc")
|
||||
assert log[2] == 2
|
||||
assert log[3] == -180
|
||||
assert log[4] == -30
|
||||
assert log[3] == 2
|
||||
assert log[4] == -180
|
||||
assert log[5] == -30
|
||||
# pass again
|
||||
d.sched.answerCard(c, 2)
|
||||
# it should by due in 10 minutes
|
||||
|
|
|
|||
|
|
@ -39,16 +39,11 @@ def setup_basic(loadDecks=None):
|
|||
deck2.addFact(f)
|
||||
deck2.reset(); deck2.sched.answerCard(deck2.sched.getCard(), 4)
|
||||
# start with same schema and sync time
|
||||
deck1.lastSync = deck2.lastSync = intTime() - 1
|
||||
deck1.scm = deck2.scm = 0
|
||||
# and same mod time, so sync does nothing
|
||||
deck1.save(); deck2.save()
|
||||
server = LocalServer(deck2)
|
||||
client = Syncer(deck1, server)
|
||||
# for testing, don't add the 10 minute padding
|
||||
def _lastSync(lsyn, rsyn):
|
||||
return min(lsyn, rsyn) - 1
|
||||
client._lastSync = _lastSync
|
||||
|
||||
def setup_modified():
|
||||
setup_basic()
|
||||
|
|
@ -76,13 +71,13 @@ def test_sync():
|
|||
assert len(d.groups.gconf) == 1
|
||||
assert len(d.tags.all()) == num
|
||||
check(1)
|
||||
origLs = deck1.lastSync
|
||||
origUsn = deck1.usn()
|
||||
assert client.sync() == "success"
|
||||
# last sync times and mod times should agree
|
||||
assert deck1.mod == deck2.mod
|
||||
assert deck1.lastSync == deck2.lastSync
|
||||
assert deck1.mod == deck1.lastSync
|
||||
assert deck1.lastSync != origLs
|
||||
assert deck1.usn() == deck2.usn()
|
||||
assert deck1.mod == deck1.ls
|
||||
assert deck1.usn() != origUsn
|
||||
# because everything was created separately it will be merged in. in
|
||||
# actual use we use a full sync to ensure initial a common starting point.
|
||||
check(2)
|
||||
|
|
@ -100,7 +95,8 @@ def test_models():
|
|||
# update model one
|
||||
cm = deck1.models.current()
|
||||
cm['name'] = "new"
|
||||
cm['mod'] = intTime() + 1
|
||||
time.sleep(1)
|
||||
deck1.models.save(cm)
|
||||
deck1.save(mod=intTime()+1)
|
||||
assert deck2.models.get(cm['id'])['name'] == "Basic"
|
||||
assert client.sync() == "success"
|
||||
|
|
@ -203,10 +199,6 @@ def test_threeway():
|
|||
deck1.reopen()
|
||||
deck3 = Deck(d3path)
|
||||
client2 = Syncer(deck3, server)
|
||||
# for testing, don't add the 10 minute padding
|
||||
def _lastSync(lsyn, rsyn):
|
||||
return min(lsyn, rsyn) - 1
|
||||
client2._lastSync = _lastSync
|
||||
assert client2.sync() == "noChanges"
|
||||
# client 1 adds a card at time 1
|
||||
time.sleep(1)
|
||||
|
|
@ -218,16 +210,12 @@ def test_threeway():
|
|||
time.sleep(1)
|
||||
deck3.save()
|
||||
assert client2.sync() == "success"
|
||||
# it now has a last sync time greater than when the card was added at time
|
||||
# 1
|
||||
assert deck3.lastSync > f.mod
|
||||
# at time 3, client 1 syncs, adding the older fact
|
||||
time.sleep(1)
|
||||
assert client.sync() == "success"
|
||||
assert deck1.factCount() == deck2.factCount()
|
||||
# syncing client2 should pick it up
|
||||
assert client2.sync() == "success"
|
||||
print deck1.factCount(), deck2.factCount(), deck3.factCount()
|
||||
assert deck1.factCount() == deck2.factCount() == deck3.factCount()
|
||||
|
||||
def _test_speed():
|
||||
|
|
|
|||
Loading…
Reference in a new issue