# -*- coding: utf-8 -*- # Copyright: Damien Elmes # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html import zlib, re, urllib, urllib2, socket, simplejson, time, shutil import os, base64, httplib, sys, tempfile, httplib, types from datetime import date import anki, anki.deck, anki.cards from anki.errors import * #from anki.models import Model, Field, Template #from anki.facts import Fact #from anki.cards import Card from anki.utils import ids2str, checksum #from anki.media import mediaFiles from anki.lang import _ from hooks import runHook if simplejson.__version__ < "1.7.3": raise Exception("SimpleJSON must be 1.7.3 or later.") CHUNK_SIZE = 32768 MIME_BOUNDARY = "Anki-sync-boundary" SYNC_HOST = os.environ.get("SYNC_HOST") or "dev.ankiweb.net" SYNC_PORT = int(os.environ.get("SYNC_PORT") or 80) SYNC_URL = "http://%s:%d/sync/" % (SYNC_HOST, SYNC_PORT) KEYS = ("models", "facts", "cards", "media") # - need to add tags table syncing ########################################################################## # Monkey-patch httplib to incrementally send instead of chewing up large # amounts of memory, and track progress. sendProgressHook = None def incrementalSend(self, strOrFile): if self.sock is None: if self.auto_open: self.connect() else: raise NotConnected() if self.debuglevel > 0: print "send:", repr(str) try: if (isinstance(strOrFile, str) or isinstance(strOrFile, unicode)): self.sock.sendall(strOrFile) else: cnt = 0 t = time.time() while 1: if sendProgressHook and time.time() - t > 1: sendProgressHook(cnt) t = time.time() data = strOrFile.read(CHUNK_SIZE) cnt += len(data) if not data: break self.sock.sendall(data) except socket.error, v: if v[0] == 32: # Broken pipe self.close() raise httplib.HTTPConnection.send = incrementalSend def fullSyncProgressHook(cnt): runHook("fullSyncProgress", "fromLocal", cnt) ########################################################################## class SyncTools(object): def __init__(self, deck=None): self.deck = deck self.diffs = {} self.serverExcludedTags = [] self.timediff = 0 self.fullThreshold = 2000 # Control ########################################################################## def setServer(self, server): self.server = server def sync(self): "Sync two decks locally. Reimplement this for finer control." if not self.prepareSync(0): return lsum = self.summary(self.deck.lastSync) if lsum: rsum = self.server.summary(self.deck.lastSync) if not lsum or not rsum: raise Exception("full sync required") payload = self.genPayload((lsum, rsum)) res = self.server.applyPayload(payload) self.applyPayloadReply(res) self.deck.reset() def prepareSync(self, timediff): "Sync setup. True if sync needed." self.localTime = self.modified() self.remoteTime = self.server.modified() if self.localTime == self.remoteTime: return False l = self._lastSync(); r = self.server._lastSync() # set lastSync to the lower of the two sides, account for slow clocks, # and assume it took up to 10 seconds for the reply to arrive self.deck.lastSync = max(0, min(l, r) - timediff - 10) return True def genPayload(self, summaries): (lsum, rsum) = summaries payload = {} # first, handle models, facts and cards for key in KEYS: diff = self.diffSummary(lsum, rsum, key) payload["added-" + key] = self.getObjsFromKey(diff[0], key) payload["deleted-" + key] = diff[1] payload["missing-" + key] = diff[2] self.deleteObjsFromKey(diff[3], key) # handle the remainder if self.localTime > self.remoteTime: payload['history'] = self.bundleHistory() payload['sources'] = self.bundleSources() # finally, set new lastSync and bundle the deck info payload['deck'] = self.bundleDeck() return payload def applyPayload(self, payload): reply = {} # model, facts and cards for key in KEYS: k = 'added-' + key # send back any requested if k in payload: reply[k] = self.getObjsFromKey( payload['missing-' + key], key) self.updateObjsFromKey(payload['added-' + key], key) self.deleteObjsFromKey(payload['deleted-' + key], key) # send back deck-related stuff if it wasn't sent to us if not 'deck' in payload: reply['history'] = self.bundleHistory() reply['sources'] = self.bundleSources() # finally, set new lastSync and bundle the deck info reply['deck'] = self.bundleDeck() else: self.updateDeck(payload['deck']) self.updateHistory(payload['history']) if 'sources' in payload: self.updateSources(payload['sources']) self.postSyncRefresh() cardIds = [x[0] for x in payload['added-cards']] self.deck.updateCardTags(cardIds) return reply def applyPayloadReply(self, reply): # model, facts and cards for key in KEYS: k = 'added-' + key # old version may not send media if k in reply: self.updateObjsFromKey(reply['added-' + key], key) # deck if 'deck' in reply: self.updateDeck(reply['deck']) self.updateHistory(reply['history']) if 'sources' in reply: self.updateSources(reply['sources']) self.postSyncRefresh() cardIds = [x[0] for x in reply['added-cards']] self.deck.updateCardTags(cardIds) if self.missingFacts() != 0: raise Exception( "Facts missing after sync. Please run Tools>Advanced>Check DB.") def missingFacts(self): return self.deck.db.scalar( "select count() from cards where factId "+ "not in (select id from facts)"); def postSyncRefresh(self): "Flush changes to DB, and reload object associations." self.deck.db.flush() self.deck.db.refresh(self.deck) self.deck.currentModel # Summaries ########################################################################## def summary(self, lastSync): "Generate a full summary of modtimes for two-way syncing." # client may have selected an earlier sync time self.deck.lastSync = lastSync # return early if there's been a schema change if self.deck.getFloat("schemaMod") > lastSync: return None d = {} cats = [ # cards ("cards", "select id, modified from cards where modified > :m"), ("delcards", "select cardId, deletedTime from cardsDeleted " "where deletedTime > :m"), # facts ("facts", "select id, modified from facts where modified > :m"), ("delfacts", "select factId, deletedTime from factsDeleted " "where deletedTime > :m"), # models ("models", "select id, modified from models where modified > :m"), ("delmodels", "select modelId, deletedTime from modelsDeleted " "where deletedTime > :m"), # media ("media", "select id, created from media where created > :m"), ("delmedia", "select mediaId, deletedTime from mediaDeleted " "where deletedTime > :m") ] for (key, sql) in cats: if self.fullThreshold: sql += " limit %d" % self.fullThreshold ret = self.deck.db.all(sql, m=lastSync) if self.fullThreshold and self.fullThreshold == len(ret): # theshold exceeded, abort early return None d[key] = self.realLists(ret) return d # Diffing ########################################################################## def diffSummary(self, localSummary, remoteSummary, key): # list of ids on both ends lexists = localSummary[key] ldeleted = localSummary["del"+key] rexists = remoteSummary[key] rdeleted = remoteSummary["del"+key] ldeletedIds = dict(ldeleted) rdeletedIds = dict(rdeleted) # to store the results locallyEdited = [] locallyDeleted = [] remotelyEdited = [] remotelyDeleted = [] # build a hash of all ids, with value (localMod, remoteMod). # deleted/nonexisting cards are marked with a modtime of None. ids = {} for (id, mod) in rexists: ids[id] = [None, mod] for (id, mod) in rdeleted: ids[id] = [None, None] for (id, mod) in lexists: if id in ids: ids[id][0] = mod else: ids[id] = [mod, None] for (id, mod) in ldeleted: if id in ids: ids[id][0] = None else: ids[id] = [None, None] # loop through the hash, determining differences for (id, (localMod, remoteMod)) in ids.items(): if localMod and remoteMod: # changed/existing on both sides if localMod < remoteMod: remotelyEdited.append(id) elif localMod > remoteMod: locallyEdited.append(id) elif localMod and not remoteMod: # if it's missing on server or newer here, sync if (id not in rdeletedIds or rdeletedIds[id] < localMod): locallyEdited.append(id) else: remotelyDeleted.append(id) elif remoteMod and not localMod: # if it's missing locally or newer there, sync if (id not in ldeletedIds or ldeletedIds[id] < remoteMod): remotelyEdited.append(id) else: locallyDeleted.append(id) else: if id in ldeletedIds and id not in rdeletedIds: locallyDeleted.append(id) elif id in rdeletedIds and id not in ldeletedIds: remotelyDeleted.append(id) return (locallyEdited, locallyDeleted, remotelyEdited, remotelyDeleted) # Models ########################################################################## def getModels(self, ids, updateModified=False): return [self.bundleModel(id, updateModified) for id in ids] def bundleModel(self, id, updateModified): "Return a model representation suitable for transport." mod = self.deck.db.query(Model).get(id) # force load of lazy attributes mod.fieldModels; mod.cardModels m = self.dictFromObj(mod) m['fieldModels'] = [self.bundleFieldModel(fm) for fm in m['fieldModels']] m['cardModels'] = [self.bundleCardModel(fm) for fm in m['cardModels']] if updateModified: m['modified'] = time.time() return m def bundleFieldModel(self, fm): d = self.dictFromObj(fm) if 'model' in d: del d['model'] return d def bundleCardModel(self, cm): d = self.dictFromObj(cm) if 'model' in d: del d['model'] return d def updateModels(self, models): for model in models: local = self.getModel(model['id']) # avoid overwriting any existing card/field models fms = model['fieldModels']; del model['fieldModels'] cms = model['cardModels']; del model['cardModels'] self.applyDict(local, model) self.mergeFieldModels(local, fms) self.mergeCardModels(local, cms) self.deck.db.execute( "delete from modelsDeleted where modelId in %s" % ids2str([m['id'] for m in models])) def getModel(self, id, create=True): "Return a local model with same ID, or create." id = int(id) for l in self.deck.models: if l.id == id: return l if not create: return m = Model() self.deck.models.append(m) return m def mergeFieldModels(self, model, fms): ids = [] for fm in fms: local = self.getFieldModel(model, fm) self.applyDict(local, fm) ids.append(fm['id']) for fm in model.fieldModels: if fm.id not in ids: self.deck.deleteFieldModel(model, fm) def getFieldModel(self, model, remote): id = int(remote['id']) for fm in model.fieldModels: if fm.id == id: return fm fm = FieldModel() model.addFieldModel(fm) return fm def mergeCardModels(self, model, cms): ids = [] for cm in cms: local = self.getCardModel(model, cm) if not 'allowEmptyAnswer' in cm or cm['allowEmptyAnswer'] is None: cm['allowEmptyAnswer'] = True self.applyDict(local, cm) ids.append(cm['id']) for cm in model.cardModels: if cm.id not in ids: self.deck.deleteCardModel(model, cm) def getCardModel(self, model, remote): id = int(remote['id']) for cm in model.cardModels: if cm.id == id: return cm cm = CardModel() model.addCardModel(cm) return cm def deleteModels(self, ids): for id in ids: model = self.getModel(id, create=False) if model: self.deck.deleteModel(model) # Facts ########################################################################## def getFacts(self, ids, updateModified=False): if updateModified: modified = time.time() else: modified = "modified" factIds = ids2str(ids) return { 'facts': self.realLists(self.deck.db.all(""" select id, modelId, created, %s, tags, spaceUntil, lastCardId from facts where id in %s""" % (modified, factIds))), 'fields': self.realLists(self.deck.db.all(""" select id, factId, fieldModelId, ordinal, value, chksum from fields where factId in %s""" % factIds)) } def updateFacts(self, factsdict): facts = factsdict['facts'] fields = factsdict['fields'] if not facts: return # update facts first dlist = [{ 'id': f[0], 'modelId': f[1], 'created': f[2], 'modified': f[3], 'tags': f[4], 'spaceUntil': f[5] or "", 'lastCardId': f[6] } for f in facts] self.deck.db.execute(""" insert or replace into facts (id, modelId, created, modified, tags, spaceUntil, lastCardId) values (:id, :modelId, :created, :modified, :tags, :spaceUntil, :lastCardId)""", dlist) # now fields def chksum(f): if len(f) > 5: return f[5] return self.deck.fieldChecksum(f[4]) dlist = [{ 'id': f[0], 'factId': f[1], 'fieldModelId': f[2], 'ordinal': f[3], 'value': f[4], 'chksum': f[5] } for f in fields] # delete local fields since ids may have changed self.deck.db.execute( "delete from fields where factId in %s" % ids2str([f[0] for f in facts])) # then update self.deck.db.execute(""" insert into fields (id, factId, fieldModelId, ordinal, value, chksum) values (:id, :factId, :fieldModelId, :ordinal, :value, :chksum)""", dlist) self.deck.db.execute( "delete from factsDeleted where factId in %s" % ids2str([f[0] for f in facts])) def deleteFacts(self, ids): self.deck.deleteFacts(ids) # Cards ########################################################################## def getCards(self, ids): return self.realLists(self.deck.db.all(""" select id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay from cards where id in %s""" % ids2str(ids))) def updateCards(self, cards): if not cards: return dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2], 'created': c[3], 'modified': c[4], 'tags': c[5], 'ordinal': c[6], 'priority': c[7], 'interval': c[8], 'lastInterval': c[9], 'due': c[10], 'lastDue': c[11], 'factor': c[12], 'firstAnswered': c[13], 'reps': c[14], 'successive': c[15], 'averageTime': c[16], 'reviewTime': c[17], 'youngEase0': c[18], 'youngEase1': c[19], 'youngEase2': c[20], 'youngEase3': c[21], 'youngEase4': c[22], 'matureEase0': c[23], 'matureEase1': c[24], 'matureEase2': c[25], 'matureEase3': c[26], 'matureEase4': c[27], 'yesCount': c[28], 'noCount': c[29], 'question': c[30], 'answer': c[31], 'lastFactor': c[32], 'spaceUntil': c[33], 'type': c[34], 'combinedDue': c[35], 'rd': c[36], } for c in cards] self.deck.db.execute(""" insert or replace into cards (id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, type, combinedDue, relativeDelay, isDue) values (:id, :factId, :cardModelId, :created, :modified, :tags, :ordinal, :priority, :interval, :lastInterval, :due, :lastDue, :factor, :firstAnswered, :reps, :successive, :averageTime, :reviewTime, :youngEase0, :youngEase1, :youngEase2, :youngEase3, :youngEase4, :matureEase0, :matureEase1, :matureEase2, :matureEase3, :matureEase4, :yesCount, :noCount, :question, :answer, :lastFactor, :spaceUntil, :type, :combinedDue, :rd, 0)""", dlist) self.deck.db.execute( "delete from cardsDeleted where cardId in %s" % ids2str([c[0] for c in cards])) def deleteCards(self, ids): self.deck.deleteCards(ids) # Deck/history ########################################################################## def bundleDeck(self): # ensure modified is not greater than server time if getattr(self, "server", None) and getattr( self.server, "timestamp", None): self.deck.modified = min(self.deck.modified,self.server.timestamp) # and ensure lastSync is greater than modified self.deck.lastSync = max(time.time(), self.deck.modified+1) d = self.dictFromObj(self.deck) for bad in ("Session", "engine", "s", "db", "path", "syncName", "version", "newQueue", "failedQueue", "revQueue", "css", "models", "currentModel"): if bad in d: del d[bad] keys = d.keys() for k in keys: if isinstance(d[k], types.MethodType): del d[k] d['meta'] = self.realLists(self.deck.db.all("select * from deckVars")) return d def updateDeck(self, deck): if 'meta' in deck: meta = deck['meta'] for (k,v) in meta: self.deck.db.execute(""" insert or replace into deckVars (key, value) values (:k, :v)""", k=k, v=v) del deck['meta'] self.applyDict(self.deck, deck) def bundleHistory(self): return self.realLists(self.deck.db.all(""" select * from revlog where time > :ls""", ls=self.deck.lastSync)) def updateHistory(self, history): dlist = [{'time': h[0], 'cardId': h[1], 'ease': h[2], 'rep': h[3], 'lastInterval': h[4], 'interval': h[5], 'factor': h[6], 'userTime': h[7], 'flags': h[8]} for h in history] if not dlist: return self.deck.db.execute(""" insert or ignore into revlog values (:time, :cardId, :ease, :rep, :lastInterval, :interval, :factor, :userTime, :flags)""", dlist) def bundleSources(self): return self.realLists(self.deck.db.all("select * from sources")) def updateSources(self, sources): for s in sources: self.deck.db.execute(""" insert or replace into sources values (:id, :name, :created, :lastSync, :syncPeriod)""", id=s[0], name=s[1], created=s[2], lastSync=s[3], syncPeriod=s[4]) # Media metadata ########################################################################## def getMedia(self, ids): return [tuple(row) for row in self.deck.db.all(""" select id, filename, size, created, originalPath, description from media where id in %s""" % ids2str(ids))] def updateMedia(self, media): meta = [] for m in media: # build meta meta.append({ 'id': m[0], 'filename': m[1], 'size': m[2], 'created': m[3], 'originalPath': m[4], 'description': m[5]}) # apply metadata if meta: self.deck.db.execute(""" insert or replace into media (id, filename, size, created, originalPath, description) values (:id, :filename, :size, :created, :originalPath, :description)""", meta) self.deck.db.execute( "delete from mediaDeleted where mediaId in %s" % ids2str([m[0] for m in media])) def deleteMedia(self, ids): sids = ids2str(ids) files = self.deck.db.column0( "select filename from media where id in %s" % sids) self.deck.db.execute(""" insert into mediaDeleted select id, :now from media where media.id in %s""" % sids, now=time.time()) self.deck.db.execute( "delete from media where id in %s" % sids) # One-way syncing (sharing) ########################################################################## def syncOneWay(self, lastSync): "Sync two decks one way." payload = self.server.genOneWayPayload(lastSync) self.applyOneWayPayload(payload) self.deck.reset() def syncOneWayDeckName(self): return (self.deck.s.scalar("select name from sources where id = :id", id=self.server.deckName) or hexifyID(int(self.server.deckName))) def prepareOneWaySync(self): "Sync setup. True if sync needed. Not used for local sync." srcID = self.server.deckName (lastSync, syncPeriod) = self.deck.s.first( "select lastSync, syncPeriod from sources where id = :id", id=srcID) if self.server.modified() <= lastSync: return self.deck.lastSync = lastSync return True def genOneWayPayload(self, lastSync): "Bundle all added or changed objects since the last sync." p = {} # facts factIds = self.deck.s.column0( "select id from facts where modified > :l", l=lastSync) p['facts'] = self.getFacts(factIds, updateModified=True) # models modelIds = self.deck.s.column0( "select id from models where modified > :l", l=lastSync) p['models'] = self.getModels(modelIds, updateModified=True) # media mediaIds = self.deck.s.column0( "select id from media where created > :l", l=lastSync) p['media'] = self.getMedia(mediaIds) # cards cardIds = self.deck.s.column0( "select id from cards where modified > :l", l=lastSync) p['cards'] = self.realLists(self.getOneWayCards(cardIds)) return p def applyOneWayPayload(self, payload): keys = [k for k in KEYS if k != "cards"] # model, facts, media for key in keys: self.updateObjsFromKey(payload[key], key) # models need their source tagged for m in payload["models"]: self.deck.s.statement("update models set source = :s " "where id = :id", s=self.server.deckName, id=m['id']) # cards last, handled differently t = time.time() try: self.updateOneWayCards(payload['cards']) except KeyError: sys.stderr.write("Subscribed to a broken deck. " "Try removing your deck subscriptions.") t = 0 # update sync time self.deck.s.statement( "update sources set lastSync = :t where id = :id", id=self.server.deckName, t=t) self.deck.modified = time.time() def getOneWayCards(self, ids): "The minimum information necessary to generate one way cards." return self.deck.s.all( "select id, factId, cardModelId, ordinal, created from cards " "where id in %s" % ids2str(ids)) def updateOneWayCards(self, cards): if not cards: return t = time.time() dlist = [{'id': c[0], 'factId': c[1], 'cardModelId': c[2], 'ordinal': c[3], 'created': c[4], 't': t} for c in cards] # add any missing cards self.deck.s.statements(""" insert or ignore into cards (id, factId, cardModelId, created, modified, tags, ordinal, priority, interval, lastInterval, due, lastDue, factor, firstAnswered, reps, successive, averageTime, reviewTime, youngEase0, youngEase1, youngEase2, youngEase3, youngEase4, matureEase0, matureEase1, matureEase2, matureEase3, matureEase4, yesCount, noCount, question, answer, lastFactor, spaceUntil, isDue, type, combinedDue, relativeDelay) values (:id, :factId, :cardModelId, :created, :t, "", :ordinal, 1, 0, 0, :created, 0, 2.5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 2.5, 0, 0, 2, :t, 2)""", dlist) # update q/as models = dict(self.deck.s.all(""" select cards.id, models.id from cards, facts, models where cards.factId = facts.id and facts.modelId = models.id and cards.id in %s""" % ids2str([c[0] for c in cards]))) self.deck.s.flush() self.deck.updateCardQACache( [(c[0], c[2], c[1], models[c[0]]) for c in cards]) # rebuild priorities on client cardIds = [c[0] for c in cards] self.deck.updateCardTags(cardIds) self.rebuildPriorities(cardIds) # Tools ########################################################################## def modified(self): return self.deck.modified def _lastSync(self): return self.deck.lastSync def unstuff(self, data): "Uncompress and convert to unicode." return simplejson.loads(unicode(zlib.decompress(data), "utf8")) def stuff(self, data): "Convert into UTF-8 and compress." return zlib.compress(simplejson.dumps(data)) def dictFromObj(self, obj): "Return a dict representing OBJ without any hidden db fields." return dict([(k,v) for (k,v) in obj.__dict__.items() if not k.startswith("_")]) def applyDict(self, obj, dict): "Apply each element in DICT to OBJ in a way the ORM notices." for (k,v) in dict.items(): setattr(obj, k, v) def realLists(self, result): "Convert an SQLAlchemy response into a list of real lists." return [list(x) for x in result] def getObjsFromKey(self, ids, key): return getattr(self, "get" + key.capitalize())(ids) def deleteObjsFromKey(self, ids, key): return getattr(self, "delete" + key.capitalize())(ids) def updateObjsFromKey(self, ids, key): return getattr(self, "update" + key.capitalize())(ids) # Full sync ########################################################################## def needFullSync(self, sums): if self.deck.lastSync <= 0: return True for sum in sums: for l in sum.values(): if len(l) > 1000: return True if self.deck.db.scalar( "select count() from revlog where time > :ls", ls=self.deck.lastSync) > 1000: return True lastDay = date.fromtimestamp(max(0, self.deck.lastSync - 60*60*24)) return False def prepareFullSync(self): t = time.time() # ensure modified is not greater than server time self.deck.modified = min(self.deck.modified, self.server.timestamp) self.deck.db.commit() self.deck.close() fields = { "p": self.server.password, "u": self.server.username, "d": self.server.deckName.encode("utf-8"), } if self.localTime > self.remoteTime: return ("fromLocal", fields, self.deck.path) else: return ("fromServer", fields, self.deck.path) def fullSync(self): ret = self.prepareFullSync() if ret[0] == "fromLocal": self.fullSyncFromLocal(ret[1], ret[2]) else: self.fullSyncFromServer(ret[1], ret[2]) def fullSyncFromLocal(self, fields, path): global sendProgressHook try: # write into a temporary file, since POST needs content-length src = open(path, "rb") (fd, name) = tempfile.mkstemp(prefix="anki") tmp = open(name, "w+b") # post vars for (key, value) in fields.items(): tmp.write('--' + MIME_BOUNDARY + "\r\n") tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key) tmp.write('\r\n') tmp.write(value) tmp.write('\r\n') # file header tmp.write('--' + MIME_BOUNDARY + "\r\n") tmp.write( 'Content-Disposition: form-data; name="deck"; filename="deck"\r\n') tmp.write('Content-Type: application/octet-stream\r\n') tmp.write('\r\n') # data comp = zlib.compressobj() while 1: data = src.read(CHUNK_SIZE) if not data: tmp.write(comp.flush()) break tmp.write(comp.compress(data)) src.close() tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n') size = tmp.tell() tmp.seek(0) # open http connection runHook("fullSyncStarted", size) headers = { 'Content-type': 'multipart/form-data; boundary=%s' % MIME_BOUNDARY, 'Content-length': str(size), 'Host': SYNC_HOST, } req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers) try: sendProgressHook = fullSyncProgressHook res = urllib2.urlopen(req).read() assert res.startswith("OK") # update lastSync c = sqlite.connect(path) c.execute("update decks set lastSync = ?", (res[3:],)) c.commit() c.close() finally: sendProgressHook = None tmp.close() os.close(fd) os.unlink(name) finally: runHook("fullSyncFinished") def fullSyncFromServer(self, fields, path): try: runHook("fullSyncStarted", 0) fields = urllib.urlencode(fields) src = urllib.urlopen(SYNC_URL + "fulldown", fields) (fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path), prefix="fullsync") tmp = open(tmpname, "wb") decomp = zlib.decompressobj() cnt = 0 while 1: data = src.read(CHUNK_SIZE) if not data: tmp.write(decomp.flush()) break tmp.write(decomp.decompress(data)) cnt += CHUNK_SIZE runHook("fullSyncProgress", "fromServer", cnt) src.close() tmp.close() os.close(fd) # if we were successful, overwrite old deck os.unlink(path) os.rename(tmpname, path) # reset the deck name c = sqlite.connect(path) c.execute("update decks set syncName = ?", [checksum(path.encode("utf-8"))]) c.commit() c.close() finally: runHook("fullSyncFinished") # Local syncing ########################################################################## class SyncServer(SyncTools): def __init__(self, deck=None): SyncTools.__init__(self, deck) class SyncClient(SyncTools): pass # HTTP proxy: act as a server and direct requests to the real server ########################################################################## class HttpSyncServerProxy(SyncServer): def __init__(self, user, passwd): SyncServer.__init__(self) self.decks = None self.deckName = None self.username = user self.password = passwd self.protocolVersion = 5 self.sourcesToCheck = [] def connect(self, clientVersion=""): "Check auth, protocol & grab deck list." if not self.decks: import socket socket.setdefaulttimeout(30) d = self.runCmd("getDecks", libanki=anki.version, client=clientVersion, sources=simplejson.dumps(self.sourcesToCheck), pversion=self.protocolVersion) socket.setdefaulttimeout(None) if d['status'] != "OK": raise SyncError(type="authFailed", status=d['status']) self.decks = d['decks'] self.timestamp = d['timestamp'] self.timediff = abs(self.timestamp - time.time()) def hasDeck(self, deckName): self.connect() return deckName in self.decks.keys() def availableDecks(self): self.connect() return self.decks.keys() def createDeck(self, deckName): ret = self.runCmd("createDeck", name=deckName.encode("utf-8")) if not ret or ret['status'] != "OK": raise SyncError(type="createFailed") self.decks[deckName] = [0, 0] def summary(self, lastSync): return self.runCmd("summary", lastSync=self.stuff(lastSync)) def genOneWayPayload(self, lastSync): return self.runCmd("genOneWayPayload", lastSync=self.stuff(lastSync)) def modified(self): self.connect() return self.decks[self.deckName][0] def _lastSync(self): self.connect() return self.decks[self.deckName][1] def applyPayload(self, payload): return self.runCmd("applyPayload", payload=self.stuff(payload)) def finish(self): assert self.runCmd("finish") == "OK" def runCmd(self, action, **args): data = {"p": self.password, "u": self.username, "v": 2} if self.deckName: data['d'] = self.deckName.encode("utf-8") else: data['d'] = None data.update(args) data = urllib.urlencode(data) try: f = urllib2.urlopen(SYNC_URL + action, data) except (urllib2.URLError, socket.error, socket.timeout, httplib.BadStatusLine), e: raise SyncError(type="connectionError", exc=`e`) ret = f.read() if not ret: raise SyncError(type="noResponse") try: return self.unstuff(ret) except Exception, e: raise SyncError(type="connectionError", exc=`e`) # HTTP server: respond to proxy requests and return data ########################################################################## class HttpSyncServer(SyncServer): def __init__(self): SyncServer.__init__(self) self.decks = {} self.deck = None def summary(self, lastSync): return self.stuff(SyncServer.summary( self, float(zlib.decompress(lastSync)))) def applyPayload(self, payload): return self.stuff(SyncServer.applyPayload(self, self.unstuff(payload))) def genOneWayPayload(self, lastSync): return self.stuff(SyncServer.genOneWayPayload( self, float(zlib.decompress(lastSync)))) def getDecks(self, libanki, client, sources, pversion): return self.stuff({ "status": "OK", "decks": self.decks, "timestamp": time.time(), }) def createDeck(self, name): "Create a deck on the server. Not implemented." return self.stuff("OK") # Local media copying ########################################################################## def copyLocalMedia(src, dst): srcDir = src.mediaDir() if not srcDir: return dstDir = dst.mediaDir(create=True) files = os.listdir(srcDir) # find media references used = {} for col in ("question", "answer"): txt = dst.s.column0(""" select %(c)s from cards where %(c)s like '%%