diff --git a/anki/exporting.py b/anki/exporting.py
index 6e6e5711f..f0efa2a5a 100644
--- a/anki/exporting.py
+++ b/anki/exporting.py
@@ -12,7 +12,7 @@ import itertools, time, re
from operator import itemgetter
from anki import DeckStorage
from anki.cards import Card
-from anki.sync import SyncClient, SyncServer, BulkMediaSyncer
+from anki.sync import SyncClient, SyncServer, copyLocalMedia
from anki.lang import _
from anki.utils import findTag, parseTags, stripHTML, ids2str
from anki.tags import tagIds
@@ -118,11 +118,7 @@ modified = :now
self.newDeck.s.statement("""
delete from stats""")
# media
- if client.mediaSyncPending:
- bulkClient = BulkMediaSyncer(client.deck)
- bulkServer = BulkMediaSyncer(server.deck)
- bulkClient.server = bulkServer
- bulkClient.sync()
+ copyLocalMedia(client.deck, server.deck)
# need to save manually
self.newDeck.rebuildCounts()
self.newDeck.updateAllPriorities()
diff --git a/anki/importing/anki10.py b/anki/importing/anki10.py
index dd5c87c82..a0d73ebf1 100644
--- a/anki/importing/anki10.py
+++ b/anki/importing/anki10.py
@@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext'
from anki import DeckStorage
from anki.importing import Importer
-from anki.sync import SyncClient, SyncServer, BulkMediaSyncer
+from anki.sync import SyncClient, SyncServer, copyLocalMedia
from anki.lang import _
from anki.utils import ids2str
from anki.deck import NEW_CARDS_RANDOM
@@ -57,12 +57,7 @@ class Anki10Importer(Importer):
res = server.applyPayload(payload)
self.deck.updateProgress()
client.applyPayloadReply(res)
- if client.mediaSyncPending:
- bulkClient = BulkMediaSyncer(client.deck)
- bulkServer = BulkMediaSyncer(server.deck)
- bulkClient.oneWay = True
- bulkClient.server = bulkServer
- bulkClient.sync()
+ copyLocalMedia(server.deck, client.deck)
# add tags
self.deck.updateProgress()
fids = [f[0] for f in res['added-facts']['facts']]
diff --git a/anki/sync.py b/anki/sync.py
index 5c14c6ac8..ca7af8f50 100644
--- a/anki/sync.py
+++ b/anki/sync.py
@@ -20,7 +20,7 @@ createDeck(name): create a deck on the server
"""
__docformat__ = 'restructuredtext'
-import zlib, re, urllib, urllib2, socket, simplejson, time
+import zlib, re, urllib, urllib2, socket, simplejson, time, shutil
import os, base64, httplib, sys, tempfile, httplib
from datetime import date
import anki, anki.deck, anki.cards
@@ -31,7 +31,6 @@ from anki.cards import Card
from anki.stats import Stats, globalStats
from anki.history import CardHistoryEntry
from anki.stats import globalStats
-from anki.media import checksum
from anki.utils import ids2str, hexifyID
from anki.lang import _
from hooks import runHook
@@ -47,6 +46,8 @@ SYNC_HOST = "anki.ichi2.net"; SYNC_PORT = 80
#SYNC_URL = "http://localhost:8001/sync/"
#SYNC_HOST = "localhost"; SYNC_PORT = 8001
+KEYS = ("models", "facts", "cards", "media")
+
##########################################################################
# Monkey-patch httplib to incrementally send instead of chewing up large
# amounts of memory, and track progress.
@@ -93,7 +94,6 @@ class SyncTools(object):
self.deck = deck
self.diffs = {}
self.serverExcludedTags = []
- self.mediaSyncPending = False
# Control
##########################################################################
@@ -109,11 +109,6 @@ class SyncTools(object):
payload = self.genPayload(sums)
res = self.server.applyPayload(payload)
self.applyPayloadReply(res)
- if self.mediaSyncPending:
- bulkClient = BulkMediaSyncer(self.deck)
- bulkServer = BulkMediaSyncer(self.server.deck)
- bulkClient.server = bulkServer
- bulkClient.sync()
def prepareSync(self):
"Sync setup. True if sync needed."
@@ -137,7 +132,7 @@ class SyncTools(object):
self.preSyncRefresh()
payload = {}
# first, handle models, facts and cards
- for key in self.keys():
+ for key in KEYS:
diff = self.diffSummary(lsum, rsum, key)
payload["added-" + key] = self.getObjsFromKey(diff[0], key)
payload["deleted-" + key] = diff[1]
@@ -156,12 +151,14 @@ class SyncTools(object):
reply = {}
self.preSyncRefresh()
# model, facts and cards
- for key in self.keys():
+ for key in KEYS:
+ k = 'added-' + key
# send back any requested
- reply['added-' + key] = self.getObjsFromKey(
+ reply[k] = self.getObjsFromKey(
payload['missing-' + key], key)
- self.updateObjsFromKey(payload['added-' + key], key)
- self.deleteObjsFromKey(payload['deleted-' + key], key)
+ if k in payload:
+ self.updateObjsFromKey(payload['added-' + key], key)
+ self.deleteObjsFromKey(payload['deleted-' + key], key)
# send back deck-related stuff if it wasn't sent to us
if not 'deck' in payload:
reply['deck'] = self.bundleDeck()
@@ -186,8 +183,11 @@ class SyncTools(object):
def applyPayloadReply(self, reply):
# model, facts and cards
- for key in self.keys():
- self.updateObjsFromKey(reply['added-' + key], key)
+ for key in KEYS:
+ k = 'added-' + key
+ # old version may not send media
+ if k in reply:
+ self.updateObjsFromKey(reply['added-' + key], key)
# deck
if 'deck' in reply:
self.updateDeck(reply['deck'])
@@ -226,12 +226,6 @@ class SyncTools(object):
'lm': len(payload['added-models']),
'rm': len(payload['missing-models']),
}
- if self.mediaSupported():
- h['lM'] = len(payload['added-media'])
- h['rM'] = len(payload['missing-media'])
- else:
- h['lM'] = _("off")
- h['rM'] = _("off")
if self.localTime > self.remoteTime:
h['ls'] = _('all')
h['rs'] = 0
@@ -249,7 +243,6 @@ class SyncTools(object):
| Cards | %(lc)d | %(rc)d |
| Facts | %(lf)d | %(rf)d |
| Models | %(lm)d | %(rm)d |
-| Media | %(lM)s | %(rM)s |
| Stats | %(ls)s | %(rs)s |
""") % p
@@ -709,22 +702,13 @@ insert or replace into sources values
lastSync=s[3],
syncPeriod=s[4])
- # Media
+ # Media metadata
##########################################################################
- def getMedia(self, ids, updateCreated=False):
- size = self.deck.s.scalar(
- "select sum(size) from media where id in %s" %
- ids2str(ids))
- if ids:
- self.mediaSyncPending = True
- if updateCreated:
- created = time.time()
- else:
- created = "created"
+ def getMedia(self, ids):
return [tuple(row) for row in self.deck.s.all("""
-select id, filename, size, %s, originalPath, description
-from media where id in %s""" % (created, ids2str(ids)))]
+select id, filename, size, created, originalPath, description
+from media where id in %s""" % ids2str(ids))]
def updateMedia(self, media):
meta = []
@@ -739,7 +723,6 @@ from media where id in %s""" % (created, ids2str(ids)))]
'description': m[5]})
# apply metadata
if meta:
- self.mediaSyncPending = True
self.deck.s.statements("""
insert or replace into media (id, filename, size, created,
originalPath, description)
@@ -759,21 +742,6 @@ select id, :now from media
where media.id in %s""" % sids, now=time.time())
self.deck.s.execute(
"delete from media where id in %s" % sids)
- for file in files:
- self.deleteMediaFile(file)
-
- # the following routines are reimplemented by the anki server so that
- # media can be shared and accounted
-
- def deleteMediaFile(self, file):
- try:
- os.unlink(self.mediaPath(file))
- except OSError:
- pass
-
- def mediaPath(self, path):
- "Return the path to store media in. Defaults to the deck media dir."
- return os.path.join(self.deck.mediaDir(create=True), path)
# One-way syncing (sharing)
##########################################################################
@@ -810,12 +778,9 @@ where media.id in %s""" % sids, now=time.time())
"select id from models where modified > :l", l=lastSync)
p['models'] = self.getModels(modelIds, updateModified=True)
# media
- if self.mediaSupported():
- mediaIds = self.deck.s.column0(
- "select id from media where created > :l", l=lastSync)
- p['media'] = self.getMedia(mediaIds, updateCreated=True)
- if p['media']:
- self.mediaSyncPending = True
+ mediaIds = self.deck.s.column0(
+ "select id from media where created > :l", l=lastSync)
+ p['media'] = self.getMedia(mediaIds)
# cards
cardIds = self.deck.s.column0(
"select id from cards where modified > :l", l=lastSync)
@@ -823,7 +788,7 @@ where media.id in %s""" % sids, now=time.time())
return p
def applyOneWayPayload(self, payload):
- keys = [k for k in self.keys() if k != "cards"]
+ keys = [k for k in KEYS if k != "cards"]
# model, facts, media
for key in keys:
self.updateObjsFromKey(payload[key], key)
@@ -833,9 +798,6 @@ where media.id in %s""" % sids, now=time.time())
"where id = :id",
s=self.server.deckName,
id=m['id'])
- # if media arrived, we'll need to download the data
- self.mediaSyncPending = (self.mediaSyncPending or
- self.mediaSupported() and payload['media'])
# cards last, handled differently
t = time.time()
try:
@@ -936,11 +898,6 @@ and cards.id in %s""" % ids2str([c[0] for c in cards])))
def updateObjsFromKey(self, ids, key):
return getattr(self, "update" + key.capitalize())(ids)
- def keys(self):
- if self.mediaSupported():
- return standardKeys + ("media",)
- return standardKeys
-
# Full sync
##########################################################################
@@ -1055,7 +1012,6 @@ and cards.id in %s""" % ids2str([c[0] for c in cards])))
# Local syncing
##########################################################################
-standardKeys = ("models", "facts", "cards")
class SyncServer(SyncTools):
@@ -1188,110 +1144,20 @@ class HttpSyncServer(SyncServer):
"Create a deck on the server. Not implemented."
return self.stuff("OK")
-# Bulk uploader/downloader
+# Local media copying
##########################################################################
-class BulkMediaSyncer(SyncTools):
-
- def __init__(self, deck):
- self.deck = deck
- self.server = None
- self.oneWay = False
-
- def missingMedia(self):
- fnames = self.deck.s.column0(
- "select filename from media")
- return [f for f in fnames if
- not os.path.exists(self.mediaPath(f))]
-
- def progressCallback(self, type, count, total, fname):
+def copyLocalMedia(src, dst):
+ src = src.mediaDir()
+ if not src:
return
-
- def sync(self):
- # upload to server
- if not self.oneWay:
- missing = self.server.missingMedia()
- total = len(missing)
- for n in range(total):
- fname = missing[n]
- self.progressCallback('up', n, total, fname)
- data = self.getFile(fname)
- if data:
- self.server.addFile(fname, data)
- n += 1
- if total > 0:
- self.progressCallback('up', total, total, missing[total-1])
-
- # download from server
- missing = self.missingMedia()
- total = len(missing)
- for n in range(total):
- fname = missing[n]
- self.progressCallback('down', n, total, fname)
- data = self.server.getFile(fname)
- if data:
- self.addFile(fname, data)
- n += 1
- if total > 0:
- self.progressCallback('down', total, total, missing[total-1])
-
- def getFile(self, fname):
- try:
- return open(self.mediaPath(fname), "rb").read()
- except (IOError, OSError):
- return None
-
- def addFile(self, fname, data):
- path = self.mediaPath(fname)
- assert not os.path.exists(path)
- size = self.deck.s.scalar(
- "select size from media where filename = :f",
- f=fname)
- # don't bother checksumming locally
- #assert size
- #assert size == len(data)
- #assert checksum(data) == os.path.splitext(fname)[0]
- open(path, "wb").write(data)
-
-class BulkMediaSyncerProxy(HttpSyncServerProxy):
-
- def missingMedia(self):
- return self.unstuff(self.runCmd("missingMedia"))
-
- def _splitMedia(self, fname):
- return "%s/%s/%s" % (fname[0:1], fname[0:2], fname)
-
- def _relativeMediaPath(self, fname):
- return "http://ankimedia.ichi2.net/%s" % (
- self._splitMedia(fname))
-
- def getFile(self, fname):
- try:
- f = urllib2.urlopen(self._relativeMediaPath(fname))
- except (urllib2.URLError, socket.error, socket.timeout):
- return ""
- ret = f.read()
- if not ret:
- return ""
- return ret
-
- def addFile(self, fname, data):
- oldsum = os.path.splitext(fname)[0]
- assert oldsum == checksum(data)
- return self.runCmd("addFile", fname=fname, data=data)
-
- def runCmd(self, action, **args):
- data = {"p": self.password,
- "u": self.username,
- "d": self.deckName.encode("utf-8")}
- data.update(args)
- data = urllib.urlencode(data)
- try:
- f = urllib2.urlopen(SYNC_URL + action, data)
- except (urllib2.URLError, socket.error, socket.timeout,
- httplib.BadStatusLine):
- raise SyncError(type="noResponse")
- ret = f.read()
- if not ret:
- raise SyncError(type="noResponse")
- return ret
+ dst = dst.mediaDir(create=True)
+ files = os.listdir(src)
+ for file in files:
+ srcfile = os.path.join(src, file)
+ dstfile = os.path.join(dst, file)
+ if not os.path.exists(dstfile):
+ try:
+ shutil.copy2(srcfile, dstfile)
+ except IOError, OSError:
+ pass
diff --git a/tests/test_sync.py b/tests/test_sync.py
index ad32a7bc1..cf04a88c0 100644
--- a/tests/test_sync.py
+++ b/tests/test_sync.py
@@ -8,6 +8,7 @@ from anki import DeckStorage
from anki.db import *
from anki.stdmodels import BasicModel
from anki.sync import SyncClient, SyncServer, HttpSyncServer, HttpSyncServerProxy
+from anki.sync import copyLocalMedia
from anki.stats import dailyStats, globalStats
from anki.facts import Fact
from anki.cards import Card
@@ -237,13 +238,20 @@ def test_localsync_media():
assert len(os.listdir(deck1media)) == 2
assert len(os.listdir(deck2media)) == 1
client.sync()
- assert len(os.listdir(deck1media)) == 3
- assert len(os.listdir(deck2media)) == 3
+ # metadata should have been copied
assert deck1.s.scalar("select count(1) from media") == 3
assert deck2.s.scalar("select count(1) from media") == 3
+ # copy local files
+ copyLocalMedia(deck1, deck2)
+ assert len(os.listdir(deck1media)) == 2
+ assert len(os.listdir(deck2media)) == 3
+ copyLocalMedia(deck2, deck1)
+ assert len(os.listdir(deck1media)) == 3
+ assert len(os.listdir(deck2media)) == 3
# check delete
os.unlink(os.path.join(deck1media, "22161b29b0c18e068038021f54eee1ee.png"))
- time.sleep(0.1)
+ os.system("sync")
+ time.sleep(0.2)
rebuildMediaDir(deck1)
client.sync()
assert deck1.s.scalar("select count(1) from media") == 2