GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-20 15:02:21 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

remove sync limit, improve bulk media

Browse source
This commit is contained in:
Damien Elmes 2008-10-18 19:10:29 +09:00
parent 06d1f713b5
commit 2afa59b24a
1 changed files with 31 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
anki/sync.py
View file

@ -38,8 +38,6 @@ from anki.lang import _
if simplejson.__version__ < "1.7.3": if simplejson.__version__ < "1.7.3":
raise "SimpleJSON must be 1.7.3 or later." raise "SimpleJSON must be 1.7.3 or later."
MEDIA_SYNC_LIMIT = 1048576
# Protocol 3 code # Protocol 3 code
########################################################################## ##########################################################################
@ -49,7 +47,7 @@ class SyncTools(object):
self.deck = deck self.deck = deck
self.diffs = {} self.diffs = {}
self.serverExcludedTags = [] self.serverExcludedTags = []
self.bundleMedia = True self.mediaSyncPending = False
# Control # Control
########################################################################## ##########################################################################
@ -603,43 +601,19 @@ values
size = self.deck.s.scalar( size = self.deck.s.scalar(
"select sum(size) from media where id in %s" % "select sum(size) from media where id in %s" %
ids2str(ids)) ids2str(ids))
if size > MEDIA_SYNC_LIMIT: if ids:
self.bundleMedia = False self.mediaSyncPending = True
else:
self.bundleMedia = True
if updateCreated: if updateCreated:
created = time.time() created = time.time()
else: else:
created = "created" created = "created"
return [(tuple(row), return [tuple(row) for row in self.deck.s.all("""
base64.b64encode(self.getMediaData(row[1])))
for row in self.deck.s.all("""
select id, filename, size, %s, originalPath, description select id, filename, size, %s, originalPath, description
from media where id in %s""" % (created, ids2str(ids)))] from media where id in %s""" % (created, ids2str(ids)))]
def getMediaData(self, fname):
if not self.bundleMedia:
return ""
try:
return open(self.mediaPath(fname), "rb").read()
except (OSError, IOError):
return ""
def updateMedia(self, media): def updateMedia(self, media):
meta = [] meta = []
for (m, data) in media: for m in media:
if data:
# ensure media is correctly checksummed and sized
fname = m[1]
size = m[2]
data = base64.b64decode(data)
assert len(data) == size
assert checksum(data) == os.path.splitext(fname)[0]
# write it out
self.addMediaFile(m, data)
else:
# missing media
self.bundleMedia = False
# build meta # build meta
meta.append({ meta.append({
'id': m[0], 'id': m[0],
@ -650,6 +624,7 @@ from media where id in %s""" % (created, ids2str(ids)))]
'description': m[5]}) 'description': m[5]})
# apply metadata # apply metadata
if meta: if meta:
self.mediaSyncPending = True
self.deck.s.statements(""" self.deck.s.statements("""
insert or replace into media (id, filename, size, created, insert or replace into media (id, filename, size, created,
originalPath, description) originalPath, description)
@ -657,7 +632,7 @@ values (:id, :filename, :size, :created, :originalPath,
:description)""", meta) :description)""", meta)
self.deck.s.statement( self.deck.s.statement(
"delete from mediaDeleted where mediaId in %s" % "delete from mediaDeleted where mediaId in %s" %
ids2str([m[0][0] for m in media])) ids2str([m[0] for m in media]))
def deleteMedia(self, ids): def deleteMedia(self, ids):
sids = ids2str(ids) sids = ids2str(ids)
@ -675,15 +650,11 @@ where media.id in %s""" % sids, now=time.time())
# the following routines are reimplemented by the anki server so that # the following routines are reimplemented by the anki server so that
# media can be shared and accounted # media can be shared and accounted
def addMediaFile(self, meta, decodedData):
fname = meta[1]
path = self.mediaPath(fname)
exists = os.path.exists(path)
if not exists:
open(path, "wb").write(decodedData)
def deleteMediaFile(self, file): def deleteMediaFile(self, file):
os.unlink(self.mediaPath(file)) try:
os.unlink(self.mediaPath(file))
except OSError:
pass
def mediaPath(self, path): def mediaPath(self, path):
"Return the path to store media in. Defaults to the deck media dir." "Return the path to store media in. Defaults to the deck media dir."
@ -745,6 +716,8 @@ where media.id in %s""" % sids, now=time.time())
"where id = :id", "where id = :id",
s=self.server.deckName, s=self.server.deckName,
id=m['id']) id=m['id'])
# if media arrived, we'll need to download the data
self.mediaSyncPending = self.mediaSupported() and payload['media']
# cards last, handled differently # cards last, handled differently
self.updateOneWayCards(payload['cards']) self.updateOneWayCards(payload['cards'])
# update sync time # update sync time
@ -975,6 +948,7 @@ class BulkMediaSyncer(SyncTools):
def __init__(self, deck): def __init__(self, deck):
self.deck = deck self.deck = deck
self.server = None self.server = None
self.oneWay = False
def missingMedia(self): def missingMedia(self):
fnames = self.deck.s.column0( fnames = self.deck.s.column0(
@ -983,20 +957,20 @@ class BulkMediaSyncer(SyncTools):
not os.path.exists(self.mediaPath(f))] not os.path.exists(self.mediaPath(f))]
def progressCallback(self, type, count, total, fname): def progressCallback(self, type, count, total, fname):
print "orig"
return return
def sync(self): def sync(self):
# upload to server # upload to server
missing = self.server.missingMedia() if not self.oneWay:
total = len(missing) missing = self.server.missingMedia()
for n in range(total): total = len(missing)
fname = missing[n] for n in range(total):
data = self.getFile(fname) fname = missing[n]
self.progressCallback('up', n, total, fname) data = self.getFile(fname)
if data: self.progressCallback('up', n, total, fname)
self.server.addFile(fname, data) if data:
n += 1 self.server.addFile(fname, data)
n += 1
# download from server # download from server
missing = self.missingMedia() missing = self.missingMedia()
total = len(missing) total = len(missing)
@ -1017,6 +991,12 @@ class BulkMediaSyncer(SyncTools):
def addFile(self, fname, data): def addFile(self, fname, data):
path = self.mediaPath(fname) path = self.mediaPath(fname)
assert not os.path.exists(path) assert not os.path.exists(path)
size = self.deck.s.scalar(
"select size from media where filename = :f",
f=fname)
assert size
assert size == len(data)
assert checksum(data) == os.path.splitext(fname)[0]
open(path, "wb").write(data) open(path, "wb").write(data)
class BulkMediaSyncerProxy(HttpSyncServerProxy): class BulkMediaSyncerProxy(HttpSyncServerProxy):
@ -1028,7 +1008,7 @@ class BulkMediaSyncerProxy(HttpSyncServerProxy):
return os.path.join(fname[0:1], fname[0:2], fname) return os.path.join(fname[0:1], fname[0:2], fname)
def _relativeMediaPath(self, fname): def _relativeMediaPath(self, fname):
return "http://anki.ichi2.net/media/%s" % ( return "http://ankimedia.ichi2.net/%s" % (
self._splitMedia(fname)) self._splitMedia(fname))
def getFile(self, fname): def getFile(self, fname):