diff --git a/pylib/anki/media.py b/pylib/anki/media.py index 9cb4b755a..be48306f5 100644 --- a/pylib/anki/media.py +++ b/pylib/anki/media.py @@ -3,8 +3,6 @@ from __future__ import annotations -import io -import json import os import re import sys @@ -12,7 +10,6 @@ import unicodedata import urllib.error import urllib.parse import urllib.request -import zipfile from typing import Any, Callable, List, Optional, Tuple, Union import anki @@ -414,109 +411,3 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0); if not v[2]: removed.append(k) return added, removed - - # Syncing-related - ########################################################################## - - def lastUsn(self) -> Any: - return self.db.scalar("select lastUsn from meta") - - def setLastUsn(self, usn) -> None: - self.db.execute("update meta set lastUsn = ?", usn) - self.db.commit() - - def syncInfo(self, fname) -> Any: - ret = self.db.first("select csum, dirty from media where fname=?", fname) - return ret or (None, 0) - - def markClean(self, fnames) -> None: - for fname in fnames: - self.db.execute("update media set dirty=0 where fname=?", fname) - - def syncDelete(self, fname) -> None: - if os.path.exists(fname): - os.unlink(fname) - self.db.execute("delete from media where fname=?", fname) - - def mediaCount(self) -> Any: - return self.db.scalar("select count() from media where csum is not null") - - def dirtyCount(self) -> Any: - return self.db.scalar("select count() from media where dirty=1") - - def forceResync(self) -> None: - self.db.execute("delete from media") - self.db.execute("update meta set lastUsn=0,dirMod=0") - self.db.commit() - self.db.setAutocommit(True) - self.db.execute("vacuum") - self.db.execute("analyze") - self.db.setAutocommit(False) - - # Media syncing: zips - ########################################################################## - - def mediaChangesZip(self) -> Tuple[bytes, list]: - f = io.BytesIO() - z = zipfile.ZipFile(f, "w", compression=zipfile.ZIP_DEFLATED) - - fnames = [] - # meta is list of (fname, zipname), where zipname of None - # is a deleted file - meta = [] - sz = 0 - - for c, (fname, csum) in enumerate( - self.db.execute( - "select fname, csum from media where dirty=1" - " limit %d" % SYNC_ZIP_COUNT - ) - ): - - fnames.append(fname) - normname = unicodedata.normalize("NFC", fname) - - if csum: - self.col.log("+media zip", fname) - z.write(fname, str(c)) - meta.append((normname, str(c))) - sz += os.path.getsize(fname) - else: - self.col.log("-media zip", fname) - meta.append((normname, "")) - - if sz >= SYNC_ZIP_SIZE: - break - - z.writestr("_meta", json.dumps(meta)) - z.close() - return f.getvalue(), fnames - - def addFilesFromZip(self, zipData) -> int: - "Extract zip data; true if finished." - f = io.BytesIO(zipData) - z = zipfile.ZipFile(f, "r") - media = [] - # get meta info first - meta = json.loads(z.read("_meta").decode("utf8")) - # then loop through all files - cnt = 0 - for i in z.infolist(): - if i.filename == "_meta": - # ignore previously-retrieved meta - continue - else: - data = z.read(i) - csum = checksum(data) - name = meta[i.filename] - # normalize name - name = unicodedata.normalize("NFC", name) - # save file - with open(name, "wb") as f: # type: ignore - f.write(data) - # update db - media.append((name, csum, self._mtime(name), 0)) - cnt += 1 - if media: - self.db.executemany("insert or replace into media values (?,?,?,?)", media) - return cnt diff --git a/pylib/anki/sync.py b/pylib/anki/sync.py index a3329128d..41d138961 100644 --- a/pylib/anki/sync.py +++ b/pylib/anki/sync.py @@ -13,12 +13,11 @@ from typing import Any, Dict, List, Optional, Tuple, Union import anki from anki.consts import * -from anki.db import DB, DBError +from anki.db import DB from anki.utils import checksum, devMode, ids2str, intTime, platDesc, versionWithBuild from . import hooks from .httpclient import HttpClient -from .lang import ngettext # add-on compat AnkiRequestsClient = HttpClient @@ -679,207 +678,3 @@ class FullSyncer(HttpSyncer): if self.req("upload", open(self.col.path, "rb")) != b"OK": return False return True - - -# Media syncing -########################################################################## -# -# About conflicts: -# - to minimize data loss, if both sides are marked for sending and one -# side has been deleted, favour the add -# - if added/changed on both sides, favour the server version on the -# assumption other syncers are in sync with the server -# - - -class MediaSyncer: - def __init__(self, col, server=None) -> None: - self.col = col - self.server = server - self.downloadCount = 0 - - def sync(self) -> Any: - # check if there have been any changes - hooks.sync_stage_did_change("findMedia") - self.col.log("findChanges") - try: - self.col.media.findChanges() - except DBError: - return "corruptMediaDB" - - # begin session and check if in sync - lastUsn = self.col.media.lastUsn() - ret = self.server.begin() - srvUsn = ret["usn"] - if lastUsn == srvUsn and not self.col.media.haveDirty(): - return "noChanges" - - # loop through and process changes from server - self.col.log("last local usn is %s" % lastUsn) - while True: - data = self.server.mediaChanges(lastUsn=lastUsn) - - self.col.log("mediaChanges resp count %d" % len(data)) - if not data: - break - - need = [] - lastUsn = data[-1][1] - for fname, rusn, rsum in data: - lsum, ldirty = self.col.media.syncInfo(fname) - self.col.log( - "check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s" - % ((lsum and lsum[0:4]), (rsum and rsum[0:4]), ldirty, rusn, fname) - ) - - if rsum: - # added/changed remotely - if not lsum or lsum != rsum: - self.col.log("will fetch") - need.append(fname) - else: - self.col.log("have same already") - if ldirty: - self.col.media.markClean([fname]) - elif lsum: - # deleted remotely - if not ldirty: - self.col.log("delete local") - self.col.media.syncDelete(fname) - else: - # conflict; local add overrides remote delete - self.col.log("conflict; will send") - else: - # deleted both sides - self.col.log("both sides deleted") - if ldirty: - self.col.media.markClean([fname]) - - self._downloadFiles(need) - - self.col.log("update last usn to %d" % lastUsn) - self.col.media.setLastUsn(lastUsn) # commits - - # at this point we're all up to date with the server's changes, - # and we need to send our own - - updateConflict = False - toSend = self.col.media.dirtyCount() - while True: - zip, fnames = self.col.media.mediaChangesZip() - if not fnames: - break - - hooks.sync_progress_did_change( - ngettext( - "%d media change to upload", "%d media changes to upload", toSend - ) - % toSend, - ) - - processedCnt, serverLastUsn = self.server.uploadChanges(zip) - self.col.media.markClean(fnames[0:processedCnt]) - - self.col.log( - "processed %d, serverUsn %d, clientUsn %d" - % (processedCnt, serverLastUsn, lastUsn) - ) - - if serverLastUsn - processedCnt == lastUsn: - self.col.log("lastUsn in sync, updating local") - lastUsn = serverLastUsn - self.col.media.setLastUsn(serverLastUsn) # commits - else: - self.col.log("concurrent update, skipping usn update") - # commit for markClean - self.col.media.db.commit() - updateConflict = True - - toSend -= processedCnt - - if updateConflict: - self.col.log("restart sync due to concurrent update") - return self.sync() - - lcnt = self.col.media.mediaCount() - ret = self.server.mediaSanity(local=lcnt) - if ret == "OK": - return "OK" - else: - self.col.media.forceResync() - return ret - - def _downloadFiles(self, fnames) -> None: - self.col.log("%d files to fetch" % len(fnames)) - while fnames: - top = fnames[0:SYNC_ZIP_COUNT] - self.col.log("fetch %s" % top) - zipData = self.server.downloadFiles(files=top) - cnt = self.col.media.addFilesFromZip(zipData) - self.downloadCount += cnt - self.col.log("received %d files" % cnt) - fnames = fnames[cnt:] - - n = self.downloadCount - hooks.sync_progress_did_change( - ngettext("%d media file downloaded", "%d media files downloaded", n) - % n, - ) - - -# Remote media syncing -########################################################################## - - -class RemoteMediaServer(HttpSyncer): - def __init__(self, col, hkey, client, hostNum) -> None: - self.col = col - HttpSyncer.__init__(self, hkey, client, hostNum=hostNum) - self.prefix = "msync/" - - def begin(self) -> Any: - self.postVars = dict( - k=self.hkey, v="ankidesktop,%s,%s" % (anki.version, platDesc()) - ) - ret = self._dataOnly( - self.req("begin", io.BytesIO(json.dumps(dict()).encode("utf8"))) - ) - self.skey = ret["sk"] - return ret - - # args: lastUsn - def mediaChanges(self, **kw) -> Any: - self.postVars = dict(sk=self.skey,) - return self._dataOnly( - self.req("mediaChanges", io.BytesIO(json.dumps(kw).encode("utf8"))) - ) - - # args: files - def downloadFiles(self, **kw) -> Any: - return self.req("downloadFiles", io.BytesIO(json.dumps(kw).encode("utf8"))) - - def uploadChanges(self, zip) -> Any: - # no compression, as we compress the zip file instead - return self._dataOnly(self.req("uploadChanges", io.BytesIO(zip), comp=0)) - - # args: local - def mediaSanity(self, **kw) -> Any: - return self._dataOnly( - self.req("mediaSanity", io.BytesIO(json.dumps(kw).encode("utf8"))) - ) - - def _dataOnly(self, resp) -> Any: - resp = json.loads(resp.decode("utf8")) - if resp["err"]: - self.col.log("error returned:%s" % resp["err"]) - raise Exception("SyncError:%s" % resp["err"]) - return resp["data"] - - # only for unit tests - def mediatest(self, cmd) -> Any: - self.postVars = dict(k=self.hkey,) - return self._dataOnly( - self.req( - "newMediaTest", io.BytesIO(json.dumps(dict(cmd=cmd)).encode("utf8")) - ) - ) diff --git a/qt/aqt/__init__.py b/qt/aqt/__init__.py index a2c3c10b7..21b3aec24 100644 --- a/qt/aqt/__init__.py +++ b/qt/aqt/__init__.py @@ -76,7 +76,7 @@ class DialogManager: "DeckStats": [stats.DeckStats, None], "About": [about.show, None], "Preferences": [preferences.Preferences, None], - "sync_log": [mediasync.MediaSyncDialog, None] + "sync_log": [mediasync.MediaSyncDialog, None], } def open(self, name, *args): diff --git a/qt/aqt/main.py b/qt/aqt/main.py index a7b7bbd46..7f71a3530 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -36,7 +36,7 @@ from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields from aqt import gui_hooks from aqt.addons import DownloadLogEntry, check_and_prompt_for_updates, show_log_to_user from aqt.legacy import install_pylib_legacy -from aqt.mediasync import MediaSyncDialog, MediaSyncer +from aqt.mediasync import MediaSyncer from aqt.profiles import ProfileManager as ProfileManagerType from aqt.qt import * from aqt.qt import sip @@ -870,14 +870,14 @@ title="%s" %s>%s""" % ( # fixme: shard # fixme: dialog # fixme: autosync -# elif evt == "mediaSanity": -# showWarning( -# _( -# """\ -# A problem occurred while syncing media. Please use Tools>Check Media, then \ -# sync again to correct the issue.""" -# ) -# ) + # elif evt == "mediaSanity": + # showWarning( + # _( + # """\ + # A problem occurred while syncing media. Please use Tools>Check Media, then \ + # sync again to correct the issue.""" + # ) + # ) def _sync_media(self): self.media_syncer.start(self.col, self.pm.sync_key(), None) diff --git a/qt/aqt/mediasync.py b/qt/aqt/mediasync.py index 5ee572078..84f3ec71a 100644 --- a/qt/aqt/mediasync.py +++ b/qt/aqt/mediasync.py @@ -7,7 +7,7 @@ import time from concurrent.futures import Future from copy import copy from dataclasses import dataclass -from typing import List, Optional, Union, Callable +from typing import Callable, List, Optional, Union import anki import aqt @@ -27,7 +27,7 @@ from anki.rsbackend import ( from anki.types import assert_impossible from anki.utils import intTime from aqt import gui_hooks -from aqt.qt import QDialog, QDialogButtonBox, QPushButton, QWidget +from aqt.qt import QDialog, QDialogButtonBox, QPushButton from aqt.taskman import TaskManager diff --git a/qt/aqt/sync.py b/qt/aqt/sync.py index dde492694..fdbb7ef1f 100644 --- a/qt/aqt/sync.py +++ b/qt/aqt/sync.py @@ -7,7 +7,7 @@ import time from anki import hooks from anki.lang import _ from anki.storage import Collection -from anki.sync import FullSyncer, MediaSyncer, RemoteMediaServer, RemoteServer, Syncer +from anki.sync import FullSyncer, RemoteServer, Syncer from aqt.qt import * from aqt.utils import askUserDialog, showInfo, showText, showWarning, tooltip