From d43d9fd87adc7b1d13492605f8f1647ecc0b9d9a Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 22 Jul 2014 08:01:19 +0900 Subject: [PATCH 1/9] disable more useful msg in disk i/o error case --- aqt/errors.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/aqt/errors.py b/aqt/errors.py index 533b37f46..1ce1421e6 100644 --- a/aqt/errors.py +++ b/aqt/errors.py @@ -65,6 +65,23 @@ Anki manual for more information.""") "other programs are not using the audio device.")) if "invalidTempFolder" in error: return showWarning(self.tempFolderMsg()) + if "disk I/O error" in error: + return showWarning(_("""\ +An error occurred while accessing the database. + +Possible causes: + +- Antivirus, firewall, backup, or synchronization software may be \ + interfering with Anki. Try disabling such software and see if the \ + problem goes away. +- Your disk may be full. +- The Documents/Anki folder may be on a network drive. +- Files in the Documents/Anki folder may not be writeable. +- Your hard disk may have errors. + +It's a good idea to run Tools>Check Database to ensure your collection \ +is not corrupt. +""")) stdText = _("""\ An error occurred. It may have been caused by a harmless bug,
or your deck may have a problem. From 354fbd33f6df7ab6ea002e78a6133ce9d00755cb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 28 Jul 2014 14:26:40 +0900 Subject: [PATCH 2/9] fix media test --- tests/test_media.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_media.py b/tests/test_media.py index 243c21516..b18969b5f 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -71,9 +71,11 @@ def test_changes(): d = getEmptyCol() assert d.media._changed() def added(): - return d.media.db.execute("select fname from log where type = 0") + return d.media.db.execute("select fname from media where csum is not null") + def removed(): + return d.media.db.execute("select fname from media where csum is null") assert not list(added()) - assert not list(d.media.removed()) + assert not list(removed()) # add a file dir = tempfile.mkdtemp(prefix="anki") path = os.path.join(dir, u"foo.jpg") @@ -83,24 +85,24 @@ def test_changes(): # should have been logged d.media.findChanges() assert list(added()) - assert not list(d.media.removed()) + assert not list(removed()) # if we modify it, the cache won't notice time.sleep(1) open(path, "w").write("world") assert len(list(added())) == 1 - assert not list(d.media.removed()) + assert not list(removed()) # but if we add another file, it will time.sleep(1) open(path+"2", "w").write("yo") d.media.findChanges() assert len(list(added())) == 2 - assert not list(d.media.removed()) + assert not list(removed()) # deletions should get noticed too time.sleep(1) os.unlink(path+"2") d.media.findChanges() assert len(list(added())) == 1 - assert len(list(d.media.removed())) == 1 + assert len(list(removed())) == 1 def test_illegal(): d = getEmptyCol() From f5d60c70e2e9c2f32849b9261d13c2b783be7e6b Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 28 Jul 2014 14:28:12 +0900 Subject: [PATCH 3/9] remove unused functions --- anki/sync.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/anki/sync.py b/anki/sync.py index cab986cae..06939f1d1 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -833,13 +833,6 @@ class MediaSyncer(object): self.col.log("received %d files"%cnt) fnames = fnames[cnt:] - def files(self): - return self.col.media.addFilesToZip() - - def addFiles(self, zip): - "True if zip is the last in set. Server returns new usn instead." - return self.col.media.addFilesFromZip(zip) - # Remote media syncing ########################################################################## From cf801e4fb4c9792347192c8c78c09bcac652ba47 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 28 Jul 2014 17:00:26 +0900 Subject: [PATCH 4/9] display more feedback when syncing media deletes in particular take some time for the server to process, but don't require much bandwidth, leading to the progress appearing to have pause when content is actually being processed this also gives the user an idea of how long the process will take to complete --- anki/media.py | 4 ++++ anki/sync.py | 13 +++++++++++++ aqt/sync.py | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/anki/media.py b/anki/media.py index 7493964b1..b0b6d1c83 100644 --- a/anki/media.py +++ b/anki/media.py @@ -439,6 +439,10 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0); return self.db.scalar( "select count() from media where csum is not null") + def dirtyCount(self): + return self.db.scalar( + "select count() from media where dirty=1") + def forceResync(self): self.db.execute("delete from media") self.db.execute("update meta set lastUsn=0,dirMod=0") diff --git a/anki/sync.py b/anki/sync.py index 06939f1d1..ded617279 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -14,6 +14,7 @@ from anki.utils import ids2str, intTime, json, isWin, isMac, platDesc, checksum from anki.consts import * from hooks import runHook import anki +from lang import ngettext # syncing vars HTTP_TIMEOUT = 90 @@ -789,11 +790,16 @@ class MediaSyncer(object): # and we need to send our own updateConflict = False + toSend = self.col.media.dirtyCount() while True: zip, fnames = self.col.media.mediaChangesZip() if not fnames: break + runHook("syncMsg", ngettext( + "%d media change to upload", "%d media changes to upload", toSend) + % toSend) + processedCnt, serverLastUsn = self.server.uploadChanges(zip) self.col.media.markClean(fnames[0:processedCnt]) @@ -811,6 +817,8 @@ class MediaSyncer(object): self.col.media.db.commit() updateConflict = True + toSend -= processedCnt + if updateConflict: self.col.log("restart sync due to concurrent update") return self.sync() @@ -826,6 +834,11 @@ class MediaSyncer(object): def _downloadFiles(self, fnames): self.col.log("%d files to fetch"%len(fnames)) while fnames: + n = len(fnames) + runHook("syncMsg", ngettext( + "%d media file to download", "%d media files to download", n) + % n) + top = fnames[0:SYNC_ZIP_COUNT] self.col.log("fetch %s"%top) zipData = self.server.downloadFiles(files=top) diff --git a/aqt/sync.py b/aqt/sync.py index f56482266..e3bc5d517 100644 --- a/aqt/sync.py +++ b/aqt/sync.py @@ -116,6 +116,9 @@ Please visit AnkiWeb, upgrade your deck, then try again.""")) if m: self.label = m self._updateLabel() + elif evt == "syncMsg": + self.label = args[0] + self._updateLabel() elif evt == "error": self._didError = True showText(_("Syncing failed:\n%s")% @@ -296,6 +299,8 @@ class SyncThread(QThread): self.byteUpdate = time.time() def syncEvent(type): self.fireEvent("sync", type) + def syncMsg(msg): + self.fireEvent("syncMsg", msg) def canPost(): if (time.time() - self.byteUpdate) > 0.1: self.byteUpdate = time.time() @@ -309,6 +314,7 @@ class SyncThread(QThread): if canPost(): self.fireEvent("recv", self.recvTotal) addHook("sync", syncEvent) + addHook("syncMsg", syncMsg) addHook("httpSend", sendEvent) addHook("httpRecv", recvEvent) # run sync and catch any errors @@ -323,6 +329,7 @@ class SyncThread(QThread): # don't bump mod time unless we explicitly save self.col.close(save=False) remHook("sync", syncEvent) + remHook("syncMsg", syncMsg) remHook("httpSend", sendEvent) remHook("httpRecv", recvEvent) From 3ee193731007ca8df28cbe9534218a70d97599fa Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Tue, 29 Jul 2014 07:37:30 +0900 Subject: [PATCH 5/9] we need to count up for downloads as we're streaming changes we don't know the total amount of downloads required --- anki/sync.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/anki/sync.py b/anki/sync.py index ded617279..2b9dc3b26 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -741,6 +741,7 @@ class MediaSyncer(object): # loop through and process changes from server self.col.log("last local usn is %s"%lastUsn) + self.downloadCount = 0 while True: data = self.server.mediaChanges(lastUsn=lastUsn) @@ -834,18 +835,19 @@ class MediaSyncer(object): def _downloadFiles(self, fnames): self.col.log("%d files to fetch"%len(fnames)) while fnames: - n = len(fnames) - runHook("syncMsg", ngettext( - "%d media file to download", "%d media files to download", n) - % n) - top = fnames[0:SYNC_ZIP_COUNT] self.col.log("fetch %s"%top) zipData = self.server.downloadFiles(files=top) cnt = self.col.media.addFilesFromZip(zipData) + self.downloadCount += cnt self.col.log("received %d files"%cnt) fnames = fnames[cnt:] + n = self.downloadCount + runHook("syncMsg", ngettext( + "%d media file downloaded", "%d media files downloaded", n) + % n) + # Remote media syncing ########################################################################## From dd2b6cb07d514fb2b2e5e33f8ad51156055bb1b8 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 30 Jul 2014 04:32:18 +0900 Subject: [PATCH 6/9] ignore >100MB files --- anki/media.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/anki/media.py b/anki/media.py index b0b6d1c83..0c39687ff 100644 --- a/anki/media.py +++ b/anki/media.py @@ -381,9 +381,13 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0); if self.hasIllegal(f): continue # empty files are invalid; clean them up and continue - if not os.path.getsize(f): + sz = os.path.getsize(f) + if not sz: os.unlink(f) continue + if sz > 100*1024*1024: + self.col.log("ignoring file over 100MB", f) + continue # check encoding if not isMac: normf = unicodedata.normalize("NFC", f) From f8bf8afe4a09f0f21db280e34ae58e18d71b9e72 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 1 Aug 2014 09:37:23 +0900 Subject: [PATCH 7/9] Revert "remove urllib.unquote() step in editor" This reverts commit 23cec2d5e994110665862815add56c4a6e7d7791. without other changes, this causes double escaping when editing --- aqt/editor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aqt/editor.py b/aqt/editor.py index 9b102b88c..bd509338c 100644 --- a/aqt/editor.py +++ b/aqt/editor.py @@ -449,6 +449,9 @@ class Editor(object): txt = self.mungeHTML(txt) # misbehaving apps may include a null byte in the text txt = txt.replace("\x00", "") + # reverse the url quoting we added to get images to display + txt = unicode(urllib2.unquote( + txt.encode("utf8")), "utf8", "replace") self.note.fields[self.currentField] = txt if not self.addMode: self.note.flush() From d53346d783906d827707fe9cc86fb073983dffa6 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 1 Aug 2014 09:42:28 +0900 Subject: [PATCH 8/9] limit url unquoting to image tags this prevents random text like %20 in a field from being converted when note is saved --- anki/media.py | 8 ++++++-- aqt/editor.py | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/anki/media.py b/anki/media.py index 0c39687ff..d19295e99 100644 --- a/anki/media.py +++ b/anki/media.py @@ -221,14 +221,18 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0); txt = re.sub(reg, "", txt) return txt - def escapeImages(self, string): + def escapeImages(self, string, unescape=False): + if unescape: + fn = urllib.unquote + else: + fn = urllib.quote def repl(match): tag = match.group(0) fname = match.group("fname") if re.match("(https?|ftp)://", fname): return tag return tag.replace( - fname, urllib.quote(fname.encode("utf-8"))) + fname, fn(fname.encode("utf-8"))) for reg in self.imgRegexps: string = re.sub(reg, repl, string) return string diff --git a/aqt/editor.py b/aqt/editor.py index bd509338c..b83c74678 100644 --- a/aqt/editor.py +++ b/aqt/editor.py @@ -450,8 +450,7 @@ class Editor(object): # misbehaving apps may include a null byte in the text txt = txt.replace("\x00", "") # reverse the url quoting we added to get images to display - txt = unicode(urllib2.unquote( - txt.encode("utf8")), "utf8", "replace") + txt = self.mw.col.media.escapeImages(txt, unescape=True) self.note.fields[self.currentField] = txt if not self.addMode: self.note.flush() From 2dd28d86a224ee4ed8852b1f1738b9ad34a2e7b1 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Mon, 4 Aug 2014 12:54:54 +0900 Subject: [PATCH 9/9] we shouldn't encode to utf8 when unquoting --- anki/media.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/anki/media.py b/anki/media.py index d19295e99..00971b729 100644 --- a/anki/media.py +++ b/anki/media.py @@ -222,17 +222,16 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0); return txt def escapeImages(self, string, unescape=False): - if unescape: - fn = urllib.unquote - else: - fn = urllib.quote def repl(match): tag = match.group(0) fname = match.group("fname") if re.match("(https?|ftp)://", fname): return tag - return tag.replace( - fname, fn(fname.encode("utf-8"))) + if unescape: + txt = urllib.unquote(fname) + else: + txt = urllib.quote(fname.encode("utf-8")) + return tag.replace(fname, txt) for reg in self.imgRegexps: string = re.sub(reg, repl, string) return string