From d7e452de1f40869769c4d47d3f9bc20f03c3a53e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 08:33:54 +0900 Subject: [PATCH 01/11] fix note types with missing reqs --- anki/collection.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/anki/collection.py b/anki/collection.py index f04ab3eaf..4f23d765e 100644 --- a/anki/collection.py +++ b/anki/collection.py @@ -695,6 +695,10 @@ select id from notes where mid not in """ + ids2str(self.models.ids())) self.remNotes(ids) # for each model for m in self.models.all(): + # model with missing req specification + if 'req' not in m: + self.models._updateRequired(m) + problems.append(_("Fixed note type: %s") % m['name']) # cards with invalid ordinal if m['type'] == MODEL_STD: ids = self.db.list(""" From 1df385db12fbba9e748bd1815345d0650b10a023 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 09:06:48 +0900 Subject: [PATCH 02/11] cards must be removed from filtered decks before they're buried if not, removeLrn() resets due=odue and odue=0, leading to an invalid delay calculation when they're later reviewed in the filtered deck to fix this we'll need to make the same changes required to support learning cards retaining their state when being emptied from a filtered deck --- anki/sched.py | 1 + 1 file changed, 1 insertion(+) diff --git a/anki/sched.py b/anki/sched.py index 9c05df4e9..6ebf71fe7 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -1279,6 +1279,7 @@ To study outside of the normal schedule, click the Custom Study button below.""" def buryCards(self, cids): self.col.log(cids) + self.remFromDyn(cids) self.removeLrn(cids) self.col.db.execute(""" update cards set queue=-2,mod=?,usn=? where id in """+ids2str(cids), From 927e618f53e56d6909ccb2b6c722feaf12b33a73 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 14:56:37 +0900 Subject: [PATCH 03/11] disable plastique theme on osx as possible crash fix --- aqt/editor.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/aqt/editor.py b/aqt/editor.py index 8fd80c6f1..4c34dd1c1 100644 --- a/aqt/editor.py +++ b/aqt/editor.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -from anki.lang import _ +import re +import os +import urllib2 +import ctypes +import urllib +from anki.lang import _ from aqt.qt import * -import re, os, urllib2, ctypes from anki.utils import stripHTML, isWin, isMac, namedtmp, json, stripHTMLMedia import anki.sound from anki.hooks import runHook, runFilter @@ -15,7 +19,6 @@ from aqt.utils import shortcut, showInfo, showWarning, getBase, getFile, \ import aqt import anki.js from BeautifulSoup import BeautifulSoup -import urllib pics = ("jpg", "jpeg", "png", "tif", "tiff", "gif", "svg") audio = ("wav", "mp3", "ogg", "flac", "mp4", "swf", "mov", "mpeg", "mkv") @@ -313,7 +316,8 @@ class Editor(object): b.setFixedHeight(20) b.setFixedWidth(20) if not native: - b.setStyle(self.plastiqueStyle) + if self.plastiqueStyle: + b.setStyle(self.plastiqueStyle) b.setFocusPolicy(Qt.NoFocus) else: b.setAutoDefault(False) @@ -333,18 +337,22 @@ class Editor(object): def setupButtons(self): self._buttons = {} # button styles for mac - self.plastiqueStyle = QStyleFactory.create("plastique") - if not self.plastiqueStyle: - # plastique was removed in qt5 - self.plastiqueStyle = QStyleFactory.create("fusion") - self.widget.setStyle(self.plastiqueStyle) + if not isMac: + self.plastiqueStyle = QStyleFactory.create("plastique") + if not self.plastiqueStyle: + # plastique was removed in qt5 + self.plastiqueStyle = QStyleFactory.create("fusion") + self.widget.setStyle(self.plastiqueStyle) + else: + self.plastiqueStyle = None # icons self.iconsBox = QHBoxLayout() if not isMac: self.iconsBox.setMargin(6) + self.iconsBox.setSpacing(0) else: self.iconsBox.setMargin(0) - self.iconsBox.setSpacing(0) + self.iconsBox.setSpacing(14) self.outerLayout.addLayout(self.iconsBox) b = self._addButton b("fields", self.onFields, "", From 9c678c32ad8bd552f7c0f8fb2e1819acf6b217a4 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 15:15:03 +0900 Subject: [PATCH 04/11] further simplify augmented httplib2 conn_request --- aqt/sync.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/aqt/sync.py b/aqt/sync.py index 24c5a1363..0d6cc51e5 100644 --- a/aqt/sync.py +++ b/aqt/sync.py @@ -421,7 +421,7 @@ class SyncThread(QThread): ###################################################################### CHUNK_SIZE = 65536 -import httplib, httplib2, errno +import httplib, httplib2 from cStringIO import StringIO from anki.hooks import runHook @@ -448,6 +448,9 @@ def _incrementalSend(self, data): httplib.HTTPConnection.send = _incrementalSend # receiving in httplib2 +# this is an augmented version of httplib's request routine that: +# - doesn't assume requests will be tried more than once +# - calls a hook for each chunk of data so we can update the gui def _conn_request(self, conn, request_uri, method, body, headers): try: if conn.sock is None: @@ -463,19 +466,11 @@ def _conn_request(self, conn, request_uri, method, body, headers): conn.close() raise except socket.error, e: - err = 0 - if hasattr(e, 'args'): - err = getattr(e, 'args')[0] - else: - err = e.errno - if err == errno.ECONNREFUSED: # Connection refused - raise + conn.close() + raise except httplib.HTTPException: - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - if conn.sock is None: - conn.close() - raise + conn.close() + raise try: response = conn.getresponse() except (socket.error, httplib.HTTPException): From 9f548ad85afe214cec9c0e3462ec67db98083cd5 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 15:29:50 +0900 Subject: [PATCH 05/11] recover from a corrupt prefs.db that fails to load at all --- aqt/profiles.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aqt/profiles.py b/aqt/profiles.py index 834560c7b..7c0e6cc74 100644 --- a/aqt/profiles.py +++ b/aqt/profiles.py @@ -6,8 +6,13 @@ # - Saves in pickles rather than json to easily store Qt window state. # - Saves in sqlite rather than a flat file so the config can't be corrupted +import os +import random +import cPickle +import locale +import re + from aqt.qt import * -import os, random, cPickle, shutil, locale, re from anki.db import DB from anki.utils import isMac, isWin, intTime, checksum from anki.lang import langs @@ -16,6 +21,7 @@ from aqt import appHelpSite import aqt.forms from send2trash import send2trash + metaConf = dict( ver=0, updates=True, @@ -186,7 +192,6 @@ documentation for information on using a flash drive.""") def _loadMeta(self): path = os.path.join(self.base, "prefs.db") new = not os.path.exists(path) - self.db = DB(path, text=str) def recover(): # if we can't load profile, start with a new one os.rename(path, path+".broken") @@ -195,6 +200,7 @@ documentation for information on using a flash drive.""") Anki's prefs.db file was corrupt and has been recreated. If you were using multiple \ profiles, please add them back using the same names to recover your cards.""") try: + self.db = DB(path, text=str) self.db.execute(""" create table if not exists profiles (name text primary key, data text not null);""") From ef9157a8ee2a61fa3ca74d43e0acdbb192255175 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 16:48:22 +0900 Subject: [PATCH 06/11] don't open log for export or upgrade, only regular+sync --- anki/collection.py | 9 ++++----- anki/storage.py | 10 +++++++--- aqt/main.py | 4 +--- aqt/sync.py | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/anki/collection.py b/anki/collection.py index 4f23d765e..4b6bb7c01 100644 --- a/anki/collection.py +++ b/anki/collection.py @@ -51,9 +51,8 @@ defaultConf = { # this is initialized by storage.Collection class _Collection(object): - debugLog = False - - def __init__(self, db, server=False): + def __init__(self, db, server=False, log=False): + self._debugLog = log self.db = db self.path = db._path self._openLog() @@ -783,7 +782,7 @@ and queue = 0""", intTime(), self.usn()) ########################################################################## def log(self, *args, **kwargs): - if not self.debugLog: + if not self._debugLog: return def customRepr(x): if isinstance(x, basestring): @@ -798,7 +797,7 @@ and queue = 0""", intTime(), self.usn()) print buf def _openLog(self): - if not self.debugLog: + if not self._debugLog: return lpath = re.sub("\.anki2$", ".log", self.path) self._logHnd = open(lpath, "ab") diff --git a/anki/storage.py b/anki/storage.py index bd2d9a1b7..d83fef1ca 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -2,7 +2,10 @@ # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import os, copy, re +import os +import copy +import re + from anki.lang import _ from anki.utils import intTime, json from anki.db import DB @@ -11,7 +14,8 @@ from anki.consts import * from anki.stdmodels import addBasicModel, addClozeModel, addForwardReverse, \ addForwardOptionalReverse -def Collection(path, lock=True, server=False, sync=True): + +def Collection(path, lock=True, server=False, sync=True, log=False): "Open a new or existing collection. Path must be unicode." assert path.endswith(".anki2") path = os.path.abspath(path) @@ -33,7 +37,7 @@ def Collection(path, lock=True, server=False, sync=True): else: db.execute("pragma synchronous = off") # add db to col and do any remaining upgrades - col = _Collection(db, server) + col = _Collection(db, server, log) if ver < SCHEMA_VERSION: _upgrade(col, ver) elif create: diff --git a/aqt/main.py b/aqt/main.py index a8133f561..2b0f840f8 100644 --- a/aqt/main.py +++ b/aqt/main.py @@ -31,8 +31,6 @@ class AnkiQt(QMainWindow): self.state = "startup" aqt.mw = self self.app = app - from anki.collection import _Collection - _Collection.debugLog = True if isWin: self._xpstyle = QStyleFactory.create("WindowsXP") self.app.setStyle(self._xpstyle) @@ -270,7 +268,7 @@ To import into a password protected profile, please open the profile before atte def loadCollection(self): self.hideSchemaMsg = True try: - self.col = Collection(self.pm.collectionPath()) + self.col = Collection(self.pm.collectionPath(), log=True) except anki.db.Error: # move back to profile manager showWarning("""\ diff --git a/aqt/sync.py b/aqt/sync.py index 0d6cc51e5..8fc9bb1f1 100644 --- a/aqt/sync.py +++ b/aqt/sync.py @@ -280,7 +280,7 @@ class SyncThread(QThread): self.syncMsg = "" self.uname = "" try: - self.col = Collection(self.path) + self.col = Collection(self.path, log=True) except: self.fireEvent("corrupt") return From c3300f733a9d470be6bbd357f8657b8fd0f8001e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 16:49:49 +0900 Subject: [PATCH 07/11] make sure we don't 'fix' req for cloze type --- anki/collection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/anki/collection.py b/anki/collection.py index 4b6bb7c01..2bda2f64f 100644 --- a/anki/collection.py +++ b/anki/collection.py @@ -694,12 +694,12 @@ select id from notes where mid not in """ + ids2str(self.models.ids())) self.remNotes(ids) # for each model for m in self.models.all(): - # model with missing req specification - if 'req' not in m: - self.models._updateRequired(m) - problems.append(_("Fixed note type: %s") % m['name']) - # cards with invalid ordinal if m['type'] == MODEL_STD: + # model with missing req specification + if 'req' not in m: + self.models._updateRequired(m) + problems.append(_("Fixed note type: %s") % m['name']) + # cards with invalid ordinal ids = self.db.list(""" select id from cards where ord not in %s and nid in ( select id from notes where mid = ?)""" % From 853faa90cd3fad80b796bc9d5613178b9410eeeb Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 16:51:40 +0900 Subject: [PATCH 08/11] don't bother logging sortCards() --- anki/sched.py | 1 - 1 file changed, 1 deletion(-) diff --git a/anki/sched.py b/anki/sched.py index 6ebf71fe7..2f52e557e 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -1415,7 +1415,6 @@ and due >= ? and queue = 0""" % scids, now, self.col.usn(), shiftby, low) d.append(dict(now=now, due=due[nid], usn=self.col.usn(), cid=id)) self.col.db.executemany( "update cards set due=:due,mod=:now,usn=:usn where id = :cid", d) - self.col.log(cids) def randomizeCards(self, did): cids = self.col.db.list("select id from cards where did = ?", did) From 9334bc37fa6644c14a95ac6febfa2cd63894db15 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 16:54:25 +0900 Subject: [PATCH 09/11] log media sanity --- anki/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/anki/sync.py b/anki/sync.py index 9ae76df61..39d7387af 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -752,6 +752,7 @@ class MediaSyncer(object): # back from sanity check to addFiles s = self.server.mediaSanity() c = self.mediaSanity() + self.col.log("mediaSanity", c, s) if c != s: # if the sanity check failed, force a resync self.col.media.forceResync() From 75f87201a22e402adeda4bb4092625f4ceb2638f Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 17:00:20 +0900 Subject: [PATCH 10/11] rotate log file when it hits 10MB --- anki/collection.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/anki/collection.py b/anki/collection.py index 2bda2f64f..706c0ff78 100644 --- a/anki/collection.py +++ b/anki/collection.py @@ -800,6 +800,11 @@ and queue = 0""", intTime(), self.usn()) if not self._debugLog: return lpath = re.sub("\.anki2$", ".log", self.path) + if os.path.exists(lpath) and os.path.getsize(lpath) > 10*1024*1024: + lpath2 = lpath + ".old" + if os.path.exists(lpath2): + os.unlink(lpath2) + os.rename(lpath, lpath2) self._logHnd = open(lpath, "ab") def _closeLog(self): From f6b9dadf135245c778124b4866a41b2b0d6ca2bf Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 13 Nov 2013 17:19:25 +0900 Subject: [PATCH 11/11] catch invalid file encodings in media check & sync --- anki/media.py | 9 ++++++++- aqt/main.py | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/anki/media.py b/anki/media.py index d44065117..822982050 100644 --- a/anki/media.py +++ b/anki/media.py @@ -213,6 +213,7 @@ class MediaManager(object): allRefs.update(noteRefs) # loop through media folder unused = [] + invalid = [] if local is None: files = os.listdir(mdir) else: @@ -225,6 +226,9 @@ class MediaManager(object): if file.startswith("_"): # leading _ says to ignore file continue + if not isinstance(file, unicode): + invalid.append(unicode(file, sys.getfilesystemencoding(), "replace")) + continue nfcFile = unicodedata.normalize("NFC", file) # we enforce NFC fs encoding on non-macs; on macs we'll have gotten # NFD so we use the above variable for comparing references @@ -242,7 +246,7 @@ class MediaManager(object): else: allRefs.discard(nfcFile) nohave = [x for x in allRefs if not x.startswith("_")] - return (nohave, unused) + return (nohave, unused, invalid) def _normalizeNoteRefs(self, nid): note = self.col.getNote(nid) @@ -336,6 +340,9 @@ class MediaManager(object): return re.sub(self._illegalCharReg, "", str) def hasIllegal(self, str): + # a file that couldn't be decoded to unicode is considered invalid + if not isinstance(str, unicode): + return False return not not re.search(self._illegalCharReg, str) # Media syncing - bundling zip files to send to server diff --git a/aqt/main.py b/aqt/main.py index 2b0f840f8..395346e72 100644 --- a/aqt/main.py +++ b/aqt/main.py @@ -913,11 +913,16 @@ will be lost. Continue?""")) def onCheckMediaDB(self): self.progress.start(immediate=True) - (nohave, unused) = self.col.media.check() + (nohave, unused, invalid) = self.col.media.check() self.progress.finish() # generate report report = "" + if invalid: + report += _("Invalid encoding; please rename:") + report += "\n" + "\n".join(invalid) if unused: + if report: + report += "\n\n\n" report += _( "In media folder but not used by any cards:") report += "\n" + "\n".join(unused)