From 240d311ca094e8803a59dddfcb23f53284091835 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Fri, 16 Jan 2009 18:17:36 +0900 Subject: [PATCH] optimize add/deleteTag(), add progress callbacks and check (media) db --- anki/deck.py | 40 ++++++++++++++++++++++++++++++++++------ anki/media.py | 11 +++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/anki/deck.py b/anki/deck.py index 9db1893c5..a6c24ebac 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -1413,10 +1413,11 @@ update facts set tags = :tags, modified = :now where id = :id""", pending) + factIds = [c['id'] for c in pending] cardIds = self.s.column0( "select id from cards where factId in %s" % - ids2str(ids)) - self.updateCardQACacheFromIds(ids, type="facts") + ids2str(factIds)) + self.updateCardQACacheFromIds(factIds, type="facts") self.updatePriorities(cardIds) self.flushMod() @@ -1441,13 +1442,27 @@ update facts set tags = :tags, modified = :now where id = :id""", pending) + factIds = [c['id'] for c in pending] cardIds = self.s.column0( "select id from cards where factId in %s" % - ids2str(ids)) - self.updateCardQACacheFromIds(ids, type="facts") + ids2str(factIds)) + self.updateCardQACacheFromIds(factIds, type="facts") self.updatePriorities(cardIds) self.flushMod() + # Progress info + ########################################################################## + + def startProgress(self, title, min, max): + runHook("startProgress", title, min, max) + + def updateProgress(self, label=None, value=None): + runHook("updateProgress", label, value) + + def finishProgress(self): + runHook("updateProgress") + runHook("finishProgress") + # File-related ########################################################################## @@ -1626,13 +1641,17 @@ Return new path, relative to media dir.""" def fixIntegrity(self): "Responsibility of caller to call rebuildQueue()" + self.startProgress(_("Check DB"), 0, 11) + self.updateProgress(_("Checking integrity...")) if self.s.scalar("pragma integrity_check") != "ok": return _("Database file damaged. Restore from backup.") # ensure correct views and indexes are available + self.updateProgress() DeckStorage._addViews(self) DeckStorage._addIndices(self) problems = [] # does the user have a model? + self.updateProgress(_("Checking schema...")) if not self.s.scalar("select count(id) from models"): self.addModel(BasicModel()) problems.append(_("Deck was missing a model")) @@ -1643,11 +1662,13 @@ decks.currentModelId = models.id"""): self.currentModelId = self.models[0].id problems.append(_("The current model didn't exist")) # forget all deletions (do this before deleting anything) + self.updateProgress() self.s.statement("delete from cardsDeleted") self.s.statement("delete from factsDeleted") self.s.statement("delete from modelsDeleted") self.s.statement("delete from mediaDeleted") # facts missing a field? + self.updateProgress() ids = self.s.column0(""" select distinct facts.id from facts, fieldModels where facts.modelId = fieldModels.modelId and fieldModels.id not in @@ -1685,6 +1706,7 @@ select id from fields where factId not in (select id from facts)""") problems.append(_("Deleted %d dangling fields") % len(ids)) self.s.flush() # fix problems with cards being scheduled when not due + self.updateProgress() self.s.statement("update cards set isDue = 0") # fix problems with conflicts on merge self.s.statement("update fields set id = random()") @@ -1694,8 +1716,10 @@ select id from fields where factId not in (select id from facts)""") "update cardModels set allowEmptyAnswer = 1, typeAnswer = 0 " "where allowEmptyAnswer is null or typeAnswer is null") # fix any priorities + self.updateProgress(_("Updating priorities...")) self.updateAllPriorities() # fix problems with stripping html + self.updateProgress(_("Rebuilding QA cache...")) fields = self.s.all("select id, value from fields") newFields = [] for (id, value) in fields: @@ -1712,14 +1736,18 @@ select id from fields where factId not in (select id from facts)""") self.s.statement("update facts set modified = :t", t=time.time()) self.s.statement("update models set modified = :t", t=time.time()) self.lastSync = 0 - # update counts + # rebuild + self.updateProgress(_("Rebuilding types...")) + self.rebuildTypes() + self.updateProgress(_("Rebuilding counts...")) self.rebuildCounts() # update deck and save self.flushMod() self.save() self.refresh() - self.rebuildTypes() + self.updateProgress(_("Rebuilding queue...")) self.rebuildQueue() + self.finishProgress() if problems: return "\n".join(problems) return "ok" diff --git a/anki/media.py b/anki/media.py index 78ca78c02..f846f769e 100644 --- a/anki/media.py +++ b/anki/media.py @@ -119,7 +119,9 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True): unusedFileCount = 0 missingFileCount = 0 deck.mediaDir(create=True) + deck.startProgress(_("Check Media DB"), 0, 6) # rename all files to checksum versions, note non-renamed ones + deck.updateProgress(_("Checksum files...")) for oldBase in os.listdir(unicode(deck.mediaDir())): oldPath = os.path.join(deck.mediaDir(), oldBase) if oldBase.startswith("."): @@ -132,6 +134,7 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True): else: renamedFiles[oldBase] = newBase # now look through all fields, and update references to files + deck.updateProgress(_("Scan fields...")) for (id, fid, val) in deck.s.all( "select id, factId, value from fields"): oldval = val @@ -158,16 +161,22 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True): if fid not in factsMissingMedia: unmodifiedFacts[fid] = 1 # update modified fields + deck.updateProgress(_("Modify fields...")) if modifiedFacts: _modifyFields(deck, updateFields, modifiedFacts, dirty) # fix tags + deck.updateProgress(_("Update tags...")) if dirty: + t = time.time() if deleteRefs: deck.deleteTags(modifiedFacts.keys(), _("Media Missing")) else: deck.addTags(factsMissingMedia.keys(), _("Media Missing")) + print time.time() - t; t = time.time() deck.deleteTags(unmodifiedFacts.keys(), _("Media Missing")) + print time.time() - t; t = time.time() # build cache of db records + deck.updateProgress(_("Delete unused files...")) mediaIds = dict(deck.s.all("select filename, id from media")) # assume latex files exist for f in deck.s.column0( @@ -186,6 +195,7 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True): else: os.unlink(path) unusedFileCount += 1 + deck.updateProgress(_("Delete stale references...")) for (fname, id) in mediaIds.items(): # maybe delete from db if id: @@ -196,6 +206,7 @@ values (:id, strftime('%s', 'now'))""", id=id) # update deck and save deck.flushMod() deck.save() + deck.finishProgress() return missingFileCount, unusedFileCount def mediaRefs(string):