optimize add/deleteTag(), add progress callbacks and check (media) db

anki/deck.py

@@ -1413,10 +1413,11 @@ update facts set
 tags = :tags,
 modified = :now
 where id = :id""", pending)
+        factIds = [c['id'] for c in pending]
         cardIds = self.s.column0(
             "select id from cards where factId in %s" %
-            ids2str(ids))
+            ids2str(factIds))
-        self.updateCardQACacheFromIds(ids, type="facts")
+        self.updateCardQACacheFromIds(factIds, type="facts")
         self.updatePriorities(cardIds)
         self.flushMod()
 
@@ -1441,13 +1442,27 @@ update facts set
 tags = :tags,
 modified = :now
 where id = :id""", pending)
+        factIds = [c['id'] for c in pending]
         cardIds = self.s.column0(
             "select id from cards where factId in %s" %
-            ids2str(ids))
+            ids2str(factIds))
-        self.updateCardQACacheFromIds(ids, type="facts")
+        self.updateCardQACacheFromIds(factIds, type="facts")
         self.updatePriorities(cardIds)
         self.flushMod()
 
+    # Progress info
+    ##########################################################################
+
+    def startProgress(self, title, min, max):
+        runHook("startProgress", title, min, max)
+
+    def updateProgress(self, label=None, value=None):
+        runHook("updateProgress", label, value)
+
+    def finishProgress(self):
+        runHook("updateProgress")
+        runHook("finishProgress")
+
     # File-related
     ##########################################################################
 
@@ -1626,13 +1641,17 @@ Return new path, relative to media dir."""
 
     def fixIntegrity(self):
         "Responsibility of caller to call rebuildQueue()"
+        self.startProgress(_("Check DB"), 0, 11)
+        self.updateProgress(_("Checking integrity..."))
         if self.s.scalar("pragma integrity_check") != "ok":
             return _("Database file damaged. Restore from backup.")
         # ensure correct views and indexes are available
+        self.updateProgress()
         DeckStorage._addViews(self)
         DeckStorage._addIndices(self)
         problems = []
         # does the user have a model?
+        self.updateProgress(_("Checking schema..."))
         if not self.s.scalar("select count(id) from models"):
             self.addModel(BasicModel())
             problems.append(_("Deck was missing a model"))
@@ -1643,11 +1662,13 @@ decks.currentModelId = models.id"""):
             self.currentModelId = self.models[0].id
             problems.append(_("The current model didn't exist"))
         # forget all deletions (do this before deleting anything)
+        self.updateProgress()
         self.s.statement("delete from cardsDeleted")
         self.s.statement("delete from factsDeleted")
         self.s.statement("delete from modelsDeleted")
         self.s.statement("delete from mediaDeleted")
         # facts missing a field?
+        self.updateProgress()
         ids = self.s.column0("""
 select distinct facts.id from facts, fieldModels where
 facts.modelId = fieldModels.modelId and fieldModels.id not in
@@ -1685,6 +1706,7 @@ select id from fields where factId not in (select id from facts)""")
             problems.append(_("Deleted %d dangling fields") % len(ids))
         self.s.flush()
         # fix problems with cards being scheduled when not due
+        self.updateProgress()
         self.s.statement("update cards set isDue = 0")
         # fix problems with conflicts on merge
         self.s.statement("update fields set id = random()")
@@ -1694,8 +1716,10 @@ select id from fields where factId not in (select id from facts)""")
             "update cardModels set allowEmptyAnswer = 1, typeAnswer = 0 "
             "where allowEmptyAnswer is null or typeAnswer is null")
         # fix any priorities
+        self.updateProgress(_("Updating priorities..."))
         self.updateAllPriorities()
         # fix problems with stripping html
+        self.updateProgress(_("Rebuilding QA cache..."))
         fields = self.s.all("select id, value from fields")
         newFields = []
         for (id, value) in fields:
@@ -1712,14 +1736,18 @@ select id from fields where factId not in (select id from facts)""")
         self.s.statement("update facts set modified = :t", t=time.time())
         self.s.statement("update models set modified = :t", t=time.time())
         self.lastSync = 0
-        # update counts
+        # rebuild
+        self.updateProgress(_("Rebuilding types..."))
+        self.rebuildTypes()
+        self.updateProgress(_("Rebuilding counts..."))
         self.rebuildCounts()
         # update deck and save
         self.flushMod()
         self.save()
         self.refresh()
-        self.rebuildTypes()
+        self.updateProgress(_("Rebuilding queue..."))
         self.rebuildQueue()
+        self.finishProgress()
         if problems:
             return "\n".join(problems)
         return "ok"

anki/media.py

@@ -119,7 +119,9 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True):
     unusedFileCount = 0
     missingFileCount = 0
     deck.mediaDir(create=True)
+    deck.startProgress(_("Check Media DB"), 0, 6)
     # rename all files to checksum versions, note non-renamed ones
+    deck.updateProgress(_("Checksum files..."))
     for oldBase in os.listdir(unicode(deck.mediaDir())):
         oldPath = os.path.join(deck.mediaDir(), oldBase)
         if oldBase.startswith("."):
@@ -132,6 +134,7 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True):
         else:
             renamedFiles[oldBase] = newBase
     # now look through all fields, and update references to files
+    deck.updateProgress(_("Scan fields..."))
     for (id, fid, val) in deck.s.all(
         "select id, factId, value from fields"):
         oldval = val
@@ -158,16 +161,22 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True):
             if fid not in factsMissingMedia:
                 unmodifiedFacts[fid] = 1
     # update modified fields
+    deck.updateProgress(_("Modify fields..."))
     if modifiedFacts:
         _modifyFields(deck, updateFields, modifiedFacts, dirty)
     # fix tags
+    deck.updateProgress(_("Update tags..."))
     if dirty:
+        t = time.time()
         if deleteRefs:
             deck.deleteTags(modifiedFacts.keys(), _("Media Missing"))
         else:
             deck.addTags(factsMissingMedia.keys(), _("Media Missing"))
+        print time.time() - t; t = time.time()
         deck.deleteTags(unmodifiedFacts.keys(), _("Media Missing"))
+        print time.time() - t; t = time.time()
     # build cache of db records
+    deck.updateProgress(_("Delete unused files..."))
     mediaIds = dict(deck.s.all("select filename, id from media"))
     # assume latex files exist
     for f in deck.s.column0(
@@ -186,6 +195,7 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True):
         else:
             os.unlink(path)
             unusedFileCount += 1
+    deck.updateProgress(_("Delete stale references..."))
     for (fname, id) in mediaIds.items():
         # maybe delete from db
         if id:
@@ -196,6 +206,7 @@ values (:id, strftime('%s', 'now'))""", id=id)
     # update deck and save
     deck.flushMod()
     deck.save()
+    deck.finishProgress()
     return missingFileCount, unusedFileCount
 
 def mediaRefs(string):