diff --git a/anki/media.py b/anki/media.py
index a7977baef..5011220e4 100644
--- a/anki/media.py
+++ b/anki/media.py
@@ -195,11 +195,11 @@ def rebuildMediaDir(deck, deleteRefs=False, dirty=True):
     # fix tags
     deck.updateProgress(_("Update tags..."))
     if dirty:
+        deck.deleteTags(unmodifiedFacts.keys(), _("MediaMissing"))
         if deleteRefs:
             deck.deleteTags(modifiedFacts.keys(), _("MediaMissing"))
         else:
             deck.addTags(factsMissingMedia.keys(), _("MediaMissing"))
-        deck.deleteTags(unmodifiedFacts.keys(), _("MediaMissing"))
     # build cache of db records
     deck.updateProgress(_("Delete unused files..."))
     mediaIds = dict(deck.s.all("select filename, id from media"))