zip64 support, and import/export improvements

- we now allow exports over 2gb/64k files - AnkiMobile and AnkiDroid
will need to be updated to support this
- avoid compressing media files in export, as in the common case of
jpg/mp3 it's much faster with no increase in size
- exports and imports now show # of files processed
- mw.progress.update() now limits # of updates per sec

anki/exporting.py

@@ -248,7 +248,7 @@ class AnkiPackageExporter(AnkiExporter):
 
     def exportInto(self, path):
         # open a zip file
-        z = zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED)
+        z = zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
         # if all decks and scheduling included, full export
         if self.includeSched and not self.did:
             media = self.exportVerbatim(z)
@@ -268,11 +268,12 @@ class AnkiPackageExporter(AnkiExporter):
         self.prepareMedia()
         media = {}
         for c, file in enumerate(self.mediaFiles):
-            c = str(c)
+            cStr = str(c)
             mpath = os.path.join(self.mediaDir, file)
             if os.path.exists(mpath):
-                z.write(mpath, c)
+                z.write(mpath, cStr, zipfile.ZIP_STORED)
-                media[c] = file
+                media[cStr] = file
+                runHook("exportedMediaFiles", c)
         # tidy up intermediate files
         os.unlink(colfile)
         p = path.replace(".apkg", ".media.db2")
@@ -294,11 +295,13 @@ class AnkiPackageExporter(AnkiExporter):
         media = {}
         mdir = self.col.media.dir()
         for c, file in enumerate(os.listdir(mdir)):
-            c = str(c)
+            cStr = str(c)
             mpath = os.path.join(mdir, file)
             if os.path.exists(mpath):
-                z.write(mpath, c)
+                z.write(mpath, cStr, zipfile.ZIP_STORED)
-                media[c] = file
+                media[cStr] = file
+                runHook("exportedMediaFiles", c)
+
         return media
 
     def prepareMedia(self):

aqt/exporting.py

@@ -9,6 +9,9 @@ import  aqt
 from aqt.utils import getSaveFile, tooltip, showWarning, askUser, \
     checkInvalidFilename
 from anki.exporting import exporters
+from anki.hooks import addHook, remHook
+from anki.lang import ngettext
+
 
 class ExportDialog(QDialog):
 
@@ -105,7 +108,13 @@ class ExportDialog(QDialog):
                 showWarning(_("Couldn't save file: %s") % unicode(e))
             else:
                 os.unlink(file)
+                exportedMedia = lambda cnt: self.mw.progress.update(
+                        label=ngettext("Exported %d media file",
+                                       "Exported %d media files", cnt) % cnt
+                        )
+                addHook("exportedMediaFiles", exportedMedia)
                 self.exporter.exportInto(file)
+                remHook("exportedMediaFiles", exportedMedia)
                 if verbatim:
                     if usingHomedir:
                         msg = _("A file called %s was saved in your home directory.")

aqt/importing.py

@@ -16,6 +16,8 @@ from anki.hooks import addHook, remHook
 import aqt.forms
 import aqt.modelchooser
 import aqt.deckchooser
+from anki.lang import ngettext
+
 
 class ChangeMap(QDialog):
     def __init__(self, mw, model, current):
@@ -391,8 +393,16 @@ def replaceWithApkg(mw, file, backup):
     # unwanted media. in the future we might also want to deduplicate this
     # step
     d = os.path.join(mw.pm.profileFolder(), "collection.media")
-    for c, file in json.loads(z.read("media")).items():
+    for n, (cStr, file) in enumerate(json.loads(z.read("media")).items()):
-        open(os.path.join(d, file), "wb").write(z.read(str(c)))
+        mw.progress.update(ngettext("Processed %d media file",
+                                    "Processed %d media files", n) % n)
+        size = z.getinfo(cStr).file_size
+        dest = os.path.join(d, file)
+        # if we have a matching file size
+        if os.path.exists(dest) and size == os.stat(dest).st_size:
+            continue
+        data = z.read(cStr)
+        open(dest, "wb").write(data)
     z.close()
     # reload
     mw.loadCollection()

aqt/progress.py

@@ -110,21 +110,22 @@ Your pysqlite2 is too old. Anki will appear frozen during long operations."""
         self._min = min
         self._max = max
         self._firstTime = time.time()
-        self._lastTime = time.time()
+        self._lastUpdate = time.time()
         self._disabled = False
 
     def update(self, label=None, value=None, process=True, maybeShow=True):
         #print self._min, self._counter, self._max, label, time.time() - self._lastTime
         if maybeShow:
             self._maybeShow()
-        self._lastTime = time.time()
+        elapsed = time.time() - self._lastUpdate
         if label:
             self._win.setLabelText(label)
         if self._max and self._shown:
             self._counter = value or (self._counter+1)
             self._win.setValue(self._counter)
-        if process:
+        if process and elapsed >= 0.2:
             self.app.processEvents(QEventLoop.ExcludeUserInputEvents)
+            self._lastUpdate = time.time()
 
     def finish(self):
         self._levels -= 1