deck and packaged deck export

anki/exporting.py

@@ -2,13 +2,12 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-import itertools, time, re, os, HTMLParser
+import itertools, time, re, os, HTMLParser, zipfile, simplejson
 from operator import itemgetter
 from anki.cards import Card
 from anki.lang import _
 from anki.utils import stripHTML, ids2str, splitFields
-
+from anki import Collection
-# remove beautifulsoup dependency
 
 class Exporter(object):
     def __init__(self, col, did=None):
@@ -39,46 +38,44 @@ class Exporter(object):
 ######################################################################
 
 class TextCardExporter(Exporter):
+    pass
 
-    key = _("Text files (*.txt)")
+#     key = _("Text files (*.txt)")
-    ext = ".txt"
+#     ext = ".txt"
 
-    # add option to strip html
+#     def __init__(self, col):
+#         Exporter.__init__(self, col)
 
+#     def doExport(self, file):
+#         ids = self.cardIds()
+#         strids = ids2str(ids)
+#         cards = self.col.db.all("""
+# select cards.question, cards.answer, cards.id from cards
+# where cards.id in %s
+# order by cards.created""" % strids)
+#         self.cardTags = dict(self.col.db.all("""
+# select cards.id, notes.tags from cards, notes
+# where cards.noteId = notes.id
+# and cards.id in %s
+# order by cards.created""" % strids))
+#         out = u"\n".join(["%s\t%s%s" % (
+#             self.escapeText(c[0], removeFields=True),
+#             self.escapeText(c[1], removeFields=True),
+#             self.tags(c[2]))
+#                           for c in cards])
+#         if out:
+#             out += "\n"
+#         file.write(out.encode("utf-8"))
 
-    def __init__(self, col):
+#     def tags(self, id):
-        Exporter.__init__(self, col)
+#         return "\t" + ", ".join(parseTags(self.cardTags[id]))
-
-    def doExport(self, file):
-        ids = self.cardIds()
-        strids = ids2str(ids)
-        cards = self.col.db.all("""
-select cards.question, cards.answer, cards.id from cards
-where cards.id in %s
-order by cards.created""" % strids)
-        self.cardTags = dict(self.col.db.all("""
-select cards.id, notes.tags from cards, notes
-where cards.noteId = notes.id
-and cards.id in %s
-order by cards.created""" % strids))
-        out = u"\n".join(["%s\t%s%s" % (
-            self.escapeText(c[0], removeFields=True),
-            self.escapeText(c[1], removeFields=True),
-            self.tags(c[2]))
-                          for c in cards])
-        if out:
-            out += "\n"
-        file.write(out.encode("utf-8"))
-
-    def tags(self, id):
-        return "\t" + ", ".join(parseTags(self.cardTags[id]))
 
 # Notes as TSV
 ######################################################################
 
 class TextNoteExporter(Exporter):
 
-    key = _("Text files (*.txt)")
+    key = _("Notes in Plain Text")
     ext = ".txt"
 
     def __init__(self, col):
@@ -108,94 +105,133 @@ where cards.id in %s)""" % ids2str(cardIds)):
         out = "\n".join(data)
         file.write(out.encode("utf-8"))
 
-    def tags(self, id):
+# Anki decks
-        if self.includeTags:
-            return "\t" + self.noteTags[id]
-        return ""
-
-# Anki collection exporter
 ######################################################################
+# media files are stored in self.mediaFiles, but not exported.
 
 class AnkiExporter(Exporter):
 
-    key = _("Anki Collection (*.anki)")
+    key = _("Anki 2.0 Deck")
-    ext = ".anki"
+    ext = ".anki2"
 
     def __init__(self, col):
         Exporter.__init__(self, col)
-        self.includeSchedulingInfo = False
+        self.includeSched = False
         self.includeMedia = True
 
     def exportInto(self, path):
-        n = 3
+        # create a new collection at the target
-        if not self.includeSchedulingInfo:
-            n += 1
         try:
             os.unlink(path)
         except (IOError, OSError):
             pass
-        self.newCol = DeckStorage.Deck(path)
+        self.dst = Collection(path)
-        client = SyncClient(self.col)
+        self.src = self.col
-        server = SyncServer(self.newDeck)
+        # find cards
-        client.setServer(server)
+        if not self.did:
-        client.localTime = self.col.modified
+            cids = self.src.db.list("select id from cards")
-        client.remoteTime = 0
+        else:
-        self.col.db.flush()
+            cids = self.src.decks.cids(self.did, children=True)
-        # set up a custom change list and sync
+        # copy cards, noting used nids
-        lsum = self.localSummary()
+        nids = {}
-        rsum = server.summary(0)
+        data = []
-        payload = client.genPayload((lsum, rsum))
+        for row in self.src.db.execute(
-        res = server.applyPayload(payload)
+            "select * from cards where id in "+ids2str(cids)):
-        if not self.includeSchedulingInfo:
+            nids[row[1]] = True
-            self.newCol.resetCards()
+            data.append(row)
-        # media
+        self.dst.db.executemany(
-        if self.includeMedia:
+            "insert into cards values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
-            server.col.mediaPrefix = ""
+            data)
-            copyLocalMedia(client.col, server.col)
-        # need to save manually
-        self.newCol.rebuildCounts()
-        # FIXME
-        #self.exportedCards = self.newCol.cardCount
-        self.newCol.crt = 0
-        self.newCol.db.commit()
-        self.newCol.close()
-
-    def localSummary(self):
-        cardIds = self.cardIds()
-        cStrIds = ids2str(cardIds)
-        cards = self.col.db.all("""
-select id, modified from cards
-where id in %s""" % cStrIds)
-        notes = self.col.db.all("""
-select notes.id, notes.modified from cards, notes where
-notes.id = cards.noteId and
-cards.id in %s""" % cStrIds)
-        models = self.col.db.all("""
-select models.id, models.modified from models, notes where
-notes.modelId = models.id and
-notes.id in %s""" % ids2str([f[0] for f in notes]))
-        media = self.col.db.all("""
-select id, modified from media""")
-        return {
-            # cards
-            "cards": cards,
-            "delcards": [],
         # notes
-            "notes": notes,
+        strnids = ids2str(nids.keys())
-            "delnotes": [],
+        notedata = self.src.db.all("select * from notes where id in "+
+                               strnids)
+        self.dst.db.executemany(
+            "insert into notes values (?,?,?,?,?,?,?,?,?,?,?,?)",
+            notedata)
+        # models used by the notes
+        mids = self.dst.db.list("select distinct mid from notes where id in "+
+                                strnids)
+        # card history and revlog
+        if self.includeSched:
+            data = self.src.db.all(
+                "select * from revlog where cid in "+ids2str(cids))
+            self.dst.db.executemany(
+                "insert into revlog values (?,?,?,?,?,?,?,?,?)",
+                data)
+        else:
+            # need to reset card state
+            self.dst.sched.forgetCards(cids)
         # models
-            "models": models,
+        for m in self.src.models.all():
-            "delmodels": [],
+            if m['id'] in mids:
-            # media
+                self.dst.models.update(m)
-            "media": media,
+        # decks
-            "delmedia": [],
+        if not self.did:
-            }
+            dids = []
+        else:
+            dids = [self.did] + self.src.decks.children(self.did)
+        dconfs = {}
+        for d in self.src.decks.all():
+            if d['id'] == 1:
+                continue
+            if dids and d['id'] not in dids:
+                continue
+            if d['conf'] != 1:
+                dconfs[d['conf']] = True
+            self.dst.decks.update(d)
+        # copy used deck confs
+        for dc in self.src.decks.allConf():
+            if dc['id'] in dconfs:
+                self.dst.decks.updateConf(dc)
+        # find used media
+        media = {}
+        if self.includeMedia:
+            for row in notedata:
+                flds = row[7]
+                mid = row[2]
+                for file in self.src.media.filesInStr(mid, flds):
+                    media[file] = True
+        self.mediaFiles = media.keys()
+        # todo: tags?
+        self.dst.setMod()
+        self.dst.close()
+
+# Packaged Anki decks
+######################################################################
+
+class AnkiPackageExporter(AnkiExporter):
+
+    key = _("Packaged Anki Deck")
+    ext = ".apkg"
+
+    def __init__(self, col):
+        AnkiExporter.__init__(self, col)
+
+    def exportInto(self, path):
+        # export into the anki2 file
+        colfile = path.replace(".apkg", ".anki2")
+        AnkiExporter.exportInto(self, colfile)
+        # zip the deck up
+        z = zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED)
+        z.write(colfile, "collection.anki2")
+        # and media
+        media = {}
+        for c, file in enumerate(self.mediaFiles):
+            c = str(c)
+            z.write(file, c)
+            media[c] = file
+        # media map
+        z.writestr("media", simplejson.dumps(media))
+        z.close()
 
 # Export modules
 ##########################################################################
 
 def exporters():
+    def id(obj):
+        return "%s (*%s)" % (obj.key, obj.ext)
     return (
-        (_("Anki Deck (*.anki)"), AnkiExporter),
+        id(TextNoteExporter),
-        (_("Cards in tab-separated text file (*.txt)"), TextCardExporter),
+        id(AnkiPackageExporter),
-        (_("Notes in tab-separated text file (*.txt)"), TextNoteExporter))
+    )

tests/test_exporting.py

@@ -17,39 +17,51 @@ def setup1():
     f = deck.newNote()
     f['Front'] = u"foo"; f['Back'] = u"bar"; f.tags = ["tag", "tag2"]
     deck.addNote(f)
+    # with a different deck
     f = deck.newNote()
     f['Front'] = u"baz"; f['Back'] = u"qux"
+    f.did = deck.decks.id("new deck")
     deck.addNote(f)
 
 ##########################################################################
 
 @nose.with_setup(setup1)
 def test_export_anki():
-    oldTime = deck.modified
     e = AnkiExporter(deck)
-    newname = unicode(tempfile.mkstemp(prefix="ankitest")[1])
+    newname = unicode(tempfile.mkstemp(prefix="ankitest", suffix=".anki2")[1])
     os.unlink(newname)
     e.exportInto(newname)
-    assert deck.modified == oldTime
     # connect to new deck
-    d2 = aopen(newname, backup=False)
+    d2 = aopen(newname)
-    assert d2.cardCount() == 4
-    # try again, limited to a tag
-    newname = unicode(tempfile.mkstemp(prefix="ankitest")[1])
-    os.unlink(newname)
-    e.limitTags = ['tag']
-    e.exportInto(newname)
-    d2 = aopen(newname, backup=False)
     assert d2.cardCount() == 2
+    # try again, limited to a deck
+    newname = unicode(tempfile.mkstemp(prefix="ankitest", suffix=".anki2")[1])
+    os.unlink(newname)
+    e.did = 1
+    e.exportInto(newname)
+    d2 = aopen(newname)
+    assert d2.cardCount() == 1
 
 @nose.with_setup(setup1)
-def test_export_textcard():
+def test_export_ankipkg():
-    e = TextCardExporter(deck)
+    # add a test file to the media folder
-    f = unicode(tempfile.mkstemp(prefix="ankitest")[1])
+    open(os.path.join(deck.media.dir(), u"今日.mp3"), "w").write("test")
-    os.unlink(f)
+    n = deck.newNote()
-    e.exportInto(f)
+    n['Front'] = u'[sound:今日.mp3]'
-    e.includeTags = True
+    deck.addNote(n)
-    e.exportInto(f)
+    e = AnkiPackageExporter(deck)
+    newname = unicode(tempfile.mkstemp(prefix="ankitest", suffix=".apkg")[1])
+    os.unlink(newname)
+    e.exportInto(newname)
+
+# @nose.with_setup(setup1)
+# def test_export_textcard():
+#     e = TextCardExporter(deck)
+#     f = unicode(tempfile.mkstemp(prefix="ankitest")[1])
+#     os.unlink(f)
+#     e.exportInto(f)
+#     e.includeTags = True
+#     e.exportInto(f)
 
 @nose.with_setup(setup1)
 def test_export_textnote():
@@ -59,3 +71,6 @@ def test_export_textnote():
     e.exportInto(f)
     e.includeTags = True
     e.exportInto(f)
+
+def test_exporters():
+    assert "*.apkg" in str(exporters())