implement full sync downloading

anki/sync.py

@@ -911,8 +911,7 @@ and cards.id in %s""" % ids2str([c[0] for c in cards])))
             return True
         for sum in sums:
             for l in sum.values():
-                if len(l) > 500:
+                if len(l) > 1000:
-                    return True
                     return True
         return False
 
@@ -921,68 +920,99 @@ and cards.id in %s""" % ids2str([c[0] for c in cards])))
         self.deck.lastSync = t
         self.deck.s.commit()
         self.deck.close()
-        if True: # self.localTime > self.remoteTime:
+        fields = {
-            return ("fromLocal",) + self._encode_formdata({
             "p": self.server.password,
             "u": self.server.username,
-                "d": self.server.deckName.encode("utf-8")
+            "d": self.server.deckName.encode("utf-8"),
-                }, open(self.deck.path, "rb"))
+            }
+        if self.localTime > self.remoteTime:
+            return ("fromLocal", fields, self.deck.path)
+        else:
+            return ("fromServer", fields, self.deck.path)
 
     def fullSync(self):
         ret = self.prepareFullSync()
         if ret[0] == "fromLocal":
+            self.fullSyncFromLocal(ret[1], ret[2])
+        else:
+            self.fullSyncFromServer(ret[1], ret[2])
+
+    def fullSyncFromLocal(self, fields, path):
         try:
-                runHook("fullSyncStarted", ret)
+            # write into a temporary file, since POST needs content-length
+            src = open(path, "rb")
+            (fd, name) = tempfile.mkstemp(prefix="anki")
+            tmp = open(name, "w+b")
+            # post vars
+            for (key, value) in fields.items():
+                tmp.write('--' + MIME_BOUNDARY + "\r\n")
+                tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
+                tmp.write('\r\n')
+                tmp.write(value)
+                tmp.write('\r\n')
+            # file header
+            tmp.write('--' + MIME_BOUNDARY + "\r\n")
+            tmp.write(
+                'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
+            tmp.write('Content-Type: application/octet-stream\r\n')
+            tmp.write('\r\n')
+            # data
+            comp = zlib.compressobj()
+            while 1:
+                data = src.read(65536)
+                if not data:
+                    tmp.write(comp.flush())
+                    break
+                tmp.write(comp.compress(data))
+            tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
+            size = tmp.tell()
+            tmp.seek(0)
+            # open http connection
+            runHook("fullSyncStarted", size)
             h = httplib.HTTP(SYNC_HOST, SYNC_PORT)
             h.putrequest('POST', "/sync/fullup")
-                h.putheader('Content-type', ret[1])
+            h.putheader('Content-type', 'multipart/form-data; boundary=%s' %
-                h.putheader('Content-length', str(ret[2]))
+                        MIME_BOUNDARY)
+            h.putheader('Content-length', str(size))
             h.endheaders()
-                f = open(ret[3], "rb")
+            dst = h._conn.sock.makefile("wb", 65536)
-                h._conn.sock.makefile("wb", 32768)
+            # dump file
             cnt = 0
-                t = time.time()
             while 1:
-                    data = f.read(32768)
+                runHook("fullSyncProgress", "fromLocal", cnt)
+                data = tmp.read(65536)
                 if not data:
                     break
-                    h._conn.sock.send(data)
+                dst.write(data)
-                    cnt += 32768
+                cnt += len(data)
-                    if time.time() - t > 0.1:
+            # wait for reply
-                        runHook("fullSyncProgress", min(cnt, ret[2] - 1))
+            dst.close()
-                        t = time.time()
+            tmp.close()
-                runHook("fullSyncProgress", ret[2] - 1)
             errcode, errmsg, headers = h.getreply()
             assert errcode == 200
         finally:
             runHook("fullSyncFinished")
 
-    def _encode_formdata(self, fields, src):
+    def fullSyncFromServer(self, fields, path):
-        (fd, name) = tempfile.mkstemp(prefix='anki')
+        try:
-        file = os.fdopen(fd, "w")
+            runHook("fullSyncStarted", 0)
-        for (key, value) in fields.items():
+            fields = urllib.urlencode(fields)
-            file.write('--' + MIME_BOUNDARY + "\r\n")
+            src = urllib.urlopen(SYNC_URL + "fulldown", fields)
-            file.write('Content-Disposition: form-data; name="%s"\r\n' % key)
+            dst = open(path, "wb")
-            file.write('\r\n')
+            decomp = zlib.decompressobj()
-            file.write(value)
+            cnt = 0
-            file.write('\r\n')
-        file.write('--' + MIME_BOUNDARY + "\r\n")
-        file.write(
-            'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
-        file.write('Content-Type: application/octet-stream\r\n')
-        file.write('\r\n')
-        comp = zlib.compressobj()
             while 1:
-            data = src.read(32768)
+                data = src.read(65536)
                 if not data:
-                file.write(comp.flush())
+                    dst.write(decomp.flush())
                     break
-            file.write(comp.compress(data))
+                dst.write(decomp.decompress(data))
-        file.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
+                cnt += 65536
-        size = file.tell()
+                runHook("fullSyncProgress", "fromServer", cnt)
-        file.close()
+            src.close()
-        type = 'multipart/form-data; boundary=%s' % MIME_BOUNDARY
+            dst.close()
-        return type, size, name
+        finally:
+            runHook("fullSyncFinished")
 
 # Local syncing
 ##########################################################################