use requests for http; add progress info back

- wrap request in AnkiRequestsClient so we can keep track of
upload/download bytes without having to monkey patch anything
- force a 64kB buffer size instead of the default 8kB
- show one decimal point in up/down so small requests still give
visual feedback
- update add-on downloading and update check to use requests
- remove the update throttling in aqt/sync.py, as it's not really
necessary anymore

anki/sync.py

@@ -2,14 +2,11 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-import urllib.request, urllib.parse, urllib.error
 import io
-import sys
 import gzip
 import random
-from io import StringIO
+import requests
 
-import httplib2
 from anki.db import DB
 from anki.utils import ids2str, intTime, json, isWin, isMac, platDesc, checksum
 from anki.consts import *
@@ -20,76 +17,7 @@ from .lang import ngettext
 # syncing vars
 HTTP_TIMEOUT = 90
 HTTP_PROXY = None
-
+HTTP_BUF_SIZE = 64*1024
-# badly named; means no retries
-httplib2.RETRIES = 1
-
-try:
-    # httplib2 >=0.7.7
-    _proxy_info_from_environment = httplib2.proxy_info_from_environment
-    _proxy_info_from_url = httplib2.proxy_info_from_url
-except AttributeError:
-    # httplib2 <0.7.7
-    _proxy_info_from_environment = httplib2.ProxyInfo.from_environment
-    _proxy_info_from_url = httplib2.ProxyInfo.from_url
-
-# Httplib2 connection object
-######################################################################
-
-def httpCon():
-    certs = os.path.join(os.path.dirname(__file__), "ankiweb.certs")
-    if not os.path.exists(certs):
-        if not isMac:
-            certs = os.path.abspath(os.path.join(
-                os.path.dirname(certs), "..", "ankiweb.certs"))
-        else:
-            certs = os.path.abspath(os.path.join(
-                os.path.dirname(os.path.abspath(sys.argv[0])),
-                "../Resources/ankiweb.certs"))
-    if not os.path.exists(certs):
-            assert 0, "Unable to locate ankiweb.certs"
-    return httplib2.Http(
-        timeout=HTTP_TIMEOUT, ca_certs=certs,
-        proxy_info=HTTP_PROXY,
-        disable_ssl_certificate_validation=not not HTTP_PROXY)
-
-# Proxy handling
-######################################################################
-
-def _setupProxy():
-    global HTTP_PROXY
-    # set in env?
-    p = _proxy_info_from_environment()
-    if not p:
-        # platform-specific fetch
-        url = None
-        if isWin:
-            print("fixme: win proxy support")
-            # r = urllib.getproxies_registry()
-            # if 'https' in r:
-            #     url = r['https']
-            # elif 'http' in r:
-            #     url = r['http']
-        elif isMac:
-            print("fixme: mac proxy support")
-            # r = urllib.getproxies_macosx_sysconf()
-            # if 'https' in r:
-            #     url = r['https']
-            # elif 'http' in r:
-            #     url = r['http']
-        if url:
-            p = _proxy_info_from_url(url, _proxyMethod(url))
-    if p:
-        p.proxy_rdns = True
-    HTTP_PROXY = p
-
-def _proxyMethod(url):
-    if url.lower().startswith("https"):
-        return "https"
-    else:
-        return "http"
-
-_setupProxy()
 
 # Incremental syncing
 ##########################################################################
@@ -526,25 +454,51 @@ class LocalServer(Syncer):
         l = json.loads; d = json.dumps
         return l(d(Syncer.applyChanges(self, l(d(changes)))))
 
+# Wrapper for requests that tracks upload/download progress
+##########################################################################
+
+class AnkiRequestsClient(object):
+
+    def __init__(self):
+        self.session = requests.Session()
+
+    def post(self, url, data, headers):
+        data = _MonitoringFile(data)
+        return self.session.post(url, data=data, headers=headers, stream=True)
+
+    def get(self, url):
+        return self.session.get(url, stream=True)
+
+    def streamContent(self, resp):
+        resp.raise_for_status()
+
+        buf = io.BytesIO()
+        for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE):
+            runHook("httpRecv", len(chunk))
+            buf.write(chunk)
+        return buf.getvalue()
+
+class _MonitoringFile(io.BufferedReader):
+    def read(self, size=-1):
+        data = io.BufferedReader.read(self, HTTP_BUF_SIZE)
+        runHook("httpSend", len(data))
+        return data
+
 # HTTP syncing tools
 ##########################################################################
 
-# Calling code should catch the following codes:
-# - 501: client needs upgrade
-# - 502: ankiweb down
-# - 503/504: server too busy
-
 class HttpSyncer(object):
 
-    def __init__(self, hkey=None, con=None):
+    def __init__(self, hkey=None, client=None):
         self.hkey = hkey
         self.skey = checksum(str(random.random()))[:8]
-        self.con = con or httpCon()
+        self.client = client or AnkiRequestsClient()
         self.postVars = {}
 
     def assertOk(self, resp):
-        if resp['status'] != '200':
+        # not using raise_for_status() as aqt expects this error msg
-            raise Exception("Unknown response code: %s" % resp['status'])
+        if resp.status_code != 200:
+            raise Exception("Unknown response code: %s" % resp.status_code)
 
     # Posting data as a file
     ######################################################################
@@ -552,7 +506,7 @@ class HttpSyncer(object):
     # costly. We could send it as a raw post, but more HTTP clients seem to
     # support file uploading, so this is the more compatible choice.
 
-    def req(self, method, fobj=None, comp=6, badAuthRaises=True):
+    def _buildPostData(self, fobj, comp):
         BOUNDARY=b"Anki-sync-boundary"
         bdry = b"--"+BOUNDARY
         buf = io.BytesIO()
@@ -590,16 +544,19 @@ Content-Type: application/octet-stream\r\n\r\n""")
             'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY.decode("utf8"),
             'Content-Length': str(size),
         }
-        body = buf.getvalue()
+        buf.seek(0)
-        buf.close()
+        return headers, buf
-        resp, cont = self.con.request(
+
-            self.syncURL()+method, "POST", headers=headers, body=body)
+    def req(self, method, fobj=None, comp=6, badAuthRaises=True):
-        if not badAuthRaises:
+        headers, body = self._buildPostData(fobj, comp)
-            # return false if bad auth instead of raising
+
-            if resp['status'] == '403':
+        r = self.client.post(self.syncURL()+method, data=body, headers=headers)
-                return False
+        if not badAuthRaises and r.status_code == 403:
-        self.assertOk(resp)
+            return False
-        return cont
+        self.assertOk(r)
+
+        buf = self.client.streamContent(r)
+        return buf
 
 # Incremental sync over HTTP
 ######################################################################
@@ -670,8 +627,8 @@ class RemoteServer(HttpSyncer):
 
 class FullSyncer(HttpSyncer):
 
-    def __init__(self, col, hkey, con):
+    def __init__(self, col, hkey, client):
-        HttpSyncer.__init__(self, hkey, con)
+        HttpSyncer.__init__(self, hkey, client)
         self.postVars = dict(
             k=self.hkey,
             v="ankidesktop,%s,%s"%(anki.version, platDesc()),
@@ -858,9 +815,9 @@ class MediaSyncer(object):
 
 class RemoteMediaServer(HttpSyncer):
 
-    def __init__(self, col, hkey, con):
+    def __init__(self, col, hkey, client):
         self.col = col
-        HttpSyncer.__init__(self, hkey, con)
+        HttpSyncer.__init__(self, hkey, client)
 
     def syncURL(self):
         if os.getenv("ANKIDEV"):

aqt/downloader.py

@@ -4,10 +4,10 @@
 
 import time, re, traceback
 from aqt.qt import *
-from anki.sync import httpCon
+from anki.sync import AnkiRequestsClient
 from aqt.utils import showWarning
 from anki.hooks import addHook, remHook
-import aqt.sync # monkey-patches httplib2
+import aqt
 
 def download(mw, code):
     "Download addon/deck from AnkiWeb. On success caller must stop progress diag."
@@ -54,19 +54,22 @@ class Downloader(QThread):
         # setup progress handler
         self.byteUpdate = time.time()
         self.recvTotal = 0
-        def canPost():
-            if (time.time() - self.byteUpdate) > 0.1:
-                self.byteUpdate = time.time()
-                return True
         def recvEvent(bytes):
             self.recvTotal += bytes
-            if canPost():
+            self.recv.emit()
-                self.recv.emit()
         addHook("httpRecv", recvEvent)
-        con =  httpCon()
+        client = AnkiRequestsClient()
         try:
-            resp, cont = con.request(
+            resp = client.get(
                 aqt.appShared + "download/%d" % self.code)
+            if resp.status_code == 200:
+                data = client.streamContent(resp)
+            elif resp.status_code in (403,404):
+                self.error = _("Invalid code")
+                return
+            else:
+                self.error = _("Error downloading: %s" % resp.status_code)
+                return
         except Exception as e:
             exc = traceback.format_exc()
             try:
@@ -76,12 +79,7 @@ class Downloader(QThread):
             return
         finally:
             remHook("httpRecv", recvEvent)
-        if resp['status'] == '200':
+
-            self.error = None
+        self.fname = re.match("attachment; filename=(.+)",
-            self.fname = re.match("attachment; filename=(.+)",
+                              resp.headers['content-disposition']).group(1)
-                                  resp['content-disposition']).group(1)
+        self.data = data
-            self.data = cont
-        elif resp['status'] == '403':
-            self.error = _("Invalid code.")
-        else:
-            self.error = _("Error downloading: %s") % resp['status']

aqt/sync.py

@@ -73,9 +73,9 @@ automatically."""))
     def _updateLabel(self):
         self.mw.progress.update(label="%s\n%s" % (
             self.label,
-            _("%(a)dkB up, %(b)dkB down") % dict(
+            _("%(a)0.1fkB up, %(b)0.1fkB down") % dict(
-                a=self.sentBytes // 1024,
+                a=self.sentBytes / 1024,
-                b=self.recvBytes // 1024)))
+                b=self.recvBytes / 1024)))
 
     def onEvent(self, evt, *args):
         pu = self.mw.progress.update
@@ -298,24 +298,16 @@ class SyncThread(QThread):
         self.client = Syncer(self.col, self.server)
         self.sentTotal = 0
         self.recvTotal = 0
-        # throttle updates; qt doesn't handle lots of posted events well
-        self.byteUpdate = time.time()
         def syncEvent(type):
             self.fireEvent("sync", type)
         def syncMsg(msg):
             self.fireEvent("syncMsg", msg)
-        def canPost():
-            if (time.time() - self.byteUpdate) > 0.1:
-                self.byteUpdate = time.time()
-                return True
         def sendEvent(bytes):
             self.sentTotal += bytes
-            if canPost():
+            self.fireEvent("send", str(self.sentTotal))
-                self.fireEvent("send", str(self.sentTotal))
         def recvEvent(bytes):
             self.recvTotal += bytes
-            if canPost():
+            self.fireEvent("recv", str(self.recvTotal))
-                self.fireEvent("recv", str(self.recvTotal))
         addHook("sync", syncEvent)
         addHook("syncMsg", syncMsg)
         addHook("httpSend", sendEvent)
@@ -395,7 +387,7 @@ class SyncThread(QThread):
             f = self.fullSyncChoice
         if f == "cancel":
             return
-        self.client = FullSyncer(self.col, self.hkey, self.server.con)
+        self.client = FullSyncer(self.col, self.hkey, self.server.client)
         if f == "upload":
             if not self.client.upload():
                 self.fireEvent("upbad")
@@ -408,7 +400,7 @@ class SyncThread(QThread):
     def _syncMedia(self):
         if not self.media:
             return
-        self.server = RemoteMediaServer(self.col, self.hkey, self.server.con)
+        self.server = RemoteMediaServer(self.col, self.hkey, self.server.client)
         self.client = MediaSyncer(self.col, self.server)
         ret = self.client.sync()
         if ret == "noChanges":
@@ -422,87 +414,3 @@ class SyncThread(QThread):
         self.event.emit(cmd, arg)
 
 
-# Monkey-patch httplib & httplib2 so we can get progress info
-######################################################################
-
-CHUNK_SIZE = 65536
-import http.client, httplib2
-from io import StringIO
-from anki.hooks import runHook
-
-print("fixme: _conn_request and _incrementalSend need updating for python3")
-
-# sending in httplib
-def _incrementalSend(self, data):
-    """Send `data' to the server."""
-    if self.sock is None:
-        if self.auto_open:
-            self.connect()
-        else:
-            raise http.client.NotConnected()
-    # if it's not a file object, make it one
-    if not hasattr(data, 'read'):
-        data = StringIO(data)
-    while 1:
-        block = data.read(CHUNK_SIZE)
-        if not block:
-            break
-        self.sock.sendall(block)
-        runHook("httpSend", len(block))
-
-#http.client.HTTPConnection.send = _incrementalSend
-
-# receiving in httplib2
-# this is an augmented version of httplib's request routine that:
-# - doesn't assume requests will be tried more than once
-# - calls a hook for each chunk of data so we can update the gui
-# - retries only when keep-alive connection is closed
-def _conn_request(self, conn, request_uri, method, body, headers):
-    for i in range(2):
-        try:
-            if conn.sock is None:
-              conn.connect()
-            conn.request(method, request_uri, body, headers)
-        except socket.timeout:
-            raise
-        except socket.gaierror:
-            conn.close()
-            raise httplib2.ServerNotFoundError(
-                "Unable to find the server at %s" % conn.host)
-        except httplib2.ssl_SSLError:
-            conn.close()
-            raise
-        except socket.error as e:
-            conn.close()
-            raise
-        except http.client.HTTPException:
-            conn.close()
-            raise
-        try:
-            response = conn.getresponse()
-        except http.client.BadStatusLine:
-            print("retry bad line")
-            conn.close()
-            conn.connect()
-            continue
-        except (socket.error, http.client.HTTPException):
-            raise
-        else:
-            content = ""
-            if method == "HEAD":
-                response.close()
-            else:
-                buf = StringIO()
-                while 1:
-                    data = response.read(CHUNK_SIZE)
-                    if not data:
-                        break
-                    buf.write(data)
-                    runHook("httpRecv", len(data))
-                content = buf.getvalue()
-            response = httplib2.Response(response)
-            if method != "HEAD":
-                content = httplib2._decompressContent(response, content)
-        return (response, content)
-
-#httplib2.Http._conn_request = _conn_request

aqt/update.py

@@ -1,17 +1,16 @@
 # Copyright: Damien Elmes <anki@ichi2.net>
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-import urllib.request, urllib.parse, urllib.error
-import urllib.request, urllib.error, urllib.parse
 import time
 
+import requests
+
 from aqt.qt import *
 import aqt
 from aqt.utils import openLink
 from anki.utils import json, platDesc
 from aqt.utils import showText
 
-
 class LatestVersionFinder(QThread):
 
     newVerAvail = pyqtSignal(str)
@@ -36,14 +35,11 @@ class LatestVersionFinder(QThread):
             return
         d = self._data()
         d['proto'] = 1
-        d = urllib.parse.urlencode(d).encode("utf8")
+
         try:
-            f = urllib.request.urlopen(aqt.appUpdate, d)
+            r = requests.post(aqt.appUpdate, data=d)
-            resp = f.read()
+            r.raise_for_status()
-            if not resp:
+            resp = r.json()
-                print("update check load failed")
-                return
-            resp = json.loads(resp.decode("utf8"))
         except:
             # behind proxy, corrupt message, etc
             print("update check failed")

requirements.txt

@@ -2,4 +2,4 @@ bs4
 send2trash
 httplib2
 pyaudio
-
+requests