diff --git a/anki/sync.py b/anki/sync.py index bc95d0dbb..450103b0a 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -2,14 +2,11 @@ # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import urllib.request, urllib.parse, urllib.error import io -import sys import gzip import random -from io import StringIO +import requests -import httplib2 from anki.db import DB from anki.utils import ids2str, intTime, json, isWin, isMac, platDesc, checksum from anki.consts import * @@ -20,76 +17,7 @@ from .lang import ngettext # syncing vars HTTP_TIMEOUT = 90 HTTP_PROXY = None - -# badly named; means no retries -httplib2.RETRIES = 1 - -try: - # httplib2 >=0.7.7 - _proxy_info_from_environment = httplib2.proxy_info_from_environment - _proxy_info_from_url = httplib2.proxy_info_from_url -except AttributeError: - # httplib2 <0.7.7 - _proxy_info_from_environment = httplib2.ProxyInfo.from_environment - _proxy_info_from_url = httplib2.ProxyInfo.from_url - -# Httplib2 connection object -###################################################################### - -def httpCon(): - certs = os.path.join(os.path.dirname(__file__), "ankiweb.certs") - if not os.path.exists(certs): - if not isMac: - certs = os.path.abspath(os.path.join( - os.path.dirname(certs), "..", "ankiweb.certs")) - else: - certs = os.path.abspath(os.path.join( - os.path.dirname(os.path.abspath(sys.argv[0])), - "../Resources/ankiweb.certs")) - if not os.path.exists(certs): - assert 0, "Unable to locate ankiweb.certs" - return httplib2.Http( - timeout=HTTP_TIMEOUT, ca_certs=certs, - proxy_info=HTTP_PROXY, - disable_ssl_certificate_validation=not not HTTP_PROXY) - -# Proxy handling -###################################################################### - -def _setupProxy(): - global HTTP_PROXY - # set in env? - p = _proxy_info_from_environment() - if not p: - # platform-specific fetch - url = None - if isWin: - print("fixme: win proxy support") - # r = urllib.getproxies_registry() - # if 'https' in r: - # url = r['https'] - # elif 'http' in r: - # url = r['http'] - elif isMac: - print("fixme: mac proxy support") - # r = urllib.getproxies_macosx_sysconf() - # if 'https' in r: - # url = r['https'] - # elif 'http' in r: - # url = r['http'] - if url: - p = _proxy_info_from_url(url, _proxyMethod(url)) - if p: - p.proxy_rdns = True - HTTP_PROXY = p - -def _proxyMethod(url): - if url.lower().startswith("https"): - return "https" - else: - return "http" - -_setupProxy() +HTTP_BUF_SIZE = 64*1024 # Incremental syncing ########################################################################## @@ -526,25 +454,51 @@ class LocalServer(Syncer): l = json.loads; d = json.dumps return l(d(Syncer.applyChanges(self, l(d(changes))))) +# Wrapper for requests that tracks upload/download progress +########################################################################## + +class AnkiRequestsClient(object): + + def __init__(self): + self.session = requests.Session() + + def post(self, url, data, headers): + data = _MonitoringFile(data) + return self.session.post(url, data=data, headers=headers, stream=True) + + def get(self, url): + return self.session.get(url, stream=True) + + def streamContent(self, resp): + resp.raise_for_status() + + buf = io.BytesIO() + for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE): + runHook("httpRecv", len(chunk)) + buf.write(chunk) + return buf.getvalue() + +class _MonitoringFile(io.BufferedReader): + def read(self, size=-1): + data = io.BufferedReader.read(self, HTTP_BUF_SIZE) + runHook("httpSend", len(data)) + return data + # HTTP syncing tools ########################################################################## -# Calling code should catch the following codes: -# - 501: client needs upgrade -# - 502: ankiweb down -# - 503/504: server too busy - class HttpSyncer(object): - def __init__(self, hkey=None, con=None): + def __init__(self, hkey=None, client=None): self.hkey = hkey self.skey = checksum(str(random.random()))[:8] - self.con = con or httpCon() + self.client = client or AnkiRequestsClient() self.postVars = {} def assertOk(self, resp): - if resp['status'] != '200': - raise Exception("Unknown response code: %s" % resp['status']) + # not using raise_for_status() as aqt expects this error msg + if resp.status_code != 200: + raise Exception("Unknown response code: %s" % resp.status_code) # Posting data as a file ###################################################################### @@ -552,7 +506,7 @@ class HttpSyncer(object): # costly. We could send it as a raw post, but more HTTP clients seem to # support file uploading, so this is the more compatible choice. - def req(self, method, fobj=None, comp=6, badAuthRaises=True): + def _buildPostData(self, fobj, comp): BOUNDARY=b"Anki-sync-boundary" bdry = b"--"+BOUNDARY buf = io.BytesIO() @@ -590,16 +544,19 @@ Content-Type: application/octet-stream\r\n\r\n""") 'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY.decode("utf8"), 'Content-Length': str(size), } - body = buf.getvalue() - buf.close() - resp, cont = self.con.request( - self.syncURL()+method, "POST", headers=headers, body=body) - if not badAuthRaises: - # return false if bad auth instead of raising - if resp['status'] == '403': - return False - self.assertOk(resp) - return cont + buf.seek(0) + return headers, buf + + def req(self, method, fobj=None, comp=6, badAuthRaises=True): + headers, body = self._buildPostData(fobj, comp) + + r = self.client.post(self.syncURL()+method, data=body, headers=headers) + if not badAuthRaises and r.status_code == 403: + return False + self.assertOk(r) + + buf = self.client.streamContent(r) + return buf # Incremental sync over HTTP ###################################################################### @@ -670,8 +627,8 @@ class RemoteServer(HttpSyncer): class FullSyncer(HttpSyncer): - def __init__(self, col, hkey, con): - HttpSyncer.__init__(self, hkey, con) + def __init__(self, col, hkey, client): + HttpSyncer.__init__(self, hkey, client) self.postVars = dict( k=self.hkey, v="ankidesktop,%s,%s"%(anki.version, platDesc()), @@ -858,9 +815,9 @@ class MediaSyncer(object): class RemoteMediaServer(HttpSyncer): - def __init__(self, col, hkey, con): + def __init__(self, col, hkey, client): self.col = col - HttpSyncer.__init__(self, hkey, con) + HttpSyncer.__init__(self, hkey, client) def syncURL(self): if os.getenv("ANKIDEV"): diff --git a/aqt/downloader.py b/aqt/downloader.py index d27e92417..66faf075a 100644 --- a/aqt/downloader.py +++ b/aqt/downloader.py @@ -4,10 +4,10 @@ import time, re, traceback from aqt.qt import * -from anki.sync import httpCon +from anki.sync import AnkiRequestsClient from aqt.utils import showWarning from anki.hooks import addHook, remHook -import aqt.sync # monkey-patches httplib2 +import aqt def download(mw, code): "Download addon/deck from AnkiWeb. On success caller must stop progress diag." @@ -54,19 +54,22 @@ class Downloader(QThread): # setup progress handler self.byteUpdate = time.time() self.recvTotal = 0 - def canPost(): - if (time.time() - self.byteUpdate) > 0.1: - self.byteUpdate = time.time() - return True def recvEvent(bytes): self.recvTotal += bytes - if canPost(): - self.recv.emit() + self.recv.emit() addHook("httpRecv", recvEvent) - con = httpCon() + client = AnkiRequestsClient() try: - resp, cont = con.request( + resp = client.get( aqt.appShared + "download/%d" % self.code) + if resp.status_code == 200: + data = client.streamContent(resp) + elif resp.status_code in (403,404): + self.error = _("Invalid code") + return + else: + self.error = _("Error downloading: %s" % resp.status_code) + return except Exception as e: exc = traceback.format_exc() try: @@ -76,12 +79,7 @@ class Downloader(QThread): return finally: remHook("httpRecv", recvEvent) - if resp['status'] == '200': - self.error = None - self.fname = re.match("attachment; filename=(.+)", - resp['content-disposition']).group(1) - self.data = cont - elif resp['status'] == '403': - self.error = _("Invalid code.") - else: - self.error = _("Error downloading: %s") % resp['status'] + + self.fname = re.match("attachment; filename=(.+)", + resp.headers['content-disposition']).group(1) + self.data = data diff --git a/aqt/sync.py b/aqt/sync.py index 4002118fd..727d7f7db 100644 --- a/aqt/sync.py +++ b/aqt/sync.py @@ -73,9 +73,9 @@ automatically.""")) def _updateLabel(self): self.mw.progress.update(label="%s\n%s" % ( self.label, - _("%(a)dkB up, %(b)dkB down") % dict( - a=self.sentBytes // 1024, - b=self.recvBytes // 1024))) + _("%(a)0.1fkB up, %(b)0.1fkB down") % dict( + a=self.sentBytes / 1024, + b=self.recvBytes / 1024))) def onEvent(self, evt, *args): pu = self.mw.progress.update @@ -298,24 +298,16 @@ class SyncThread(QThread): self.client = Syncer(self.col, self.server) self.sentTotal = 0 self.recvTotal = 0 - # throttle updates; qt doesn't handle lots of posted events well - self.byteUpdate = time.time() def syncEvent(type): self.fireEvent("sync", type) def syncMsg(msg): self.fireEvent("syncMsg", msg) - def canPost(): - if (time.time() - self.byteUpdate) > 0.1: - self.byteUpdate = time.time() - return True def sendEvent(bytes): self.sentTotal += bytes - if canPost(): - self.fireEvent("send", str(self.sentTotal)) + self.fireEvent("send", str(self.sentTotal)) def recvEvent(bytes): self.recvTotal += bytes - if canPost(): - self.fireEvent("recv", str(self.recvTotal)) + self.fireEvent("recv", str(self.recvTotal)) addHook("sync", syncEvent) addHook("syncMsg", syncMsg) addHook("httpSend", sendEvent) @@ -395,7 +387,7 @@ class SyncThread(QThread): f = self.fullSyncChoice if f == "cancel": return - self.client = FullSyncer(self.col, self.hkey, self.server.con) + self.client = FullSyncer(self.col, self.hkey, self.server.client) if f == "upload": if not self.client.upload(): self.fireEvent("upbad") @@ -408,7 +400,7 @@ class SyncThread(QThread): def _syncMedia(self): if not self.media: return - self.server = RemoteMediaServer(self.col, self.hkey, self.server.con) + self.server = RemoteMediaServer(self.col, self.hkey, self.server.client) self.client = MediaSyncer(self.col, self.server) ret = self.client.sync() if ret == "noChanges": @@ -422,87 +414,3 @@ class SyncThread(QThread): self.event.emit(cmd, arg) -# Monkey-patch httplib & httplib2 so we can get progress info -###################################################################### - -CHUNK_SIZE = 65536 -import http.client, httplib2 -from io import StringIO -from anki.hooks import runHook - -print("fixme: _conn_request and _incrementalSend need updating for python3") - -# sending in httplib -def _incrementalSend(self, data): - """Send `data' to the server.""" - if self.sock is None: - if self.auto_open: - self.connect() - else: - raise http.client.NotConnected() - # if it's not a file object, make it one - if not hasattr(data, 'read'): - data = StringIO(data) - while 1: - block = data.read(CHUNK_SIZE) - if not block: - break - self.sock.sendall(block) - runHook("httpSend", len(block)) - -#http.client.HTTPConnection.send = _incrementalSend - -# receiving in httplib2 -# this is an augmented version of httplib's request routine that: -# - doesn't assume requests will be tried more than once -# - calls a hook for each chunk of data so we can update the gui -# - retries only when keep-alive connection is closed -def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): - try: - if conn.sock is None: - conn.connect() - conn.request(method, request_uri, body, headers) - except socket.timeout: - raise - except socket.gaierror: - conn.close() - raise httplib2.ServerNotFoundError( - "Unable to find the server at %s" % conn.host) - except httplib2.ssl_SSLError: - conn.close() - raise - except socket.error as e: - conn.close() - raise - except http.client.HTTPException: - conn.close() - raise - try: - response = conn.getresponse() - except http.client.BadStatusLine: - print("retry bad line") - conn.close() - conn.connect() - continue - except (socket.error, http.client.HTTPException): - raise - else: - content = "" - if method == "HEAD": - response.close() - else: - buf = StringIO() - while 1: - data = response.read(CHUNK_SIZE) - if not data: - break - buf.write(data) - runHook("httpRecv", len(data)) - content = buf.getvalue() - response = httplib2.Response(response) - if method != "HEAD": - content = httplib2._decompressContent(response, content) - return (response, content) - -#httplib2.Http._conn_request = _conn_request diff --git a/aqt/update.py b/aqt/update.py index b36f9046f..06f6fec5e 100644 --- a/aqt/update.py +++ b/aqt/update.py @@ -1,17 +1,16 @@ # Copyright: Damien Elmes # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse import time +import requests + from aqt.qt import * import aqt from aqt.utils import openLink from anki.utils import json, platDesc from aqt.utils import showText - class LatestVersionFinder(QThread): newVerAvail = pyqtSignal(str) @@ -36,14 +35,11 @@ class LatestVersionFinder(QThread): return d = self._data() d['proto'] = 1 - d = urllib.parse.urlencode(d).encode("utf8") + try: - f = urllib.request.urlopen(aqt.appUpdate, d) - resp = f.read() - if not resp: - print("update check load failed") - return - resp = json.loads(resp.decode("utf8")) + r = requests.post(aqt.appUpdate, data=d) + r.raise_for_status() + resp = r.json() except: # behind proxy, corrupt message, etc print("update check failed") diff --git a/requirements.txt b/requirements.txt index dfda5431d..cd37c7b9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ bs4 send2trash httplib2 pyaudio - +requests