mirror of
https://github.com/ankitects/anki.git
synced 2025-09-18 22:12:21 -04:00
use requests for http; add progress info back
- wrap request in AnkiRequestsClient so we can keep track of upload/download bytes without having to monkey patch anything - force a 64kB buffer size instead of the default 8kB - show one decimal point in up/down so small requests still give visual feedback - update add-on downloading and update check to use requests - remove the update throttling in aqt/sync.py, as it's not really necessary anymore
This commit is contained in:
Damien Elmes
parent
147e09a6cb
commit
f6245cdfd1
5 changed files with 86 additions and 227 deletions
151
anki/sync.py
151
anki/sync.py
|
|
@ -2,14 +2,11 @@
|
|||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
import urllib.request, urllib.parse, urllib.error
|
||||
import io
|
||||
import sys
|
||||
import gzip
|
||||
import random
|
||||
from io import StringIO
|
||||
import requests
|
||||
|
||||
import httplib2
|
||||
from anki.db import DB
|
||||
from anki.utils import ids2str, intTime, json, isWin, isMac, platDesc, checksum
|
||||
from anki.consts import *
|
||||
|
|
@ -20,76 +17,7 @@ from .lang import ngettext
|
|||
# syncing vars
|
||||
HTTP_TIMEOUT = 90
|
||||
HTTP_PROXY = None
|
||||
|
||||
# badly named; means no retries
|
||||
httplib2.RETRIES = 1
|
||||
|
||||
try:
|
||||
# httplib2 >=0.7.7
|
||||
_proxy_info_from_environment = httplib2.proxy_info_from_environment
|
||||
_proxy_info_from_url = httplib2.proxy_info_from_url
|
||||
except AttributeError:
|
||||
# httplib2 <0.7.7
|
||||
_proxy_info_from_environment = httplib2.ProxyInfo.from_environment
|
||||
_proxy_info_from_url = httplib2.ProxyInfo.from_url
|
||||
|
||||
# Httplib2 connection object
|
||||
######################################################################
|
||||
|
||||
def httpCon():
|
||||
certs = os.path.join(os.path.dirname(__file__), "ankiweb.certs")
|
||||
if not os.path.exists(certs):
|
||||
if not isMac:
|
||||
certs = os.path.abspath(os.path.join(
|
||||
os.path.dirname(certs), "..", "ankiweb.certs"))
|
||||
else:
|
||||
certs = os.path.abspath(os.path.join(
|
||||
os.path.dirname(os.path.abspath(sys.argv[0])),
|
||||
"../Resources/ankiweb.certs"))
|
||||
if not os.path.exists(certs):
|
||||
assert 0, "Unable to locate ankiweb.certs"
|
||||
return httplib2.Http(
|
||||
timeout=HTTP_TIMEOUT, ca_certs=certs,
|
||||
proxy_info=HTTP_PROXY,
|
||||
disable_ssl_certificate_validation=not not HTTP_PROXY)
|
||||
|
||||
# Proxy handling
|
||||
######################################################################
|
||||
|
||||
def _setupProxy():
|
||||
global HTTP_PROXY
|
||||
# set in env?
|
||||
p = _proxy_info_from_environment()
|
||||
if not p:
|
||||
# platform-specific fetch
|
||||
url = None
|
||||
if isWin:
|
||||
print("fixme: win proxy support")
|
||||
# r = urllib.getproxies_registry()
|
||||
# if 'https' in r:
|
||||
# url = r['https']
|
||||
# elif 'http' in r:
|
||||
# url = r['http']
|
||||
elif isMac:
|
||||
print("fixme: mac proxy support")
|
||||
# r = urllib.getproxies_macosx_sysconf()
|
||||
# if 'https' in r:
|
||||
# url = r['https']
|
||||
# elif 'http' in r:
|
||||
# url = r['http']
|
||||
if url:
|
||||
p = _proxy_info_from_url(url, _proxyMethod(url))
|
||||
if p:
|
||||
p.proxy_rdns = True
|
||||
HTTP_PROXY = p
|
||||
|
||||
def _proxyMethod(url):
|
||||
if url.lower().startswith("https"):
|
||||
return "https"
|
||||
else:
|
||||
return "http"
|
||||
|
||||
_setupProxy()
|
||||
HTTP_BUF_SIZE = 64*1024
|
||||
|
||||
# Incremental syncing
|
||||
##########################################################################
|
||||
|
|
@ -526,25 +454,51 @@ class LocalServer(Syncer):
|
|||
l = json.loads; d = json.dumps
|
||||
return l(d(Syncer.applyChanges(self, l(d(changes)))))
|
||||
|
||||
# Wrapper for requests that tracks upload/download progress
|
||||
##########################################################################
|
||||
|
||||
class AnkiRequestsClient(object):
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
|
||||
def post(self, url, data, headers):
|
||||
data = _MonitoringFile(data)
|
||||
return self.session.post(url, data=data, headers=headers, stream=True)
|
||||
|
||||
def get(self, url):
|
||||
return self.session.get(url, stream=True)
|
||||
|
||||
def streamContent(self, resp):
|
||||
resp.raise_for_status()
|
||||
|
||||
buf = io.BytesIO()
|
||||
for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE):
|
||||
runHook("httpRecv", len(chunk))
|
||||
buf.write(chunk)
|
||||
return buf.getvalue()
|
||||
|
||||
class _MonitoringFile(io.BufferedReader):
|
||||
def read(self, size=-1):
|
||||
data = io.BufferedReader.read(self, HTTP_BUF_SIZE)
|
||||
runHook("httpSend", len(data))
|
||||
return data
|
||||
|
||||
# HTTP syncing tools
|
||||
##########################################################################
|
||||
|
||||
# Calling code should catch the following codes:
|
||||
# - 501: client needs upgrade
|
||||
# - 502: ankiweb down
|
||||
# - 503/504: server too busy
|
||||
|
||||
class HttpSyncer(object):
|
||||
|
||||
def __init__(self, hkey=None, con=None):
|
||||
def __init__(self, hkey=None, client=None):
|
||||
self.hkey = hkey
|
||||
self.skey = checksum(str(random.random()))[:8]
|
||||
self.con = con or httpCon()
|
||||
self.client = client or AnkiRequestsClient()
|
||||
self.postVars = {}
|
||||
|
||||
def assertOk(self, resp):
|
||||
if resp['status'] != '200':
|
||||
raise Exception("Unknown response code: %s" % resp['status'])
|
||||
# not using raise_for_status() as aqt expects this error msg
|
||||
if resp.status_code != 200:
|
||||
raise Exception("Unknown response code: %s" % resp.status_code)
|
||||
|
||||
# Posting data as a file
|
||||
######################################################################
|
||||
|
|
@ -552,7 +506,7 @@ class HttpSyncer(object):
|
|||
# costly. We could send it as a raw post, but more HTTP clients seem to
|
||||
# support file uploading, so this is the more compatible choice.
|
||||
|
||||
def req(self, method, fobj=None, comp=6, badAuthRaises=True):
|
||||
def _buildPostData(self, fobj, comp):
|
||||
BOUNDARY=b"Anki-sync-boundary"
|
||||
bdry = b"--"+BOUNDARY
|
||||
buf = io.BytesIO()
|
||||
|
|
@ -590,16 +544,19 @@ Content-Type: application/octet-stream\r\n\r\n""")
|
|||
'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY.decode("utf8"),
|
||||
'Content-Length': str(size),
|
||||
}
|
||||
body = buf.getvalue()
|
||||
buf.close()
|
||||
resp, cont = self.con.request(
|
||||
self.syncURL()+method, "POST", headers=headers, body=body)
|
||||
if not badAuthRaises:
|
||||
# return false if bad auth instead of raising
|
||||
if resp['status'] == '403':
|
||||
buf.seek(0)
|
||||
return headers, buf
|
||||
|
||||
def req(self, method, fobj=None, comp=6, badAuthRaises=True):
|
||||
headers, body = self._buildPostData(fobj, comp)
|
||||
|
||||
r = self.client.post(self.syncURL()+method, data=body, headers=headers)
|
||||
if not badAuthRaises and r.status_code == 403:
|
||||
return False
|
||||
self.assertOk(resp)
|
||||
return cont
|
||||
self.assertOk(r)
|
||||
|
||||
buf = self.client.streamContent(r)
|
||||
return buf
|
||||
|
||||
# Incremental sync over HTTP
|
||||
######################################################################
|
||||
|
|
@ -670,8 +627,8 @@ class RemoteServer(HttpSyncer):
|
|||
|
||||
class FullSyncer(HttpSyncer):
|
||||
|
||||
def __init__(self, col, hkey, con):
|
||||
HttpSyncer.__init__(self, hkey, con)
|
||||
def __init__(self, col, hkey, client):
|
||||
HttpSyncer.__init__(self, hkey, client)
|
||||
self.postVars = dict(
|
||||
k=self.hkey,
|
||||
v="ankidesktop,%s,%s"%(anki.version, platDesc()),
|
||||
|
|
@ -858,9 +815,9 @@ class MediaSyncer(object):
|
|||
|
||||
class RemoteMediaServer(HttpSyncer):
|
||||
|
||||
def __init__(self, col, hkey, con):
|
||||
def __init__(self, col, hkey, client):
|
||||
self.col = col
|
||||
HttpSyncer.__init__(self, hkey, con)
|
||||
HttpSyncer.__init__(self, hkey, client)
|
||||
|
||||
def syncURL(self):
|
||||
if os.getenv("ANKIDEV"):
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@
|
|||
|
||||
import time, re, traceback
|
||||
from aqt.qt import *
|
||||
from anki.sync import httpCon
|
||||
from anki.sync import AnkiRequestsClient
|
||||
from aqt.utils import showWarning
|
||||
from anki.hooks import addHook, remHook
|
||||
import aqt.sync # monkey-patches httplib2
|
||||
import aqt
|
||||
|
||||
def download(mw, code):
|
||||
"Download addon/deck from AnkiWeb. On success caller must stop progress diag."
|
||||
|
|
@ -54,19 +54,22 @@ class Downloader(QThread):
|
|||
# setup progress handler
|
||||
self.byteUpdate = time.time()
|
||||
self.recvTotal = 0
|
||||
def canPost():
|
||||
if (time.time() - self.byteUpdate) > 0.1:
|
||||
self.byteUpdate = time.time()
|
||||
return True
|
||||
def recvEvent(bytes):
|
||||
self.recvTotal += bytes
|
||||
if canPost():
|
||||
self.recv.emit()
|
||||
addHook("httpRecv", recvEvent)
|
||||
con = httpCon()
|
||||
client = AnkiRequestsClient()
|
||||
try:
|
||||
resp, cont = con.request(
|
||||
resp = client.get(
|
||||
aqt.appShared + "download/%d" % self.code)
|
||||
if resp.status_code == 200:
|
||||
data = client.streamContent(resp)
|
||||
elif resp.status_code in (403,404):
|
||||
self.error = _("Invalid code")
|
||||
return
|
||||
else:
|
||||
self.error = _("Error downloading: %s" % resp.status_code)
|
||||
return
|
||||
except Exception as e:
|
||||
exc = traceback.format_exc()
|
||||
try:
|
||||
|
|
@ -76,12 +79,7 @@ class Downloader(QThread):
|
|||
return
|
||||
finally:
|
||||
remHook("httpRecv", recvEvent)
|
||||
if resp['status'] == '200':
|
||||
self.error = None
|
||||
|
||||
self.fname = re.match("attachment; filename=(.+)",
|
||||
resp['content-disposition']).group(1)
|
||||
self.data = cont
|
||||
elif resp['status'] == '403':
|
||||
self.error = _("Invalid code.")
|
||||
else:
|
||||
self.error = _("Error downloading: %s") % resp['status']
|
||||
resp.headers['content-disposition']).group(1)
|
||||
self.data = data
|
||||
|
|
|
|||
102
aqt/sync.py
102
aqt/sync.py
|
|
@ -73,9 +73,9 @@ automatically."""))
|
|||
def _updateLabel(self):
|
||||
self.mw.progress.update(label="%s\n%s" % (
|
||||
self.label,
|
||||
_("%(a)dkB up, %(b)dkB down") % dict(
|
||||
a=self.sentBytes // 1024,
|
||||
b=self.recvBytes // 1024)))
|
||||
_("%(a)0.1fkB up, %(b)0.1fkB down") % dict(
|
||||
a=self.sentBytes / 1024,
|
||||
b=self.recvBytes / 1024)))
|
||||
|
||||
def onEvent(self, evt, *args):
|
||||
pu = self.mw.progress.update
|
||||
|
|
@ -298,23 +298,15 @@ class SyncThread(QThread):
|
|||
self.client = Syncer(self.col, self.server)
|
||||
self.sentTotal = 0
|
||||
self.recvTotal = 0
|
||||
# throttle updates; qt doesn't handle lots of posted events well
|
||||
self.byteUpdate = time.time()
|
||||
def syncEvent(type):
|
||||
self.fireEvent("sync", type)
|
||||
def syncMsg(msg):
|
||||
self.fireEvent("syncMsg", msg)
|
||||
def canPost():
|
||||
if (time.time() - self.byteUpdate) > 0.1:
|
||||
self.byteUpdate = time.time()
|
||||
return True
|
||||
def sendEvent(bytes):
|
||||
self.sentTotal += bytes
|
||||
if canPost():
|
||||
self.fireEvent("send", str(self.sentTotal))
|
||||
def recvEvent(bytes):
|
||||
self.recvTotal += bytes
|
||||
if canPost():
|
||||
self.fireEvent("recv", str(self.recvTotal))
|
||||
addHook("sync", syncEvent)
|
||||
addHook("syncMsg", syncMsg)
|
||||
|
|
@ -395,7 +387,7 @@ class SyncThread(QThread):
|
|||
f = self.fullSyncChoice
|
||||
if f == "cancel":
|
||||
return
|
||||
self.client = FullSyncer(self.col, self.hkey, self.server.con)
|
||||
self.client = FullSyncer(self.col, self.hkey, self.server.client)
|
||||
if f == "upload":
|
||||
if not self.client.upload():
|
||||
self.fireEvent("upbad")
|
||||
|
|
@ -408,7 +400,7 @@ class SyncThread(QThread):
|
|||
def _syncMedia(self):
|
||||
if not self.media:
|
||||
return
|
||||
self.server = RemoteMediaServer(self.col, self.hkey, self.server.con)
|
||||
self.server = RemoteMediaServer(self.col, self.hkey, self.server.client)
|
||||
self.client = MediaSyncer(self.col, self.server)
|
||||
ret = self.client.sync()
|
||||
if ret == "noChanges":
|
||||
|
|
@ -422,87 +414,3 @@ class SyncThread(QThread):
|
|||
self.event.emit(cmd, arg)
|
||||
|
||||
|
||||
# Monkey-patch httplib & httplib2 so we can get progress info
|
||||
######################################################################
|
||||
|
||||
CHUNK_SIZE = 65536
|
||||
import http.client, httplib2
|
||||
from io import StringIO
|
||||
from anki.hooks import runHook
|
||||
|
||||
print("fixme: _conn_request and _incrementalSend need updating for python3")
|
||||
|
||||
# sending in httplib
|
||||
def _incrementalSend(self, data):
|
||||
"""Send `data' to the server."""
|
||||
if self.sock is None:
|
||||
if self.auto_open:
|
||||
self.connect()
|
||||
else:
|
||||
raise http.client.NotConnected()
|
||||
# if it's not a file object, make it one
|
||||
if not hasattr(data, 'read'):
|
||||
data = StringIO(data)
|
||||
while 1:
|
||||
block = data.read(CHUNK_SIZE)
|
||||
if not block:
|
||||
break
|
||||
self.sock.sendall(block)
|
||||
runHook("httpSend", len(block))
|
||||
|
||||
#http.client.HTTPConnection.send = _incrementalSend
|
||||
|
||||
# receiving in httplib2
|
||||
# this is an augmented version of httplib's request routine that:
|
||||
# - doesn't assume requests will be tried more than once
|
||||
# - calls a hook for each chunk of data so we can update the gui
|
||||
# - retries only when keep-alive connection is closed
|
||||
def _conn_request(self, conn, request_uri, method, body, headers):
|
||||
for i in range(2):
|
||||
try:
|
||||
if conn.sock is None:
|
||||
conn.connect()
|
||||
conn.request(method, request_uri, body, headers)
|
||||
except socket.timeout:
|
||||
raise
|
||||
except socket.gaierror:
|
||||
conn.close()
|
||||
raise httplib2.ServerNotFoundError(
|
||||
"Unable to find the server at %s" % conn.host)
|
||||
except httplib2.ssl_SSLError:
|
||||
conn.close()
|
||||
raise
|
||||
except socket.error as e:
|
||||
conn.close()
|
||||
raise
|
||||
except http.client.HTTPException:
|
||||
conn.close()
|
||||
raise
|
||||
try:
|
||||
response = conn.getresponse()
|
||||
except http.client.BadStatusLine:
|
||||
print("retry bad line")
|
||||
conn.close()
|
||||
conn.connect()
|
||||
continue
|
||||
except (socket.error, http.client.HTTPException):
|
||||
raise
|
||||
else:
|
||||
content = ""
|
||||
if method == "HEAD":
|
||||
response.close()
|
||||
else:
|
||||
buf = StringIO()
|
||||
while 1:
|
||||
data = response.read(CHUNK_SIZE)
|
||||
if not data:
|
||||
break
|
||||
buf.write(data)
|
||||
runHook("httpRecv", len(data))
|
||||
content = buf.getvalue()
|
||||
response = httplib2.Response(response)
|
||||
if method != "HEAD":
|
||||
content = httplib2._decompressContent(response, content)
|
||||
return (response, content)
|
||||
|
||||
#httplib2.Http._conn_request = _conn_request
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
||||
|
||||
import urllib.request, urllib.parse, urllib.error
|
||||
import urllib.request, urllib.error, urllib.parse
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
from aqt.qt import *
|
||||
import aqt
|
||||
from aqt.utils import openLink
|
||||
from anki.utils import json, platDesc
|
||||
from aqt.utils import showText
|
||||
|
||||
|
||||
class LatestVersionFinder(QThread):
|
||||
|
||||
newVerAvail = pyqtSignal(str)
|
||||
|
|
@ -36,14 +35,11 @@ class LatestVersionFinder(QThread):
|
|||
return
|
||||
d = self._data()
|
||||
d['proto'] = 1
|
||||
d = urllib.parse.urlencode(d).encode("utf8")
|
||||
|
||||
try:
|
||||
f = urllib.request.urlopen(aqt.appUpdate, d)
|
||||
resp = f.read()
|
||||
if not resp:
|
||||
print("update check load failed")
|
||||
return
|
||||
resp = json.loads(resp.decode("utf8"))
|
||||
r = requests.post(aqt.appUpdate, data=d)
|
||||
r.raise_for_status()
|
||||
resp = r.json()
|
||||
except:
|
||||
# behind proxy, corrupt message, etc
|
||||
print("update check failed")
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@ bs4
|
|||
send2trash
|
||||
httplib2
|
||||
pyaudio
|
||||
|
||||
requests
|
||||
|
|
|
|||
Loading…
Reference in a new issue