Permalink
Browse files

use requests for http; add progress info back

- wrap request in AnkiRequestsClient so we can keep track of
upload/download bytes without having to monkey patch anything
- force a 64kB buffer size instead of the default 8kB
- show one decimal point in up/down so small requests still give
visual feedback
- update add-on downloading and update check to use requests
- remove the update throttling in aqt/sync.py, as it's not really
necessary anymore
  • Loading branch information...
1 parent 147e09a commit f6245cdfd1e81fecb581a17d3ee314ed0d72698d @dae committed Jan 8, 2017
Showing with 86 additions and 227 deletions.
  1. +55 −98 anki/sync.py
  2. +17 −19 aqt/downloader.py
  3. +7 −99 aqt/sync.py
  4. +6 −10 aqt/update.py
  5. +1 −1 requirements.txt
View
@@ -2,14 +2,11 @@
# Copyright: Damien Elmes <anki@ichi2.net>
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-import urllib.request, urllib.parse, urllib.error
import io
-import sys
import gzip
import random
-from io import StringIO
+import requests
-import httplib2
from anki.db import DB
from anki.utils import ids2str, intTime, json, isWin, isMac, platDesc, checksum
from anki.consts import *
@@ -20,76 +17,7 @@
# syncing vars
HTTP_TIMEOUT = 90
HTTP_PROXY = None
-
-# badly named; means no retries
-httplib2.RETRIES = 1
-
-try:
- # httplib2 >=0.7.7
- _proxy_info_from_environment = httplib2.proxy_info_from_environment
- _proxy_info_from_url = httplib2.proxy_info_from_url
-except AttributeError:
- # httplib2 <0.7.7
- _proxy_info_from_environment = httplib2.ProxyInfo.from_environment
- _proxy_info_from_url = httplib2.ProxyInfo.from_url
-
-# Httplib2 connection object
-######################################################################
-
-def httpCon():
- certs = os.path.join(os.path.dirname(__file__), "ankiweb.certs")
- if not os.path.exists(certs):
- if not isMac:
- certs = os.path.abspath(os.path.join(
- os.path.dirname(certs), "..", "ankiweb.certs"))
- else:
- certs = os.path.abspath(os.path.join(
- os.path.dirname(os.path.abspath(sys.argv[0])),
- "../Resources/ankiweb.certs"))
- if not os.path.exists(certs):
- assert 0, "Unable to locate ankiweb.certs"
- return httplib2.Http(
- timeout=HTTP_TIMEOUT, ca_certs=certs,
- proxy_info=HTTP_PROXY,
- disable_ssl_certificate_validation=not not HTTP_PROXY)
-
-# Proxy handling
-######################################################################
-
-def _setupProxy():
- global HTTP_PROXY
- # set in env?
- p = _proxy_info_from_environment()
- if not p:
- # platform-specific fetch
- url = None
- if isWin:
- print("fixme: win proxy support")
- # r = urllib.getproxies_registry()
- # if 'https' in r:
- # url = r['https']
- # elif 'http' in r:
- # url = r['http']
- elif isMac:
- print("fixme: mac proxy support")
- # r = urllib.getproxies_macosx_sysconf()
- # if 'https' in r:
- # url = r['https']
- # elif 'http' in r:
- # url = r['http']
- if url:
- p = _proxy_info_from_url(url, _proxyMethod(url))
- if p:
- p.proxy_rdns = True
- HTTP_PROXY = p
-
-def _proxyMethod(url):
- if url.lower().startswith("https"):
- return "https"
- else:
- return "http"
-
-_setupProxy()
+HTTP_BUF_SIZE = 64*1024
# Incremental syncing
##########################################################################
@@ -526,33 +454,59 @@ def applyChanges(self, changes):
l = json.loads; d = json.dumps
return l(d(Syncer.applyChanges(self, l(d(changes)))))
-# HTTP syncing tools
+# Wrapper for requests that tracks upload/download progress
##########################################################################
-# Calling code should catch the following codes:
-# - 501: client needs upgrade
-# - 502: ankiweb down
-# - 503/504: server too busy
+class AnkiRequestsClient(object):
+
+ def __init__(self):
+ self.session = requests.Session()
+
+ def post(self, url, data, headers):
+ data = _MonitoringFile(data)
+ return self.session.post(url, data=data, headers=headers, stream=True)
+
+ def get(self, url):
+ return self.session.get(url, stream=True)
+
+ def streamContent(self, resp):
+ resp.raise_for_status()
+
+ buf = io.BytesIO()
+ for chunk in resp.iter_content(chunk_size=HTTP_BUF_SIZE):
+ runHook("httpRecv", len(chunk))
+ buf.write(chunk)
+ return buf.getvalue()
+
+class _MonitoringFile(io.BufferedReader):
+ def read(self, size=-1):
+ data = io.BufferedReader.read(self, HTTP_BUF_SIZE)
+ runHook("httpSend", len(data))
+ return data
+
+# HTTP syncing tools
+##########################################################################
class HttpSyncer(object):
- def __init__(self, hkey=None, con=None):
+ def __init__(self, hkey=None, client=None):
self.hkey = hkey
self.skey = checksum(str(random.random()))[:8]
- self.con = con or httpCon()
+ self.client = client or AnkiRequestsClient()
self.postVars = {}
def assertOk(self, resp):
- if resp['status'] != '200':
- raise Exception("Unknown response code: %s" % resp['status'])
+ # not using raise_for_status() as aqt expects this error msg
+ if resp.status_code != 200:
+ raise Exception("Unknown response code: %s" % resp.status_code)
# Posting data as a file
######################################################################
# We don't want to post the payload as a form var, as the percent-encoding is
# costly. We could send it as a raw post, but more HTTP clients seem to
# support file uploading, so this is the more compatible choice.
- def req(self, method, fobj=None, comp=6, badAuthRaises=True):
+ def _buildPostData(self, fobj, comp):
BOUNDARY=b"Anki-sync-boundary"
bdry = b"--"+BOUNDARY
buf = io.BytesIO()
@@ -590,16 +544,19 @@ def req(self, method, fobj=None, comp=6, badAuthRaises=True):
'Content-Type': 'multipart/form-data; boundary=%s' % BOUNDARY.decode("utf8"),
'Content-Length': str(size),
}
- body = buf.getvalue()
- buf.close()
- resp, cont = self.con.request(
- self.syncURL()+method, "POST", headers=headers, body=body)
- if not badAuthRaises:
- # return false if bad auth instead of raising
- if resp['status'] == '403':
- return False
- self.assertOk(resp)
- return cont
+ buf.seek(0)
+ return headers, buf
+
+ def req(self, method, fobj=None, comp=6, badAuthRaises=True):
+ headers, body = self._buildPostData(fobj, comp)
+
+ r = self.client.post(self.syncURL()+method, data=body, headers=headers)
+ if not badAuthRaises and r.status_code == 403:
+ return False
+ self.assertOk(r)
+
+ buf = self.client.streamContent(r)
+ return buf
# Incremental sync over HTTP
######################################################################
@@ -670,8 +627,8 @@ def _run(self, cmd, data):
class FullSyncer(HttpSyncer):
- def __init__(self, col, hkey, con):
- HttpSyncer.__init__(self, hkey, con)
+ def __init__(self, col, hkey, client):
+ HttpSyncer.__init__(self, hkey, client)
self.postVars = dict(
k=self.hkey,
v="ankidesktop,%s,%s"%(anki.version, platDesc()),
@@ -858,9 +815,9 @@ def _downloadFiles(self, fnames):
class RemoteMediaServer(HttpSyncer):
- def __init__(self, col, hkey, con):
+ def __init__(self, col, hkey, client):
self.col = col
- HttpSyncer.__init__(self, hkey, con)
+ HttpSyncer.__init__(self, hkey, client)
def syncURL(self):
if os.getenv("ANKIDEV"):
View
@@ -4,10 +4,10 @@
import time, re, traceback
from aqt.qt import *
-from anki.sync import httpCon
+from anki.sync import AnkiRequestsClient
from aqt.utils import showWarning
from anki.hooks import addHook, remHook
-import aqt.sync # monkey-patches httplib2
+import aqt
def download(mw, code):
"Download addon/deck from AnkiWeb. On success caller must stop progress diag."
@@ -54,19 +54,22 @@ def run(self):
# setup progress handler
self.byteUpdate = time.time()
self.recvTotal = 0
- def canPost():
- if (time.time() - self.byteUpdate) > 0.1:
- self.byteUpdate = time.time()
- return True
def recvEvent(bytes):
self.recvTotal += bytes
- if canPost():
- self.recv.emit()
+ self.recv.emit()
addHook("httpRecv", recvEvent)
- con = httpCon()
+ client = AnkiRequestsClient()
try:
- resp, cont = con.request(
+ resp = client.get(
aqt.appShared + "download/%d" % self.code)
+ if resp.status_code == 200:
+ data = client.streamContent(resp)
+ elif resp.status_code in (403,404):
+ self.error = _("Invalid code")
+ return
+ else:
+ self.error = _("Error downloading: %s" % resp.status_code)
+ return
except Exception as e:
exc = traceback.format_exc()
try:
@@ -76,12 +79,7 @@ def recvEvent(bytes):
return
finally:
remHook("httpRecv", recvEvent)
- if resp['status'] == '200':
- self.error = None
- self.fname = re.match("attachment; filename=(.+)",
- resp['content-disposition']).group(1)
- self.data = cont
- elif resp['status'] == '403':
- self.error = _("Invalid code.")
- else:
- self.error = _("Error downloading: %s") % resp['status']
+
+ self.fname = re.match("attachment; filename=(.+)",
+ resp.headers['content-disposition']).group(1)
+ self.data = data
Oops, something went wrong.

0 comments on commit f6245cd

Please sign in to comment.