Port away from urlgrabber to requests

Fixes #45, #43, #19

liveusb-creator.spec

@@ -32,7 +32,7 @@ Requires:       PyQt5
 Requires:       qt5-qtgraphicaleffects
 Requires:       qt5-qtquickcontrols
 Requires:       isomd5sum
-Requires:       python-urlgrabber
+Requires:       python-requests
 Requires:       pyparted >= 2.0
 Requires:       syslinux-extlinux
 Requires:       udisks2
@@ -56,7 +56,6 @@ make gui
 %install
 rm -rf %{buildroot}
 %{__python} setup.py install -O1 --skip-build --root %{buildroot}
-%{__rm} -r liveusb/urlgrabber
 
 #Adding the AppStream metadata file
 install -Dm 0644 -p liveusb-creator.appdata.xml \
@@ -105,6 +104,9 @@ rm -rf %{buildroot}
 %{_datadir}/appdata/liveusb-creator.appdata.xml
 
 %changelog
+* Wed Apr 20 2016 Martin Briza <mbriza@redhat.com> - 3.15.0-0.3.newui
+- Changed urlgrabber to requests
+
 * Wed Apr 20 2016 Jiri Eischmann <eischmann@redhat.com> - 3.15.0-0.2.newui
 - Adding AppStream metadata file
 

liveusb/grabber.py

@@ -0,0 +1,107 @@
+from liveusb import LiveUSBError
+
+import subprocess
+import os
+import sys
+import requests
+import tempfile
+
+from PyQt5.QtCore import QStandardPaths
+
+
+def find_downloads():
+    # todo look into SUDO_UID and PKEXEC_UID for the original user
+    if sys.platform.startswith("linux"):
+        import pwd
+        uid = 0
+        if 'SUDO_UID' in os.environ:
+            uid = int(os.environ['SUDO_UID'])
+        elif 'PKEXEC_UID' in os.environ:
+            uid = int(os.environ['PKEXEC_UID'])
+
+        pw = pwd.getpwuid(uid)
+        p = subprocess.Popen(['sudo', '-u', pw[0], 'xdg-user-dir', 'DOWNLOAD'], stdout=subprocess.PIPE)
+        p.wait()
+        path = p.stdout.readline().strip()
+    else:
+        path = QStandardPaths.writableLocation(QStandardPaths.DownloadLocation)
+
+    return path
+
+def chown_file(path):
+    if sys.platform.startswith("linux"):
+        import pwd
+        uid = 0
+        gid = 0
+        if 'SUDO_UID' in os.environ:
+            uid = int(os.environ['SUDO_UID'])
+            gid = int(os.environ['SUDO_GID'])
+        elif 'PKEXEC_UID' in os.environ:
+            uid = int(os.environ['PKEXEC_UID'])
+            pw = pwd.getpwuid(uid)
+            gid = pw[3]
+        else:
+            pass
+
+        os.chown(path, uid, gid)
+    else:
+        pass
+
+
+def download(url, target_folder=find_downloads(), update_maximum = None, update_current = None):
+    CHUNK_SIZE = 1024 * 1024
+    current_size = 0
+
+    file_name = os.path.basename(url)
+    full_path = os.path.join(target_folder, file_name)
+    if os.path.exists(full_path):
+        current_size = os.path.getsize(full_path)
+    bytes_read = current_size
+
+    if current_size > 0:
+        resume_header = {'Range': 'bytes=%d-' % current_size}
+    else:
+        resume_header = {}
+
+    try:
+        r = requests.get(url, headers=resume_header, stream=True, allow_redirects=True, timeout=(30.0, 30.0))
+
+        if r.status_code == 200:
+            mode = "ab"
+        elif r.status_code == 206:
+            mode = "wb"
+        elif r.status_code == 416:
+            return full_path
+        else:
+            raise LiveUSBError("Couldn't download the file: %s (%d)" % (r.reason, r.status_code))
+
+        if update_maximum:
+            update_maximum(current_size + int(r.headers['Content-Length']))
+
+        with open(full_path, mode) as f:
+            chown_file(full_path)
+
+            for chunk in r.iter_content(CHUNK_SIZE):
+                f.write(chunk)
+                bytes_read += len(chunk)
+                if update_current:
+                    update_current(bytes_read)
+
+    except requests.exceptions.ReadTimeout as e:
+        raise LiveUSBError("Your internet connection seems to be broken")
+    except requests.exceptions.ConnectTimeout as e:
+        raise LiveUSBError("Your internet connection seems to be broken")
+    except Exception as e:
+        raise LiveUSBError("Couldn't download the file: %s (%d): %s" % (r.reason, r.status_code, e.message))
+
+    return full_path
+
+
+
+def __print(val):
+    print(val)
+
+if __name__ == '__main__':
+    import pprint
+    #download("https://download.fedoraproject.org/pub/fedora/linux/releases/23/Workstation/x86_64/iso/Fedora-Live-Workstation-x86_64-23-10.iso", update_maximum=__print, update_current=__print)
+    print find_downloads()

liveusb/gui.py

@@ -49,17 +49,11 @@
 
 import resources_rc
 import qml_rc
+import grabber
 
 from liveusb import LiveUSBCreator, LiveUSBError, _
 from liveusb.releases import releases, get_fedora_flavors
 
-if sys.platform == 'win32':
-    from liveusb.urlgrabber.grabber import URLGrabber, URLGrabError
-    from liveusb.urlgrabber.progress import BaseMeter
-else:
-    from urlgrabber.grabber import URLGrabber, URLGrabError
-    from urlgrabber.progress import BaseMeter
-
 try:
     import dbus.mainloop.pyqt5
     dbus.mainloop.pyqt5.DBusQtMainLoop(set_as_default=True)
@@ -81,25 +75,14 @@ def __init__(self, progress, proxies):
         self.proxies = proxies
 
     def run(self):
-        self.grabber = URLGrabber(progress_obj=self.progress, proxies=self.proxies)
-        home = os.getenv('HOME', 'USERPROFILE')
-        filename = os.path.basename(urlparse.urlparse(self.progress.release.url).path)
-        for folder in ('Downloads', 'My Documents'):
-            if os.path.isdir(os.path.join(home, folder)):
-                filename = os.path.join(home, folder, filename)
-                break
         try:
-            iso = self.grabber.urlgrab(self.progress.release.url, filename=filename, reget='simple')
-        except URLGrabError, e:
-            # TODO find out if this errno is _really_ benign
-            if e.errno == 9: # Requested byte range not satisfiable.
-                self.downloadFinished.emit(filename)
-            else:
-                self.downloadError.emit(e.strerror)
-        else:
-            self.downloadFinished.emit(iso)
+            filename = grabber.download(self.progress.release.url, update_maximum=self.progress.start, update_current=self.progress.update)
+            self.progress.end()
+            self.downloadFinished.emit(filename)
+        except LiveUSBError as e:
+            self.downloadError.emit(e.args[0])
 
-class ReleaseDownload(QObject, BaseMeter):
+class ReleaseDownload(QObject):
     """ Wrapper for the iso download process.
     It exports properties to track the percentage and the file with the result.
     """
@@ -128,13 +111,13 @@ def reset(self):
         self.currentChanged.emit()
         self.maximumChanged.emit()
 
-    def start(self, filename=None, url=None, basename=None, size=None, now=None, text=None):
+    def start(self, size=None):
         self._maximum = size
         self._running = True
         self.maximumChanged.emit()
         self.runningChanged.emit()
 
-    def update(self, amount_read, now=None):
+    def update(self, amount_read):
         """ Update our download progressbar.
 
         :read: the number of bytes read so far
@@ -143,9 +126,11 @@ def update(self, amount_read, now=None):
             self._current = amount_read
             self.currentChanged.emit()
 
-    def end(self, amount_read):
-        self._current = amount_read
+    def end(self):
+        self._current = self._maximum
         self.currentChanged.emit()
+        self._running = False
+        self.runningChanged.emit()
 
     @pyqtSlot(str)
     def childFinished(self, iso):

liveusb/releases.py

@@ -3,6 +3,7 @@
 import re
 import traceback
 
+#todo port away from urlgrabber too
 from urlgrabber import urlread
 from urlgrabber.grabber import URLGrabError