Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Progress on --download.

  • Loading branch information...
commit 9b2a293e6efdda98aada896d0eac1cf10c331194 1 parent b0dd463
@jakubroztocil authored
Showing with 133 additions and 91 deletions.
  1. +12 −6 httpie/core.py
  2. +120 −85 httpie/downloads.py
  3. +1 −0  httpie/humanize.py
View
18 httpie/core.py
@@ -80,18 +80,22 @@ def error(msg, *args, **kwargs):
download = None
if args.download:
- args.follow = True
+ args.follow = True # --download implies --follow.
download = Download(
output_file=args.output_file,
progress_file=env.stderr,
resume=args.download_resume
)
- download.alter_request_headers(args.headers)
+ download.pre_request(args.headers)
response = get_response(args, config_dir=env.config.directory)
- if args.check_status:
- exit_status = get_exit_status(response.status_code, args.follow)
+ if args.check_status or download:
+
+ exit_status = get_exit_status(
+ http_status=response.status_code,
+ follow=args.follow
+ )
if not env.stdout_isatty and exit_status != ExitStatus.OK:
error('HTTP %s %s',
@@ -116,7 +120,7 @@ def error(msg, *args, **kwargs):
else:
write(**write_kwargs)
- if download:
+ if download and exit_status == ExitStatus.OK:
# Response body download.
download_stream, download_to = download.start(response)
write(
@@ -124,7 +128,9 @@ def error(msg, *args, **kwargs):
outfile=download_to,
flush=False,
)
- download.finished()
+ download.finish()
+ if download.interrupted:
+ exit_status = ExitStatus.ERROR
except IOError as e:
if not traceback and e.errno == errno.EPIPE:
View
205 httpie/downloads.py
@@ -15,12 +15,6 @@
from .compat import urlsplit
-CLEAR_LINE = '\r\033[K'
-TPL_PROGRESS = '{percentage:0.2f} % ({downloaded}) of {total} ({speed}/s)'
-TPL_PROGRESS_NO_CONTENT_LENGTH = '{downloaded} ({speed}/s)'
-TPL_SUMMARY = '{downloaded} of {total} in {time:0.5f}s ({speed}/s)\n'
-
-
class Download(object):
def __init__(self, output_file=None,
@@ -38,31 +32,29 @@ def __init__(self, output_file=None,
:type progress_file: file
"""
- self.output_file = output_file
- self.progress_file = progress_file
- self.resume = resume
+ self._output_file = output_file
+ self._resume = resume
+ self._progress = Progress(output=progress_file)
+ self._resumed_from = 0
- self.bytes_resumed_from = 0
- self.content_length = None
- self.bytes_downloaded = 0
- self.bytes_downloaded_prev = 0
- self.bytes_total_humanized = ''
- self.time_started = None
- self.time_prev = None
- self.speed = 0
-
- def alter_request_headers(self, headers):
- """Called just before a download request is sent."""
+ def pre_request(self, headers):
+ """Called just before the HTTP request is sent.
+
+ Might alter `headers`.
+
+ :type headers: dict
+
+ """
# Disable content encoding so that we can resume, etc.
headers['Accept-Encoding'] = ''
- if self.resume:
+ if self._resume:
try:
- bytes_have = os.path.getsize(self.output_file.name)
+ bytes_have = os.path.getsize(self._output_file.name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
- self.bytes_resumed_from = self.bytes_downloaded = bytes_have
+ self._resumed_from = bytes_have
# Set ``Range`` header to resume the download
# TODO: detect Range support first?
headers['Range'] = '%d-' % bytes_have
@@ -78,18 +70,21 @@ def start(self, response):
:return: RawStream, output_file
"""
- assert not self.time_started
+ assert not self._progress._time_started
- if self.output_file:
- if not self.resume:
- self.output_file.seek(0)
- self.output_file.truncate()
+ content_length = response.headers.get('Content-Length')
+ if content_length:
+ content_length = int(content_length)
+
+ if self._output_file:
+ if not self._resume:
+ self._output_file.seek(0)
+ self._output_file.truncate()
else:
# TODO: should we take the filename from response.history[0].url?
- # TODO: --download implies --follow
# Output file not specified. Pick a name that doesn't exist yet.
content_type = response.headers.get('Content-Type', '')
- self.output_file = open(
+ self._output_file = open(
self._get_unique_output_filename(
url=response.url,
content_type=content_type,
@@ -97,72 +92,37 @@ def start(self, response):
mode='a+b'
)
- self.content_length = response.headers.get('Content-Length')
- if self.content_length:
- self.content_length = int(self.content_length)
-
- self.bytes_total_humanized = (humanize_bytes(self.content_length)
- if self.content_length else '?')
-
- self.time_started = time()
- self.time_prev = self.time_started
+ self._progress.start(
+ resumed_from=self._resumed_from,
+ content_length=content_length
+ )
stream = RawStream(
msg=HTTPResponse(response),
with_headers=False,
with_body=True,
on_body_chunk_downloaded=self._on_progress,
- # FIXME: large chunks & chunked response freezes
+ # FIXME: large chunks & chunked response => freezes
chunk_size=1
)
- self.progress_file.write('Saving to %s\n' % self.output_file.name)
- self.report_status()
+ self._progress.output.write('Saving to %s\n' % self._output_file.name)
+ self._progress.report()
- return stream, self.output_file
+ return stream, self._output_file
- def report_status(self, interval=.6):
- now = time()
+ def finish(self):
+ assert not self._output_file.closed
+ self._output_file.close()
+ self._progress.finished()
- # Update the reported speed on the first chunk and once in a while.
- if self.bytes_downloaded_prev and now - self.time_prev < interval:
- return
-
- self.speed = (
- (self.bytes_downloaded - self.bytes_downloaded_prev)
- / (now - self.time_prev)
+ @property
+ def interrupted(self):
+ return (
+ self._output_file.closed
+ and self._progress.content_length
+ and self._progress.content_length != self._progress.downloaded
)
- self.time_prev = now
- self.bytes_downloaded_prev = self.bytes_downloaded
-
- if self.content_length:
- template = TPL_PROGRESS
- percentage = self.bytes_downloaded / self.content_length * 100
- else:
- template = TPL_PROGRESS_NO_CONTENT_LENGTH
- percentage = None
-
- self.progress_file.write(CLEAR_LINE + template.format(
- percentage=percentage,
- downloaded=humanize_bytes(self.bytes_downloaded),
- total=self.bytes_total_humanized,
- speed=humanize_bytes(self.speed)
- ))
-
- self.progress_file.flush()
-
- def finished(self):
- self.output_file.close()
-
- bytes_downloaded = self.bytes_downloaded - self.bytes_resumed_from
- time_taken = time() - self.time_started
- self.progress_file.write(CLEAR_LINE + TPL_SUMMARY.format(
- downloaded=humanize_bytes(bytes_downloaded),
- total=humanize_bytes(self.bytes_downloaded),
- speed=humanize_bytes(bytes_downloaded / time_taken),
- time=time_taken,
- ))
- self.progress_file.flush()
def _on_progress(self, chunk):
"""
@@ -173,8 +133,8 @@ def _on_progress(self, chunk):
:type chunk: bytes
"""
- self.bytes_downloaded += len(chunk)
- self.report_status()
+ self._progress.chunk_downloaded(len(chunk))
+ self._progress.report()
def _get_unique_output_filename(self, url, content_type):
suffix = 0
@@ -198,3 +158,78 @@ def _get_output_filename(self, url, content_type, suffix=None):
ext = mimetypes.guess_extension(content_type.split(';')[0]) or ''
return base + suffix + ext
+
+
+class Progress(object):
+
+ CLEAR_LINE = '\r\033[K'
+ PROGRESS = '{percentage:0.2f} % ({downloaded}) of {total} ({speed}/s)'
+ PROGRESS_NO_CONTENT_LENGTH = '{downloaded} ({speed}/s)'
+ SUMMARY = '{downloaded} of {total} in {time:0.5f}s ({speed}/s)\n'
+
+ def __init__(self, output):
+ """
+ :type output: file
+
+ """
+ self.output = output
+ self.downloaded = 0
+ self.content_length = None
+ self._resumed_from = 0
+ self._downloaded_prev = 0
+ self._content_length_humanized = '?'
+ self._time_started = None
+ self._time_prev = None
+ self._speed = 0
+
+ def start(self, resumed_from=0, content_length=None):
+ assert self._time_started is None
+ if content_length is not None:
+ self._content_length_humanized = humanize_bytes(content_length)
+ self.content_length = content_length
+ self.downloaded = self._resumed_from = resumed_from
+ self._time_started = time()
+ self._time_prev = self._time_started
+
+ def chunk_downloaded(self, size):
+ self.downloaded += size
+
+ def report(self, interval=.6):
+ now = time()
+
+ # Update the reported speed on the first chunk and once in a while.
+ if self._downloaded_prev and now - self._time_prev < interval:
+ return
+
+ self._speed = (
+ (self.downloaded - self._downloaded_prev)
+ / (now - self._time_prev)
+ )
+ self._time_prev = now
+ self._downloaded_prev = self.downloaded
+
+ if self.content_length:
+ template = self.PROGRESS
+ percentage = self.downloaded / self.content_length * 100
+ else:
+ template = self.PROGRESS_NO_CONTENT_LENGTH
+ percentage = None
+
+ self.output.write(self.CLEAR_LINE + template.format(
+ percentage=percentage,
+ downloaded=humanize_bytes(self.downloaded),
+ total=self._content_length_humanized,
+ speed=humanize_bytes(self._speed)
+ ))
+
+ self.output.flush()
+
+ def finished(self):
+ downloaded = self.downloaded - self._resumed_from
+ time_taken = time() - self._time_started
+ self.output.write(self.CLEAR_LINE + self.SUMMARY.format(
+ downloaded=humanize_bytes(downloaded),
+ total=humanize_bytes(self.downloaded),
+ speed=humanize_bytes(downloaded / time_taken),
+ time=time_taken,
+ ))
View
1  httpie/humanize.py
@@ -4,6 +4,7 @@
URL: http://code.activestate.com/recipes/577081/
"""
+from __future__ import division
import doctest
Please sign in to comment.
Something went wrong with that request. Please try again.