Take advantage of streaming.

It's now possible to download huge files with HTTPie, and it's often faster than curl and wget!

README.rst

@@ -183,7 +183,7 @@ Note that when the **output is redirected** (like the examples above), HTTPie
 applies a different set of defaults than for a console output. Namely, colors
 aren't used (unless ``--pretty`` is set) and only the response body
 is printed (unless ``--print`` options specified). It is a convenience
-that allows for things like the one above or downloading (smallish) binary
+that allows for things like the one above or downloading binary
 files without having to set any flags:
 
 .. code-block:: shell
@@ -373,6 +373,7 @@ Changelog
 =========
 
 * `0.2.7dev`_
+    * Support for efficient large file downloads.
     * Response body is fetched only when needed (e.g., not with ``--headers``).
     * Updated Solarized color scheme.
     * Windows: Added ``--output FILE`` to store output into a file

httpie/core.py

@@ -16,7 +16,7 @@
 from requests.compat import str
 
 from .models import HTTPRequest, HTTPResponse, Environment
-from .output import OutputProcessor, format
+from .output import OutputProcessor, formatted_stream
 from .input import (OUT_REQ_BODY, OUT_REQ_HEAD,
                     OUT_RESP_HEAD, OUT_RESP_BODY)
 from .cli import parser
@@ -51,45 +51,35 @@ def get_response(args, env):
             # the `Content-Type` for us.
             args.headers['Content-Type'] = FORM
 
-    try:
-        credentials = None
-        if args.auth:
-            credentials = {
-                'basic': requests.auth.HTTPBasicAuth,
-                'digest': requests.auth.HTTPDigestAuth,
-            }[args.auth_type](args.auth.key, args.auth.value)
-
-        if not (args.url.startswith(HTTP) or args.url.startswith(HTTPS)):
-            scheme = HTTPS if env.progname == 'https' else HTTP
-            url = scheme + args.url
-        else:
-            url = args.url
-
-        return requests.request(
-            method=args.method.lower(),
-            url=url,
-            headers=args.headers,
-            data=args.data,
-            verify={'yes': True, 'no': False}.get(args.verify, args.verify),
-            timeout=args.timeout,
-            auth=credentials,
-            proxies=dict((p.key, p.value) for p in args.proxy),
-            files=args.files,
-            allow_redirects=args.allow_redirects,
-            params=args.params,
-        )
-
-    except (KeyboardInterrupt, SystemExit):
-        env.stderr.write('\n')
-        sys.exit(1)
-    except Exception as e:
-        if args.debug:
-            raise
-        env.stderr.write(str(repr(e) + '\n'))
-        sys.exit(1)
+    credentials = None
+    if args.auth:
+        credentials = {
+            'basic': requests.auth.HTTPBasicAuth,
+            'digest': requests.auth.HTTPDigestAuth,
+        }[args.auth_type](args.auth.key, args.auth.value)
 
-
-def get_output(args, env, request, response):
+    if not (args.url.startswith(HTTP) or args.url.startswith(HTTPS)):
+        scheme = HTTPS if env.progname == 'https' else HTTP
+        url = scheme + args.url
+    else:
+        url = args.url
+
+    return requests.request(
+        method=args.method.lower(),
+        url=url,
+        headers=args.headers,
+        data=args.data,
+        verify={'yes': True, 'no': False}.get(args.verify, args.verify),
+        timeout=args.timeout,
+        auth=credentials,
+        proxies=dict((p.key, p.value) for p in args.proxy),
+        files=args.files,
+        allow_redirects=args.allow_redirects,
+        params=args.params,
+    )
+
+
+def output_stream(args, env, request, response):
     """Format parts of the `request`-`response` exchange
      according to `args` and `env` and return `bytes`.
 
@@ -99,32 +89,38 @@ def get_output(args, env, request, response):
     prettifier = (OutputProcessor(env, pygments_style=args.style)
                   if args.prettify else None)
 
-    if (OUT_REQ_HEAD in args.output_options
-        or OUT_REQ_BODY in args.output_options):
-        exchange.append(format(
+    with_request = (OUT_REQ_HEAD in args.output_options
+                    or OUT_REQ_BODY in args.output_options)
+    with_response = (OUT_RESP_HEAD in args.output_options
+                     or OUT_RESP_BODY in args.output_options)
+
+    if with_request:
+        request_iter = formatted_stream(
             msg=HTTPRequest(request),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_REQ_HEAD in args.output_options,
-            with_body=OUT_REQ_BODY in args.output_options
-        ))
+            with_body=OUT_REQ_BODY in args.output_options)
 
-    if (OUT_RESP_HEAD in args.output_options
-        or OUT_RESP_BODY in args.output_options):
-        exchange.append(format(
+        for chunk in request_iter:
+            yield chunk
+
+        if with_response:
+            yield b'\n\n\n'
+
+    if with_response:
+        response_iter = formatted_stream(
             msg=HTTPResponse(response),
             env=env,
             prettifier=prettifier,
             with_headers=OUT_RESP_HEAD in args.output_options,
             with_body=OUT_RESP_BODY in args.output_options)
-        )
 
-    output = b'\n\n\n'.join(exchange)
+        for chunk in response_iter:
+            yield chunk
 
     if env.stdout_isatty:
-        output += b'\n\n'
-
-    return output
+        yield b'\n\n'
 
 
 def get_exist_status(code, allow_redirects=False):
@@ -155,25 +151,37 @@ def main(args=sys.argv[1:], env=Environment()):
             ' Please use `--output FILE\' instead.\n')
         return 1
 
-    args = parser.parse_args(args=args, env=env)
-
-    response = get_response(args, env)
-
-    status = 0
-
-    if args.check_status:
-        status = get_exist_status(response.status_code,
-                                  args.allow_redirects)
-        if status and not env.stdout_isatty:
-            err = 'http error: %s %s\n' % (
-                response.raw.status, response.raw.reason)
-            env.stderr.write(err)
-
-    output = get_output(args, env, response.request, response)
-
     try:
-        env.stdout.buffer.write(output)
-    except AttributeError:
-        env.stdout.write(output)
+        args = parser.parse_args(args=args, env=env)
+        response = get_response(args, env)
+        status = 0
+
+        if args.check_status:
+            status = get_exist_status(response.status_code,
+                                      args.allow_redirects)
+            if status and not env.stdout_isatty:
+                err = 'http error: %s %s\n' % (
+                    response.raw.status, response.raw.reason)
+                env.stderr.write(err)
+
+        try:
+            # We are writing bytes so we use buffer on Python 3
+            buffer = env.stdout.buffer
+        except AttributeError:
+            buffer = env.stdout
+
+        for chunk in output_stream(args, env, response.request, response):
+            buffer.write(chunk)
+            if env.stdout_isatty:
+                env.stdout.flush()
+
+    except (KeyboardInterrupt, SystemExit):
+        env.stderr.write('\n')
+        return 1
+    except Exception as e:
+        if '--debug' in args:
+            raise
+        env.stderr.write(str(repr(e) + '\n'))
+        return 1
 
     return status

httpie/input.py

@@ -8,6 +8,7 @@
 import argparse
 import mimetypes
 import getpass
+from io import BytesIO
 
 try:
     from collections import OrderedDict
@@ -424,8 +425,9 @@ def parse_items(items, data=None, headers=None, files=None, params=None):
             target = params
         elif item.sep == SEP_FILES:
             try:
-                value = (os.path.basename(value),
-                         open(os.path.expanduser(value), 'rb'))
+                with open(os.path.expanduser(value), 'rb') as f:
+                    value = (os.path.basename(value),
+                             BytesIO(f.read()))
             except IOError as e:
                 raise ParseError(
                     'Invalid argument "%s": %s' % (item.orig, e))

httpie/models.py

@@ -57,6 +57,10 @@ def content_type(self):
 class HTTPResponse(HTTPMessage):
     """A `requests.models.Response` wrapper."""
 
+    def __iter__(self):
+        mb = 1024 * 1000
+        return self._orig.iter_content(chunk_size=2 * mb)
+
     @property
     def line(self):
         """Return Status-Line"""
@@ -85,6 +89,9 @@ def body(self):
 class HTTPRequest(HTTPMessage):
     """A `requests.models.Request` wrapper."""
 
+    def __iter__(self):
+        yield self.body
+
     @property
     def line(self):
         """Return Request-Line"""

httpie/output.py

@@ -26,11 +26,12 @@
 )
 
 
-def format(msg, prettifier=None, with_headers=True, with_body=True,
-           env=Environment()):
-    """Return `bytes` representation of a `models.HTTPMessage`.
+def formatted_stream(msg, prettifier=None, with_headers=True, with_body=True,
+                     env=Environment()):
+    """Return an iterator yielding `bytes` representing `msg`
+    (a `models.HTTPMessage` subclass).
 
-    Sometimes the body contains binary data so we always return `bytes`.
+    The body can be binary so we always yield `bytes`.
 
     If `prettifier` is set or the output is a terminal then a binary
     body is not included in the output and is replaced with notice.
@@ -41,7 +42,6 @@ def format(msg, prettifier=None, with_headers=True, with_body=True,
     then we prefer readability over precision.
 
     """
-
     # Output encoding.
     if env.stdout_isatty:
         # Use encoding suitable for the terminal. Unsupported characters
@@ -59,46 +59,42 @@ def format(msg, prettifier=None, with_headers=True, with_body=True,
     if prettifier:
         env.init_colors()
 
-    #noinspection PyArgumentList
-    output = bytearray()
-
     if with_headers:
         headers = '\n'.join([msg.line, msg.headers])
 
         if prettifier:
             headers = prettifier.process_headers(headers)
 
-        output.extend(
-            headers.encode(output_encoding, errors).strip())
+        yield headers.encode(output_encoding, errors).strip()
 
-        if with_body and msg.body:
-            output.extend(b'\n\n')
+    if with_body:
 
-    if with_body and msg.body:
-
-        body = msg.body
+        prefix = b'\n\n' if with_headers else None
 
         if not (env.stdout_isatty or prettifier):
             # Verbatim body even if it's binary.
-            pass
-        else:
+            for body_chunk in msg:
+                if prefix:
+                    yield prefix
+                    prefix = None
+                yield body_chunk
+        elif msg.body:
             try:
-                body = body.decode(msg.encoding)
+                body = msg.body.decode(msg.encoding)
             except UnicodeDecodeError:
                 # Suppress binary data.
                 body = BINARY_SUPPRESSED_NOTICE.encode(output_encoding)
                 if not with_headers:
-                    output.extend(b'\n')
+                    yield b'\n'
             else:
                 if prettifier and msg.content_type:
                     body = prettifier.process_body(
                         body, msg.content_type).strip()
 
                 body = body.encode(output_encoding, errors)
-
-        output.extend(body)
-
-    return bytes(output)
+            if prefix:
+                yield prefix
+            yield body
 
 
 class HTTPLexer(lexer.RegexLexer):