initial usage of generator file object in requests

src/quorum/__init__.py

@@ -106,7 +106,7 @@
 from .model import Model, LocalModel, Field, link, operation, view, field
 from .mongodb import MongoMap, MongoEncoder
 from .observer import Observable
-from .structures import OrderedDict, LazyDict, LazyValue, lazy_dict, lazy
+from .structures import OrderedDict, LazyDict, LazyValue, GeneratorFile, lazy_dict, lazy
 from .template import render_template, template_resolve
 from .typesf import AbstractType, Type, File, Files, ImageFile, ImageFiles, image, images, Reference,\
     reference, References, references, Encrypted, encrypted, secure

src/quorum/httpc.py

@@ -50,6 +50,7 @@
 from . import config
 from . import common
 from . import exceptions
+from . import structures
 
 TIMEOUT = 60
 """ The timeout in seconds to be used for the blocking
@@ -561,11 +562,11 @@ def _resolve_requests(url, method, headers, data, silent, timeout, **kwargs):
 
     # verifies if the provided data is a generator, assumes that if the
     # data is not invalid and not of types string then it's a generator
-    # and then if that's the case skips the first iteration (data size)
-    # and joins the rest of the buffer (in the generator), this must be
-    # done because requests is not compatible with generator data input
+    # and then if that's the case encapsulates this size based generator
+    # into a generator based file-like object so that it can be used inside
+    # the request infra-structure (as it accepts only file objects)
     is_generator = not data == None and not legacy.is_string(data)
-    if is_generator: next(data); data = b"".join(data)
+    if is_generator: data = structures.GeneratorFile(data)
 
     method = method.lower()
     caller = getattr(requests, method)

src/quorum/structures.py

@@ -37,6 +37,8 @@
 __license__ = "Apache License, Version 2.0"
 """ The license for the module """
 
+import os
+
 class OrderedDict(dict):
 
     def __init__(self, value = None, *args, **kwargs):
@@ -238,5 +240,40 @@ def resolve(self, force = False):
     def call(self):
         return self.resolve()
 
+class GeneratorFile(object):
+    """
+    File like class that encapsulates an underlying
+    stream generator (first yield is size) into a
+    file, to be used as a normal file.
+
+    Notice that there are certain limitation to this
+    strategy like the fact that the read operation
+    (chunk) size parameter is not respected.
+    """
+
+    def __init__(self, generator):
+        self._generator = generator
+        self._size = next(generator)
+        self._position = 0
+
+    def seek(self, offset, whence = os.SEEK_SET):
+        if whence == os.SEEK_SET:
+            self._position = offset
+        if whence == os.SEEK_CUR:
+            self._position += offset
+        if whence == os.SEEK_END:
+            self._position = self._size
+
+    def tell(self):
+        return self._position
+
+    def read(self, size):
+        try: data = next(self._generator)
+        except StopIteration: data = b""
+        return data
+
+    def close(self):
+        self._generator.close()
+
 lazy_dict = LazyDict
 lazy = LazyValue