Merge pull request #63 from python-lz4/development

Development

lz4/__init__.py

@@ -2,11 +2,6 @@
 from .version import version as __version__
 VERSION = __version__
 
-try:
-    from .lz4version import LZ4_VERSION
-except ImportError:
-    LZ4_VERSION = None
-
 from ._version import lz4version
 
 # The following definitions are for backwards compatibility, and will

lz4/frame/__init__.py

@@ -165,11 +165,15 @@ class LZ4FrameDecompressor(object):
         return_bytearray (bool): When ``False`` a bytes object is returned from the
             calls to methods of this class. When ``True`` a bytearray object will be
             returned. The default is ``False``.
+        return_bytes_read (bool): When ``True``, calls to ``decompress`` will return
+            the number of bytes read from the input data as well as the uncompressed
+            data. The default is ``False``.
 
     """
 
-    def __init__(self, return_bytearray=False):
+    def __init__(self, return_bytearray=False, return_bytes_read=False):
         self.return_bytearray = return_bytearray
+        self.return_bytes_read = return_bytes_read
         self._context = create_decompression_context()
 
     def __enter__(self):
@@ -193,19 +197,23 @@ def decompress(self, data, full_frame=False):
 
         Returns:
             bytes or bytearray: decompressed data
-            int: Number of bytes consumed from input data
+            int: Number of bytes consumed from input data. This is only returned if
+                ``return_bytes_read`` is ``True`` when ``LZ4FrameDecompressor`` is
+                instantiated.
 
         """
 
         if full_frame is True:
             return decompress(
                 self._context,
                 data,
-                return_bytearray=self.return_bytearray
+                return_bytearray=self.return_bytearray,
+                return_bytes_read=self.return_bytes_read,
             )
         else:
             return decompress_chunk(
                 self._context,
                 data,
-                return_bytearray=self.return_bytearray
+                return_bytearray=self.return_bytearray,
+                return_bytes_read=self.return_bytes_read,
             )

lz4/frame/_frame.c

@@ -876,7 +876,7 @@ create_decompression_context (PyObject * Py_UNUSED (self))
 
 static inline PyObject *
 __decompress(LZ4F_dctx * context, char * source, size_t source_size,
-             int full_frame, int return_bytearray)
+             int full_frame, int return_bytearray, int return_bytes_read)
 {
   size_t source_remain;
   size_t source_read;
@@ -1062,7 +1062,14 @@ __decompress(LZ4F_dctx * context, char * source, size_t source_size,
       Py_SIZE (py_destination) = destination_written;
     }
 
-  return Py_BuildValue ("Oi", py_destination, source_cursor - source);
+  if (return_bytes_read)
+    {
+      return Py_BuildValue ("Oi", py_destination, source_cursor - source);
+    }
+  else
+    {
+      return Py_BuildValue ("O", py_destination);
+    }
 }
 
 /**************
@@ -1079,23 +1086,27 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args,
   size_t source_size;
   PyObject * decompressed;
   int return_bytearray = 0;
+  int return_bytes_read = 0;
   static char *kwlist[] = { "data",
                             "return_bytearray",
+                            "return_bytes_read",
                             NULL
                           };
 
 #if IS_PY3
-  if (!PyArg_ParseTupleAndKeywords (args, keywds, "y*|p", kwlist,
+  if (!PyArg_ParseTupleAndKeywords (args, keywds, "y*|pp", kwlist,
                                     &py_source,
-                                    &return_bytearray
+                                    &return_bytearray,
+                                    &return_bytes_read
                                     ))
     {
       return NULL;
     }
 #else
-  if (!PyArg_ParseTupleAndKeywords (args, keywds, "s*|i", kwlist,
+  if (!PyArg_ParseTupleAndKeywords (args, keywds, "s*|ii", kwlist,
                                     &py_source,
-                                    &return_bytearray
+                                    &return_bytearray,
+                                    &return_bytes_read
                                     ))
     {
       return NULL;
@@ -1120,7 +1131,8 @@ decompress (PyObject * Py_UNUSED (self), PyObject * args,
   source = (char *) py_source.buf;
   source_size = py_source.len;
 
-  decompressed = __decompress (context, source, source_size, 1, return_bytearray);
+  decompressed = __decompress (context, source, source_size, 1, return_bytearray,
+                               return_bytes_read);
 
   PyBuffer_Release(&py_source);
 
@@ -1145,26 +1157,30 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args,
   char * source;
   size_t source_size;
   int return_bytearray = 0;
+  int return_bytes_read = 0;
   static char *kwlist[] = { "context",
                             "data",
                             "return_bytearray",
+                            "return_bytes_read",
                             NULL
                           };
 
 #if IS_PY3
-  if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|p", kwlist,
+  if (!PyArg_ParseTupleAndKeywords (args, keywds, "Oy*|pp", kwlist,
                                     &py_context,
                                     &py_source,
-                                    &return_bytearray
+                                    &return_bytearray,
+                                    &return_bytes_read
                                     ))
     {
       return NULL;
     }
 #else
-  if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|i", kwlist,
+  if (!PyArg_ParseTupleAndKeywords (args, keywds, "Os*|ii", kwlist,
                                     &py_context,
                                     &py_source,
-                                    &return_bytearray
+                                    &return_bytearray,
+                                    &return_bytes_read
                                     ))
     {
       return NULL;
@@ -1186,7 +1202,8 @@ decompress_chunk (PyObject * Py_UNUSED (self), PyObject * args,
   source = (char *) py_source.buf;
   source_size = py_source.len;
 
-  decompressed = __decompress (context, source, source_size, 0, return_bytearray);
+  decompressed = __decompress (context, source, source_size, 0, return_bytearray,
+                               return_bytes_read);
 
   PyBuffer_Release(&py_source);
 
@@ -1354,9 +1371,14 @@ PyDoc_STRVAR
  "       This should contain a complete LZ4 frame of compressed data.\n\n" \
  "Keyword Args:\n"                                                      \
  "    return_bytearray (bool): If True a bytearray object will be returned.\n" \
- "        If False, a string of bytes is returned. The default is False.\n\n" \
+ "        If False, a string of bytes is returned. The default is False.\n" \
+ "    return_bytes_read (bool): If ``True`` then the number of bytes read\n" \
+ "        from ``data`` will also be returned.\n"                        \
+ "\n"                                                                   \
  "Returns:\n"                                                           \
  "    str or bytearray: Uncompressed data\n"                            \
+ "    int: (Optional) Number of bytes consumed from source. See\n"      \
+ "        ``return_bytes_read`` keyword argument\n"
  );
 
 PyDoc_STRVAR
@@ -1373,9 +1395,13 @@ PyDoc_STRVAR
  "Keyword Args:\n"                                                      \
  "    return_bytearray (bool): If True a bytearray object will be returned.\n" \
  "        If False, a string of bytes is returned. The default is False.\n\n" \
+ "    return_bytes_read (bool): If ``True`` then the number of bytes read\n" \
+ "        from ``data`` will also be returned.\n"                        \
+ "\n"                                                                   \
  "Returns:\n"                                                           \
  "    str or bytearray: Uncompressed data\n"                            \
- "    int: Number of bytes consumed from source\n"
+ "    int: (Optional) Number of bytes consumed from source. See\n"      \
+ "        ``return_bytes_read`` keyword argument\n"
  );
 
 static PyMethodDef module_methods[] =

setup.py

@@ -5,11 +5,6 @@
 import sys
 from distutils import ccompiler
 
-# This is the version of the bundled LZ4 library files. This variable was
-# historically set by this library, so we set it for backwards compatibility
-# for now. This neds removing before 1.0 release.
-LZ4_VERSION = "1.7.4.2"
-
 def pkgconfig_cmd(cmd, libname):
     try:
         pkg_config_exe = os.environ.get('PKG_CONFIG', None) or 'pkg-config'
@@ -139,14 +134,6 @@ def get_ldflags(libname):
                      include_dirs=include_dirs,
 )
 
-# If we're building against the bundled lz4 libs, we can set LZ4_VERSION. This
-# is present for backward compatibility, but should be removed before 10.
-if liblz4_found is False:
-    with open('lz4/lz4version.py', 'w+') as f:
-        f.write('# File autogenerated during install.\n')
-        f.write('# Do not change. Do not store in version control.\n')
-        f.write('LZ4_VERSION = \'' + LZ4_VERSION + '\'\n')
-
 # Finally call setup with the extension modules as defined above.
 setup(
     name='lz4',
@@ -177,7 +164,6 @@ def get_ldflags(libname):
         'Intended Audience :: Developers',
         'Programming Language :: C',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',

tests/frame/helpers.py

@@ -53,7 +53,7 @@ def roundtrip_1(data,
         block_linked,
         content_checksum
     )
-    decompressed, bytes_read = lz4frame.decompress(compressed)
+    decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True)
     assert bytes_read == len(compressed)
     assert decompressed == data
 
@@ -93,7 +93,7 @@ def roundtrip_2(data,
         block_linked,
         content_checksum
     )
-    decompressed, bytes_read = lz4frame.decompress(compressed)
+    decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True)
     assert bytes_read == len(compressed)
     assert decompressed == data
 
@@ -164,15 +164,19 @@ def roundtrip_chunked(data,
     bytes_read = 0
     try:
         while True:
-            d, b = lz4frame.decompress_chunk(d_context, next(compressed_in))
+            d, b = lz4frame.decompress_chunk(
+                d_context,
+                next(compressed_in),
+                return_bytes_read=True
+            )
             decompressed += d
             bytes_read += b
     except StopIteration:
         pass
     finally:
         del compressed_in
 
-    #assert bytes_read == len(compressed)
+    assert bytes_read == len(compressed)
     assert decompressed == data
 
 
@@ -221,7 +225,7 @@ def do_compress():
         content_checksum
     )
 
-    decompressed, bytes_read = lz4frame.decompress(compressed)
+    decompressed, bytes_read = lz4frame.decompress(compressed, return_bytes_read=True)
     assert data == decompressed
     assert bytes_read == len(compressed)
 
@@ -270,7 +274,7 @@ def do_compress():
         content_checksum
     )
 
-    with lz4frame.LZ4FrameDecompressor() as decompressor:
+    with lz4frame.LZ4FrameDecompressor(return_bytes_read=True) as decompressor:
         decompressed = b''
         bytes_read = 0
         for chunk in get_chunked(compressed, chunks):

tests/frame/test_frame_0.py

@@ -7,3 +7,99 @@ def test_create_compression_context():
 def test_create_decompression_context():
     context = lz4frame.create_decompression_context()
     assert context != None
+
+def test_compress_return_type_1():
+    r = lz4frame.compress(b'', return_bytearray=False)
+    assert isinstance(r, bytes)
+
+def test_compress_return_type_2():
+    r = lz4frame.compress(b'', return_bytearray=True)
+    assert isinstance(r, bytearray)
+
+def test_decompress_return_type_1():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    r = lz4frame.decompress(
+        c,
+        return_bytearray=False,
+        return_bytes_read=False
+    )
+    assert isinstance(r, bytes)
+
+def test_decompress_return_type_2():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    r = lz4frame.decompress(
+        c,
+        return_bytearray=True,
+        return_bytes_read=False
+    )
+    assert isinstance(r, bytearray)
+
+def test_decompress_return_type_3():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    r = lz4frame.decompress(
+        c,
+        return_bytearray=False,
+        return_bytes_read=True
+    )
+    assert isinstance(r, tuple)
+    assert isinstance(r[0], bytes)
+    assert isinstance(r[1], int)
+
+def test_decompress_return_type_4():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    r = lz4frame.decompress(
+        c,
+        return_bytearray=True,
+        return_bytes_read=True
+    )
+    assert isinstance(r, tuple)
+    assert isinstance(r[0], bytearray)
+    assert isinstance(r[1], int)
+
+def test_decompress_chunk_return_type_1():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    d = lz4frame.create_decompression_context()
+    r = lz4frame.decompress_chunk(
+        d,
+        c,
+        return_bytearray=False,
+        return_bytes_read=False
+    )
+    assert isinstance(r, bytes)
+
+def test_decompress_chunk_return_type_2():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    d = lz4frame.create_decompression_context()
+    r = lz4frame.decompress_chunk(
+        d,
+        c,
+        return_bytearray=True,
+        return_bytes_read=False
+    )
+    assert isinstance(r, bytearray)
+
+def test_decompress_chunk_return_type_3():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    d = lz4frame.create_decompression_context()
+    r = lz4frame.decompress_chunk(
+        d,
+        c,
+        return_bytearray=False,
+        return_bytes_read=True
+    )
+    assert isinstance(r, tuple)
+    assert isinstance(r[0], bytes)
+    assert isinstance(r[1], int)
+
+def test_decompress_chunk_return_type_4():
+    c = lz4frame.compress(b'', return_bytearray=False)
+    d = lz4frame.create_decompression_context()
+    r = lz4frame.decompress_chunk(
+        d,
+        c,
+        return_bytearray=True,
+        return_bytes_read=True
+    )
+    assert isinstance(r, tuple)
+    assert isinstance(r[0], bytearray)
+    assert isinstance(r[1], int)