Add LZMA support; bump version (#27)

* [requires.io] dependency update on master branch (#7) * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update on master branch (#8) * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update on master branch (#9) * [requires.io] dependency update * Add support for XZ compressed mar files * Fix verification to fail if a signature block exists with no signatures * Add usage examples * Use same LZMA compressor options as C version
mozilla-releng · Jul 27, 2017 · fe0d679 · fe0d679
1 parent 17df9e0
commit fe0d679
Show file tree

Hide file tree

Showing 15 changed files with 315 additions and 47 deletions.
diff --git a/README.rst b/README.rst
@@ -66,6 +66,41 @@ Package for handling Mozilla Archive files. MAR file format is documented at htt
 
 * Free software: MPL 2.0 license
 
+Usage
+=====
+
+To list the contents of a mar::
+
+    mar -t complete.mar
+
+To list the contents of a mar with extra detail::
+
+    mar -T complete.mar
+
+To extract a mar::
+
+    mar -x complete.mar
+
+To extract, and uncompress a bz2 compressed mar::
+
+    mar -j -x complete.mar
+
+To verify a mar::
+
+    mar -k :mozilla-nightly -v complete.mar
+
+To create a mar, using bz2 compression::
+
+    mar -j -c complete.mar *
+
+To create a mar, using xz compression::
+
+    mar -J -c complete.mar *
+
+To create a signed mar::
+
+    mar -J -c complete.mar -k private.key -H nightly -V 123 tests
+
 Installation
 ============
 

diff --git a/requirements.in b/requirements.in
@@ -1,4 +1,5 @@
 cryptography
 construct
-enum34
+enum34; python_version < '3.4'
 click
+backports.lzma; python_version < '3.3'
diff --git a/requirements.txt b/requirements.txt
@@ -4,18 +4,14 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
-appdirs==1.4.3            # via setuptools
 asn1crypto==0.22.0        # via cryptography
-cffi==1.10.0               # via cryptography
+backports.lzma==0.0.6 ; python_version < "3.3"
+cffi==1.10.0              # via cryptography
 click==6.7
-construct==2.8.10
-cryptography==1.8.1
-enum34==1.1.6
+construct==2.8.12
+cryptography==2.0
+enum34==1.1.6 ; python_version < "3.4"
 idna==2.5                 # via cryptography
-packaging==16.8           # via cryptography, setuptools
-pycparser==2.17           # via cffi
-pyparsing==2.2.0          # via packaging
-six==1.10.0               # via cryptography, packaging, setuptools
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools                # via cryptography
+ipaddress==1.0.18         # via cryptography
+pycparser==2.18           # via cffi
+six==1.10.0               # via cryptography
diff --git a/setup.py b/setup.py
@@ -27,7 +27,7 @@ def read(*names, **kwargs):
 
 setup(
     name='mar',
-    version='2.0',
+    version='2.1.0',
     license='MPL 2.0',
     description='Package for handling Mozilla Archive files.',
     long_description='%s\n%s' % (
@@ -53,7 +53,6 @@ def read(*names, **kwargs):
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',

diff --git a/src/mardor/__init__.py b/src/mardor/__init__.py
@@ -8,5 +8,5 @@
 
 The primary modules of interest are `mardor.reader` and `mardor.writer`
 """
-version = (2, 0, 0)
-version_str = "2.0.0"
+version = (2, 1, 0)
+version_str = "2.1.0"
diff --git a/src/mardor/cli.py b/src/mardor/cli.py
@@ -44,8 +44,10 @@ def build_argparser():
     verify_group.add_argument("-v", "--verify", metavar="MARFILE",
                               help="verify the marfile")
 
-    parser.add_argument("-j", "--bzip2", action="store_true", dest="bz2",
-                        help="compress/decompress members with BZ2")
+    parser.add_argument("-j", "--bzip2", action="store_const", dest="compression",
+                        const="bz2", help="compress/decompress members with BZ2")
+    parser.add_argument("-J", "--xz", action="store_const", dest="compression",
+                        const="xz", help="compress/decompress archive with XZ")
 
     parser.add_argument("-k", "--keyfiles", dest="keyfiles", action='append',
                         help="sign/verify with given key(s)")
@@ -104,8 +106,13 @@ def do_list(marfile, detailed=False):
     with open(marfile, 'rb') as f:
         with MarReader(f) as m:
             if detailed:
+                if m.mardata.is_compressed:
+                    yield "MAR data is XZ compressed"
                 if m.mardata.signatures:
                     yield "Signature block found with {} signature".format(m.mardata.signatures.count)
+                    for s in m.mardata.signatures.sigs:
+                        yield "- Signature {} size {}".format(s.algorithm_id, s.size)
+                    yield ""
                 if m.mardata.additional:
                     yield "{} additional block found:".format(len(m.mardata.additional.sections))
                     for s in m.mardata.additional.sections:
@@ -124,10 +131,12 @@ def do_list(marfile, detailed=False):
 def do_create(marfile, files, compress, productversion=None, channel=None,
               signing_key=None, signing_algorithm=None):
     """Create a new MAR file."""
+    xz_compression = compress == mardor.writer.Compression.xz
     with open(marfile, 'w+b') as f:
         with MarWriter(f, productversion=productversion, channel=channel,
                        signing_key=signing_key,
-                       signing_algorithm=signing_algorithm) as m:
+                       signing_algorithm=signing_algorithm,
+                       xz_compression=xz_compression) as m:
             for f in files:
                 m.add(f, compress=compress)
 
@@ -152,7 +161,13 @@ def main(argv=None):
 
     if args.extract:
         marfile = os.path.abspath(args.extract)
-        decompress = mardor.reader.Decompression.bz2 if args.bz2 else None
+        if args.compression == 'bz2':
+            decompress = mardor.reader.Decompression.bz2
+        elif args.compression == 'xz':
+            decompress = mardor.reader.Decompression.xz
+        else:
+            decompress = None
+
         if args.chdir:
             os.chdir(args.chdir)
         do_extract(marfile, os.getcwd(), decompress)
@@ -172,7 +187,13 @@ def main(argv=None):
         print("\n".join(do_list(args.list_detailed, detailed=True)))
 
     elif args.create:
-        compress = mardor.writer.Compression.bz2 if args.bz2 else None
+        if args.compression == 'bz2':
+            compress = mardor.writer.Compression.bz2
+        elif args.compression == 'xz':
+            compress = mardor.writer.Compression.xz
+        else:
+            compress = None
+
         marfile = os.path.abspath(args.create)
         if args.keyfiles:
             signing_key = open(args.keyfiles[0], 'rb').read()

diff --git a/src/mardor/format.py b/src/mardor/format.py
@@ -10,6 +10,7 @@
 
 from construct import Array
 from construct import Bytes
+from construct import Computed
 from construct import Const
 from construct import CString
 from construct import GreedyRange
@@ -107,4 +108,8 @@ def _has_sigs(ctx):
     # Only add them if the earliest entry offset is greater than 8
     "signatures" / If(_has_sigs, sigs_header),
     "additional" / If(_has_sigs, extras_header),
+    "data_offset" / Computed(lambda ctx: ctx.index.entries[0].offset),
+    "data_length" / Computed(this.header.index_offset - this.data_offset),
+    "data_header" / Pointer(this.data_offset, Bytes(6)),
+    "is_compressed" / Computed(this.data_header == b'\xfd7zXZ\x00'),
 )
diff --git a/src/mardor/reader.py b/src/mardor/reader.py
@@ -8,6 +8,7 @@
 """
 
 import os
+import tempfile
 from enum import Enum
 
 from cryptography.exceptions import InvalidSignature
@@ -24,6 +25,7 @@
 from mardor.utils import safejoin
 from mardor.utils import takeexactly
 from mardor.utils import write_to_file
+from mardor.utils import xz_decompress_stream
 
 
 class Decompression(Enum):
@@ -33,11 +35,13 @@ class Decompression(Enum):
     none: don't decompress
     auto: automatically decompress depending on specific format
     bz2: decompress using bz2
+    xz: decompress using xz
     """
 
     none = None
     auto = 1
     bz2 = 2
+    xz = 3
 
 
 class MarReader(object):
@@ -59,8 +63,37 @@ def __init__(self, fileobj):
                 the MAR data will be read from. This object must also be
                 seekable (i.e.  support .seek() and .tell()).
         """
-        self.fileobj = fileobj
-        self.mardata = mar.parse_stream(self.fileobj)
+        self._raw_fileobj = fileobj
+        self._decompressed_fileobj = None
+
+        self.mardata = mar.parse_stream(self._raw_fileobj)
+        self.is_compressed = self.mardata.is_compressed
+
+    @property
+    def fileobj(self):
+        if not self.is_compressed:
+            return self._raw_fileobj
+
+        if not self._decompressed_fileobj:
+            self._decompressed_fileobj = self.decompress()
+        return self._decompressed_fileobj
+
+    def decompress(self):
+        """Decompress the compressed data section of the mar file
+        Return a fileobject pointing to the decompressed data.
+        """
+        dst = tempfile.TemporaryFile()
+        dst.seek(self.mardata.data_offset)
+
+        self._raw_fileobj.seek(self.mardata.data_offset)
+
+        stream = takeexactly(file_iter(self._raw_fileobj), self.mardata.data_length)
+        stream = xz_decompress_stream(stream)
+
+        write_to_file(stream, dst)
+
+        dst.seek(0)
+        return dst
 
     def __enter__(self):
         """Support the context manager protocol."""
@@ -91,6 +124,9 @@ def extract_entry(self, e, decompress=Decompression.auto):
             stream = auto_decompress_stream(stream)
         elif decompress == Decompression.bz2:
             stream = bz2_decompress_stream(stream)
+        elif decompress == Decompression.xz:
+            # Nothing to do here since we've already decompressed the XZ chunk
+            pass
 
         for block in stream:
             yield block
@@ -124,7 +160,7 @@ def verify(self, verify_key):
             True if the MAR file's signature matches its contents
             False otherwise; this includes cases where there is no signature.
         """
-        if not self.mardata.signatures:
+        if not self.mardata.signatures or not self.mardata.signatures.sigs:
             # This MAR file can't be verified since it has no signatures
             return False
 

diff --git a/src/mardor/utils.py b/src/mardor/utils.py
@@ -7,6 +7,13 @@
 from functools import partial
 from itertools import chain
 
+import six
+
+if six.PY2:
+    from backports import lzma
+else:
+    import lzma
+
 
 def mkdir(path):
     """Make a directory and its parents.
@@ -129,6 +136,48 @@ def bz2_decompress_stream(src):
             yield decoded
 
 
+def xz_compress_stream(src):
+    """Compress data from `src`.
+
+    Args:
+        src (iterable): iterable that yields blocks of data to compress
+
+    Yields:
+        blocks of compressed data
+    """
+    compressor = lzma.LZMACompressor(
+        check=lzma.CHECK_CRC64,
+        filters=[
+            {"id": lzma.FILTER_X86},
+            {"id": lzma.FILTER_LZMA2,
+             "preset": lzma.PRESET_DEFAULT},
+        ])
+    for block in src:
+        encoded = compressor.compress(block)
+        if encoded:
+            yield encoded
+    yield compressor.flush()
+
+
+def xz_decompress_stream(src):
+    """Decompress data from `src`.
+
+    Args:
+        src (iterable): iterable that yields blocks of compressed data
+
+    Yields:
+        blocks of uncompressed data
+    """
+    dec = lzma.LZMADecompressor()
+    for block in src:
+        decoded = dec.decompress(block)
+        if decoded:
+            yield decoded
+
+    if dec.unused_data:
+        raise IOError('Read unused data at end of compressed stream')
+
+
 def auto_decompress_stream(src):
     """Decompress data from `src` if required.
 
@@ -152,6 +201,20 @@ def auto_decompress_stream(src):
         yield block
 
 
+def path_is_inside(path, dirname):
+    """Returns True if path is under dirname"""
+    path = os.path.abspath(path)
+    dirname = os.path.abspath(dirname)
+    while len(path) >= len(dirname):
+        if path == dirname:
+            return True
+        newpath = os.path.dirname(path)
+        if newpath == path:
+            return False
+        path = newpath
+    return False
+
+
 def safejoin(base, *elements):
     """Safely joins paths together.
 
@@ -169,6 +232,6 @@ def safejoin(base, *elements):
     base = os.path.abspath(base)
     path = os.path.join(base, *elements)
     path = os.path.normpath(path)
-    if not path.startswith(base):
-        raise ValueError('result is outside of base')
+    if not path_is_inside(path, base):
+        raise ValueError('target path is outside of the base path')
     return path