extmod: Add uzipfile

This supports decompressing stored files, and if MICROPY_PY_ZLIB is enabled then DEFLATED files (the default compression that zip uses) can be decompressed.
dhylands · Feb 1, 2016 · 88cf919 · 88cf919
1 parent faeb8e1
commit 88cf919
Show file tree

Hide file tree

Showing 9 changed files with 850 additions and 0 deletions.
diff --git a/extmod/moduzipfile.c b/extmod/moduzipfile.c
diff --git a/extmod/moduzipfile.h b/extmod/moduzipfile.h
@@ -0,0 +1,40 @@
+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Dave Hylands
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__
+#define __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__
+
+#include <stdbool.h>
+#include "py/lexer.h"
+
+// C API for importing archives from zipfiles.
+
+mp_import_stat_t zipfile_import_stat(const char *zip_archive_path);
+mp_obj_t zipfile_import_open_archive(const char *zip_archive_path);
+
+#endif  // __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__
+
+
diff --git a/py/builtin.h b/py/builtin.h
@@ -95,6 +95,7 @@ extern const mp_obj_dict_t mp_module_builtins_globals;
 
 // extmod modules
 extern const mp_obj_module_t mp_module_uctypes;
+extern const mp_obj_module_t mp_module_uzipfile;
 extern const mp_obj_module_t mp_module_uzlib;
 extern const mp_obj_module_t mp_module_ujson;
 extern const mp_obj_module_t mp_module_ure;

diff --git a/py/mpconfig.h b/py/mpconfig.h
@@ -788,6 +788,10 @@ typedef double mp_float_t;
 #define MICROPY_PY_UBINASCII (0)
 #endif
 
+#ifndef MICROPY_PY_UZIPFILE
+#define MICROPY_PY_UZIPFILE (0)
+#endif
+
 #ifndef MICROPY_PY_URANDOM
 #define MICROPY_PY_URANDOM (0)
 #endif

diff --git a/py/objmodule.c b/py/objmodule.c
@@ -166,6 +166,9 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
 #if MICROPY_PY_UCTYPES
     { MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) },
 #endif
+#if MICROPY_PY_UZIPFILE
+    { MP_ROM_QSTR(MP_QSTR_uzipfile), MP_ROM_PTR(&mp_module_uzipfile) },
+#endif
 #if MICROPY_PY_UZLIB
     { MP_ROM_QSTR(MP_QSTR_uzlib), MP_ROM_PTR(&mp_module_uzlib) },
 #endif

diff --git a/py/py.mk b/py/py.mk
@@ -163,6 +163,7 @@ PY_O_BASENAME = \
 	../extmod/moductypes.o \
 	../extmod/modujson.o \
 	../extmod/modure.o \
+	../extmod/moduzipfile.o \
 	../extmod/moduzlib.o \
 	../extmod/moduheapq.o \
 	../extmod/moduhashlib.o \

diff --git a/py/qstrdefs.h b/py/qstrdefs.h
@@ -628,6 +628,17 @@ Q(a2b_base64)
 Q(b2a_base64)
 #endif
 
+#if MICROPY_PY_UZIPFILE
+Q(uzipfile)
+Q(ZIP_STORED)
+Q(ZIP_DEFLATED)
+Q(is_zipfile)
+Q(rb)
+Q(ZipFile)
+Q(namelist)
+Q(ZipExtFile)
+#endif
+
 #if MICROPY_PY_MACHINE
 Q(umachine)
 Q(mem)

diff --git a/tests/extmod/zipfile1.py b/tests/extmod/zipfile1.py
@@ -0,0 +1,121 @@
+try:
+    import uzipfile as zipfile
+    import ubinascii as binascii
+except ImportError:
+    import zipfile as zipfile
+    import binascii
+
+EINVAL = 22
+
+files = {
+    'empty.zip':    b'504b0506000000000000000000000000000000000000',
+    'stored.zip':   b'504b03040a000000000090933748fbbdefe6150000001500000012001c007a697066696c655f73746f7265642e7478745554090003cf36a456cf36a45675780b000104d588000004e90300007072696e74282748656c6c6f20576f726c6427290a504b01021e030a000000000090933748fbbdefe61500000015000000120018000000000001000000b481000000007a697066696c655f73746f7265642e7478745554050003cf36a45675780b000104d588000004e9030000504b0506000000000100010058000000610000000000',
+    'deflated.zip': b'504b0304140000000800f293374836763c6d270000002c00000014001c007a697066696c655f6465666c617465642e74787455540900038837a4560238a45675780b000104d588000004e90300000bc9c82c5600a2448592d4e21285b4cc9c54ae10a8584946aa42716a727e5e8a424e665e2a1700504b01021e03140000000800f293374836763c6d270000002c000000140018000000000001000000b481000000007a697066696c655f6465666c617465642e74787455540500038837a45675780b000104d588000004e9030000504b050600000000010001005a000000750000000000',
+    'deflated.txt': b'54686973206973206120746573742066696c650a5468697320697320746865207365636f6e64206c696e650a',
+    'stored.txt':   b'7072696e74282748656c6c6f20576f726c6427290a',
+}
+
+for filename, data in sorted(files.items()):
+    # with doesn't work in conjunction with --emit native (part of the test run)
+    f = open(filename, 'wb')
+    f.write(binascii.unhexlify(data))
+    f.close()
+
+for filename in sorted(files):
+    print(filename, 'is_zipfile:', zipfile.is_zipfile(filename))
+    if zipfile.is_zipfile(filename):
+        zf = zipfile.ZipFile(filename)
+        zf_names = zf.namelist()
+        print(filename, 'namelist:', zf_names)
+        zf.close()
+
+def check_file(zip_filename, data_filename, data):
+    if type(zip_filename) is str:
+        print('check_file zip filename:', zip_filename, 'data file', data_filename)
+    else:
+        print('check_file zip file', zip_filename.name, 'data file', data_filename)
+    zf = zipfile.ZipFile(zip_filename)
+    zef = zf.open(data_filename)
+    buf = zef.read(len(data)//2 + 10)
+    cmp = binascii.unhexlify(data)
+    print('file compare:', buf == cmp)
+    zef.close()
+    zf.close()
+
+check_file('stored.zip', 'zipfile_stored.txt', files['stored.txt'])
+check_file('deflated.zip', 'zipfile_deflated.txt', files['deflated.txt'])
+
+
+class ByteFile:
+    """Class which treats a byestring as a file-like object."""
+
+    def __init__(self, bytes, name=None):
+        self.bytes = bytes
+        self.posn = 0
+        if name is None:
+            name = 'ByteFile object'
+        self.name = name
+
+    def close(self):
+        pass
+
+    def read(self, num_bytes=-1):
+        """CPython's zipfile implementation needs read to be implemented."""
+        #print('read', num_bytes)
+        if num_bytes < 0:
+            return self.readall()
+        size = num_bytes
+        if self.posn + size > len(self.bytes):
+            size = len(self.bytes) - self.posn
+        if size <= 0:
+            return b''
+        buf = bytearray(self.bytes[self.posn:self.posn+size])
+        #print('read @', self.posn, binascii.hexlify(buf))
+        self.posn += len(buf)
+        return buf
+
+    def readall(self):
+        buf = self.bytes[self.posn:]
+        #print('readall @', self.posn, binascii.hexlify(buf))
+        self.posn = len(self.bytes)
+        return buf
+
+    def readinto(self, buf):
+        size = len(buf)
+        if self.posn + size > len(self.bytes):
+            size = len(self.bytes) - self.posn
+        if size <= 0:
+            return 0
+        buf[0:size] = self.bytes[self.posn:self.posn+size]
+        #print('readinto @', self.posn, binascii.hexlify(buf[0:size]))
+        self.posn += size
+        return size
+
+    def seek(self, offset, whence):
+        if whence == 0:
+            self.posn = offset
+        elif whence == 1:
+            self.posn += offset
+        elif whence == 2:
+            self.posn = len(self.bytes) + offset
+        #print('seek', offset, whence, 'got', self.posn)
+        if self.posn < 0 or self.posn > len(self.bytes):
+            raise OSError(EINVAL)
+        return self.posn
+
+    def tell(self):
+        return self.posn
+
+print('===== Checking ByteFile =====')
+
+for filename, data in sorted(files.items()):
+    bf = ByteFile(binascii.unhexlify(data), name=filename)
+    print(filename, 'is_zipfile:', zipfile.is_zipfile(bf))
+    if zipfile.is_zipfile(bf):
+        zf = zipfile.ZipFile(bf)
+        zf_names = zf.namelist()
+        print(filename, 'namelist:', zf_names)
+        zf.close()
+
+check_file(ByteFile(binascii.unhexlify(files['stored.zip'])), 'zipfile_stored.txt', files['stored.txt'])
+check_file(ByteFile(binascii.unhexlify(files['deflated.zip'])), 'zipfile_deflated.txt', files['deflated.txt'])
diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h
@@ -108,6 +108,8 @@
 #ifndef MICROPY_PY_USELECT
 #define MICROPY_PY_USELECT          (1)
 #endif
+#define MICROPY_PY_UZIPFILE         (1)
+#define MICROPY_PY_FILE_LIKE        (1)
 #define MICROPY_PY_MACHINE          (1)
 #define MICROPY_MACHINE_MEM_GET_READ_ADDR   mod_machine_mem_get_addr
 #define MICROPY_MACHINE_MEM_GET_WRITE_ADDR  mod_machine_mem_get_addr