Skip to content
Permalink
Browse files

extmod: Add uzipfile

This supports decompressing stored files, and if
MICROPY_PY_ZLIB is enabled then DEFLATED files (the default
compression that zip uses) can be decompressed.
  • Loading branch information
dhylands committed Jan 22, 2016
1 parent faeb8e1 commit 88cf919b7d73ffa3b87191338843511560211af3
Showing with 850 additions and 0 deletions.
  1. +667 −0 extmod/moduzipfile.c
  2. +40 −0 extmod/moduzipfile.h
  3. +1 −0 py/builtin.h
  4. +4 −0 py/mpconfig.h
  5. +3 −0 py/objmodule.c
  6. +1 −0 py/py.mk
  7. +11 −0 py/qstrdefs.h
  8. +121 −0 tests/extmod/zipfile1.py
  9. +2 −0 unix/mpconfigport.h

Large diffs are not rendered by default.

@@ -0,0 +1,40 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2016 Dave Hylands
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

#ifndef __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__
#define __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__

#include <stdbool.h>
#include "py/lexer.h"

// C API for importing archives from zipfiles.

mp_import_stat_t zipfile_import_stat(const char *zip_archive_path);
mp_obj_t zipfile_import_open_archive(const char *zip_archive_path);

#endif // __MICROPY_INCLUDED_EXTMOD_MODUZIPFILE_H__


@@ -95,6 +95,7 @@ extern const mp_obj_dict_t mp_module_builtins_globals;

// extmod modules
extern const mp_obj_module_t mp_module_uctypes;
extern const mp_obj_module_t mp_module_uzipfile;
extern const mp_obj_module_t mp_module_uzlib;
extern const mp_obj_module_t mp_module_ujson;
extern const mp_obj_module_t mp_module_ure;
@@ -788,6 +788,10 @@ typedef double mp_float_t;
#define MICROPY_PY_UBINASCII (0)
#endif

#ifndef MICROPY_PY_UZIPFILE
#define MICROPY_PY_UZIPFILE (0)
#endif

#ifndef MICROPY_PY_URANDOM
#define MICROPY_PY_URANDOM (0)
#endif
@@ -166,6 +166,9 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
#if MICROPY_PY_UCTYPES
{ MP_ROM_QSTR(MP_QSTR_uctypes), MP_ROM_PTR(&mp_module_uctypes) },
#endif
#if MICROPY_PY_UZIPFILE
{ MP_ROM_QSTR(MP_QSTR_uzipfile), MP_ROM_PTR(&mp_module_uzipfile) },
#endif
#if MICROPY_PY_UZLIB
{ MP_ROM_QSTR(MP_QSTR_uzlib), MP_ROM_PTR(&mp_module_uzlib) },
#endif
@@ -163,6 +163,7 @@ PY_O_BASENAME = \
../extmod/moductypes.o \
../extmod/modujson.o \
../extmod/modure.o \
../extmod/moduzipfile.o \
../extmod/moduzlib.o \
../extmod/moduheapq.o \
../extmod/moduhashlib.o \
@@ -628,6 +628,17 @@ Q(a2b_base64)
Q(b2a_base64)
#endif

#if MICROPY_PY_UZIPFILE
Q(uzipfile)
Q(ZIP_STORED)
Q(ZIP_DEFLATED)
Q(is_zipfile)
Q(rb)
Q(ZipFile)
Q(namelist)
Q(ZipExtFile)
#endif

#if MICROPY_PY_MACHINE
Q(umachine)
Q(mem)
@@ -0,0 +1,121 @@
try:
import uzipfile as zipfile
import ubinascii as binascii
except ImportError:
import zipfile as zipfile
import binascii

EINVAL = 22

files = {
'empty.zip': b'504b0506000000000000000000000000000000000000',
'stored.zip': b'504b03040a000000000090933748fbbdefe6150000001500000012001c007a697066696c655f73746f7265642e7478745554090003cf36a456cf36a45675780b000104d588000004e90300007072696e74282748656c6c6f20576f726c6427290a504b01021e030a000000000090933748fbbdefe61500000015000000120018000000000001000000b481000000007a697066696c655f73746f7265642e7478745554050003cf36a45675780b000104d588000004e9030000504b0506000000000100010058000000610000000000',
'deflated.zip': b'504b0304140000000800f293374836763c6d270000002c00000014001c007a697066696c655f6465666c617465642e74787455540900038837a4560238a45675780b000104d588000004e90300000bc9c82c5600a2448592d4e21285b4cc9c54ae10a8584946aa42716a727e5e8a424e665e2a1700504b01021e03140000000800f293374836763c6d270000002c000000140018000000000001000000b481000000007a697066696c655f6465666c617465642e74787455540500038837a45675780b000104d588000004e9030000504b050600000000010001005a000000750000000000',
'deflated.txt': b'54686973206973206120746573742066696c650a5468697320697320746865207365636f6e64206c696e650a',
'stored.txt': b'7072696e74282748656c6c6f20576f726c6427290a',
}

for filename, data in sorted(files.items()):
# with doesn't work in conjunction with --emit native (part of the test run)
f = open(filename, 'wb')
f.write(binascii.unhexlify(data))
f.close()

for filename in sorted(files):
print(filename, 'is_zipfile:', zipfile.is_zipfile(filename))
if zipfile.is_zipfile(filename):
zf = zipfile.ZipFile(filename)
zf_names = zf.namelist()
print(filename, 'namelist:', zf_names)
zf.close()

def check_file(zip_filename, data_filename, data):
if type(zip_filename) is str:
print('check_file zip filename:', zip_filename, 'data file', data_filename)
else:
print('check_file zip file', zip_filename.name, 'data file', data_filename)
zf = zipfile.ZipFile(zip_filename)
zef = zf.open(data_filename)
buf = zef.read(len(data)//2 + 10)
cmp = binascii.unhexlify(data)
print('file compare:', buf == cmp)
zef.close()
zf.close()

check_file('stored.zip', 'zipfile_stored.txt', files['stored.txt'])
check_file('deflated.zip', 'zipfile_deflated.txt', files['deflated.txt'])


class ByteFile:
"""Class which treats a byestring as a file-like object."""

def __init__(self, bytes, name=None):
self.bytes = bytes
self.posn = 0
if name is None:
name = 'ByteFile object'
self.name = name

def close(self):
pass

def read(self, num_bytes=-1):
"""CPython's zipfile implementation needs read to be implemented."""
#print('read', num_bytes)
if num_bytes < 0:
return self.readall()
size = num_bytes
if self.posn + size > len(self.bytes):
size = len(self.bytes) - self.posn
if size <= 0:
return b''
buf = bytearray(self.bytes[self.posn:self.posn+size])
#print('read @', self.posn, binascii.hexlify(buf))
self.posn += len(buf)
return buf

def readall(self):
buf = self.bytes[self.posn:]
#print('readall @', self.posn, binascii.hexlify(buf))
self.posn = len(self.bytes)
return buf

def readinto(self, buf):
size = len(buf)
if self.posn + size > len(self.bytes):
size = len(self.bytes) - self.posn
if size <= 0:
return 0
buf[0:size] = self.bytes[self.posn:self.posn+size]
#print('readinto @', self.posn, binascii.hexlify(buf[0:size]))
self.posn += size
return size

def seek(self, offset, whence):
if whence == 0:
self.posn = offset
elif whence == 1:
self.posn += offset
elif whence == 2:
self.posn = len(self.bytes) + offset
#print('seek', offset, whence, 'got', self.posn)
if self.posn < 0 or self.posn > len(self.bytes):
raise OSError(EINVAL)
return self.posn

def tell(self):
return self.posn

print('===== Checking ByteFile =====')

for filename, data in sorted(files.items()):
bf = ByteFile(binascii.unhexlify(data), name=filename)
print(filename, 'is_zipfile:', zipfile.is_zipfile(bf))
if zipfile.is_zipfile(bf):
zf = zipfile.ZipFile(bf)
zf_names = zf.namelist()
print(filename, 'namelist:', zf_names)
zf.close()

check_file(ByteFile(binascii.unhexlify(files['stored.zip'])), 'zipfile_stored.txt', files['stored.txt'])
check_file(ByteFile(binascii.unhexlify(files['deflated.zip'])), 'zipfile_deflated.txt', files['deflated.txt'])
@@ -108,6 +108,8 @@
#ifndef MICROPY_PY_USELECT
#define MICROPY_PY_USELECT (1)
#endif
#define MICROPY_PY_UZIPFILE (1)
#define MICROPY_PY_FILE_LIKE (1)
#define MICROPY_PY_MACHINE (1)
#define MICROPY_MACHINE_MEM_GET_READ_ADDR mod_machine_mem_get_addr
#define MICROPY_MACHINE_MEM_GET_WRITE_ADDR mod_machine_mem_get_addr

0 comments on commit 88cf919

Please sign in to comment.
You can’t perform that action at this time.