diff --git a/atmel-samd/Makefile b/atmel-samd/Makefile index f557c88541dab..42071d3aeefd6 100644 --- a/atmel-samd/Makefile +++ b/atmel-samd/Makefile @@ -290,6 +290,7 @@ SRC_SHARED_MODULE = \ os/__init__.c \ random/__init__.c \ storage/__init__.c \ + struct/__init__.c \ uheap/__init__.c \ ustack/__init__.c diff --git a/atmel-samd/mpconfigport.h b/atmel-samd/mpconfigport.h index a871dfd7909ea..f468b51be6ad3 100644 --- a/atmel-samd/mpconfigport.h +++ b/atmel-samd/mpconfigport.h @@ -56,7 +56,7 @@ #define MICROPY_PY_IO (0) #define MICROPY_PY_URANDOM (0) #define MICROPY_PY_URANDOM_EXTRA_FUNCS (0) -#define MICROPY_PY_STRUCT (1) +#define MICROPY_PY_STRUCT (0) #define MICROPY_PY_SYS (1) // If you change MICROPY_LONGINT_IMPL, also change MPY_TOOL_LONGINT_IMPL in mpconfigport.mk. #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_NONE) @@ -157,6 +157,7 @@ extern const struct _mp_obj_module_t board_module; extern const struct _mp_obj_module_t os_module; extern const struct _mp_obj_module_t random_module; extern const struct _mp_obj_module_t storage_module; +extern const struct _mp_obj_module_t struct_module; extern const struct _mp_obj_module_t time_module; extern const struct _mp_obj_module_t cpy_nvm_module; extern const struct _mp_obj_module_t neopixel_write_module; @@ -219,6 +220,7 @@ extern const struct _mp_obj_module_t usb_hid_module; { MP_OBJ_NEW_QSTR(MP_QSTR_microcontroller), (mp_obj_t)µcontroller_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_os), (mp_obj_t)&os_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_random), (mp_obj_t)&random_module }, \ + { MP_OBJ_NEW_QSTR(MP_QSTR_struct), (mp_obj_t)&struct_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_time), (mp_obj_t)&time_module }, EXTRA_BUILTIN_MODULES diff --git a/esp8266/Makefile b/esp8266/Makefile index bfaefb4651a21..1083f36a140aa 100644 --- a/esp8266/Makefile +++ b/esp8266/Makefile @@ -143,7 +143,8 @@ SRC_SHARED_MODULE = \ multiterminal/__init__.c \ os/__init__.c \ random/__init__.c \ - storage/__init__.c + storage/__init__.c \ + struct/__init__.c SRC_SHARED_MODULE_EXPANDED = $(addprefix shared-bindings/, $(SRC_SHARED_MODULE)) \ $(addprefix shared-module/, $(SRC_SHARED_MODULE)) diff --git a/esp8266/mpconfigport.h b/esp8266/mpconfigport.h index 19f80fbb6b89a..5e8b524a32223 100644 --- a/esp8266/mpconfigport.h +++ b/esp8266/mpconfigport.h @@ -55,7 +55,7 @@ #define MICROPY_PY_CMATH (0) #define MICROPY_PY_IO (1) #define MICROPY_PY_IO_FILEIO (1) -#define MICROPY_PY_STRUCT (1) +#define MICROPY_PY_STRUCT (0) #define MICROPY_PY_SYS (1) #define MICROPY_PY_SYS_MAXSIZE (1) #define MICROPY_PY_SYS_EXIT (1) @@ -159,6 +159,7 @@ extern const struct _mp_obj_module_t network_module; extern const struct _mp_obj_module_t os_module; extern const struct _mp_obj_module_t random_module; extern const struct _mp_obj_module_t storage_module; +extern const struct _mp_obj_module_t struct_module; extern const struct _mp_obj_module_t mp_module_lwip; extern const struct _mp_obj_module_t mp_module_machine; extern const struct _mp_obj_module_t mp_module_onewire; @@ -189,6 +190,7 @@ extern const struct _mp_obj_module_t multiterminal_module; { MP_OBJ_NEW_QSTR(MP_QSTR_bitbangio), (mp_obj_t)&bitbangio_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_storage), (mp_obj_t)&storage_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_random), (mp_obj_t)&random_module }, \ + { MP_OBJ_NEW_QSTR(MP_QSTR_struct), (mp_obj_t)&struct_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_time), (mp_obj_t)&time_module }, \ { MP_OBJ_NEW_QSTR(MP_QSTR_multiterminal), (mp_obj_t)&multiterminal_module }, \ diff --git a/shared-bindings/index.rst b/shared-bindings/index.rst index 556626268bbce..9b1acc29f47c6 100644 --- a/shared-bindings/index.rst +++ b/shared-bindings/index.rst @@ -30,6 +30,7 @@ Module / Port SAMD21 SAMD21 Express ESP8266 `pulseio` No **Yes** No `random` **Yes** **Yes** **Yes** `storage` **Yes** **Yes** **Yes** +`struct` **Yes** **Yes** **Yes** `time` **Yes** **Yes** **Yes** `touchio` **Yes** **Yes** No `uheap` Debug Debug Debug diff --git a/shared-bindings/struct/__init__.c b/shared-bindings/struct/__init__.c new file mode 100644 index 0000000000000..add91fa618060 --- /dev/null +++ b/shared-bindings/struct/__init__.c @@ -0,0 +1,168 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2017 Michael McWethy + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "py/runtime.h" +#include "py/builtin.h" +#include "py/objtuple.h" +#include "py/binary.h" +#include "py/parsenum.h" +#include "shared-bindings/struct/__init__.h" +#include "shared-module/struct/__init__.h" + +//| :mod:`struct` --- manipulation of c-style data +//| ======================================================== +//| +//| .. module:: struct +//| :synopsis: byte data control +//| :platform: SAMD21 +//| +//| This module implements a subset of the corresponding CPython module, +//| as described below. For more information, refer to the original CPython +//| documentation: struct. +//| +//| Supported size/byte order prefixes: *@*, *<*, *>*, *!*. +//| +//| Supported format codes: *b*, *B*, *h*, *H*, *i*, *I*, *l*, *L*, *q*, *Q*, +//| *s*, *P*, *f*, *d* (the latter 2 depending on the floating-point support). + + + //| .. function:: calcsize(fmt) + //| + //| Return the number of bytes needed to store the given fmt. + //| + +STATIC mp_obj_t struct_calcsize(mp_obj_t fmt_in) { + + return MP_OBJ_NEW_SMALL_INT(shared_modules_struct_calcsize(fmt_in)); +} +MP_DEFINE_CONST_FUN_OBJ_1(struct_calcsize_obj, struct_calcsize); + +//| .. function:: pack(fmt, v1, v2, ...) +//| +//| Pack the values v1, v2, ... according to the format string fmt. +//| The return value is a bytes object encoding the values. +//| + +STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) { + // TODO: "The arguments must match the values required by the format exactly." + mp_int_t size = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0])); + vstr_t vstr; + vstr_init_len(&vstr, size); + byte *p = (byte*)vstr.buf; + memset(p, 0, size); + byte *end_p = &p[size]; + shared_modules_struct_pack_into(args[0], p, end_p, n_args - 1, &args[1]); + return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_obj, 1, MP_OBJ_FUN_ARGS_MAX, struct_pack); + + +//| .. function:: pack_into(fmt, buffer, offset, v1, v2, ...) +//| +//| Pack the values v1, v2, ... according to the format string fmt into a buffer +//| starting at offset. offset may be negative to count from the end of buffer. +//| + +STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) { + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE); + mp_int_t offset = mp_obj_get_int(args[2]); + if (offset < 0) { + // negative offsets are relative to the end of the buffer + offset = (mp_int_t)bufinfo.len + offset; + if (offset < 0) { + mp_raise_RuntimeError("buffer too small"); + } + } + byte *p = (byte *)bufinfo.buf; + byte *end_p = &p[bufinfo.len]; + p += offset; + + shared_modules_struct_pack_into(args[0], p, end_p, n_args - 3, &args[3]); + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_into_obj, 3, MP_OBJ_FUN_ARGS_MAX, struct_pack_into); + +//| .. function:: unpack(fmt, data) +//| +//| Unpack from the data according to the format string fmt. The return value +//| is a tuple of the unpacked values. +//| + +//| .. function:: unpack_from(fmt, data, offset) +//| +//| Unpack from the data starting at offset according to the format string fmt. +//| offset may be negative to count from the end of buffer. The return value is +//| a tuple of the unpacked values. +//| + +STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { + // unpack requires that the buffer be exactly the right size. + // unpack_from requires that the buffer be "big enough". + // Since we implement unpack and unpack_from using the same function + // we relax the "exact" requirement, and only implement "big enough". + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); + byte *p = bufinfo.buf; + byte *end_p = &p[bufinfo.len]; + + if (n_args > 2) { + mp_int_t offset = mp_obj_get_int(args[2]); + // offset arg provided + if (offset < 0) { + // negative offsets are relative to the end of the buffer + offset = bufinfo.len + offset; + if (offset < 0) { + mp_raise_RuntimeError("buffer too small"); + } + } + p += offset; + } + + return MP_OBJ_FROM_PTR(shared_modules_struct_unpack_from(args[0] , p, end_p)); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_from); + +STATIC const mp_rom_map_elem_t mp_module_struct_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_struct) }, + { MP_ROM_QSTR(MP_QSTR_calcsize), MP_ROM_PTR(&struct_calcsize_obj) }, + { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&struct_pack_obj) }, + { MP_ROM_QSTR(MP_QSTR_pack_into), MP_ROM_PTR(&struct_pack_into_obj) }, + { MP_ROM_QSTR(MP_QSTR_unpack), MP_ROM_PTR(&struct_unpack_from_obj) }, + { MP_ROM_QSTR(MP_QSTR_unpack_from), MP_ROM_PTR(&struct_unpack_from_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(mp_module_struct_globals, mp_module_struct_globals_table); + +const mp_obj_module_t struct_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t*)&mp_module_struct_globals, +}; diff --git a/shared-bindings/struct/__init__.h b/shared-bindings/struct/__init__.h new file mode 100644 index 0000000000000..c4e867aaaf3a6 --- /dev/null +++ b/shared-bindings/struct/__init__.h @@ -0,0 +1,34 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_STRUCT___INIT___H +#define MICROPY_INCLUDED_SHARED_BINDINGS_STRUCT___INIT___H + +void shared_modules_struct_pack_into(mp_obj_t fmt_in, byte *p, byte* end_p, size_t n_args, const mp_obj_t *args); +mp_uint_t shared_modules_struct_calcsize(mp_obj_t fmt_in); +mp_obj_tuple_t * shared_modules_struct_unpack_from(mp_obj_t fmt_in, byte *p, byte *end_p); + +#endif // MICROPY_INCLUDED_SHARED_BINDINGS_RANDOM___INIT___H diff --git a/shared-module/struct/__init__.c b/shared-module/struct/__init__.c new file mode 100644 index 0000000000000..e944f773876ba --- /dev/null +++ b/shared-module/struct/__init__.c @@ -0,0 +1,190 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2016 Paul Sokolovsky + * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries + * Copyright (c) 2017 Michael McWethy + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include +#include + +#include "py/runtime.h" +#include "py/binary.h" +#include "py/parsenum.h" + +void struct_validate_format(char fmt) { + if( fmt == 'S' || fmt == 'O') { + mp_raise_RuntimeError("'S' and 'O' are not supported format types"); + } +} + +char get_fmt_type(const char **fmt) { + char t = **fmt; + switch (t) { + case '!': + t = '>'; + break; + case '@': + case '=': + case '<': + case '>': + break; + default: + return '@'; + } + // Skip type char + (*fmt)++; + return t; +} + +mp_uint_t get_fmt_num(const char **p) { + const char *num = *p; + uint len = 1; + while (unichar_isdigit(*++num)) { + len++; + } + mp_uint_t val = (mp_uint_t)MP_OBJ_SMALL_INT_VALUE(mp_parse_num_integer(*p, len, 10, NULL)); + *p = num; + return val; +} + + +void shared_modules_struct_pack_into(mp_obj_t fmt_in, byte *p, byte* end_p, size_t n_args, const mp_obj_t *args) { + const char *fmt = mp_obj_str_get_str(fmt_in); + char fmt_type = get_fmt_type(&fmt); + + size_t i; + for (i = 0; i < n_args;) { + mp_uint_t sz = 1; + if (*fmt == '\0') { + // more arguments given than used by format string; CPython raises struct.error here + mp_raise_RuntimeError("too many arguments provided with the given format"); + } + struct_validate_format(*fmt); + + if (unichar_isdigit(*fmt)) { + sz = get_fmt_num(&fmt); + } + if (p + sz > end_p) { + mp_raise_RuntimeError("buffer too small"); + } + + if (*fmt == 's') { + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[i++], &bufinfo, MP_BUFFER_READ); + mp_uint_t to_copy = sz; + if (bufinfo.len < to_copy) { + to_copy = bufinfo.len; + } + memcpy(p, bufinfo.buf, to_copy); + memset(p + to_copy, 0, sz - to_copy); + p += sz; + } else { + while (sz--) { + mp_binary_set_val(fmt_type, *fmt, args[i++], &p); + } + } + fmt++; + } +} + +mp_uint_t calcsize_items(const char *fmt) { + mp_uint_t cnt = 0; + while (*fmt) { + int num = 1; + if (unichar_isdigit(*fmt)) { + num = get_fmt_num(&fmt); + if (*fmt == 's') { + num = 1; + } + } + cnt += num; + fmt++; + } + return cnt; +} + +mp_uint_t shared_modules_struct_calcsize(mp_obj_t fmt_in) { + const char *fmt = mp_obj_str_get_str(fmt_in); + char fmt_type = get_fmt_type(&fmt); + + mp_uint_t size; + for (size = 0; *fmt; fmt++) { + + struct_validate_format(*fmt); + + mp_uint_t cnt = 1; + if (unichar_isdigit(*fmt)) { + cnt = get_fmt_num(&fmt); + } + + if (*fmt == 's') { + size += cnt; + } else { + mp_uint_t align; + size_t sz = mp_binary_get_size(fmt_type, *fmt, &align); + while (cnt--) { + // Apply alignment + size = (size + align - 1) & ~(align - 1); + size += sz; + } + } + } + return size; +} + + +mp_obj_tuple_t * shared_modules_struct_unpack_from(mp_obj_t fmt_in, byte *p, byte *end_p) { + + const char *fmt = mp_obj_str_get_str(fmt_in); + char fmt_type = get_fmt_type(&fmt); + mp_uint_t num_items = calcsize_items(fmt); + mp_obj_tuple_t *res = MP_OBJ_TO_PTR(mp_obj_new_tuple(num_items, NULL)); + + for (uint i = 0; i < num_items;) { + mp_uint_t sz = 1; + + struct_validate_format(*fmt); + + if (unichar_isdigit(*fmt)) { + sz = get_fmt_num(&fmt); + } + if (p + sz > end_p) { + mp_raise_RuntimeError("buffer too small"); + } + mp_obj_t item; + if (*fmt == 's') { + item = mp_obj_new_bytes(p, sz); + p += sz; + res->items[i++] = item; + } else { + while (sz--) { + item = mp_binary_get_val(fmt_type, *fmt, &p); + res->items[i++] = item; + } + } + fmt++; + } + return res; + +} diff --git a/shared-module/struct/__init__.h b/shared-module/struct/__init__.h new file mode 100644 index 0000000000000..637080b29200e --- /dev/null +++ b/shared-module/struct/__init__.h @@ -0,0 +1,34 @@ + +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_SHARED_MODULE_STRUCT___INIT___H +#define MICROPY_INCLUDED_SHARED_MODULE_STRUCT___INIT___H + +char get_fmt_type(const char **fmt); +mp_uint_t get_fmt_num(const char **p); +mp_uint_t calcsize_items(const char *fmt); + +#endif diff --git a/tests/misc/non_compliant.py b/tests/misc/non_compliant.py index b4c90e9fcf439..e1ea66e69e71b 100644 --- a/tests/misc/non_compliant.py +++ b/tests/misc/non_compliant.py @@ -2,7 +2,7 @@ try: import array - import ustruct + import struct except ImportError: print("SKIP") raise SystemExit @@ -118,10 +118,10 @@ print('NotImplementedError') # struct pack with too many args, not checked by uPy -print(ustruct.pack('bb', 1, 2, 3)) +print(struct.pack('bb', 1, 2, 3)) # struct pack with too few args, not checked by uPy -print(ustruct.pack('bb', 1)) +print(struct.pack('bb', 1)) # array slice assignment with unsupported RHS try: