ENH Add Ctypes support

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 defaults: &defaults
   working_directory: ~/repo
   docker:
-    - image: pyodide/pyodide-env:17
+    - image: hoodmane/pyodide-env:18
   environment:
     - EMSDK_NUM_CORES: 3
       EMCC_CORES: 3

.gitignore

@@ -19,6 +19,7 @@ build
 ccache
 cpython/downloads
 cpython/installs
+cpython/build.log
 docs/_build/
 emsdk/emsdk
 geckodriver.log

Dockerfile

@@ -3,11 +3,13 @@ FROM python:3.9.5-slim-buster
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
-                  # building packages
-                  bzip2 ccache clang-format-6.0 cmake f2c g++ gfortran git make \
-                  patch pkg-config swig unzip wget xz-utils \
-                  # testing packages: libgconf-2-4 is necessary for running chromium
-                  libgconf-2-4 "chromium=90.*" \
+        # building packages
+        bzip2 ccache clang-format-6.0 cmake f2c g++ gfortran git make \
+        patch pkg-config swig unzip wget xz-utils \
+        autoconf autotools-dev automake texinfo \
+        build-essential prelink autoconf libtool libltdl-dev \
+        # testing packages: libgconf-2-4 is necessary for running chromium
+        libgconf-2-4 "chromium=90.*" \
   && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 --no-cache-dir install \

Makefile.envs

@@ -42,14 +42,17 @@ export LDFLAGS_BASE=\
 	-s WASM=1 \
 	-std=c++14 \
 	-s LZ4=1 \
+	-L $(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/ \
 	$(EXTRA_LDFLAGS)
 
 export CXXFLAGS_BASE=
 
 export SIDE_MODULE_LDFLAGS=	$(LDFLAGS_BASE) -s SIDE_MODULE=1
 export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) -s MAIN_MODULE=1 \
 	-s EXPORTED_FUNCTIONS='["___cxa_guard_acquire", "__ZNSt3__28ios_base4initEPv", "_main"]' \
-	$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
+	-lpython$(PYMINOR) \
+	-lffi \
+	-s LLD_REPORT_UNDEFINED \
 	-s TOTAL_MEMORY=20971520 \
 	-s ALLOW_MEMORY_GROWTH=1 \
     --use-preload-plugins \
@@ -59,6 +62,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) -s MAIN_MODULE=1 \
 	$(wildcard $(CPYTHONROOT)/build/bzip2*/libbz2.a) \
 	-lstdc++ \
 	--memory-init-file 0 \
+	-s WASM_BIGINT \
 
 export SIDE_MODULE_CXXFLAGS = $(CXXFLAGS_BASE)
 
@@ -73,5 +77,6 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \
 		-I$(PYTHONINCLUDE)
 
 
+
 .output_vars:
 	set

cpython/Makefile

@@ -25,8 +25,10 @@ BZIP2TARBALL=$(ROOT)/downloads/bzip2-1.0.2.tar.gz
 BZIP2BUILD=$(ROOT)/build/bzip2-1.0.2
 BZIP2URL=https://sourceware.org/pub/bzip2/bzip2-1.0.2.tar.gz
 
+FFIBUILD=$(ROOT)/build/libffi
+LIBFFIREPO=https://github.com/hoodmane/libffi-emscripten
 
-all: $(INSTALL)/lib/$(LIB)
+all: $(INSTALL)/lib/$(LIB) $(INSTALL)/lib/libffi.a
 
 
 $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) remove_modules.txt
@@ -106,6 +108,14 @@ $(BZIP2BUILD)/libbz2.a: $(BZIP2TARBALL)
 		emmake make -j $${PYODIDE_JOBS:-3} CC=emcc CFLAGS="$(PYTHON_CFLAGS) -D_FILE_OFFSET_BITS=64" AR=emar RANLIB=emranlib libbz2.a; \
 	)
 
+$(FFIBUILD)/target/lib/libffi.a :
+	rm -rf $(FFIBUILD)
+	git clone $(LIBFFIREPO) $(FFIBUILD) --depth 1
+	source $(PYODIDE_ROOT)/emsdk/emsdk/emsdk_env.sh
+	cd  $(FFIBUILD) ; bash ./build.sh ; make install
+
+$(INSTALL)/lib/libffi.a : $(INSTALL)/lib/$(LIB) $(FFIBUILD)/target/lib/libffi.a
+	cp $(FFIBUILD)/target/lib/libffi.a $(INSTALL)/lib/
 
 $(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.configured $(SQLITEBUILD)/libsqlite3.la $(BZIP2BUILD)/libbz2.a
 	cp config.site $(BUILD)/
@@ -125,9 +135,13 @@ $(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.configured $(SQLITEBUILD)/lib
 	)
 
 
-$(BUILD)/$(LIB): $(BUILD)/Makefile Setup.local
+$(BUILD)/Modules/Setup.local : Setup.local
 	cp Setup.local $(BUILD)/Modules/
-	cat pyconfig.undefs.h >> $(BUILD)/pyconfig.h
+
+$(BUILD)/pyconfig.h : pyconfig.undefs.h
+	cp pyconfig.undefs.h $(BUILD)/pyconfig.h
+
+$(BUILD)/$(LIB): $(BUILD)/Makefile $(BUILD)/pyconfig.h $(BUILD)/Modules/Setup.local
 	( \
 		cd $(BUILD); \
 		emmake make CROSS_COMPILE=yes $(LIB) -j $${PYODIDE_JOBS:-3} \

cpython/Setup.local

@@ -18,6 +18,10 @@ _datetime _datetimemodule.c
 _heapq _heapqmodule.c
 _json _json.c
 _csv _csv.c
+
+CTYPES_FLAGS=-DPy_BUILD_CORE_BUILTIN -DPy_BUILD_CORE_MODULE -DHAVE_FFI_PREP_CIF_VAR=1 -DHAVE_FFI_PREP_CLOSURE_LOC=1 -DHAVE_FFI_CLOSURE_ALLOC=1 -I/src/cpython/build/libffi/target/include
+_ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/cfield.c _ctypes/stgdict.c $(CTYPES_FLAGS)
+
 unicodedata unicodedata.c
 _pickle _pickle.c
 parser parsermodule.c

cpython/patches/0001-Remove-duplicate-symbols-from-cfield.c.patch

@@ -0,0 +1,48 @@
+From fc3b69f9afa779185a60834cf1817c22706edcd1 Mon Sep 17 00:00:00 2001
+From: Hood <hood@mit.edu>
+Date: Tue, 22 Jun 2021 20:12:45 -0700
+Subject: [PATCH] Remove duplicate symbols from cfield.c
+
+---
+ Modules/_ctypes/cfield.c | 26 --------------------------
+ 1 file changed, 26 deletions(-)
+
+diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c
+index 7ebd4ba..7a63ab7 100644
+--- a/Modules/_ctypes/cfield.c
++++ b/Modules/_ctypes/cfield.c
+@@ -1635,31 +1635,5 @@ typedef struct _ffi_type
+ } ffi_type;
+ */
+
+-/* align and size are bogus for void, but they must not be zero */
+-ffi_type ffi_type_void = { 1, 1, FFI_TYPE_VOID };
+-
+-ffi_type ffi_type_uint8 = { 1, 1, FFI_TYPE_UINT8 };
+-ffi_type ffi_type_sint8 = { 1, 1, FFI_TYPE_SINT8 };
+-
+-ffi_type ffi_type_uint16 = { 2, 2, FFI_TYPE_UINT16 };
+-ffi_type ffi_type_sint16 = { 2, 2, FFI_TYPE_SINT16 };
+-
+-ffi_type ffi_type_uint32 = { 4, INT_ALIGN, FFI_TYPE_UINT32 };
+-ffi_type ffi_type_sint32 = { 4, INT_ALIGN, FFI_TYPE_SINT32 };
+-
+-ffi_type ffi_type_uint64 = { 8, LONG_LONG_ALIGN, FFI_TYPE_UINT64 };
+-ffi_type ffi_type_sint64 = { 8, LONG_LONG_ALIGN, FFI_TYPE_SINT64 };
+-
+-ffi_type ffi_type_float = { sizeof(float), FLOAT_ALIGN, FFI_TYPE_FLOAT };
+-ffi_type ffi_type_double = { sizeof(double), DOUBLE_ALIGN, FFI_TYPE_DOUBLE };
+-
+-#ifdef ffi_type_longdouble
+-#undef ffi_type_longdouble
+-#endif
+-  /* This is already defined on OSX */
+-ffi_type ffi_type_longdouble = { sizeof(long double), LONGDOUBLE_ALIGN,
+-                                 FFI_TYPE_LONGDOUBLE };
+-
+-ffi_type ffi_type_pointer = { sizeof(void *), VOID_P_ALIGN, FFI_TYPE_POINTER };
+
+ /*---------------- EOF ----------------*/
+-- 
+2.17.1
+

cpython/patches/ctypes-dont-deref-function-pointer.patch

@@ -0,0 +1,35 @@
+From 3f11694d7587e782530118d176306eeb6eebf5d1 Mon Sep 17 00:00:00 2001
+From: Hood <hood@mit.edu>
+Date: Wed, 23 Jun 2021 13:47:30 -0700
+Subject: [PATCH] Don't dereference function pointer
+
+Ctypes thinks that the result of dlsym is a pointer to the function pointer, so
+it should call it like `result = (*f)(args)`. Probably this is true for the
+native dlsym, but our dlsym returns an index into the indirect call table
+"wasmTable", in particular it isn't even aligned like a pointer should be.
+This patch fixes it so that it calls it like `result = f(args)` instead.
+
+---
+ Modules/_ctypes/_ctypes.c | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
+index ceae67e..44f2d76 100644
+--- a/Modules/_ctypes/_ctypes.c
++++ b/Modules/_ctypes/_ctypes.c
+@@ -771,7 +771,11 @@ CDataType_in_dll(PyObject *type, PyObject *args)
+         return NULL;
+     }
+ #endif
+-    return PyCData_AtAddress(type, address);
++    CDataObject *ob = (CDataObject *)GenericPyCData_new(type, NULL, NULL);
++    if (ob == NULL)
++        return NULL;
++    *(void **)ob->b_ptr = address;
++    return (PyObject*)ob;
+ }
+
+ static const char from_param_doc[] =
+-- 
+2.17.1
+

cpython/remove_modules.txt

@@ -1,5 +1,4 @@
 _osx_support.py
-ctypes
 curses
 dbm
 ensurepip

emsdk/patches/0001-Dummy-out-failing-terser-call.patch

@@ -0,0 +1,33 @@
+From 2d6b0f7986afabcf17b049a199b0af795faf5db2 Mon Sep 17 00:00:00 2001
+From: Hood <hood@mit.edu>
+Date: Thu, 24 Jun 2021 04:08:02 -0700
+Subject: [PATCH] Dummy out failing terser call
+
+---
+ emcc.py | 10 +++++-----
+ 1 file changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/emsdk/upstream/emscripten/eemcc.py b/emsdk/upstream/emscripten/emcc.py
+index 839f791b3..b3eef03ef 100755
+--- a/emsdk/upstream/emscripten/eemcc.py
++++ b/emsdk/upstream/emscripten/eemcc.py
+@@ -2901,11 +2901,11 @@ def do_binaryen(target, options, wasm_target):
+       # Closure can print out readable error messages (Closure will then
+       # minify whitespace afterwards)
+       save_intermediate_with_wasm('preclean', wasm_target)
+-      final_js = building.minify_wasm_js(js_file=final_js,
+-                                         wasm_file=wasm_target,
+-                                         expensive_optimizations=will_metadce(),
+-                                         minify_whitespace=minify_whitespace() and not options.use_closure_compiler,
+-                                         debug_info=intermediate_debug_info)
++      # final_js = building.minify_wasm_js(js_file=final_js,
++      #                                    wasm_file=wasm_target,
++      #                                    expensive_optimizations=will_metadce(),
++      #                                    minify_whitespace=minify_whitespace() and not options.use_closure_compiler,
++      #                                    debug_info=intermediate_debug_info)
+       save_intermediate_with_wasm('postclean', wasm_target)
+
+   if shared.Settings.ASYNCIFY_LAZY_LOAD_CODE:
+-- 
+2.17.1
+

packages/imageio/meta.yaml

@@ -4,8 +4,6 @@ package:
 source:
   sha256: 52ddbaeca2dccf53ba2d6dec5676ca7bc3b2403ef8b37f7da78b7654bb3e10f0
   url: https://files.pythonhosted.org/packages/c3/73/f37f428748c4f10a7991ac5bff00f113a34bcc0d0a78957d6e1cdc29a94e/imageio-2.9.0.tar.gz
-  patches:
-  - patches/setitup.patch
 requirements:
   run:
   - numpy

packages/joblib/meta.yaml

@@ -12,7 +12,6 @@ source:
 
   patches:
     - patches/use-setuptools.patch
-    - patches/ctypes-import.patch
 
 test:
   imports:

packages/numpy/meta.yaml

@@ -20,7 +20,6 @@ source:
     - patches/make-int-return-values.patch
     - patches/fix-ieee754.patch
     - patches/rm-duplicate-symbols-link.patch
-    - patches/delay-ctypes-import.patch
 
 build:
   skip_host: False