From 6255cdf5a96a38d53204d191ab6c34625984084b Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 23 Sep 2025 20:53:54 +0200 Subject: [PATCH 01/30] some cleanup --- pglite-wasm/pgl_stubs.h | 45 ----------------------------------------- 1 file changed, 45 deletions(-) diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index 2ad262c7f0096..642274083203c 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -30,53 +30,8 @@ PostgresMain(const char *dbname, const char *username) { } -void -startup_hacks(const char *progname) { -#ifdef PG16 - SpinLockInit(&dummy_spinlock); -#endif -} - - // embedded initdb requirements -void -get_restricted_token(void) { - // stub -} - -void * -pg_malloc(size_t size) { - return malloc(size); -} - -void * -pg_malloc_extended(size_t size, int flags) { - return malloc(size); -} - -void * -pg_realloc(void *ptr, size_t size) { - return realloc(ptr, size); -} - -char * -pg_strdup(const char *in) { - char *tmp; - - if (!in) { - fprintf(stderr, _("cannot duplicate null pointer (internal error)\n")); - exit(EXIT_FAILURE); - } - tmp = strdup(in); - if (!tmp) { - fprintf(stderr, _("out of memory\n")); - exit(EXIT_FAILURE); - } - return tmp; -} - - char * simple_prompt(const char *prompt, bool echo) { return pg_strdup(""); From 97dbb550e58b9d4d5e3fb562b26c6d5e8acb2846 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 10:25:13 +0200 Subject: [PATCH 02/30] do not run ./configure if not needed; specify portname in clean script --- build-pglite.sh | 22 +++++++++++++++++++++- clean-pglite.sh | 9 +++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/build-pglite.sh b/build-pglite.sh index 7fa05c3900d95..bd3e87d5147c4 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -21,8 +21,28 @@ fi echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS" +# run ./configure only if config.status is older than this file +REF_FILE="build-pglite.sh" +CONFIG_STATUS="config.status" +RUN_CONFIGURE=false + +if [ ! -f "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS does not exist, need to run ./configure" + RUN_CONFIGURE=true +elif [ "$REF_FILE" -nt "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS is older than $REF_FILE. Need to run ./configure." + RUN_CONFIGURE=true +else + echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run." +fi + # Step 1: configure the project -LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } +if [ "$RUN_CONFIGURE" = true ] +then + LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } +else + echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" +fi # Step 2: make and install all except pglite emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; } diff --git a/clean-pglite.sh b/clean-pglite.sh index 3d6c86458baa6..538f63b2150de 100755 --- a/clean-pglite.sh +++ b/clean-pglite.sh @@ -1,6 +1,7 @@ #!/bin/bash -emmake make -C src/backend uninstall; emmake make -C src/backend clean; -emmake make -C pglite/ clean; emmake make -C pglite/ uninstall; -emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall; -emmake make clean; emmake make uninstal \ No newline at end of file +emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=emscripten -C src/backend clean; +emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall; +emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; +emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall; +emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall \ No newline at end of file From bd5acf4a2908bb6123e7d547a19786485b64be4c Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 11:18:28 +0200 Subject: [PATCH 03/30] cleanup --- clean-pglite.sh | 2 +- src/backend/Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clean-pglite.sh b/clean-pglite.sh index 538f63b2150de..70fa5f819d756 100755 --- a/clean-pglite.sh +++ b/clean-pglite.sh @@ -4,4 +4,4 @@ emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=e emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall; emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall; -emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall \ No newline at end of file +emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall; \ No newline at end of file diff --git a/src/backend/Makefile b/src/backend/Makefile index a79301f587fc3..4c436f5d8e887 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -75,9 +75,9 @@ endif ifeq ($(PORTNAME), emscripten) AR ?= llvm-ar -LIBPGCORE ?= $(top_builddir)/libpgcore.a -LIBPG = $(top_builddir)/libpostgres.a -LIBPGMAIN = $(top_builddir)/libpgmain.a +LIBPGCORE ?= libpgcore.a +LIBPG = libpostgres.a +LIBPGMAIN = libpgmain.a PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) PGMAIN = main/main.o tcop/postgres.o PGLITE_MAIN = pglite-wasm/pg_main.c From 7e4257c0366b8e97391773eaa87c4aca1bb5c6ef Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 13:31:51 +0200 Subject: [PATCH 04/30] remove startup_hacks --- pglite-wasm/pg_main.c | 1 - pglite-wasm/pgl_stubs.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index a3b77ece6cae6..d6fae41af0dae 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -564,7 +564,6 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif progname = get_progname(argv[0]); - startup_hacks(progname); g_argv = argv; g_argc = argc; diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index 642274083203c..f21142062bd2c 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -14,8 +14,6 @@ // option_parse_int parse_sync_method #include "../src/fe_utils/option_utils.c" - - static void init_locale(const char *categoryname, int category, const char *locale) { if (pg_perm_setlocale(category, locale) == NULL && From 174398816c4f71b7649c6ed583b3a333b22e0197 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 13:46:50 +0200 Subject: [PATCH 05/30] readd initdb requirements --- pglite-wasm/pgl_stubs.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index f21142062bd2c..5367add820685 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -29,6 +29,41 @@ PostgresMain(const char *dbname, const char *username) { // embedded initdb requirements +void +get_restricted_token(void) { + // stub +} + +void * +pg_malloc(size_t size) { + return malloc(size); +} + +void * +pg_malloc_extended(size_t size, int flags) { + return malloc(size); +} + +void * +pg_realloc(void *ptr, size_t size) { + return realloc(ptr, size); +} + +char * +pg_strdup(const char *in) { + char *tmp; + + if (!in) { + fprintf(stderr, _("cannot duplicate null pointer (internal error)\n")); + exit(EXIT_FAILURE); + } + tmp = strdup(in); + if (!tmp) { + fprintf(stderr, _("out of memory\n")); + exit(EXIT_FAILURE); + } + return tmp; +} char * simple_prompt(const char *prompt, bool echo) { From 09a56666044322b3b29b1e294cbf5c33552d1b9f Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 13:47:18 +0200 Subject: [PATCH 06/30] add linker search path for backend --- src/backend/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/Makefile b/src/backend/Makefile index 4c436f5d8e887..d765a273e5bae 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -120,7 +120,7 @@ pglite: pglite-exported-functions -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 \ -sEXPORTED_RUNTIME_METHODS=$(EXPORTED_RUNTIME_METHODS) \ -I./src/include -I./src/ -I./src/interfaces/libpq \ - -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ \ + -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ -L. \ pglite.o \ -Ldict_snowball \ -lxslt \ From c57040479cbd88007e41be22431c97a9b51a7012 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 14:18:28 +0200 Subject: [PATCH 07/30] remove get_buffer_size and get_buffer_addr" --- pglite-wasm/included.pglite.imports | 2 -- pglite-wasm/interactive_one.c | 53 ----------------------------- 2 files changed, 55 deletions(-) diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index 63cbc5e039b90..e5a61007b2442 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -1,8 +1,6 @@ close fcntl free -get_buffer_addr -get_buffer_size get_channel getpid gettimeofday diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 124e99c5059f4..c4224004a721d 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -17,19 +17,6 @@ volatile int cma_wsize = 0; volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm volatile bool sockfiles = false; // also defined in postgres.c for pqcomm -__attribute__((export_name("get_buffer_size"))) -int -get_buffer_size(int fd) { - return (CMA_MB * 1024 * 1024) / CMA_FD; -} - -// TODO add query size -__attribute__((export_name("get_buffer_addr"))) -int -get_buffer_addr(int fd) { - return 1 + ( get_buffer_size(fd) *fd); -} - __attribute__((export_name("get_channel"))) int get_channel() { @@ -55,46 +42,6 @@ extern void ClientAuthentication(Port *port); extern FILE* SOCKET_FILE; extern int SOCKET_DATA; - - -/* -init sequence -___________________________________ -SubPostmasterMain / (forkexec) - InitPostmasterChild - shm attach - preload - - BackendInitialize(Port *port) -> collect initial packet - - pq_init(); - whereToSendOutput = DestRemote; - status = ProcessStartupPacket(port, false, false); - pq_startmsgread - pq_getbytes from pq_recvbuf - TODO: place PqRecvBuffer (8K) in lower mem for zero copy - - PerformAuthentication - ClientAuthentication(port) - CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) - InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores - - BackendRun(port) - PostgresMain - - --> pq_flush() is synchronous - - -buffer sizes: - - https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 - - https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 - - -*/ - extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); extern void pq_recvbuf_fill(FILE* fp, int packetlen); From 8f1bad613caee84196e32d78d86f7ae94ea24fc4 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 14:42:32 +0200 Subject: [PATCH 08/30] remove recv_password_packet in interactive_one --- pglite-wasm/interactive_one.c | 67 ++--------------------------------- 1 file changed, 3 insertions(+), 64 deletions(-) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index c4224004a721d..0aab77cfc403a 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -46,71 +46,10 @@ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); extern void pq_recvbuf_fill(FILE* fp, int packetlen); #define PG_MAX_AUTH_TOKEN_LENGTH 65535 -static char * -recv_password_packet(Port *port) { - StringInfoData buf; - int mtype; - - pq_startmsgread(); - - /* Expect 'p' message type */ - mtype = pq_getbyte(); - if (mtype != 'p') - { - /* - * If the client just disconnects without offering a password, don't - * make a log entry. This is legal per protocol spec and in fact - * commonly done by psql, so complaining just clutters the log. - */ - if (mtype != EOF) - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("expected password response, got message type %d", - mtype))); - return NULL; /* EOF or bad message type */ - } - - initStringInfo(&buf); - if (pq_getmessage(&buf, PG_MAX_AUTH_TOKEN_LENGTH)) /* receive password */ - { - /* EOF - pq_getmessage already logged a suitable message */ - pfree(buf.data); - return NULL; - } - - /* - * Apply sanity check: password packet length should agree with length of - * contained string. Note it is safe to use strlen here because - * StringInfo is guaranteed to have an appended '\0'. - */ - if (strlen(buf.data) + 1 != buf.len) - ereport(ERROR, - (errcode(ERRCODE_PROTOCOL_VIOLATION), - errmsg("invalid password packet size"))); - - /* - * Don't allow an empty password. Libpq treats an empty password the same - * as no password at all, and won't even try to authenticate. But other - * clients might, so allowing it would be confusing. - * - * Note that this only catches an empty password sent by the client in - * plaintext. There's also a check in CREATE/ALTER USER that prevents an - * empty string from being stored as a user's password in the first place. - * We rely on that for MD5 and SCRAM authentication, but we still need - * this check here, to prevent an empty password from being used with - * authentication methods that check the password against an external - * system, like PAM, LDAP and RADIUS. - */ - if (buf.len == 1) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PASSWORD), - errmsg("empty password returned by client"))); - - /* Do not echo password to logs, for security. */ - elog(DEBUG5, "received password packet"); - return buf.data; -} +// defined as static char *recv_password_packet(Port *port); in auth.c +// we still need access to it from here, so break the encapsulation by marking it extern +extern char *recv_password_packet(Port *port); int md5Salt_len = 4; char md5Salt[4]; From bb1befd6e74e12327a85c1eabe91b26e61366dba Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 15:07:34 +0200 Subject: [PATCH 09/30] removed pglite-modpython.c; removed __PYDK__ --- pglite-wasm/builder/Dockerfile | 2 +- pglite-wasm/pglite-modpython.c | 218 --------------------------------- src/backend/Makefile | 2 +- 3 files changed, 2 insertions(+), 220 deletions(-) delete mode 100644 pglite-wasm/pglite-modpython.c diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile index d8f5d931f75b1..a4b1e36982f01 100644 --- a/pglite-wasm/builder/Dockerfile +++ b/pglite-wasm/builder/Dockerfile @@ -21,7 +21,7 @@ WORKDIR /install/libs WORKDIR /src -# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" +# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" ENV EMCC_COMMON_FLAGS="-O2 -fPIC" WORKDIR /src diff --git a/pglite-wasm/pglite-modpython.c b/pglite-wasm/pglite-modpython.c deleted file mode 100644 index 8dc492eab99ad..0000000000000 --- a/pglite-wasm/pglite-modpython.c +++ /dev/null @@ -1,218 +0,0 @@ -#include -#include -#include - -#ifdef __wii__ -# include -# include -# include -#endif - -#include "w2c2/w2c2_base.h" -#include "wasi/wasi.h" -#include "${WASM2C}.h" - -void -trap(Trap trap) { - fprintf(stderr, "TRAP: %s\n", trapDescription(trap)); -#ifdef __wii__ - VIDEO_WaitVSync(); -#endif - abort(); -} - -U32 -wasi_snapshot_preview1__fd_renumber(void* ctx, U32 from, U32 to) { - return -1; -} - -wasmMemory* -wasiMemory(void* instance) { - return ${WASM2C}_memory((${WASM2C}Instance*)instance); -} - -extern char **environ; -volatile int skip_main = 0; -//char** __environ = NULL; -${WASM2C}Instance instance; - -#include "${WASM2C}.c" - -#define WASM_PREFIX "/tmp/pglite" -#define WASM_USERNAME "postgres" -#define PGDB WASM_PREFIX "/base" - - -char **g_argv; -int g_argc; - -char **copy_argv(int argc, char *argv[]) { - // calculate the contiguous argv buffer size - int length=0; - size_t ptr_args = argc + 1; - for (int i = 0; i < argc; i++) { - length += (strlen(argv[i]) + 1); - } - char** new_argv = (char**)malloc((ptr_args) * sizeof(char*) + length); - - // copy argv into the contiguous buffer - length = 0; - for (int i = 0; i < argc; i++) { - new_argv[i] = &(((char*)new_argv)[(ptr_args * sizeof(char*)) + length]); - strcpy(new_argv[i], argv[i]); - length += (strlen(argv[i]) + 1); - } - - // insert NULL terminating ptr at the end of the ptr array - new_argv[ptr_args-1] = NULL; - return (new_argv); -} - -int pre_main(int tmp_argc, char* tmp_argv[]) { - puts("# 84: pre_main"); - // __environ = environ; - - setenv("EMBED", "wasi", 1); - setenv("REPL", "N", 1); - - setenv("PGSYSCONFDIR", WASM_PREFIX, 1); - setenv("PGCLIENTENCODING", "UTF8", 1); - - setenv("TZ", "UTC", 1); - setenv("PGTZ", "UTC", 1); - setenv("PGUSER", WASM_USERNAME , 1); - setenv("PGDATA", PGDB , 1); - setenv("PGDATABASE", "template1" , 1); - setenv("PG_COLOR", "always", 0); - puts("# 87: global argc/argv"); - g_argv = copy_argv(tmp_argc, tmp_argv); - g_argc = sizeof(g_argv) / sizeof(char*) - 1; - - -# if defined(__MWERKS__) && defined(macintosh) - MaxApplZone(); - MoreMasters(); - MoreMasters(); - argc = ccommand(&argv); -# elif defined(__wii__) - /* TODO: get interactive console working */ - wiiInitVideo(); - fatInitDefault(); -# endif - - /* Initialize WASI */ - if (!wasiInit(g_argc, g_argv, environ)) { - fprintf(stderr, "failed to init WASI\n"); - return 1; - } - - if (!wasiFileDescriptorAdd(-1, "/", NULL)) { - fprintf(stderr, "failed to add preopen\n"); - return 1; - } - -# ifdef __MSL__ - SIOUXSetTitle("\p${WASM2C}"); -# endif - - ${WASM2C}Instantiate(&instance, NULL); - - return 0; -} - -// #define REACTOR - -#if defined(REACTOR) - #define STARTPROC(i) ${WASM2C}_setup(i) - // #define STARTPROC(i) ${WASM2C}_pg_initdb(i) -#else - #define STARTPROC(i) ${WASM2C}__start(&instance); -#endif - - -int main(int argc, char* argv[]); - -void do_main() { - puts("# 128: do_main Begin"); - STARTPROC(&instance); - puts("# 134: do_main End"); -} - -#if defined(__PYDK__) -# include "Python.h" - - static PyObject * ${WASM2C}_info(PyObject *self, PyObject *args, PyObject *kwds) - { - puts("${WASM2C} test function : return 42"); - return Py_BuildValue("i", 42); - } - - static PyObject * Begin(PyObject *self, PyObject *args, PyObject *kwds) { - puts("PyInit_${WASM2C}"); - - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; -puts("167"); - pre_main(tmp_argc, tmp_argv); -puts("169"); - -//${WASM2C}__start(&instance); - Py_RETURN_NONE; - } - - static PyObject * End(PyObject *self, PyObject *args, PyObject *kwds) { -puts("164: do main"); - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - Py_RETURN_NONE; - } - -# include "${WASM2C}.pymod" - - static PyMethodDef mod_${WASM2C}_methods[] = { -#include "${WASM2C}.def" - {NULL, NULL, 0, NULL} - }; - - static struct PyModuleDef mod_${WASM2C} = { - PyModuleDef_HEAD_INIT, - "${WASM2C}", - NULL, - -1, - mod_${WASM2C}_methods, - NULL, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free - }; - - - PyMODINIT_FUNC - PyInit_${WASM2C}(void) { - PyObject *${WASM2C}_mod = PyModule_Create(&mod_${WASM2C}); -# ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(${WASM2C}_mod, Py_MOD_GIL_NOT_USED); -# endif - return ${WASM2C}_mod; - } -#endif - - -int main(int argc, char* argv[]) { - if (!skip_main) { -puts("216"); - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; - skip_main = pre_main(tmp_argc, tmp_argv); -puts("220"); - if (!skip_main) { - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - } - } else { - puts(" -- main skipped --"); - } - return skip_main; -} diff --git a/src/backend/Makefile b/src/backend/Makefile index d765a273e5bae..a5c4ef3fda18b 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -109,7 +109,7 @@ pglite: pglite-exported-functions $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ $(PGPRELOAD) \ -sGLOBAL_BASE=12MB -ferror-limit=1 \ From 068679e1ae053c9c59a46e6ef1eaa28115e33dcd Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 15:13:57 +0200 Subject: [PATCH 10/30] remove specifing MIN_SAFARI_VERSION since that is the default value anyway --- src/backend/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/Makefile b/src/backend/Makefile index a5c4ef3fda18b..4030ccf046289 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -109,7 +109,7 @@ pglite: pglite-exported-functions $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -fPIC -sWASM_BIGINT -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ $(PGPRELOAD) \ -sGLOBAL_BASE=12MB -ferror-limit=1 \ From ee194e65fae653e05b8ebc1cffc40d2d067d2743 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:38:43 +0200 Subject: [PATCH 11/30] moved some of the emscripten flags to bash script to allow a native build --- build-pglite.sh | 17 ++++++++++++++--- src/backend/Makefile | 12 ++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/build-pglite.sh b/build-pglite.sh index bd3e87d5147c4..3b8e4c36e5efd 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -22,6 +22,7 @@ fi echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS" # run ./configure only if config.status is older than this file +# TODO: we should ALSO check if any of the PGLITE_CFLAGS have changed and trigger a ./configure if they did!!! REF_FILE="build-pglite.sh" CONFIG_STATUS="config.status" RUN_CONFIGURE=false @@ -37,8 +38,7 @@ else fi # Step 1: configure the project -if [ "$RUN_CONFIGURE" = true ] -then +if [ "$RUN_CONFIGURE" = true ] then LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } else echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" @@ -62,5 +62,16 @@ emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pgli PATH=$SAVE_PATH # Step 5: make and install pglite +# we define here "all" emscripten flags in order to allow native builds (like libpglite) +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" +PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ +-sSUPPORT_LONGJMP=emscripten \ +-sGLOBAL_BASE=12MB -sFORCE_FILESYSTEM=1 \ +-sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ +-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ +-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ +-sERROR_ON_UNDEFINED_SYMBOLS=1 \ +-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS" + # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. -PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } +PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } diff --git a/src/backend/Makefile b/src/backend/Makefile index 4030ccf046289..f3e7d2119ee13 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -81,7 +81,6 @@ LIBPGMAIN = libpgmain.a PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) PGMAIN = main/main.o tcop/postgres.o PGLITE_MAIN = pglite-wasm/pg_main.c -EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" PGROOT=/install/pglite PGPRELOAD=--preload-file ${PGROOT}/share/postgresql@/tmp/pglite/share/postgresql --preload-file ${PGROOT}/lib/postgresql@/tmp/pglite/lib/postgresql --preload-file $(top_builddir)/other/password@/tmp/pglite/password --preload-file $(top_builddir)/other/PGPASSFILE@/home/web_user/.pgpass --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/postgres --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/initdb postgres: $(OBJS) @@ -105,20 +104,17 @@ pglite-exported-functions: # -sDYLINK_DEBUG=2 use this for debugging missing exported symbols (ex when an extension calls a pgcore function that hasn't been exported) # PGLITE_CFLAGS is something like "-O2" (aka release version) or "-g -gsource-map --no-wasm-opt" (aka debug version) +# PGLITE_EMSCRIPTEN_FLAGS are the emscripten flags to be passed to the build. Ideally they would be completely transparent, such that we +# could build a libpglite with or without those flags (to switch between native and wasm builds) pglite: pglite-exported-functions $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -sWASM_BIGINT -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ $(PGPRELOAD) \ - -sGLOBAL_BASE=12MB -ferror-limit=1 \ - -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ - -sMAIN_MODULE=2 \ + -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ - -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 \ - -sEXPORTED_RUNTIME_METHODS=$(EXPORTED_RUNTIME_METHODS) \ -I./src/include -I./src/ -I./src/interfaces/libpq \ -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ -L. \ pglite.o \ From e57f54cc8b79193b94a4a66020300a0c04fe7d4c Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:39:30 +0200 Subject: [PATCH 12/30] docker run under current user --- build-with-docker.sh | 4 +++- pglite-wasm/builder/Dockerfile | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build-with-docker.sh b/build-with-docker.sh index d3d31a17fdc75..902f26f322548 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -3,11 +3,13 @@ DOCKER_WORKSPACE=$(pwd) docker run $@ \ + -u $(id -u):$(id -g) \ + -it \ --rm \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ -v .:${DOCKER_WORKSPACE}:rw \ -v ./dist:/install/pglite:rw \ - electricsql/pglite-builder:3.1.74_1 \ + electricsql/pglite-builder:3.1.74_2 \ ./build-pglite.sh diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile index a4b1e36982f01..cf613dd2ddc85 100644 --- a/pglite-wasm/builder/Dockerfile +++ b/pglite-wasm/builder/Dockerfile @@ -3,7 +3,7 @@ ARG BUILDPLATFORM # we only need to build on one platform, since we're interested in the WASM output # building on the native (BUILDPLATFORM) is much faster, so use that # remove "-arm64" suffix if building on x86_64 -FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER}-arm64 AS builder +FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER} AS builder ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm @@ -60,6 +60,8 @@ RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | se WORKDIR /src RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz +# COPY . . +# RUN tar xvf uuid-1.6.2.tar.gz WORKDIR /src/uuid-1.6.2 RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm @@ -85,5 +87,8 @@ ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR} COPY --from=builder /install/libs ${INSTALL_PREFIX} COPY --from=builder /install/exports ${LIB_EXPORTS_DIR} +# allow access to anyone +RUN chmod -R 777 /install + # needed in building pglite.wasm ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm \ No newline at end of file From bc1a2c8f30d518fa28b7bea7db94f461dc09e6bb Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:39:45 +0200 Subject: [PATCH 13/30] improved cleanup --- clean-pglite.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clean-pglite.sh b/clean-pglite.sh index 70fa5f819d756..4f72e3fa60a8b 100755 --- a/clean-pglite.sh +++ b/clean-pglite.sh @@ -4,4 +4,7 @@ emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=e emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall; emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall; -emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall; \ No newline at end of file +emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall; + +echo "removing config.status" +rm config.status \ No newline at end of file From 7c73e9c21660fec05ee534368d41d0b98df83306 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:40:06 +0200 Subject: [PATCH 14/30] readd recv_password_packet to interactive_one.c --- pglite-wasm/interactive_one.c | 67 +++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 0aab77cfc403a..c4224004a721d 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -46,10 +46,71 @@ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); extern void pq_recvbuf_fill(FILE* fp, int packetlen); #define PG_MAX_AUTH_TOKEN_LENGTH 65535 +static char * +recv_password_packet(Port *port) { + StringInfoData buf; + int mtype; + + pq_startmsgread(); + + /* Expect 'p' message type */ + mtype = pq_getbyte(); + if (mtype != 'p') + { + /* + * If the client just disconnects without offering a password, don't + * make a log entry. This is legal per protocol spec and in fact + * commonly done by psql, so complaining just clutters the log. + */ + if (mtype != EOF) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("expected password response, got message type %d", + mtype))); + return NULL; /* EOF or bad message type */ + } + + initStringInfo(&buf); + if (pq_getmessage(&buf, PG_MAX_AUTH_TOKEN_LENGTH)) /* receive password */ + { + /* EOF - pq_getmessage already logged a suitable message */ + pfree(buf.data); + return NULL; + } + + /* + * Apply sanity check: password packet length should agree with length of + * contained string. Note it is safe to use strlen here because + * StringInfo is guaranteed to have an appended '\0'. + */ + if (strlen(buf.data) + 1 != buf.len) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid password packet size"))); + + /* + * Don't allow an empty password. Libpq treats an empty password the same + * as no password at all, and won't even try to authenticate. But other + * clients might, so allowing it would be confusing. + * + * Note that this only catches an empty password sent by the client in + * plaintext. There's also a check in CREATE/ALTER USER that prevents an + * empty string from being stored as a user's password in the first place. + * We rely on that for MD5 and SCRAM authentication, but we still need + * this check here, to prevent an empty password from being used with + * authentication methods that check the password against an external + * system, like PAM, LDAP and RADIUS. + */ + if (buf.len == 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PASSWORD), + errmsg("empty password returned by client"))); + + /* Do not echo password to logs, for security. */ + elog(DEBUG5, "received password packet"); + return buf.data; +} -// defined as static char *recv_password_packet(Port *port); in auth.c -// we still need access to it from here, so break the encapsulation by marking it extern -extern char *recv_password_packet(Port *port); int md5Salt_len = 4; char md5Salt[4]; From 58724c5692e33d08e73ea3df4d8bda1d2d2b6d6e Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 17:49:12 +0200 Subject: [PATCH 15/30] remove -it when running docker --- build-with-docker.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build-with-docker.sh b/build-with-docker.sh index 902f26f322548..ea851f425ce2e 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -4,7 +4,6 @@ DOCKER_WORKSPACE=$(pwd) docker run $@ \ -u $(id -u):$(id -g) \ - -it \ --rm \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ From b4b9af3a5514d39096c1aa26237a2ef71912e373 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 18:03:08 +0200 Subject: [PATCH 16/30] script fix --- build-pglite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-pglite.sh b/build-pglite.sh index 3b8e4c36e5efd..c5d695d22712a 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -38,7 +38,7 @@ else fi # Step 1: configure the project -if [ "$RUN_CONFIGURE" = true ] then +if [ "$RUN_CONFIGURE" = true ]; then LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } else echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" From 433ce1eb8729011b777bf467a2170927e9cd4426 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 19:37:59 +0200 Subject: [PATCH 17/30] fix build-with-docker.sh script for GH actions; cleanup target for extensions --- build-with-docker.sh | 11 ++++++++++- pglite/Makefile | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/build-with-docker.sh b/build-with-docker.sh index ea851f425ce2e..58c3a5eced11f 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -1,9 +1,18 @@ # although we could use any path inside docker, using the same path as on the host # allows the DWARF info (when building in DEBUG) to contain the correct file paths DOCKER_WORKSPACE=$(pwd) +GITHUB_ACTIONS=${GITHUB_ACTIONS:-"false"} +ADDITIONAL_DOCKER_PARAMS="" +if [ "$GITHUB_ACTIONS" == "true" ]; then + echo "Running inside GitHub Actions" + ADDITIONAL_DOCKER_PARAMS="" +else + echo "Not running inside GitHub Actions" + ADDITIONAL_DOCKER_PARAMS="-it -u $(id -u):$(id -g)" +fi docker run $@ \ - -u $(id -u):$(id -g) \ + $ADDITIONAL_DOCKER_PARAMS \ --rm \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \ diff --git a/pglite/Makefile b/pglite/Makefile index 4f4ef50406dd8..fc1f84c45bdc1 100644 --- a/pglite/Makefile +++ b/pglite/Makefile @@ -31,5 +31,9 @@ dist: $(addsuffix .tar.gz,$(EXTENSIONS)) .PHONY: dist +archive_clean: + @echo "=== Cleaning $* ===" + rm -rf $(EXTENSIONS_BUILD_ROOT)/$* + rm -rf $(ARCHIVE_DIR) -$(recurse) \ No newline at end of file +$(recurse) From 4b7eafe64580fd8d409c2dfcb0104a4254dfecad Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 24 Sep 2025 19:40:32 +0200 Subject: [PATCH 18/30] fix --- build-with-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-with-docker.sh b/build-with-docker.sh index 58c3a5eced11f..17521a8575690 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -3,7 +3,7 @@ DOCKER_WORKSPACE=$(pwd) GITHUB_ACTIONS=${GITHUB_ACTIONS:-"false"} ADDITIONAL_DOCKER_PARAMS="" -if [ "$GITHUB_ACTIONS" == "true" ]; then +if [ "$GITHUB_ACTIONS" = "true" ]; then echo "Running inside GitHub Actions" ADDITIONAL_DOCKER_PARAMS="" else From dd474bc987871b54ab3145943b532db98c75b0b5 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 1 Oct 2025 20:36:38 +0200 Subject: [PATCH 19/30] cb data transport --- pglite-wasm/included.pglite.imports | 3 +- pglite-wasm/interactive_one.c | 402 +++++++++++++++++----------- src/backend/libpq/pqcomm.c | 273 +++++++++++-------- src/backend/tcop/postgres.c | 6 + 4 files changed, 420 insertions(+), 264 deletions(-) diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index e5a61007b2442..39931facdab90 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -33,4 +33,5 @@ strcmp strftime strlen strtoul -use_wire \ No newline at end of file +use_wire +set_read_write_cbs \ No newline at end of file diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index c4224004a721d..b704f16305ec6 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -17,12 +17,45 @@ volatile int cma_wsize = 0; volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm volatile bool sockfiles = false; // also defined in postgres.c for pqcomm +__attribute__((export_name("get_buffer_size"))) +int +get_buffer_size(int fd) { + return (CMA_MB * 1024 * 1024) / CMA_FD; +} + +// TODO add query size +__attribute__((export_name("get_buffer_addr"))) +int +get_buffer_addr(int fd) { + return 1 + ( get_buffer_size(fd) *fd); +} + __attribute__((export_name("get_channel"))) int get_channel() { return channel; } +// read FROM JS +// (i guess return number of bytes written) +// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); +//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len); +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +// write TO JS +// (i guess return number of bytes read) +// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +typedef ssize_t (*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +__attribute__((export_name("set_read_write_cbs"))) +void +set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { + pglite_read = read_cb; + pglite_write = write_cb; +} __attribute__((export_name("interactive_read"))) int @@ -42,6 +75,46 @@ extern void ClientAuthentication(Port *port); extern FILE* SOCKET_FILE; extern int SOCKET_DATA; + + +/* +init sequence +___________________________________ +SubPostmasterMain / (forkexec) + InitPostmasterChild + shm attach + preload + + BackendInitialize(Port *port) -> collect initial packet + + pq_init(); + whereToSendOutput = DestRemote; + status = ProcessStartupPacket(port, false, false); + pq_startmsgread + pq_getbytes from pq_recvbuf + TODO: place PqRecvBuffer (8K) in lower mem for zero copy + + PerformAuthentication + ClientAuthentication(port) + CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) + InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores + + BackendRun(port) + PostgresMain + + +-> pq_flush() is synchronous + + +buffer sizes: + + https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 + + https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 + + +*/ + extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); extern void pq_recvbuf_fill(FILE* fp, int packetlen); @@ -183,11 +256,11 @@ clear_error() { idle_in_transaction_timeout_enabled = false; idle_session_timeout_enabled = false; DoingCommandRead = false; -puts("# 239:" __FILE__ ); + pq_comm_reset(); EmitErrorReport(); debug_query_string = NULL; -puts("# 243:" __FILE__ ); + AbortCurrentTransaction(); if (am_walsender) @@ -315,21 +388,19 @@ PDEBUG("# 330: TODO: set a pgl started flag"); extern void pg_startcma(); __attribute__((export_name("interactive_one"))) void -interactive_one() { - int peek = -1; /* preview of firstchar with no pos change */ +interactive_one(int packetlen, int peek) { + // int peek = -1; /* preview of firstchar with no pos change */ int firstchar = 0; /* character read from getc() */ bool pipelining = true; StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; FILE *fp = NULL; - int packetlen; + // int packetlen; bool had_notification = notifyInterruptPending; bool notified = false; // send_ready_for_query = false; -if (cma_rsize<0) - goto resume_on_error; if (!MyProcPort) { PDEBUG("# 353: client created"); @@ -337,6 +408,7 @@ if (cma_rsize<0) } #if PGDEBUG + puts("\n\n# 369: interactive_one"); if (notifyInterruptPending) PDEBUG("# 371: has notification !"); #endif @@ -404,94 +476,105 @@ if (cma_rsize<0) * TODO: allow to redirect stdout for fully external repl. */ - peek = IO[0]; - packetlen = cma_rsize; - - if (packetlen) { - sockfiles = false; - if (!is_repl) { - whereToSendOutput = DestRemote; - if (!is_wire) - PDEBUG("# 439: repl message in cma buffer !"); - } else { - if (is_wire) - PDEBUG("# 442: wire message in cma buffer for REPL !"); - whereToSendOutput = DestDebug; - } + if (!is_repl) { + whereToSendOutput = DestRemote; + if (!is_wire) + PDEBUG("# 439: repl message in cma buffer !"); } else { - fp = fopen(PGS_IN, "r"); -puts("# 475:" PGS_IN "\r\n"); - // read file in socket buffer for SocketBackend to consumme. - if (fp) { - fseek(fp, 0L, SEEK_END); - packetlen = ftell(fp); - if (packetlen) { - // set to always true if no REPL. -// is_wire = true; - resetStringInfo(inBuf); - rewind(fp); - /* peek on first char */ - peek = getc(fp); - rewind(fp); - if (is_repl && !is_wire) { - // sql in buffer - for (int i=0; i socketfiles -> repl +// packetlen = cma_rsize; + +// if (cma_rsize) { +// peek = IO[0]; +// sockfiles = false; +// if (!is_repl) { +// whereToSendOutput = DestRemote; +// if (!is_wire) +// PDEBUG("# 439: repl message in cma buffer !"); +// } else { +// if (is_wire) +// PDEBUG("# 442: wire message in cma buffer for REPL !"); +// whereToSendOutput = DestDebug; +// } +// } else { +// fp = fopen(PGS_IN, "r"); +// PDEBUG("# 452:" PGS_IN "\n"); +// // read file in socket buffer for SocketBackend to consumme. +// if (fp) { +// fseek(fp, 0L, SEEK_END); +// packetlen = ftell(fp); +// if (packetlen) { +// // set to always true if no REPL. +// // is_wire = true; +// resetStringInfo(inBuf); +// rewind(fp); +// /* peek on first char */ +// peek = getc(fp); +// rewind(fp); +// if (is_repl && !is_wire) { +// // sql in buffer +// for (int i=0; i true : %c\n", peek); +// force_echo = true; +// } + +// #endif +// firstchar = peek; +// goto incoming; +// } // wire msg +// PDEBUG("# 507: NO DATA:" PGS_IN "\n"); +// } // fp data read + +// // is it REPL in cma ? +// if (!peek) +// goto return_early; + +// firstchar = peek ; + +// //REPL mode in zero copy buffer ( lowest wasm memory segment ) +// packetlen = strlen(IO); + +// } // !cma_rsize -> socketfiles -> repl #if PGDEBUG if (packetlen) @@ -501,11 +584,11 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); resetStringInfo(inBuf); // when cma buffer is used to fake stdin, data is not read by socket/wire backend. - if (is_repl) { - for (int i=0; i0; +#if PGDEBUG + if (!pipelining) { + printf("# 556: end of wire, rfq=%d\n", send_ready_for_query); + } else { + printf("# 558: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); + } +#endif } else { /* nowire */ // pipelining = false; @@ -569,6 +659,15 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); } DoingCommandRead = false; +#if 0 // PGDEBUG + if (!pipelining) { + printf("# 573: wire=%d 1stchar=%c Q: %s\n", is_wire, firstchar, inBuf->data); + force_echo = false; + } else { + printf("# 576: PIPELINING [%c]!\n", firstchar); + } +#endif + if (!ignore_till_sync) { /* initially, or after error */ // send_ready_for_query = true; @@ -588,12 +687,13 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); if (pipelining) { pipelining = pq_buffer_remaining_data()>0; if (pipelining && send_ready_for_query) { +puts("# 631: PIPELINING + rfq"); ReadyForQuery(whereToSendOutput); send_ready_for_query = false; } } } -resume_on_error: + if (!is_repl) { wire_flush: if (!ClientAuthInProgress) { @@ -602,7 +702,7 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); ProcessNotifyInterrupt(false); if (send_ready_for_query) { -// PDEBUG("# 602: end packet - sending rfq\n"); + PDEBUG("# 602: end packet - sending rfq\n"); ReadyForQuery(DestRemote); //done at postgres.c 4623 send_ready_for_query = false; @@ -613,50 +713,50 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } - if (SOCKET_DATA>0) { - if (sockfiles) { - channel = -1; - if (cma_wsize) { - puts("ERROR: cma was not flushed before socketfile interface"); - } - } else { - /* wsize may have increased with previous rfq so assign here */ - cma_wsize = SOCKET_DATA; - channel = cma_rsize + 2; - } - if (SOCKET_FILE) { - int outb = SOCKET_DATA; - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - SOCKET_DATA = 0; - - if (cma_wsize) { - PDEBUG("# 672: cma and sockfile ???\n"); - } - - if (sockfiles) { -#if PGDEBUG - printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); -#endif - rename(PGS_OLOCK, PGS_OUT); - } - } else { -#if PGDEBUG - printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); -#endif - SOCKET_DATA = 0; - } - - } else { - cma_wsize = 0; - PDEBUG("# 698: no data, send empty ?"); -// TODO: dedup 739 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } - } +// if (SOCKET_DATA>0) { +// if (sockfiles) { +// channel = -1; +// if (cma_wsize) { +// puts("ERROR: cma was not flushed before socketfile interface"); +// } +// } else { +// /* wsize may have increased with previous rfq so assign here */ +// cma_wsize = SOCKET_DATA; +// channel = cma_rsize + 2; +// } +// if (SOCKET_FILE) { +// int outb = SOCKET_DATA; +// fclose(SOCKET_FILE); +// SOCKET_FILE = NULL; +// SOCKET_DATA = 0; + +// if (cma_wsize) { +// PDEBUG("# 672: cma and sockfile ???\n"); +// } + +// if (sockfiles) { +// #if PGDEBUG +// printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); +// #endif +// rename(PGS_OLOCK, PGS_OUT); +// } +// } else { +// #if PGDEBUG +// printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); +// #endif +// SOCKET_DATA = 0; +// } + +// } else { +// cma_wsize = 0; +// PDEBUG("# 698: no data, send empty ?"); +// // TODO: dedup 739 +// if (sockfiles) { +// fclose(SOCKET_FILE); +// SOCKET_FILE = NULL; +// rename(PGS_OLOCK, PGS_OUT); +// } +// } } else { pg_prompt(); #if PGDEBUG @@ -679,15 +779,15 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); } return_early:; /* always FD CLEANUP */ - if (fp) { - fclose(fp); - unlink(PGS_IN); - } + // if (fp) { + // fclose(fp); + // unlink(PGS_IN); + // } // always free kernel buffer !!! - cma_rsize = 0; - IO[0] = 0; + // cma_rsize = 0; + // IO[0] = 0; #undef IO @@ -695,4 +795,4 @@ return_early:; canary_ex = 0; } -#undef PGL_LOOP +#undef PGL_LOOP \ No newline at end of file diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 47fc44bf618e5..5b954aaf9a063 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -137,6 +137,53 @@ static int PqRecvPointer; static int PqRecvLength; volatile int querylen = 0; volatile FILE* queryfp = NULL; +extern FILE* SOCKET_FILE; + +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +int EMSCRIPTEN_KEEPALIVE fcntl (int __fd, int __cmd, ...) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE setsockopt (int __fd, int __level, int __optname, + const void *__optval, socklen_t __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockopt (int __fd, int __level, int __optname, + void *__restrict __optval, + socklen_t *__restrict __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockname (int __fd, struct sockaddr * __addr, + socklen_t *__restrict __len) { + // dummy + return 0; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + recv (int __fd, void *__buf, size_t __n, int __flags) { + // ssize_t got = fread(__buf, 1, __n, queryfp); + ssize_t got = pglite_read(__buf, n); + return got; + } + + +ssize_t EMSCRIPTEN_KEEPALIVE + send (int __fd, const void *__buf, size_t __n, int __flags) { + // ssize_t wrote = fwrite(__buf, 1, __n, SOCKET_FILE); + ssize_t wrote = pglite_write(__buf, __n); + return wrote; + } + #endif /* pglite specific */ @@ -200,7 +247,7 @@ pq_init(ClientSocket *client_sock) port->sock = client_sock->sock; memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); port->raddr.salen = client_sock->raddr.salen; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +// #if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, @@ -292,7 +339,7 @@ pq_init(ClientSocket *client_sock) (void) pq_setkeepalivescount(tcp_keepalives_count, port); (void) pq_settcpusertimeout(tcp_user_timeout, port); } -#endif /* WASM */ +// #endif /* WASM */ /* initialize state variables */ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; @@ -300,7 +347,7 @@ pq_init(ClientSocket *client_sock) PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; PqCommBusy = false; PqCommReadingMsg = false; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) +// #if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* set up process-exit hook to close the socket */ on_proc_exit(socket_close, 0); @@ -330,10 +377,10 @@ pq_init(ClientSocket *client_sock) MyLatch, NULL); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); -#else /* WASM */ - /* because we may fill before starting reading message */ - PqRecvBuffer = &PqRecvBuffer_static[0]; -#endif /* WASM */ +// #else /* WASM */ +// /* because we may fill before starting reading message */ +// PqRecvBuffer = &PqRecvBuffer_static[0]; +// #endif /* WASM */ /* * The event positions match the order we added them, but let's sanity * check them to be sure. @@ -932,20 +979,20 @@ pq_recvbuf(void) else PqRecvLength = PqRecvPointer = 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (queryfp && querylen) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); - querylen -= got; - PqRecvLength += got; - if (querylen<=0) { - PDEBUG("# 931: could close fp early here " __FILE__); - queryfp = NULL; - } - if (got>0) - return 0; - } - return EOF; -#endif +// #if defined(__EMSCRIPTEN__) || defined(__wasi__) +// if (queryfp && querylen) { +// int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); +// querylen -= got; +// PqRecvLength += got; +// if (querylen<=0) { +// PDEBUG("# 931: could close fp early here " __FILE__); +// queryfp = NULL; +// } +// if (got>0) +// return 0; +// } +// return EOF; +// #endif /* Ensure that we're in blocking mode */ socket_set_nonblocking(false); @@ -1048,9 +1095,9 @@ pq_getbyte_if_available(unsigned char *c) *c = PqRecvBuffer[PqRecvPointer++]; return 1; } -#if defined(__EMSCRIPTEN__) || (__wasi__) -puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); -#else +// #if defined(__EMSCRIPTEN__) || (__wasi__) +// puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); +// #else /* Put the socket into non-blocking mode */ socket_set_nonblocking(true); @@ -1087,7 +1134,7 @@ puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); /* EOF detected */ r = EOF; } -#endif +// #endif return r; } @@ -1177,20 +1224,23 @@ pq_buffer_remaining_data(void) * -------------------------------- */ #if defined(__EMSCRIPTEN__) || defined(__wasi__) -EMSCRIPTEN_KEEPALIVE void +void EMSCRIPTEN_KEEPALIVE pq_recvbuf_fill(FILE* fp, int packetlen) { - if (packetlen>PQ_RECV_BUFFER_SIZE) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); - queryfp = fp; - querylen = packetlen - got; - PqRecvLength = got; - } else { - fread( PqRecvBuffer, packetlen, 1, fp); - PqRecvLength = packetlen; - queryfp = NULL; - querylen = 0; - } - PqRecvPointer = 0; + + queryfp = fp; + + // if (packetlen>PQ_RECV_BUFFER_SIZE) { + // int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); + // queryfp = fp; + // querylen = packetlen - got; + // PqRecvLength = got; + // } else { + // fread( PqRecvBuffer, packetlen, 1, fp); + // PqRecvLength = packetlen; + // queryfp = NULL; + // querylen = 0; + // } + // PqRecvPointer = 0; } #endif @@ -1207,29 +1257,29 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (!pq_buffer_remaining_data()) { - if (sockfiles) { - PqRecvBuffer = &PqRecvBuffer_static[0]; - if (PqSendBuffer_save) - PqSendBuffer=PqSendBuffer_save; - PqSendBufferSize = PQ_SEND_BUFFER_SIZE; - } else { - PqRecvPointer = 0; - PqRecvLength = cma_rsize; - PqRecvBuffer = (char*)0x1; - - PqSendPointer = 0; - PqSendBuffer_save = PqSendBuffer; - PqSendBuffer = 2 + (char*)(cma_rsize); - PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); - } - } -#if PDEBUG - printf("# 1225: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -#endif - -#endif +// #if defined(__EMSCRIPTEN__) || defined(__wasi__) +// if (!pq_buffer_remaining_data()) { +// if (sockfiles) { +// PqRecvBuffer = &PqRecvBuffer_static[0]; +// if (PqSendBuffer_save) +// PqSendBuffer=PqSendBuffer_save; +// PqSendBufferSize = PQ_SEND_BUFFER_SIZE; +// } else { +// PqRecvPointer = 0; +// PqRecvLength = cma_rsize; +// PqRecvBuffer = (char*)0x1; + +// PqSendPointer = 0; +// PqSendBuffer_save = PqSendBuffer; +// PqSendBuffer = 2 + (char*)(cma_rsize); +// PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); +// } +// } +// #if PDEBUG +// printf("# 1225: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); +// #endif + +// #endif PqCommReadingMsg = true; } @@ -1352,56 +1402,55 @@ pq_getmessage(StringInfo s, int maxlen) return 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -static int -internal_putbytes(const char *s, size_t len) { - size_t amount; - if (!sockfiles) { - while (len > 0) { - /* If buffer is full, then flush it out from cma to file and continue from there */ - if (PqSendPointer >= PqSendBufferSize) { - int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); - sockfiles = true; -#if PGDEBUG - fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); -#endif - break; - } - amount = PqSendBufferSize - PqSendPointer; - if (amount > len) - amount = len; - memcpy(PqSendBuffer + PqSendPointer, s, amount); - PqSendPointer += amount; - s += amount; - len -= amount; - SOCKET_DATA+=amount; - } - } - - if (sockfiles) { - int wc= fwrite(s, 1, len, SOCKET_FILE); - SOCKET_DATA+=wc; - } - return 0; -} - -static int -socket_flush(void) { - return internal_flush(); -} - -static int -internal_flush(void) { - /* no flush for raw wire */ - if (sockfiles) { - PqSendStart = PqSendPointer = 0; - } - return 0; -} - -#else +// #if defined(__EMSCRIPTEN__) || defined(__wasi__) +// extern int SOCKET_DATA; +// static int +// internal_putbytes(const char *s, size_t len) { +// size_t amount; +// if (!sockfiles) { +// while (len > 0) { +// /* If buffer is full, then flush it out from cma to file and continue from there */ +// if (PqSendPointer >= PqSendBufferSize) { +// int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); +// sockfiles = true; +// #if PGDEBUG +// fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); +// #endif +// break; +// } +// amount = PqSendBufferSize - PqSendPointer; +// if (amount > len) +// amount = len; +// memcpy(PqSendBuffer + PqSendPointer, s, amount); +// PqSendPointer += amount; +// s += amount; +// len -= amount; +// SOCKET_DATA+=amount; +// } +// } + +// if (sockfiles) { +// int wc= fwrite(s, 1, len, SOCKET_FILE); +// SOCKET_DATA+=wc; +// } +// return 0; +// } + +// static int +// socket_flush(void) { +// return internal_flush(); +// } + +// static int +// internal_flush(void) { +// /* no flush for raw wire */ +// if (sockfiles) { +// PqSendStart = PqSendPointer = 0; +// } +// return 0; +// } + +// #else static inline int internal_putbytes(const char *s, size_t len) @@ -1552,7 +1601,7 @@ internal_flush_buffer(const char *buf, size_t *start, size_t *end) *start = *end = 0; return 0; } -#endif /* wasm */ +// #endif /* wasm */ /* -------------------------------- * pq_flush_if_writable - flush pending output if writable without blocking * diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 72481df9bd16a..bbe5b702ac1f5 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -115,6 +115,12 @@ int restrict_nonsystem_relation_kind; bool quote_all_identifiers = false; FILE* SOCKET_FILE = NULL; int SOCKET_DATA = 0; + +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +pglite_read_t pglite_read = NULL; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +pglite_write_t pglite_write = NULL; #endif // WASM From 6cf2d5fea350853045d1c50c274da689693c63d9 Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 1 Oct 2025 21:54:42 +0200 Subject: [PATCH 20/30] cleanup --- build-pglite.sh | 2 +- pglite-wasm/interactive_one.c | 1 - src/backend/libpq/pqcomm.c | 149 +++------------------------------- 3 files changed, 12 insertions(+), 140 deletions(-) diff --git a/build-pglite.sh b/build-pglite.sh index c5d695d22712a..f50233ec01a50 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -63,7 +63,7 @@ PATH=$SAVE_PATH # Step 5: make and install pglite # we define here "all" emscripten flags in order to allow native builds (like libpglite) -EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sSUPPORT_LONGJMP=emscripten \ -sGLOBAL_BASE=12MB -sFORCE_FILESYSTEM=1 \ diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index b704f16305ec6..1bff143e4f0f8 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -116,7 +116,6 @@ buffer sizes: */ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); -extern void pq_recvbuf_fill(FILE* fp, int packetlen); #define PG_MAX_AUTH_TOKEN_LENGTH 65535 static char * diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 5b954aaf9a063..f0d1f8277ea3b 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -126,15 +126,10 @@ static char *PqSendBuffer; static int PqSendBufferSize; /* Size send buffer */ static size_t PqSendPointer; /* Next index to store a byte in PqSendBuffer */ static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ static int PqRecvLength; /* End of data available in PqRecvBuffer */ -#else -static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; -static char *PqRecvBuffer; -static int PqRecvPointer; -static int PqRecvLength; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) volatile int querylen = 0; volatile FILE* queryfp = NULL; extern FILE* SOCKET_FILE; @@ -171,15 +166,12 @@ int EMSCRIPTEN_KEEPALIVE getsockname (int __fd, struct sockaddr * __addr, ssize_t EMSCRIPTEN_KEEPALIVE recv (int __fd, void *__buf, size_t __n, int __flags) { - // ssize_t got = fread(__buf, 1, __n, queryfp); - ssize_t got = pglite_read(__buf, n); + ssize_t got = pglite_read(__buf, __n); return got; } - ssize_t EMSCRIPTEN_KEEPALIVE send (int __fd, const void *__buf, size_t __n, int __flags) { - // ssize_t wrote = fwrite(__buf, 1, __n, SOCKET_FILE); ssize_t wrote = pglite_write(__buf, __n); return wrote; } @@ -199,7 +191,6 @@ static bool PqCommReadingMsg; /* in the middle of reading a message */ /* Internal functions */ - static void socket_comm_reset(void); static void socket_close(int code, Datum arg); static void socket_set_nonblocking(bool nonblocking); @@ -213,6 +204,9 @@ static inline int internal_flush(void); static pg_noinline int internal_flush_buffer(const char *buf, size_t *start, size_t *end); +static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); +static int Setup_AF_UNIX(const char *sock_path); + static const PQcommMethods PqCommSocketMethods = { .comm_reset = socket_comm_reset, .flush = socket_flush, @@ -222,10 +216,6 @@ static const PQcommMethods PqCommSocketMethods = { .putmessage_noblock = socket_putmessage_noblock }; -static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); -static int Setup_AF_UNIX(const char *sock_path); - - const PQcommMethods *PqCommMethods = &PqCommSocketMethods; WaitEventSet *FeBeWaitSet; @@ -247,7 +237,6 @@ pq_init(ClientSocket *client_sock) port->sock = client_sock->sock; memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); port->raddr.salen = client_sock->raddr.salen; -// #if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, @@ -339,7 +328,6 @@ pq_init(ClientSocket *client_sock) (void) pq_setkeepalivescount(tcp_keepalives_count, port); (void) pq_settcpusertimeout(tcp_user_timeout, port); } -// #endif /* WASM */ /* initialize state variables */ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; @@ -347,7 +335,6 @@ pq_init(ClientSocket *client_sock) PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; PqCommBusy = false; PqCommReadingMsg = false; -// #if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* set up process-exit hook to close the socket */ on_proc_exit(socket_close, 0); @@ -377,10 +364,6 @@ pq_init(ClientSocket *client_sock) MyLatch, NULL); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); -// #else /* WASM */ -// /* because we may fill before starting reading message */ -// PqRecvBuffer = &PqRecvBuffer_static[0]; -// #endif /* WASM */ /* * The event positions match the order we added them, but let's sanity * check them to be sure. @@ -800,7 +783,7 @@ Setup_AF_UNIX(const char *sock_path) Assert(Unix_socket_group); if (Unix_socket_group[0] != '\0') { -#if defined(WIN32) || defined(__wasi__) +#ifdef WIN32 elog(WARNING, "configuration item \"unix_socket_group\" is not supported on this platform"); #else char *endptr; @@ -979,20 +962,6 @@ pq_recvbuf(void) else PqRecvLength = PqRecvPointer = 0; } -// #if defined(__EMSCRIPTEN__) || defined(__wasi__) -// if (queryfp && querylen) { -// int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); -// querylen -= got; -// PqRecvLength += got; -// if (querylen<=0) { -// PDEBUG("# 931: could close fp early here " __FILE__); -// queryfp = NULL; -// } -// if (got>0) -// return 0; -// } -// return EOF; -// #endif /* Ensure that we're in blocking mode */ socket_set_nonblocking(false); @@ -1095,9 +1064,7 @@ pq_getbyte_if_available(unsigned char *c) *c = PqRecvBuffer[PqRecvPointer++]; return 1; } -// #if defined(__EMSCRIPTEN__) || (__wasi__) -// puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); -// #else + /* Put the socket into non-blocking mode */ socket_set_nonblocking(true); @@ -1134,7 +1101,7 @@ pq_getbyte_if_available(unsigned char *c) /* EOF detected */ r = EOF; } -// #endif + return r; } @@ -1201,7 +1168,6 @@ pq_discardbytes(size_t len) return 0; } - /* -------------------------------- * pq_buffer_remaining_data - return number of bytes in receive buffer * @@ -1223,29 +1189,6 @@ pq_buffer_remaining_data(void) * This must be called before any of the pq_get* functions. * -------------------------------- */ -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -void EMSCRIPTEN_KEEPALIVE -pq_recvbuf_fill(FILE* fp, int packetlen) { - - queryfp = fp; - - // if (packetlen>PQ_RECV_BUFFER_SIZE) { - // int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); - // queryfp = fp; - // querylen = packetlen - got; - // PqRecvLength = got; - // } else { - // fread( PqRecvBuffer, packetlen, 1, fp); - // PqRecvLength = packetlen; - // queryfp = NULL; - // querylen = 0; - // } - // PqRecvPointer = 0; -} - -#endif - -static char * PqSendBuffer_save; void pq_startmsgread(void) { @@ -1257,29 +1200,7 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); -// #if defined(__EMSCRIPTEN__) || defined(__wasi__) -// if (!pq_buffer_remaining_data()) { -// if (sockfiles) { -// PqRecvBuffer = &PqRecvBuffer_static[0]; -// if (PqSendBuffer_save) -// PqSendBuffer=PqSendBuffer_save; -// PqSendBufferSize = PQ_SEND_BUFFER_SIZE; -// } else { -// PqRecvPointer = 0; -// PqRecvLength = cma_rsize; -// PqRecvBuffer = (char*)0x1; - -// PqSendPointer = 0; -// PqSendBuffer_save = PqSendBuffer; -// PqSendBuffer = 2 + (char*)(cma_rsize); -// PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); -// } -// } -// #if PDEBUG -// printf("# 1225: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -// #endif - -// #endif + PqCommReadingMsg = true; } @@ -1402,55 +1323,7 @@ pq_getmessage(StringInfo s, int maxlen) return 0; } -// #if defined(__EMSCRIPTEN__) || defined(__wasi__) -// extern int SOCKET_DATA; -// static int -// internal_putbytes(const char *s, size_t len) { -// size_t amount; -// if (!sockfiles) { -// while (len > 0) { -// /* If buffer is full, then flush it out from cma to file and continue from there */ -// if (PqSendPointer >= PqSendBufferSize) { -// int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); -// sockfiles = true; -// #if PGDEBUG -// fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); -// #endif -// break; -// } -// amount = PqSendBufferSize - PqSendPointer; -// if (amount > len) -// amount = len; -// memcpy(PqSendBuffer + PqSendPointer, s, amount); -// PqSendPointer += amount; -// s += amount; -// len -= amount; -// SOCKET_DATA+=amount; -// } -// } - -// if (sockfiles) { -// int wc= fwrite(s, 1, len, SOCKET_FILE); -// SOCKET_DATA+=wc; -// } -// return 0; -// } - -// static int -// socket_flush(void) { -// return internal_flush(); -// } - -// static int -// internal_flush(void) { -// /* no flush for raw wire */ -// if (sockfiles) { -// PqSendStart = PqSendPointer = 0; -// } -// return 0; -// } - -// #else + static inline int internal_putbytes(const char *s, size_t len) @@ -1601,7 +1474,7 @@ internal_flush_buffer(const char *buf, size_t *start, size_t *end) *start = *end = 0; return 0; } -// #endif /* wasm */ + /* -------------------------------- * pq_flush_if_writable - flush pending output if writable without blocking * From fd02949cd5ac26ecfb0a15ef8f78bb54ca803cb4 Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 11:54:40 +0200 Subject: [PATCH 21/30] nothing --- src/backend/libpq/pqcomm.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index f0d1f8277ea3b..0bae02f554fae 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -140,38 +140,38 @@ extern pglite_read_t pglite_read; typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); extern pglite_write_t pglite_write; -int EMSCRIPTEN_KEEPALIVE fcntl (int __fd, int __cmd, ...) { +int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) { // dummy return 0; } -int EMSCRIPTEN_KEEPALIVE setsockopt (int __fd, int __level, int __optname, +int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname, const void *__optval, socklen_t __optlen) { // dummy return 0; } -int EMSCRIPTEN_KEEPALIVE getsockopt (int __fd, int __level, int __optname, +int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname, void *__restrict __optval, socklen_t *__restrict __optlen) { // dummy return 0; } -int EMSCRIPTEN_KEEPALIVE getsockname (int __fd, struct sockaddr * __addr, +int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr, socklen_t *__restrict __len) { // dummy return 0; } ssize_t EMSCRIPTEN_KEEPALIVE - recv (int __fd, void *__buf, size_t __n, int __flags) { + recv(int __fd, void *__buf, size_t __n, int __flags) { ssize_t got = pglite_read(__buf, __n); return got; } ssize_t EMSCRIPTEN_KEEPALIVE - send (int __fd, const void *__buf, size_t __n, int __flags) { + send(int __fd, const void *__buf, size_t __n, int __flags) { ssize_t wrote = pglite_write(__buf, __n); return wrote; } From 3df81f39cb26c4f0b38a931abe902dd766541d56 Mon Sep 17 00:00:00 2001 From: tudor Date: Thu, 2 Oct 2025 14:44:26 +0200 Subject: [PATCH 22/30] more cleanup --- build-pglite.sh | 4 +- pglite-wasm/included.pglite.imports | 3 - pglite-wasm/interactive_one.c | 94 ++++++----------------------- pglite-wasm/pg_main.c | 5 +- src/backend/Makefile | 4 +- 5 files changed, 24 insertions(+), 86 deletions(-) diff --git a/build-pglite.sh b/build-pglite.sh index f50233ec01a50..8fd4b25572087 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -39,7 +39,7 @@ fi # Step 1: configure the project if [ "$RUN_CONFIGURE" = true ]; then - LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } + LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } else echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" fi @@ -66,7 +66,7 @@ PATH=$SAVE_PATH EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sSUPPORT_LONGJMP=emscripten \ --sGLOBAL_BASE=12MB -sFORCE_FILESYSTEM=1 \ +-sFORCE_FILESYSTEM=1 \ -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ -sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ -sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index 39931facdab90..eb43cc8c7a0e6 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -1,13 +1,10 @@ close fcntl free -get_channel getpid gettimeofday gmtime interactive_one -interactive_read -interactive_write ioctl isalnum isxdigit diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 1bff143e4f0f8..f8c50a8319aa1 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -9,33 +9,10 @@ volatile sigjmp_buf local_sigjmp_buf; // track back how many ex raised in steps of the loop until sucessfull clear_error volatile int canary_ex = 0; -// track back mode used for last reply <0 socketfiles , 0== repl , > 0 cma addr -volatile int channel = 0; - /* TODO : prevent multiple write and write while reading ? */ -volatile int cma_wsize = 0; volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm volatile bool sockfiles = false; // also defined in postgres.c for pqcomm -__attribute__((export_name("get_buffer_size"))) -int -get_buffer_size(int fd) { - return (CMA_MB * 1024 * 1024) / CMA_FD; -} - -// TODO add query size -__attribute__((export_name("get_buffer_addr"))) -int -get_buffer_addr(int fd) { - return 1 + ( get_buffer_size(fd) *fd); -} - -__attribute__((export_name("get_channel"))) -int -get_channel() { - return channel; -} - // read FROM JS // (i guess return number of bytes written) // ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); @@ -57,14 +34,6 @@ set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { pglite_write = write_cb; } -__attribute__((export_name("interactive_read"))) -int -interactive_read() { - /* should cma_rsize should be reset here ? */ - return cma_wsize; -} - - static void pg_prompt() { fprintf(stdout,"pg> %c\n", 4); } @@ -213,16 +182,8 @@ static void io_init(bool in_auth, bool out_auth) { volatile bool is_wire = true; -extern char * cma_port; extern void pq_startmsgread(void); -__attribute__((export_name("interactive_write"))) -void -interactive_write(int size) { - cma_rsize = size; - cma_wsize = 0; -} - __attribute__((export_name("use_wire"))) void use_wire(int state) { @@ -298,16 +259,6 @@ clear_error() { send_ready_for_query = true; } -void discard_input(){ - if (!cma_rsize) - return; - pq_startmsgread(); - for (int i = 0; i < cma_rsize; i++) { - pq_getbyte(); - } - pq_endmsgread(); -} - void startup_auth() { /* code is in handshake/auth domain so read whole msg now */ @@ -320,7 +271,6 @@ startup_auth() { sf_connected++; PDEBUG("# 273: sending auth request"); //ClientAuthentication(MyProcPort); - discard_input(); ClientAuthInProgress = true; md5Salt[0]=0x01; @@ -358,7 +308,6 @@ startup_pass(bool check) { pfree(passwd); } else { PDEBUG("# 310: auth skip"); - discard_input(); } ClientAuthInProgress = false; @@ -414,24 +363,24 @@ interactive_one(int packetlen, int peek) { // this could be pg_flush in sync mode. // but in fact we are writing socket data that was piled up previous frame async. - if (SOCKET_DATA>0) { - puts("# 361: ERROR flush after frame"); - goto wire_flush; - } + // if (SOCKET_DATA>0) { + // puts("# 361: ERROR flush after frame"); + // goto wire_flush; + // } - if (!cma_rsize) { - // no cma : reading from file. writing to file. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - MyProcPort->sock = fileno(SOCKET_FILE); - } - } else { - // prepare file reply queue, just in case of cma overflow - // if unused the file will be kept open till next query. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - } - } + // if (!cma_rsize) { + // // no cma : reading from file. writing to file. + // if (!SOCKET_FILE) { + // SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + // MyProcPort->sock = fileno(SOCKET_FILE); + // } + // } else { + // // prepare file reply queue, just in case of cma overflow + // // if unused the file will be kept open till next query. + // if (!SOCKET_FILE) { + // SOCKET_FILE = fopen(PGS_OLOCK, "w") ; + // } + // } MemoryContextSwitchTo(MessageContext); MemoryContextReset(MessageContext); @@ -460,13 +409,6 @@ interactive_one(int packetlen, int peek) { // postgres.c 4627 DoingCommandRead = true; -#if defined(EMUL_CMA) - // temp fix for -O0 but less efficient than literal - #define IO ((char *)(1+(int)cma_port)) -#else - #define IO ((char *)(1)) -#endif - /* * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. @@ -788,7 +730,7 @@ return_early:; // cma_rsize = 0; // IO[0] = 0; - #undef IO + // #undef IO // reset EX counter canary_ex = 0; diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index d6fae41af0dae..4274bb0741f76 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -62,7 +62,6 @@ volatile int async_restart = 1; #define WASM_PGDATA WASM_PREFIX "/base" -#define CMA_FD 1 extern bool IsPostmasterEnvironment; @@ -174,8 +173,8 @@ main_pre(int argc, char *argv[]) { #if defined(__EMSCRIPTEN__) EM_ASM( { Module.is_worker = (typeof WorkerGlobalScope !== 'undefined') && self instanceof WorkerGlobalScope; - Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} - , (CMA_MB * 1024 * 1024) / CMA_FD); /* ( global mem start / num fd max ) */ + Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} // tdrz: what is FD_BUFFER_MAX? + , (12 * 1024 * 1024)); /* ( global mem start / num fd max ) */ if (is_node) { setenv("ENVIRONMENT", "node", 1); diff --git a/src/backend/Makefile b/src/backend/Makefile index f3e7d2119ee13..986529ad6b17c 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -107,11 +107,11 @@ pglite-exported-functions: # PGLITE_EMSCRIPTEN_FLAGS are the emscripten flags to be passed to the build. Ideally they would be completely transparent, such that we # could build a libpglite with or without those flags (to switch between native and wasm builds) pglite: pglite-exported-functions - $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers + $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ - -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ + -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ $(PGPRELOAD) \ -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ From a4013b504e9cfad59fc2d6db3d41267aba326c3c Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 12:04:02 +0200 Subject: [PATCH 23/30] cleanup is_repl and others --- pglite-wasm/interactive_one.c | 277 ++-------------------------------- pglite-wasm/pg_main.c | 35 +---- pglite-wasm/pg_proto.c | 1 - pglite-wasm/pgl_initdb.c | 5 - pglite-wasm/pgl_sjlj.c | 5 - src/backend/libpq/pqcomm.c | 5 - src/backend/tcop/postgres.c | 4 - 7 files changed, 21 insertions(+), 311 deletions(-) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index f8c50a8319aa1..2de2bb8d04126 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -9,10 +9,6 @@ volatile sigjmp_buf local_sigjmp_buf; // track back how many ex raised in steps of the loop until sucessfull clear_error volatile int canary_ex = 0; -/* TODO : prevent multiple write and write while reading ? */ -volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm -volatile bool sockfiles = false; // also defined in postgres.c for pqcomm - // read FROM JS // (i guess return number of bytes written) // ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); @@ -195,13 +191,11 @@ use_wire(int state) { printf("\n\n# 194: PACKET START: wire mode, repl off, echo %d\n", force_echo); #endif is_wire = true; - is_repl = false; } else { #if PGDEBUG printf("\n\n# 200: PACKET START: repl mode, no wire, echo %d\n", force_echo); #endif is_wire = false; - is_repl = true; } } @@ -333,8 +327,6 @@ PDEBUG("# 330: TODO: set a pgl started flag"); volatile int sf_connected = 0; } -extern void pg_startcma(); - __attribute__((export_name("interactive_one"))) void interactive_one(int packetlen, int peek) { // int peek = -1; /* preview of firstchar with no pos change */ @@ -361,27 +353,6 @@ interactive_one(int packetlen, int peek) { PDEBUG("# 371: has notification !"); #endif - // this could be pg_flush in sync mode. - // but in fact we are writing socket data that was piled up previous frame async. - // if (SOCKET_DATA>0) { - // puts("# 361: ERROR flush after frame"); - // goto wire_flush; - // } - - // if (!cma_rsize) { - // // no cma : reading from file. writing to file. - // if (!SOCKET_FILE) { - // SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - // MyProcPort->sock = fileno(SOCKET_FILE); - // } - // } else { - // // prepare file reply queue, just in case of cma overflow - // // if unused the file will be kept open till next query. - // if (!SOCKET_FILE) { - // SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - // } - // } - MemoryContextSwitchTo(MessageContext); MemoryContextReset(MessageContext); @@ -406,138 +377,21 @@ interactive_one(int packetlen, int peek) { } send_ready_for_query = false; } -// postgres.c 4627 - DoingCommandRead = true; -/* - * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results - * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. - * in repl mode (cma_rsize==0) output is on stdout not cma/socketfiles wire. - * repl mode is the simpleset mode where stdin is just copied into input buffer (limited by CMA size). - * TODO: allow to redirect stdout for fully external repl. - */ - - if (!is_repl) { - whereToSendOutput = DestRemote; - if (!is_wire) - PDEBUG("# 439: repl message in cma buffer !"); - } else { - if (is_wire) - PDEBUG("# 442: wire message in cma buffer for REPL !"); - whereToSendOutput = DestDebug; - } + DoingCommandRead = true; -// packetlen = cma_rsize; - -// if (cma_rsize) { -// peek = IO[0]; -// sockfiles = false; -// if (!is_repl) { -// whereToSendOutput = DestRemote; -// if (!is_wire) -// PDEBUG("# 439: repl message in cma buffer !"); -// } else { -// if (is_wire) -// PDEBUG("# 442: wire message in cma buffer for REPL !"); -// whereToSendOutput = DestDebug; -// } -// } else { -// fp = fopen(PGS_IN, "r"); -// PDEBUG("# 452:" PGS_IN "\n"); -// // read file in socket buffer for SocketBackend to consumme. -// if (fp) { -// fseek(fp, 0L, SEEK_END); -// packetlen = ftell(fp); -// if (packetlen) { -// // set to always true if no REPL. -// // is_wire = true; -// resetStringInfo(inBuf); -// rewind(fp); -// /* peek on first char */ -// peek = getc(fp); -// rewind(fp); -// if (is_repl && !is_wire) { -// // sql in buffer -// for (int i=0; i true : %c\n", peek); -// force_echo = true; -// } - -// #endif -// firstchar = peek; -// goto incoming; -// } // wire msg -// PDEBUG("# 507: NO DATA:" PGS_IN "\n"); -// } // fp data read - -// // is it REPL in cma ? -// if (!peek) -// goto return_early; - -// firstchar = peek ; - -// //REPL mode in zero copy buffer ( lowest wasm memory segment ) -// packetlen = strlen(IO); - -// } // !cma_rsize -> socketfiles -> repl + whereToSendOutput = DestRemote; #if PGDEBUG if (packetlen) IO[packetlen]=0; // wire blocks are not zero terminated - printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); + printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen, peek, IO); #endif resetStringInfo(inBuf); - // when cma buffer is used to fake stdin, data is not read by socket/wire backend. - // if (is_repl) { - // for (int i=0; idata); - force_echo = false; - } else { - printf("# 576: PIPELINING [%c]!\n", firstchar); - } -#endif - if (!ignore_till_sync) { /* initially, or after error */ // send_ready_for_query = true; @@ -635,103 +469,24 @@ puts("# 631: PIPELINING + rfq"); } } - if (!is_repl) { wire_flush: - if (!ClientAuthInProgress) { - /* process notifications (SYNC) */ - if (notifyInterruptPending) - ProcessNotifyInterrupt(false); - - if (send_ready_for_query) { - PDEBUG("# 602: end packet - sending rfq\n"); - ReadyForQuery(DestRemote); - //done at postgres.c 4623 - send_ready_for_query = false; - } else { - PDEBUG("# 606: end packet - with no rfq\n"); - } + if (!ClientAuthInProgress) { + /* process notifications (SYNC) */ + if (notifyInterruptPending) + ProcessNotifyInterrupt(false); + if (send_ready_for_query) { + PDEBUG("# 602: end packet - sending rfq\n"); + ReadyForQuery(DestRemote); + //done at postgres.c 4623 + send_ready_for_query = false; } else { - PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); + PDEBUG("# 606: end packet - with no rfq\n"); } - -// if (SOCKET_DATA>0) { -// if (sockfiles) { -// channel = -1; -// if (cma_wsize) { -// puts("ERROR: cma was not flushed before socketfile interface"); -// } -// } else { -// /* wsize may have increased with previous rfq so assign here */ -// cma_wsize = SOCKET_DATA; -// channel = cma_rsize + 2; -// } -// if (SOCKET_FILE) { -// int outb = SOCKET_DATA; -// fclose(SOCKET_FILE); -// SOCKET_FILE = NULL; -// SOCKET_DATA = 0; - -// if (cma_wsize) { -// PDEBUG("# 672: cma and sockfile ???\n"); -// } - -// if (sockfiles) { -// #if PGDEBUG -// printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); -// #endif -// rename(PGS_OLOCK, PGS_OUT); -// } -// } else { -// #if PGDEBUG -// printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); -// #endif -// SOCKET_DATA = 0; -// } - -// } else { -// cma_wsize = 0; -// PDEBUG("# 698: no data, send empty ?"); -// // TODO: dedup 739 -// if (sockfiles) { -// fclose(SOCKET_FILE); -// SOCKET_FILE = NULL; -// rename(PGS_OLOCK, PGS_OUT); -// } -// } } else { - pg_prompt(); -#if PGDEBUG - puts("# 683: repl output"); - if (SOCKET_DATA>0) { - puts("# 686: socket has data"); - if (sockfiles) - printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); - } else { -// TODO: dedup 723 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } - } - if (cma_wsize) - puts("ERROR: cma was not flushed before socketfile interface"); -#endif + PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } -return_early:; - /* always FD CLEANUP */ - // if (fp) { - // fclose(fp); - // unlink(PGS_IN); - // } - - - // always free kernel buffer !!! - // cma_rsize = 0; - // IO[0] = 0; - - // #undef IO +return_early:; // reset EX counter canary_ex = 0; } diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index 4274bb0741f76..a76aabee6374c 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -45,7 +45,6 @@ volatile char *PGUSER; const char *progname; -volatile bool is_repl = true; volatile bool is_node = true; volatile bool is_embed = false; volatile int pgl_idb_status; @@ -193,7 +192,6 @@ main_pre(int argc, char *argv[]) { console.warn("prerun(C-web) worker=", Module.is_worker);} ); # endif - is_repl = true; } // *INDENT-OFF* EM_ASM({ @@ -562,34 +560,11 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" printf("# 550: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif - progname = get_progname(argv[0]); - g_argv = argv; - g_argc = argc; + progname = get_progname(argv[0]); + g_argv = argv; + g_argc = argc; + is_embed = true; - is_repl = strlen(getenv("REPL")) && getenv("REPL")[0] != 'N'; - is_embed = true; - - if (!is_repl) { - PDEBUG("# 562: exit with live runtime (nodb)"); - return 0; - } -#if defined(__wasi__) - - -#else - /* - main_post(); - - PDEBUG("# 565: repl"); - // so it is repl - main_repl(); - - if (is_node) { - PDEBUG("# 570: node repl"); - pg_repl_raf(); - } - */ - emscripten_force_exit(exit_code); -#endif + emscripten_force_exit(exit_code); return exit_code; } diff --git a/pglite-wasm/pg_proto.c b/pglite-wasm/pg_proto.c index fd21210f420d3..2ba14a846e5db 100644 --- a/pglite-wasm/pg_proto.c +++ b/pglite-wasm/pg_proto.c @@ -313,7 +313,6 @@ if (sf_connected) { PDEBUG("ERROR: more exits than connections"); } PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); - is_repl = false; ignore_till_sync = false; send_ready_for_query = false; diff --git a/pglite-wasm/pgl_initdb.c b/pglite-wasm/pgl_initdb.c index 19bf7fd405457..9536b257e7cea 100644 --- a/pglite-wasm/pgl_initdb.c +++ b/pglite-wasm/pgl_initdb.c @@ -49,10 +49,5 @@ pg_chmod(const char * path, int mode_t) { #include "bin/initdb/initdb.c" -void use_socketfile(void) { - is_repl = true; - is_embed = false; -} - diff --git a/pglite-wasm/pgl_sjlj.c b/pglite-wasm/pgl_sjlj.c index 6c05dafa9253f..469053a2705b2 100644 --- a/pglite-wasm/pgl_sjlj.c +++ b/pglite-wasm/pgl_sjlj.c @@ -57,11 +57,6 @@ #endif #if !defined(INITDB_SINGLE) -#if PGDEBUG - if (is_repl) - pg_prompt(); -#endif - if (pq_buffer_remaining_data()>0) { if (canary_ex++ > 8) abort(); diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 0bae02f554fae..26c66ed5d8376 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -178,11 +178,6 @@ ssize_t EMSCRIPTEN_KEEPALIVE #endif -/* pglite specific */ -extern int cma_rsize; -extern bool sockfiles; - - /* * Message status */ diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index bbe5b702ac1f5..64f3dd8b259ba 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -108,10 +108,6 @@ int client_connection_check_interval = 0; int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) -#if !defined(PGL_MAIN) - volatile int cma_rsize = 0; - volatile bool sockfiles = false; -#endif // PGL_MAIN bool quote_all_identifiers = false; FILE* SOCKET_FILE = NULL; int SOCKET_DATA = 0; From d512ff5bc269479f536baa7e7f92bea44312541d Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 13:59:10 +0200 Subject: [PATCH 24/30] sort included imports --- pglite-wasm/included.pglite.imports | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index eb43cc8c7a0e6..72bc4cf1862de 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -24,11 +24,11 @@ read readstoplist realloc searchstoplist +set_read_write_cbs socket srand strcmp strftime strlen strtoul -use_wire -set_read_write_cbs \ No newline at end of file +use_wire \ No newline at end of file From 5452dfc6f86092d775c6599c40e3c491ab547d98 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 14:00:23 +0200 Subject: [PATCH 25/30] add a pq_flush() at the end of interactive_one(). this is needed because of the new frontend <-> backend data transfer mechanism because before the write was not buffered. --- pglite-wasm/interactive_one.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 2de2bb8d04126..f12719728e51e 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -489,6 +489,7 @@ puts("# 631: PIPELINING + rfq"); return_early:; // reset EX counter canary_ex = 0; + pq_flush(); } #undef PGL_LOOP \ No newline at end of file From e6ac5c2eac7f5e1aa0607895eb010fc446070d11 Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 20:06:11 +0200 Subject: [PATCH 26/30] more cleanup --- pglite-wasm/interactive_one.c | 48 ----------- src/backend/libpq/pqcomm.c | 1 - src/backend/tcop/postgres.c | 2 - src/include/port/wasm_common.h | 23 ----- wasm-build/sdk_port-wasi/sdk_port-wasi.c | 105 ----------------------- 5 files changed, 179 deletions(-) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index f12719728e51e..d92d0380dcd57 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -30,55 +30,9 @@ set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { pglite_write = write_cb; } -static void pg_prompt() { - fprintf(stdout,"pg> %c\n", 4); -} - extern void AbortTransaction(void); extern void CleanupTransaction(void); extern void ClientAuthentication(Port *port); -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; - - - -/* -init sequence -___________________________________ -SubPostmasterMain / (forkexec) - InitPostmasterChild - shm attach - preload - - BackendInitialize(Port *port) -> collect initial packet - - pq_init(); - whereToSendOutput = DestRemote; - status = ProcessStartupPacket(port, false, false); - pq_startmsgread - pq_getbytes from pq_recvbuf - TODO: place PqRecvBuffer (8K) in lower mem for zero copy - - PerformAuthentication - ClientAuthentication(port) - CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) - InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores - - BackendRun(port) - PostgresMain - - --> pq_flush() is synchronous - - -buffer sizes: - - https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 - - https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 - - -*/ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); @@ -171,8 +125,6 @@ static void io_init(bool in_auth, bool out_auth) { MyProcPort->canAcceptConnections = CAC_OK; #endif ClientAuthInProgress = out_auth; - SOCKET_FILE = NULL; - SOCKET_DATA = 0; PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 26c66ed5d8376..856e9d2a3623c 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -132,7 +132,6 @@ static int PqRecvLength; /* End of data available in PqRecvBuffer */ #if defined(__EMSCRIPTEN__) || defined(__wasi__) volatile int querylen = 0; volatile FILE* queryfp = NULL; -extern FILE* SOCKET_FILE; typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); extern pglite_read_t pglite_read; diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 64f3dd8b259ba..bbcf4112aead6 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -109,8 +109,6 @@ int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) bool quote_all_identifiers = false; -FILE* SOCKET_FILE = NULL; -int SOCKET_DATA = 0; typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); pglite_read_t pglite_read = NULL; diff --git a/src/include/port/wasm_common.h b/src/include/port/wasm_common.h index d8ade8cc7523e..1801fb0468c01 100644 --- a/src/include/port/wasm_common.h +++ b/src/include/port/wasm_common.h @@ -102,30 +102,7 @@ extern int pg_valid_server_encoding_id_private(int encoding); #define proc_exit(arg) pg_proc_exit(arg) -/* -extern FILE* IDB_PIPE_FP; -extern int IDB_STAGE; -*/ -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -extern int pgl_pclose(FILE *stream); -/* -#if !defined(PGL_MAIN) && !defined(PGL_INITDB_MAIN) -# if !defined(PG_EXEC) extern int pgl_pclose(FILE *stream); -#define pclose(stream) pg_pclose(stream) -# else -#if 1 -#include -int -pg_pclose(FILE *stream) { - puts("# 118:" __FILE__ " int pg_pclose(FILE *stream) STUB"); - return 0; -} -#endif -# endif // PG_EXEC -#endif // pgl -*/ /* * OpenPipeStream : another kind of pipe open in fd.c diff --git a/wasm-build/sdk_port-wasi/sdk_port-wasi.c b/wasm-build/sdk_port-wasi/sdk_port-wasi.c index 5dbab909264d7..f01d519d91075 100644 --- a/wasm-build/sdk_port-wasi/sdk_port-wasi.c +++ b/wasm-build/sdk_port-wasi/sdk_port-wasi.c @@ -316,11 +316,8 @@ int shm_unlink(const char *name) { #include // FILE+fprintf extern FILE* IDB_PIPE_FP; -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; extern int IDB_STAGE; - static inline int ends_with(const char *str, const char *suffix) { @@ -404,106 +401,4 @@ volatile int fd_sock = 0; volatile int fd_out = 2; volatile int fd_queue = 0; - - - -#if 0 -#define AF_UNIX 1 // PF_LOCAL - - - - -volatile bool web_warned = false; - -void sock_flush() { - if (fd_queue) { - printf(" -- 441 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); - if (!fd_FILE) { - if (!web_warned) { - puts("# 444: WARNING: fd_FILE not set but queue not empty, assuming web"); - web_warned = true; - } - - } else { - printf("# 449: SENT=%ld/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); - fclose(fd_FILE); - rename(PGS_ILOCK, PGS_IN); -//freopen(PGS_ILOCK, "w+", fd_FILE); - fd_FILE = fopen(PGS_ILOCK, "w"); - fd_out = fileno(fd_FILE); - printf("# 455: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); - } - fd_queue = 0; - sched_yield(); - return; - } - - printf(" -- 462 sockflush[%d] : NO YIELD --\n",stage); - - // limit inf loops - if (stage++ > 1024) { - puts("# 466 sock_flush : busy looping ?"); - abort(); - } -} - - -volatile int fd_current_pos = 0; -volatile int fd_filesize = 0; - - -ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { -// int busy = 0; - int rcv = -1; - sock_flush(); -/* - while (access(PGS_OUT, F_OK) != 0) { - if (!(++busy % 555111)) { - printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); - } - if (busy>1665334) { - errno = EINTR; - return -1; - } - } -*/ - FILE *sock_in = fopen(PGS_OUT,"r"); - if (sock_in) { - if (!fd_filesize) { - fseek(sock_in, 0L, SEEK_END); - fd_filesize = ftell(sock_in); - } - fseek(sock_in, fd_current_pos, SEEK_SET); - - char *buf = buffer; - buf[0] = 0; - rcv = fread(buf, 1, length, sock_in); - - if (rcv Date: Mon, 6 Oct 2025 20:58:41 +0200 Subject: [PATCH 27/30] remove repl.html --- pglite-wasm/repl.html | 1226 ----------------------------------------- 1 file changed, 1226 deletions(-) delete mode 100644 pglite-wasm/repl.html diff --git a/pglite-wasm/repl.html b/pglite-wasm/repl.html deleted file mode 100644 index 15e9ed2d20e1c..0000000000000 --- a/pglite-wasm/repl.html +++ /dev/null @@ -1,1226 +0,0 @@ - - - - - - - PG SHELL TEST - - - - - - - - - - - - - - - - - - -
-
emscripten
-
Downloading...
-
- -
- - - -
- - -
-pglite.wasi: Download file
-
-pglite.wasi + filesystem: Download file
-
-pg_dump.wasm: Download file
-
-

- -
- -
- -
- - - -#if MODULARIZE - - -#endif - - - - - From da471685dffb2792a0123233695b296beda5547d Mon Sep 17 00:00:00 2001 From: tudor Date: Mon, 6 Oct 2025 20:58:56 +0200 Subject: [PATCH 28/30] more cleanup --- pglite-wasm/included.pglite.imports | 3 +-- pglite-wasm/interactive_one.c | 19 ------------------- pglite-wasm/pg_main.c | 18 ------------------ src/backend/Makefile | 2 +- 4 files changed, 2 insertions(+), 40 deletions(-) diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index 72bc4cf1862de..7a5d30d895111 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -30,5 +30,4 @@ srand strcmp strftime strlen -strtoul -use_wire \ No newline at end of file +strtoul \ No newline at end of file diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index d92d0380dcd57..2c2fa478c43ad 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -132,25 +132,6 @@ static void io_init(bool in_auth, bool out_auth) { volatile bool is_wire = true; extern void pq_startmsgread(void); -__attribute__((export_name("use_wire"))) -void -use_wire(int state) { -#if PGDEBUG - force_echo=true; -#endif - if (state>0) { -#if PGDEBUG - printf("\n\n# 194: PACKET START: wire mode, repl off, echo %d\n", force_echo); -#endif - is_wire = true; - } else { -#if PGDEBUG - printf("\n\n# 200: PACKET START: repl mode, no wire, echo %d\n", force_echo); -#endif - is_wire = false; - } -} - __attribute__((export_name("clear_error"))) void clear_error() { diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index a76aabee6374c..bbfa698606dcb 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -92,21 +92,8 @@ volatile bool send_ready_for_query = true; volatile bool idle_in_transaction_timeout_enabled = false; volatile bool idle_session_timeout_enabled = false; - -void -pg_free(void *ptr) { - free(ptr); -} - #include "../backend/tcop/postgres.c" - -// initdb + start on fd (pipe emulation) - - -static bool force_echo = false; - - #include "pgl_mains.c" #include "pgl_stubs.h" @@ -115,16 +102,11 @@ static bool force_echo = false; #include "pgl_initdb.c" - // interactive_one, heart of the async loop. - #include "./interactive_one.c" - static void main_pre(int argc, char *argv[]) { - - char key[256]; int i = 0; // extra env is always after normal args diff --git a/src/backend/Makefile b/src/backend/Makefile index 986529ad6b17c..0bc6a84c3b10a 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -111,7 +111,7 @@ pglite: pglite-exported-functions $(CC) \ $(PGLITE_CFLAGS) \ -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ - -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ + -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \ $(PGPRELOAD) \ -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ From d7e78325804374017d1f91cfcdb7d55263cc5d09 Mon Sep 17 00:00:00 2001 From: tudor Date: Tue, 7 Oct 2025 08:49:44 +0200 Subject: [PATCH 29/30] :) --- other/empty | 1 + 1 file changed, 1 insertion(+) diff --git a/other/empty b/other/empty index e69de29bb2d1d..646d10863717a 100644 --- a/other/empty +++ b/other/empty @@ -0,0 +1 @@ +PGlite is the best! \ No newline at end of file From b57063866ec6b6f3a6505ca71d019a1501734bcc Mon Sep 17 00:00:00 2001 From: tudor Date: Wed, 8 Oct 2025 11:00:52 +0200 Subject: [PATCH 30/30] remove things not working --- build-with-docker.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/build-with-docker.sh b/build-with-docker.sh index 17521a8575690..6a849b52ab863 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -1,18 +1,8 @@ # although we could use any path inside docker, using the same path as on the host # allows the DWARF info (when building in DEBUG) to contain the correct file paths DOCKER_WORKSPACE=$(pwd) -GITHUB_ACTIONS=${GITHUB_ACTIONS:-"false"} -ADDITIONAL_DOCKER_PARAMS="" -if [ "$GITHUB_ACTIONS" = "true" ]; then - echo "Running inside GitHub Actions" - ADDITIONAL_DOCKER_PARAMS="" -else - echo "Not running inside GitHub Actions" - ADDITIONAL_DOCKER_PARAMS="-it -u $(id -u):$(id -g)" -fi docker run $@ \ - $ADDITIONAL_DOCKER_PARAMS \ --rm \ -e DEBUG=${DEBUG:-false} \ --workdir=${DOCKER_WORKSPACE} \