Loadable extensions

.github/config/extension_config_wasm.cmake

@@ -0,0 +1,29 @@
+################################################################################
+# DuckDB-Wasm extension base config
+################################################################################
+#
+duckdb_extension_load(json)
+duckdb_extension_load(parquet)
+duckdb_extension_load(autocomplete)
+
+duckdb_extension_load(excel DONT_LINK)
+duckdb_extension_load(fts DONT_LINK)
+duckdb_extension_load(inet DONT_LINK)
+duckdb_extension_load(icu DONT_LINK)
+duckdb_extension_load(sqlsmith DONT_LINK)
+duckdb_extension_load(tpcds DONT_LINK)
+duckdb_extension_load(tpch DONT_LINK)
+duckdb_extension_load(visualizer DONT_LINK)
+
+#duckdb_extension_load(httpfs DONT_LINK)
+
+duckdb_extension_load(substrait
+    LOAD_TESTS DONT_LINK
+    GIT_URL https://github.com/duckdblabs/substrait
+    GIT_TAG 5d621b1d7d16fe86f8b1930870c8e6bf05bcb92a
+    )
+duckdb_extension_load(sqlite_scanner
+        DONT_LINK LOAD_TESTS
+        GIT_URL https://github.com/duckdblabs/sqlite_scanner
+        GIT_TAG 9c38a30be2237456cdcd423d527b96c944158c77
+        )

.github/workflows/main.yml

@@ -499,6 +499,94 @@ jobs:
                       ./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm
                   retention-days: 1
 
+    wasm_mvp_loadable:
+        name: Wasm / mvp (loadable version)
+        runs-on: ubuntu-latest
+        needs:
+            - dataprep
+            - tpchgen
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  submodules: 'recursive'
+                  fetch-depth: 0
+
+            - name: Prepare repository
+              run: |
+                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"
+
+            - uses: mymindstorm/setup-emsdk@v12
+              with:
+                  version: 'latest'
+
+            - name: Setup Ccache
+              uses: hendrikmuhs/ccache-action@main
+              with:
+                  key: ${{ github.job }}
+
+            - name: Git submodule status
+              run: |
+                  git submodule status > git_submodule_status.txt
+
+            - name: Build Wasm module
+              run: |
+                  cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
+                  DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
+                  bash ./scripts/build_loadable.sh relsize mvp
+
+            - name: Upload artifact
+              uses: actions/upload-artifact@v3
+              with:
+                  name: wasm-mvp-loadable
+                  path: |
+                      ./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
+                      ./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
+                  retention-days: 1
+
+    wasm_eh_loadable:
+        name: Wasm / eh (loadable version)
+        runs-on: ubuntu-latest
+        needs:
+            - dataprep
+            - tpchgen
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  submodules: 'recursive'
+                  fetch-depth: 0
+
+            - name: Prepare repository
+              run: |
+                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"
+
+            - uses: mymindstorm/setup-emsdk@v12
+              with:
+                  version: 'latest'
+
+            - name: Setup Ccache
+              uses: hendrikmuhs/ccache-action@main
+              with:
+                  key: ${{ github.job }}
+
+            - name: Git submodule status
+              run: |
+                  git submodule status > git_submodule_status.txt
+
+            - name: Build Wasm module
+              run: |
+                  cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
+                  DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
+                  bash ./scripts/build_loadable.sh relsize eh
+
+            - name: Upload artifact
+              uses: actions/upload-artifact@v3
+              with:
+                  name: wasm-eh-loadable
+                  path: |
+                      ./packages/duckdb-wasm/src/bindings/duckdb-eh.js
+                      ./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
+                  retention-days: 1
+
     js_libs:
         name: Js / Libraries
         runs-on: ubuntu-latest
@@ -837,6 +925,157 @@ jobs:
                       ./reports/benchmark_system_tpch_05_lovefield.json
                   retention-days: 1
 
+    js_libs_loadable:
+        name: Js / Libraries (loadable version)
+        runs-on: ubuntu-latest
+        needs:
+            - wasm_mvp_loadable
+            - wasm_eh_loadable
+            - wasm_coi
+            - duckdb_shell
+            - native_debug
+            - native_release
+            - clang_format
+            - eslint
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  submodules: 'recursive'
+                  fetch-depth: 0
+
+            - uses: actions/setup-node@v3
+              with:
+                  node-version: '18.x'
+                  registry-url: 'https://registry.npmjs.org'
+
+            - name: Git submodule status
+              run: |
+                  git submodule status > git_submodule_status.txt
+
+            - name: Cache rust build
+              uses: actions/cache@v3
+              with:
+                  path: |
+                      ./.cargo/bin/
+                      ./.cargo/registry/index/
+                      ./.cargo/registry/cache/
+                      ./.cargo/git/db/
+                      ./target
+                  key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
+                  restore-keys: |
+                      ${{ runner.os }}-shell-
+
+            - name: Cache node_modules
+              uses: actions/cache@v3
+              with:
+                  path: |
+                      ./node_modules
+                      ./packages/benchmarks/node_modules
+                      ./packages/duckdb-wasm/node_modules
+                      ./packages/duckdb-wasm-shell/node_modules
+                  key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
+                  restore-keys: |
+                      ${{ runner.os }}-yarn-
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: dataprep
+                  path: ./target/release/
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: tpch-dbgen
+                  path: ./submodules/tpch-dbgen/dbgen/
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: duckdb-shell
+                  path: ./submodules/duckdb/build/Release/
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: wasm-mvp-loadable
+                  path: ./packages/duckdb-wasm/src/bindings/
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: wasm-eh-loadable
+                  path: ./packages/duckdb-wasm/src/bindings/
+
+            - uses: actions/download-artifact@v3
+              with:
+                  name: wasm-coi
+                  path: ./packages/duckdb-wasm/src/bindings/
+
+            - name: Prepare repository
+              run: |
+                  git fetch --tags --no-recurse-submodules -f
+                  (cd ./submodules/duckdb && git fetch --all --tags)
+
+            - name: Prepare environment
+              uses: duckdb/duckdb-wasm-ci-env@v0.11
+              with:
+                  script: |-
+                      git config --global --add safe.directory '*'
+                      mkdir -p ./lib/build/wasm/release ./reports
+                      yarn install --frozen-lockfile
+                      ./scripts/generate_tpch_tbl.sh 0.01
+                      ./scripts/generate_tpch_arrow.sh 0.01
+                      ./scripts/generate_tpch_duckdb.sh 0.01
+                      ./scripts/generate_tpch_sqlite.sh 0.01
+                      ./scripts/generate_uni.sh
+                      ./scripts/npm_version.sh
+
+            - name: Build @duckdb/duckdb-wasm
+              shell: bash
+              run: |
+                      rm -rf ./packages/duckdb-wasm/dist/
+                      yarn workspace @duckdb/duckdb-wasm build:release
+                      yarn workspace @duckdb/duckdb-wasm docs
+
+            - name: Build @duckdb/duckdb-wasm-shell
+              shell: bash
+              run: |
+                      rm -rf ./packages/duckdb-wasm-shell/dist/
+                      yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
+                      yarn workspace @duckdb/duckdb-wasm-shell build:release
+
+            - name: Build @duckdb/duckdb-wasm-app
+              shell: bash
+              run: |
+                      rm -rf ./packages/duckdb-wasm-app/build/
+                      yarn workspace @duckdb/duckdb-wasm-app build:release
+
+            - name: Test @duckdb/duckdb-wasm on Chrome
+              shell: bash
+              run: |
+                      CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome || true
+
+            #            - name: Test @duckdb/duckdb-wasm on Firefox
+            #              uses: duckdb/duckdb-wasm-ci-env@v0.11
+            #              with:
+            #                  script: |-
+            #                      yarn workspace @duckdb/duckdb-wasm test:firefox
+
+            #- name: Test @duckdb/duckdb-wasm on Node.js
+            #  shell: bash
+            #  run: |
+            #          yarn workspace @duckdb/duckdb-wasm test:node || treu
+
+            #- name: Coverage @duckdb/duckdb-wasm
+            #  shell: bash
+            #  run: |
+            #           CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage
+
+            - name: Package
+              shell: bash
+              run: zip -r duckdb-wasm-packages.zip ./packages
+
+            - uses: actions/upload-artifact@v3
+              with:
+                  name: duckdb-wasm-packages-loadable
+                  path: duckdb-wasm-packages.zip
+
     merge_reports:
         name: Merge benchmark reports
         runs-on: ubuntu-latest

Makefile

@@ -15,7 +15,7 @@ LIB_RELWITHDEBINFO_DIR="${ROOT_DIR}/build/RelWithDebInfo"
 LIB_XRAY_DIR="${ROOT_DIR}/build/Xray"
 DUCKDB_WASM_DIR="${ROOT_DIR}/packages/duckdb/src/wasm"
 
-DUCKDB_HASH=${shell cat .git/modules/submodules/duckdb/refs/heads/main | head -c 10}
+DUCKDB_HASH=${shell cd submodules/duckdb && git reflog -n 1 | head -c 10}
 
 CACHE_DIRS=${ROOT_DIR}/.ccache/ ${ROOT_DIR}/.emscripten_cache/
 DOCKER_EXEC_ENVIRONMENT=docker compose run duckdb-wasm-ci
@@ -360,7 +360,13 @@ app: wasm wasmpack shell docs js_tests_release
 	yarn workspace @duckdb/duckdb-wasm-app build:release
 
 build_loadable:
-	WASM_LOADABLE_EXTENSIONS=1 GEN=ninja USE_MERGED_VCPKG_MANIFEST=1 VCPKG_TOOLCHAIN_PATH="~/vcpkg/scripts/buildsystems/vcpkg.cmake" DUCKDB_WASM_LOADABLE_EXTENSIONS=1 ./scripts/wasm_build_lib.sh relsize eh
+	cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
+	DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
+	bash ./scripts/build_loadable.sh relsize eh
+
+build_loadable_unsigned:
+	cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
+	DUCKDB_WASM_LOADABLE_EXTENSIONS="unsigned" GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
 	bash ./scripts/build_loadable.sh relsize eh
 
 serve_loadable: wasmpack shell docs

duckdb.patch

@@ -1,24 +1,12 @@
-diff --git a/src/common/arrow/arrow_converter.cpp b/src/common/arrow/arrow_converter.cpp
-index 26db9840ac..7d904aa202 100644
---- a/src/common/arrow/arrow_converter.cpp
-+++ b/src/common/arrow/arrow_converter.cpp
-@@ -138,6 +138,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
- 		child.format = "tdD";
- 		break;
- 	case LogicalTypeId::TIME:
-+	case LogicalTypeId::TIME_TZ:
- 		child.format = "ttu";
- 		break;
- 	case LogicalTypeId::TIMESTAMP:
 diff --git a/src/function/table/system/test_all_types.cpp b/src/function/table/system/test_all_types.cpp
-index a0b856266c..59761509a4 100644
+index a0b856266c..6674cf8baf 100644
 --- a/src/function/table/system/test_all_types.cpp
 +++ b/src/function/table/system/test_all_types.cpp
 @@ -204,13 +204,14 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
  	auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
  	result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
 
-+#ifndef DUCKDB_FROM_DUCKDB_WASM
++#if 0
  	// union
  	child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
  	auto union_type = LogicalType::UNION(members);

lib/CMakeLists.txt

@@ -9,18 +9,29 @@ cmake_minimum_required(VERSION 3.10)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_FLAGS -std=c++17)
 set(ignoreMe "${DUCKDB_WASM_VERSION}")
 
-option(DUCKDUB_WASM_LOADABLE_EXTENSIONS "Build with loadable extensions" OFF)
-option(DUCKDUB_WASM_LOADABLE_EXTENSIONS_UNSIGNED "Build with loadable extensions" OFF)
+option(DUCKDB_WASM_LOADABLE_EXTENSIONS "Build with loadable extensions" OFF)
+option(DUCKDB_WASM_LOADABLE_EXTENSIONS_UNSIGNED "Build with loadable extensions" OFF)
 
-if(DUCKDUB_WASM_LOADABLE_EXTENSIONS)
-  set(CMAKE_CXX_FLAGS "-std=c++17 -fPIC -DWASM_LOADABLE_EXTENSIONS=1")
+if(DEFINED ENV{DUCKDB_WASM_LOADABLE_EXTENSIONS})
+  set(DUCKDB_WASM_LOADABLE_EXTENSIONS ON)
+  if("$ENV{DUCKDB_WASM_LOADABLE_EXTENSIONS}" STREQUAL "unsigned")
+    set(DUCKDB_WASM_LOADABLE_EXTENSIONS_UNSIGNED ON)
+  endif()
+endif()
+
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -DDUCKDB_WASM=1")
+
+if(DUCKDB_WASM_LOADABLE_EXTENSIONS)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWASM_LOADABLE_EXTENSIONS=1 -DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1 -fPIC")
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDISABLE_EXTENSION_LOAD=1")
 endif()
 
-if(DUCKDUB_WASM_LOADABLE_EXTENSIONS_UNSIGNED)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWASM_LOADABLE_EXTENSIONS_UNSIGNED")
+if(DUCKDB_WASM_LOADABLE_EXTENSIONS_UNSIGNED)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWASM_LOADABLE_EXTENSIONS_UNSIGNED=1")
 endif()
 
 if(NOT EMSCRIPTEN)
@@ -151,7 +162,7 @@ else()
       "${CMAKE_CXX_FLAGS} -DWEBDB_FAST_EXCEPTIONS=1 -DWEBDB_THREADS=1")
 endif()
 
-if (DUCKDUB_WASM_LOADABLE_EXTENSIONS)
+if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
   set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s MAIN_MODULE=1 -s  -s EXPORT_ALL=1 -s FILESYSTEM=1 -s ENVIRONMENT='web,worker' -s ALLOW_TABLE_GROWTH")
 else()
   set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -s FILESYSTEM=0 -s ENVIRONMENT='web,node,worker'")