test(qe-wasm): WASM query engine benchmarks isolated from DB

.github/workflows/wasm-benhmarks.yml

@@ -0,0 +1,107 @@
+name: Wasm perf
+on:
+  pull_request:
+    paths-ignore:
+      - ".github/**"
+      - "!.github/workflows/wasm-benchmarks.yml"
+      - ".buildkite/**"
+      - "*.md"
+      - "LICENSE"
+      - "CODEOWNERS"
+      - "renovate.json"
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    env: # Set environment variables for the whole job
+      PROFILE: release
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+
+      - name: "Setup Node.js"
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node_version }}
+
+      - name: "Setup pnpm"
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: "Login to Docker Hub"
+        uses: docker/login-action@v3
+        continue-on-error: true
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        if: "${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}"
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: cachix/install-nix-action@v24
+
+      - name: Setup benchmark
+        run: make setup-pg-bench
+
+      - name: Run benchmarks
+        id: bench
+        run: make run-bench | tee out.txt
+
+      - name: Read benchmark results
+        id: read_results
+        run: |
+          {
+            echo 'bench_output<<EOF'
+            cat out.txt
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Summarize results
+        id: summarize_results
+        run: |
+          input="${{ steps.read_results.outputs.bench_output }}"
+          regressed=$(echo "$input" | grep "slower than Web Assembly: Latest" | cut -f1 -d'x' | awk '$1 > 1.01' | wc -l )          
+          if [ "$regressed" -gt 0 ]; then
+              message="🚨 WASM query-engine: $regressed benchmark(s) have regressed at least 1%"
+              status=failed
+          else
+              message="✅ WASM query-engine: no benchmarks have regressed"
+              status=passed
+          fi
+          echo "summary=$message" >> $GITHUB_OUTPUT
+          echo "status=$status" >> $GITHUB_OUTPUT
+
+      - name: Find past report comment
+        uses: peter-evans/find-comment@v2
+        id: findReportComment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: "<!-- wasm-engine-perf -->"
+
+      - name: Create or update report
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.findReportComment.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            <!-- wasm-engine-perf -->
+            #### ${{ steps.summarize_results.outputs.summary }}
+
+            <details>
+            <summary>Full benchmark report</summary>
+
+            ```
+            ${{ steps.read_results.outputs.bench_output }}
+            ```
+            </details>
+
+            After changes in ${{ github.event.pull_request.head.sha }}
+          edit-mode: replace
+
+      - name: Fail workflow if regression detected
+        if: steps.summarize_results.outputs.status == 'failed'
+        run: |
+          echo "Workflow failed due to benchmark regression."
+          exit 1

Makefile

@@ -12,6 +12,8 @@ LIBRARY_EXT := $(shell                            \
         (*)                    echo "so"    ;;    \
     esac)
 
+PROFILE ?= dev
+
 default: build
 
 ###################
@@ -88,14 +90,14 @@ start-sqlite:
 dev-sqlite:
 	cp $(CONFIG_PATH)/sqlite $(CONFIG_FILE)
 
-dev-libsql-js: build-qe-napi build-connector-kit-js
+dev-libsql-js: build-qe-napi build-driver-adapters-kit
 	cp $(CONFIG_PATH)/libsql-js $(CONFIG_FILE)
 
 test-libsql-js: dev-libsql-js test-qe-st
 
 test-driver-adapter-libsql: test-libsql-js
 
-dev-libsql-wasm: build-qe-wasm build-connector-kit-js
+dev-libsql-wasm: build-qe-wasm build-driver-adapters-kit
 	cp $(CONFIG_PATH)/libsql-wasm $(CONFIG_FILE)
 
 test-libsql-wasm: dev-libsql-wasm test-qe-st
@@ -133,28 +135,39 @@ dev-postgres13: start-postgres13
 
 start-pg-js: start-postgres13
 
-dev-pg-js: start-pg-js build-qe-napi build-connector-kit-js
+dev-pg-js: start-pg-js build-qe-napi build-driver-adapters-kit
 	cp $(CONFIG_PATH)/pg-js $(CONFIG_FILE)
 
 test-pg-js: dev-pg-js test-qe-st
 
-dev-pg-wasm: start-pg-js build-qe-wasm build-connector-kit-js
+dev-pg-wasm: start-pg-js build-qe-wasm build-driver-adapters-kit
 	cp $(CONFIG_PATH)/pg-wasm $(CONFIG_FILE)
 
 test-pg-wasm: dev-pg-wasm test-qe-st
 
 test-driver-adapter-pg: test-pg-js
 test-driver-adapter-pg-wasm: test-pg-wasm
 
+start-pg-bench:
+	docker compose -f query-engine/driver-adapters/executor/bench/docker-compose.yml up --wait -d --remove-orphans postgres
+
+setup-pg-bench: start-pg-bench build-qe-napi build-qe-wasm build-driver-adapters-kit
+
+run-bench:
+	DATABASE_URL="postgresql://postgres:postgres@localhost:5432/bench?schema=imdb_bench&sslmode=disable" \
+	node --experimental-wasm-modules query-engine/driver-adapters/executor/dist/bench.mjs
+
+bench-pg-js: setup-pg-bench run-bench
+
 start-neon-js:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-proxy
 
-dev-neon-js: start-neon-js build-qe-napi build-connector-kit-js
+dev-neon-js: start-neon-js build-qe-napi build-driver-adapters-kit
 	cp $(CONFIG_PATH)/neon-js $(CONFIG_FILE)
 
 test-neon-js: dev-neon-js test-qe-st
 
-dev-neon-wasm: start-neon-js build-qe-wasm build-connector-kit-js
+dev-neon-wasm: start-neon-js build-qe-wasm build-driver-adapters-kit
 	cp $(CONFIG_PATH)/neon-wasm $(CONFIG_FILE)
 
 test-neon-wasm: dev-neon-wasm test-qe-st
@@ -293,12 +306,12 @@ dev-vitess_8_0: start-vitess_8_0
 start-planetscale-js:
 	docker compose -f docker-compose.yml up -d --remove-orphans planetscale-proxy
 
-dev-planetscale-js: start-planetscale-js build-qe-napi build-connector-kit-js
+dev-planetscale-js: start-planetscale-js build-qe-napi build-driver-adapters-kit
 	cp $(CONFIG_PATH)/planetscale-js $(CONFIG_FILE)
 
 test-planetscale-js: dev-planetscale-js test-qe-st
 
-dev-planetscale-wasm: start-planetscale-js build-qe-wasm build-connector-kit-js
+dev-planetscale-wasm: start-planetscale-js build-qe-wasm build-driver-adapters-kit
 	cp $(CONFIG_PATH)/planetscale-wasm $(CONFIG_FILE)
 
 test-planetscale-wasm: dev-planetscale-wasm test-qe-st
@@ -311,7 +324,7 @@ test-driver-adapter-planetscale-wasm: test-planetscale-wasm
 ######################
 
 build-qe-napi:
-	cargo build --package query-engine-node-api
+	cargo build --package query-engine-node-api --profile $(PROFILE)
 
 build-qe-wasm:
 ifndef $(NIX)
@@ -322,7 +335,7 @@ else
 	cd query-engine/query-engine-wasm && ./build.sh
 endif
 
-build-connector-kit-js: build-driver-adapters
+build-driver-adapters-kit: build-driver-adapters
 	cd query-engine/driver-adapters && pnpm i && pnpm build
 
 build-driver-adapters: ensure-prisma-present
@@ -384,7 +397,7 @@ otel:
 
 # Build the debug version of Query Engine Node-API library ready to be consumed by Node.js
 .PHONY: qe-node-api
-qe-node-api: build target/debug/libquery_engine.node
+qe-node-api: build target/debug/libquery_engine.node --profile=$(PROFILE)
 
 %.node: %.$(LIBRARY_EXT)
 # Remove the file first to work around a macOS bug: https://openradar.appspot.com/FB8914243

nix/all-engines.nix

@@ -115,6 +115,8 @@ in
     })
     { profile = "release"; };
 
+
+
   packages.query-engine-wasm = lib.makeOverridable
     ({ profile }: stdenv.mkDerivation {
       name = "query-engine-wasm";

query-engine/connector-test-kit-rs/README.md

@@ -89,7 +89,7 @@ To run tests through a driver adapters, you should also configure the following
 Example:
 
 ```shell
-export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh"
+export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/executor/script/testd.sh"
 export DRIVER_ADAPTER=neon
 export ENGINE=wasm
 export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }'

query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs

@@ -183,8 +183,7 @@ impl TestConfig {
     }
 
     pub fn external_test_executor_path(&self) -> Option<String> {
-        const DEFAULT_TEST_EXECUTOR: &str =
-            "query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh";
+        const DEFAULT_TEST_EXECUTOR: &str = "query-engine/driver-adapters/executor/script/testd.sh";
         self.external_test_executor
             .as_ref()
             .and_then(|_| {

query-engine/driver-adapters/executor/bench/Dockerfile

@@ -0,0 +1,4 @@
+FROM postgres:15
+COPY seed.sql.gz .
+RUN gunzip seed.sql.gz && \
+    mv seed.sql /docker-entrypoint-initdb.d/seed.sql

query-engine/driver-adapters/executor/bench/docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.8'
+
+name: bench
+
+services:  
+  postgres:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    shm_size: 1g
+    restart: unless-stopped
+    # Uncomment the following line to enable query logging
+    # Then restart the container.
+    # command: ['postgres', '-c', 'log_statement=all']
+    environment:
+      - POSTGRES_DB=bench
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres    
+    ports:
+      - '5432:5432'
+    healthcheck:
+      test: ['CMD', 'pg_isready']
+      interval: 5s
+      timeout: 2s
+      retries: 20