Merge remote-tracking branch 'origin/main' into mff/tmp-planetscale-t…

…ests * origin/main: qe: Allow to run query engine test suite against wasm engines (#4467) query-engine-wasm: Fix build & ensure it stays fixed (#4488) feat(driver-adapters): enable Wasm on `request-handlers` (#4455)
prisma · Nov 22, 2023 · 27eff23 · 27eff23
2 parents 9a1b4cf + 2c867f4
commit 27eff23
Show file tree

Hide file tree

Showing 25 changed files with 345 additions and 164 deletions.
diff --git a/.envrc b/.envrc
@@ -23,7 +23,7 @@ export QE_LOG_LEVEL=debug # Set it to "trace" to enable query-graph debugging lo
 # export FMT_SQL=1 # Uncomment it to enable logging formatted SQL queries
 
 ### Uncomment to run driver adapters tests. See query-engine-driver-adapters.yml workflow for how tests run in CI.
-# export EXTERNAL_TEST_EXECUTOR="$(pwd)/query-engine/driver-adapters/js/connector-test-kit-executor/script/start_node.sh"
+# export EXTERNAL_TEST_EXECUTOR="napi"
 # export DRIVER_ADAPTER=pg # Set to pg, neon or planetscale
 # export PRISMA_DISABLE_QUAINT_EXECUTORS=1 # Disable quaint executors for driver adapters
 # export DRIVER_ADAPTER_URL_OVERRIDE ="postgres://USER:PASSWORD@DATABASExxxx" # Override the database url for the driver adapter tests

diff --git a/.github/workflows/qe-wasm-check.yml b/.github/workflows/qe-wasm-check.yml
@@ -0,0 +1,27 @@
+name: WASM engine compile check
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - '.github/**'
+      - '!.github/workflows/qe-wasm-check.yml'
+      - '.buildkite/**'
+      - '*.md'
+      - 'LICENSE'
+      - 'CODEOWNERS'
+      - 'renovate.json'
+
+jobs:
+  build:
+    name: 'Compilation check for query-engine-wasm'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install wasm-pack
+        run: cargo install wasm-pack
+      - name: Build wasm query engine
+        run: ./build.sh
+        working-directory: ./query-engine/query-engine-wasm
diff --git a/.github/workflows/query-engine-driver-adapters.yml b/.github/workflows/query-engine-driver-adapters.yml
@@ -26,13 +26,23 @@ jobs:
       matrix:
         adapter:
           - name: 'planetscale'
-            setup_task: 'dev-planetscale'
-          - name: 'pg'
+            setup_task: 'dev-planetscale-vitess8'
+          - name: 'pg (napi)'
             setup_task: 'dev-pg-postgres13'
-          - name: 'neon:ws'
+          - name: 'neon:ws (napi)'
             setup_task: 'dev-neon-ws-postgres13'
-          - name: 'libsql'
+          - name: 'libsql (napi)'
             setup_task: 'dev-libsql-sqlite'
+          # TODO: uncomment when WASM engine is functional
+          # - name: 'pg (wasm)'
+          #   setup_task: 'dev-pg-postgres13-wasm'
+          #   needs_wasm_pack: true
+          # - name: 'neon:ws (wasm)'
+          #   setup_task: 'dev-neon-ws-postgres13-wasm'
+          #   needs_wasm_pack: true
+          # - name: 'libsql (wasm)'
+          #   setup_task: 'dev-libsql-sqlite-wasm'
+          #   needs_wasm_pack: true
         node_version: ['18']
     env:
       LOG_LEVEL: 'info' # Set to "debug" to trace the query engine and node process running the driver adapter
@@ -87,9 +97,13 @@ jobs:
             echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
           fi
 
-      - run: make ${{ matrix.adapter.setup_task }}
-
       - uses: dtolnay/rust-toolchain@stable
 
+      - name: 'Install wasm-pack'
+        if: ${{ matrix.adapter.needs_wasm_pack }}
+        run: cargo install wasm-pack
+
+      - run: make ${{ matrix.adapter.setup_task }}
+
       - name: 'Run tests'
         run: cargo test --package query-engine-tests -- --test-threads=1
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Makefile b/Makefile
@@ -49,8 +49,11 @@ ifndef DRIVER_ADAPTER
 	cargo test --package query-engine-tests
 else
 	@echo "Executing query engine tests with $(DRIVER_ADAPTER) driver adapter"; \
-	# Add your actual command for the "test-driver-adapter" task here
-	$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER);
+	if [ "$(ENGINE)" = "wasm" ]; then \
+		$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER)-wasm; \
+	else \
+		$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER); \
+	fi
 endif
 
 test-qe-verbose:
@@ -91,6 +94,12 @@ test-libsql-sqlite: dev-libsql-sqlite test-qe-st
 
 test-driver-adapter-libsql: test-libsql-sqlite
 
+dev-libsql-sqlite-wasm: build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/libsql-sqlite-wasm $(CONFIG_FILE)
+
+test-libsql-sqlite-wasm: dev-libsql-sqlite-wasm test-qe-st
+test-driver-adapter-libsql-sqlite-wasm: test-libsql-sqlite-wasm
+
 start-postgres9:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9
 
@@ -121,14 +130,20 @@ start-postgres13:
 dev-postgres13: start-postgres13
 	cp $(CONFIG_PATH)/postgres13 $(CONFIG_FILE)
 
-start-pg-postgres13: build-qe-napi build-connector-kit-js start-postgres13
+start-pg-postgres13: start-postgres13
 
-dev-pg-postgres13: start-pg-postgres13
+dev-pg-postgres13: start-pg-postgres13 build-qe-napi build-connector-kit-js
 	cp $(CONFIG_PATH)/pg-postgres13 $(CONFIG_FILE)
 
 test-pg-postgres13: dev-pg-postgres13 test-qe-st
 
+dev-pg-postgres13-wasm: start-pg-postgres13 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/pg-postgres13-wasm $(CONFIG_FILE)
+
+test-pg-postgres13-wasm: dev-pg-postgres13-wasm test-qe-st
+
 test-driver-adapter-pg: test-pg-postgres13
+test-driver-adapter-pg-wasm: test-pg-postgres13-wasm
 
 start-neon-postgres13:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-postgres13
@@ -138,7 +153,13 @@ dev-neon-ws-postgres13: start-neon-postgres13 build-qe-napi build-connector-kit-
 
 test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st
 
+dev-neon-ws-postgres13-wasm: start-neon-postgres13 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/neon-ws-postgres13-wasm $(CONFIG_FILE)
+
+test-neon-ws-postgres13-wasm: dev-neon-ws-postgres13-wasm test-qe-st
+
 test-driver-adapter-neon: test-neon-ws-postgres13
+test-driver-adapter-neon-wasm: test-neon-ws-postgres13-wasm
 
 start-postgres14:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres14
@@ -262,15 +283,21 @@ start-vitess_8_0:
 dev-vitess_8_0: start-vitess_8_0
 	cp $(CONFIG_PATH)/vitess_8_0 $(CONFIG_FILE)
 
-start-planetscale:
-	docker compose -f docker-compose.yml up -d --remove-orphans planetscale
+start-planetscale-vitess8:
+	docker compose -f docker-compose.yml up -d --remove-orphans planetscale-vitess8
+
+dev-planetscale-vitess8: start-planetscale-vitess8 build-qe-napi build-connector-kit-js
+	cp $(CONFIG_PATH)/planetscale-vitess8 $(CONFIG_FILE)
 
-dev-planetscale: start-planetscale build-qe-napi build-connector-kit-js
-	cp $(CONFIG_PATH)/planetscale $(CONFIG_FILE)
+test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st
 
-test-planetscale: dev-planetscale test-qe-st
+dev-planetscale-vitess8-wasm: start-planetscale-vitess8 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/planetscale-vitess8-wasm $(CONFIG_FILE)
 
-test-driver-adapter-planetscale: test-planetscale
+test-planetscale-vitess8-wasm: dev-planetscale-vitess8-wasm test-qe-st
+
+test-driver-adapter-planetscale: test-planetscale-vitess8
+test-driver-adapter-planetscale-wasm: test-planetscale-vitess8-wasm
 
 ######################
 # Local dev commands #
@@ -279,13 +306,16 @@ test-driver-adapter-planetscale: test-planetscale
 build-qe-napi:
 	cargo build --package query-engine-node-api
 
-build-connector-kit-js: install-driver-adapters
+build-qe-wasm:
+	cd query-engine/query-engine-wasm && ./build.sh
+
+build-connector-kit-js: build-driver-adapters
 	cd query-engine/driver-adapters && pnpm i && pnpm build
 
-install-driver-adapters: ensure-prisma-present
-	@echo "Installing driver adapters dependencies..."
+build-driver-adapters: ensure-prisma-present
+	@echo "Building driver adapters..."
 	@cd ../prisma && pnpm --filter "*adapter*" i
-	@echo "Driver adapters dependencies installed.";
+	@echo "Driver adapters build completed.";
 
 ensure-prisma-present:
 	@if [ -d ../prisma ]; then \

diff --git a/query-engine/connector-test-kit-rs/README.md b/query-engine/connector-test-kit-rs/README.md
@@ -82,23 +82,24 @@ drivers the code that actually communicates with the databases. See [`adapter-*`
 
 To run tests through a driver adapters, you should also configure the following environment variables:
 
-* `EXTERNAL_TEST_EXECUTOR`: tells the query engine test kit to use an external process to run the queries, this is a node process running a program that will read the queries to run from STDIN, and return responses to STDOUT. The connector kit follows a protocol over JSON RPC for this communication.
 * `DRIVER_ADAPTER`: tells the test executor to use a particular driver adapter. Set to `neon`, `planetscale` or any other supported adapter.
 * `DRIVER_ADAPTER_CONFIG`: a json string with the configuration for the driver adapter. This is adapter specific. See the [github workflow for driver adapter tests](.github/workflows/query-engine-driver-adapters.yml) for examples on how to configure the driver adapters.
+* `ENGINE`: can be used to run either `wasm` or `napi` version of the engine.
 
 Example:
 
 ```shell
 export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh"
 export DRIVER_ADAPTER=neon
+export ENGINE=wasm
 export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }'
 ````
 
 We have provided helpers to run the query-engine tests with driver adapters, these helpers set all the required environment
 variables for you:
 
 ```shell
-DRIVER_ADAPTER=$adapter make test-qe
+DRIVER_ADAPTER=$adapter ENGINE=$engine make test-qe
 ```
 
 Where `$adapter` is one of the supported adapters: `neon`, `planetscale`, `libsql`.

diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs
@@ -3,10 +3,25 @@ use crate::{
     PostgresConnectorTag, SqlServerConnectorTag, SqliteConnectorTag, TestResult, VitessConnectorTag,
 };
 use serde::{Deserialize, Serialize};
-use std::{convert::TryFrom, env, fs::File, io::Read, path::PathBuf};
+use std::{convert::TryFrom, env, fmt::Display, fs::File, io::Read, path::PathBuf};
 
 static TEST_CONFIG_FILE_NAME: &str = ".test_config";
 
+#[derive(Debug, Deserialize, Clone)]
+pub enum TestExecutor {
+    Napi,
+    Wasm,
+}
+
+impl Display for TestExecutor {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TestExecutor::Napi => f.write_str("Napi"),
+            TestExecutor::Wasm => f.write_str("Wasm"),
+        }
+    }
+}
+
 /// The central test configuration.
 #[derive(Debug, Default, Deserialize)]
 pub struct TestConfig {
@@ -24,8 +39,9 @@ pub struct TestConfig {
     /// Used when testing driver adapters, this process is expected to be a javascript process
     /// loading the library engine (as a library, or WASM modules) and providing it with a
     /// driver adapter.
+    /// Possible values: Napi, Wasm
     /// Env key: `EXTERNAL_TEST_EXECUTOR`
-    pub(crate) external_test_executor: Option<String>,
+    pub(crate) external_test_executor: Option<TestExecutor>,
 
     /// The driver adapter to use when running tests, will be forwarded to the external test
     /// executor by setting the `DRIVER_ADAPTER` env var when spawning the executor process
@@ -90,12 +106,11 @@ fn exit_with_message(msg: &str) -> ! {
 impl TestConfig {
     /// Loads a configuration. File-based config has precedence over env config.
     pub(crate) fn load() -> Self {
-        let mut config = match Self::from_file().or_else(Self::from_env) {
+        let config = match Self::from_file().or_else(Self::from_env) {
             Some(config) => config,
             None => exit_with_message(CONFIG_LOAD_FAILED),
         };
 
-        config.fill_defaults();
         config.validate();
         config.log_info();
 
@@ -112,8 +127,8 @@ impl TestConfig {
             self.connector_version().unwrap_or_default()
         );
         println!("* CI? {}", self.is_ci);
-        if self.external_test_executor.as_ref().is_some() {
-            println!("* External test executor: {}", self.external_test_executor().unwrap_or_default());
+        if let Some(external_test_executor) = self.external_test_executor.as_ref() {
+            println!("* External test executor: {}", external_test_executor);
             println!("* Driver adapter: {}", self.driver_adapter().unwrap_or_default());
             println!("* Driver adapter config: {}", self.json_stringify_driver_adapter_config());
         }
@@ -123,7 +138,10 @@ impl TestConfig {
     fn from_env() -> Option<Self> {
         let connector = std::env::var("TEST_CONNECTOR").ok();
         let connector_version = std::env::var("TEST_CONNECTOR_VERSION").ok();
-        let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR").ok();
+        let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR")
+            .map(|value| serde_json::from_str::<TestExecutor>(&value).ok())
+            .unwrap_or_default();
+
         let driver_adapter = std::env::var("DRIVER_ADAPTER").ok();
         let driver_adapter_config = std::env::var("DRIVER_ADAPTER_CONFIG")
             .map(|config| serde_json::from_str::<DriverAdapterConfig>(config.as_str()).ok())
@@ -160,31 +178,24 @@ impl TestConfig {
         })
     }
 
-    /// if the loaded value for external_test_executor is "default" (case insensitive),
-    /// and the workspace_root is set, then use the default external test executor.
-    fn fill_defaults(&mut self) {
+    fn workspace_root() -> Option<PathBuf> {
+        env::var("WORKSPACE_ROOT").ok().map(PathBuf::from)
+    }
+
+    pub fn external_test_executor_path(&self) -> Option<String> {
         const DEFAULT_TEST_EXECUTOR: &str =
             "query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh";
-
-        if self
-            .external_test_executor
+        self.external_test_executor
             .as_ref()
-            .filter(|s| s.eq_ignore_ascii_case("default"))
-            .is_some()
-        {
-            self.external_test_executor = Self::workspace_root()
-                .map(|path| path.join(DEFAULT_TEST_EXECUTOR))
-                .or_else(|| {
+            .and_then(|_| {
+                Self::workspace_root().or_else(|| {
                     exit_with_message(
                         "WORKSPACE_ROOT needs to be correctly set to the root of the prisma-engines repository",
                     )
                 })
-                .and_then(|path| path.to_str().map(|s| s.to_owned()));
-        }
-    }
-
-    fn workspace_root() -> Option<PathBuf> {
-        env::var("WORKSPACE_ROOT").ok().map(PathBuf::from)
+            })
+            .map(|path| path.join(DEFAULT_TEST_EXECUTOR))
+            .and_then(|path| path.to_str().map(|s| s.to_owned()))
     }
 
     fn validate(&self) {
@@ -212,7 +223,7 @@ impl TestConfig {
             Err(err) => exit_with_message(&err.to_string()),
         }
 
-        if let Some(file) = self.external_test_executor.as_ref() {
+        if let Some(file) = self.external_test_executor_path().as_ref() {
             let path = PathBuf::from(file);
             let md = path.metadata();
             if !path.exists() || md.is_err() || !md.unwrap().is_file() {
@@ -265,8 +276,8 @@ impl TestConfig {
         self.is_ci
     }
 
-    pub fn external_test_executor(&self) -> Option<&str> {
-        self.external_test_executor.as_deref()
+    pub fn external_test_executor(&self) -> Option<TestExecutor> {
+        self.external_test_executor.clone()
     }
 
     pub fn driver_adapter(&self) -> Option<&str> {
@@ -297,11 +308,16 @@ impl TestConfig {
         vec!(
             (
                 "DRIVER_ADAPTER".to_string(), 
-                self.driver_adapter.clone().unwrap_or_default()),
+                self.driver_adapter.clone().unwrap_or_default()
+            ),
             (
                 "DRIVER_ADAPTER_CONFIG".to_string(),
                 self.json_stringify_driver_adapter_config()
             ),
+            (
+                "EXTERNAL_TEST_EXECUTOR".to_string(),
+                self.external_test_executor.clone().unwrap_or(TestExecutor::Napi).to_string(),
+            ),
             (
                 "PRISMA_DISABLE_QUAINT_EXECUTORS".to_string(),
                 "1".to_string(),