qe: Allow to run query engine test suite against wasm engines (#4467)

* qe: Allow to run query engine test suite against wasm engines Intoroduces new environment variable/test config option, `EXTERNAL_TEST_EXECUTOR_ENGINE` that can be either `NAPI` (default) or `WASM` that affects which engine external test executor will start with. Adds a bunch of additional test configs for driver adapters that duplicate existing ones, but run against WASM engine. Close prisma/team-orm#551 * Fixes & clippy * Add GH Actions jobs * Install wasm-pack * Comment out WASM tests * Remove `external_test_executor_engine` option Specify engine option in `external_test_executor` * fix native tests * Adjust makefile and fix wasm build * Docs * Uncomment dependency
prisma · Nov 22, 2023 · 2c867f4 · 2c867f4
1 parent 63a4fd9
commit 2c867f4
Show file tree

Hide file tree

Showing 20 changed files with 209 additions and 72 deletions.
diff --git a/.envrc b/.envrc
@@ -23,7 +23,7 @@ export QE_LOG_LEVEL=debug # Set it to "trace" to enable query-graph debugging lo
 # export FMT_SQL=1 # Uncomment it to enable logging formatted SQL queries
 
 ### Uncomment to run driver adapters tests. See query-engine-driver-adapters.yml workflow for how tests run in CI.
-# export EXTERNAL_TEST_EXECUTOR="$(pwd)/query-engine/driver-adapters/js/connector-test-kit-executor/script/start_node.sh"
+# export EXTERNAL_TEST_EXECUTOR="napi"
 # export DRIVER_ADAPTER=pg # Set to pg, neon or planetscale
 # export PRISMA_DISABLE_QUAINT_EXECUTORS=1 # Disable quaint executors for driver adapters
 # export DRIVER_ADAPTER_URL_OVERRIDE ="postgres://USER:PASSWORD@DATABASExxxx" # Override the database url for the driver adapter tests

diff --git a/.github/workflows/query-engine-driver-adapters.yml b/.github/workflows/query-engine-driver-adapters.yml
@@ -25,12 +25,22 @@ jobs:
       fail-fast: false
       matrix:
         adapter:
-          - name: 'pg'
+          - name: 'pg (napi)'
             setup_task: 'dev-pg-postgres13'
-          - name: 'neon:ws'
+          - name: 'neon:ws (napi)'
             setup_task: 'dev-neon-ws-postgres13'
-          - name: 'libsql'
+          - name: 'libsql (napi)'
             setup_task: 'dev-libsql-sqlite'
+          # TODO: uncomment when WASM engine is functional
+          # - name: 'pg (wasm)'
+          #   setup_task: 'dev-pg-postgres13-wasm'
+          #   needs_wasm_pack: true
+          # - name: 'neon:ws (wasm)'
+          #   setup_task: 'dev-neon-ws-postgres13-wasm'
+          #   needs_wasm_pack: true
+          # - name: 'libsql (wasm)'
+          #   setup_task: 'dev-libsql-sqlite-wasm'
+          #   needs_wasm_pack: true
         node_version: ['18']
     env:
       LOG_LEVEL: 'info' # Set to "debug" to trace the query engine and node process running the driver adapter
@@ -85,9 +95,13 @@ jobs:
             echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
           fi
 
-      - run: make ${{ matrix.adapter.setup_task }}
-
       - uses: dtolnay/rust-toolchain@stable
 
+      - name: 'Install wasm-pack'
+        if: ${{ matrix.adapter.needs_wasm_pack }}
+        run: cargo install wasm-pack
+
+      - run: make ${{ matrix.adapter.setup_task }}
+
       - name: 'Run tests'
         run: cargo test --package query-engine-tests -- --test-threads=1
diff --git a/Makefile b/Makefile
@@ -49,8 +49,11 @@ ifndef DRIVER_ADAPTER
 	cargo test --package query-engine-tests
 else
 	@echo "Executing query engine tests with $(DRIVER_ADAPTER) driver adapter"; \
-	# Add your actual command for the "test-driver-adapter" task here
-	$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER);
+	if [ "$(ENGINE)" = "wasm" ]; then \
+		$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER)-wasm; \
+	else \
+		$(MAKE) test-driver-adapter-$(DRIVER_ADAPTER); \
+	fi
 endif
 
 test-qe-verbose:
@@ -91,6 +94,12 @@ test-libsql-sqlite: dev-libsql-sqlite test-qe-st
 
 test-driver-adapter-libsql: test-libsql-sqlite
 
+dev-libsql-sqlite-wasm: build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/libsql-sqlite-wasm $(CONFIG_FILE)
+
+test-libsql-sqlite-wasm: dev-libsql-sqlite-wasm test-qe-st
+test-driver-adapter-libsql-sqlite-wasm: test-libsql-sqlite-wasm
+
 start-postgres9:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9
 
@@ -121,14 +130,20 @@ start-postgres13:
 dev-postgres13: start-postgres13
 	cp $(CONFIG_PATH)/postgres13 $(CONFIG_FILE)
 
-start-pg-postgres13: build-qe-napi build-connector-kit-js start-postgres13
+start-pg-postgres13: start-postgres13
 
-dev-pg-postgres13: start-pg-postgres13
+dev-pg-postgres13: start-pg-postgres13 build-qe-napi build-connector-kit-js
 	cp $(CONFIG_PATH)/pg-postgres13 $(CONFIG_FILE)
 
 test-pg-postgres13: dev-pg-postgres13 test-qe-st
 
+dev-pg-postgres13-wasm: start-pg-postgres13 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/pg-postgres13-wasm $(CONFIG_FILE)
+
+test-pg-postgres13-wasm: dev-pg-postgres13-wasm test-qe-st
+
 test-driver-adapter-pg: test-pg-postgres13
+test-driver-adapter-pg-wasm: test-pg-postgres13-wasm
 
 start-neon-postgres13:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-postgres13
@@ -138,7 +153,13 @@ dev-neon-ws-postgres13: start-neon-postgres13 build-qe-napi build-connector-kit-
 
 test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st
 
+dev-neon-ws-postgres13-wasm: start-neon-postgres13 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/neon-ws-postgres13-wasm $(CONFIG_FILE)
+
+test-neon-ws-postgres13-wasm: dev-neon-ws-postgres13-wasm test-qe-st
+
 test-driver-adapter-neon: test-neon-ws-postgres13
+test-driver-adapter-neon-wasm: test-neon-ws-postgres13-wasm
 
 start-postgres14:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres14
@@ -270,7 +291,13 @@ dev-planetscale-vitess8: start-planetscale-vitess8 build-qe-napi build-connector
 
 test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st
 
+dev-planetscale-vitess8-wasm: start-planetscale-vitess8 build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/planetscale-vitess8-wasm $(CONFIG_FILE)
+
+test-planetscale-vitess8-wasm: dev-planetscale-vitess8-wasm test-qe-st
+
 test-driver-adapter-planetscale: test-planetscale-vitess8
+test-driver-adapter-planetscale-wasm: test-planetscale-vitess8-wasm
 
 ######################
 # Local dev commands #
@@ -279,6 +306,9 @@ test-driver-adapter-planetscale: test-planetscale-vitess8
 build-qe-napi:
 	cargo build --package query-engine-node-api
 
+build-qe-wasm:
+	cd query-engine/query-engine-wasm && ./build.sh
+
 build-connector-kit-js: build-driver-adapters
 	cd query-engine/driver-adapters && pnpm i && pnpm build
 

diff --git a/query-engine/connector-test-kit-rs/README.md b/query-engine/connector-test-kit-rs/README.md
@@ -82,23 +82,24 @@ drivers the code that actually communicates with the databases. See [`adapter-*`
 
 To run tests through a driver adapters, you should also configure the following environment variables:
 
-* `EXTERNAL_TEST_EXECUTOR`: tells the query engine test kit to use an external process to run the queries, this is a node process running a program that will read the queries to run from STDIN, and return responses to STDOUT. The connector kit follows a protocol over JSON RPC for this communication.
 * `DRIVER_ADAPTER`: tells the test executor to use a particular driver adapter. Set to `neon`, `planetscale` or any other supported adapter.
 * `DRIVER_ADAPTER_CONFIG`: a json string with the configuration for the driver adapter. This is adapter specific. See the [github workflow for driver adapter tests](.github/workflows/query-engine-driver-adapters.yml) for examples on how to configure the driver adapters.
+* `ENGINE`: can be used to run either `wasm` or `napi` version of the engine.
 
 Example:
 
 ```shell
 export EXTERNAL_TEST_EXECUTOR="$WORKSPACE_ROOT/query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh"
 export DRIVER_ADAPTER=neon
+export ENGINE=wasm
 export DRIVER_ADAPTER_CONFIG ='{ "proxyUrl": "127.0.0.1:5488/v1" }'
 ````
 
 We have provided helpers to run the query-engine tests with driver adapters, these helpers set all the required environment
 variables for you:
 
 ```shell
-DRIVER_ADAPTER=$adapter make test-qe
+DRIVER_ADAPTER=$adapter ENGINE=$engine make test-qe
 ```
 
 Where `$adapter` is one of the supported adapters: `neon`, `planetscale`, `libsql`.

diff --git a/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs b/query-engine/connector-test-kit-rs/query-tests-setup/src/config.rs
@@ -3,10 +3,25 @@ use crate::{
     PostgresConnectorTag, SqlServerConnectorTag, SqliteConnectorTag, TestResult, VitessConnectorTag,
 };
 use serde::Deserialize;
-use std::{convert::TryFrom, env, fs::File, io::Read, path::PathBuf};
+use std::{convert::TryFrom, env, fmt::Display, fs::File, io::Read, path::PathBuf};
 
 static TEST_CONFIG_FILE_NAME: &str = ".test_config";
 
+#[derive(Debug, Deserialize, Clone)]
+pub enum TestExecutor {
+    Napi,
+    Wasm,
+}
+
+impl Display for TestExecutor {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TestExecutor::Napi => f.write_str("Napi"),
+            TestExecutor::Wasm => f.write_str("Wasm"),
+        }
+    }
+}
+
 /// The central test configuration.
 #[derive(Debug, Default, Deserialize)]
 pub struct TestConfig {
@@ -24,8 +39,9 @@ pub struct TestConfig {
     /// Used when testing driver adapters, this process is expected to be a javascript process
     /// loading the library engine (as a library, or WASM modules) and providing it with a
     /// driver adapter.
+    /// Possible values: Napi, Wasm
     /// Env key: `EXTERNAL_TEST_EXECUTOR`
-    external_test_executor: Option<String>,
+    external_test_executor: Option<TestExecutor>,
 
     /// The driver adapter to use when running tests, will be forwarded to the external test
     /// executor by setting the `DRIVER_ADAPTER` env var when spawning the executor process
@@ -85,12 +101,11 @@ fn exit_with_message(msg: &str) -> ! {
 impl TestConfig {
     /// Loads a configuration. File-based config has precedence over env config.
     pub(crate) fn load() -> Self {
-        let mut config = match Self::from_file().or_else(Self::from_env) {
+        let config = match Self::from_file().or_else(Self::from_env) {
             Some(config) => config,
             None => exit_with_message(CONFIG_LOAD_FAILED),
         };
 
-        config.fill_defaults();
         config.validate();
         config.log_info();
 
@@ -107,8 +122,8 @@ impl TestConfig {
             self.connector_version().unwrap_or_default()
         );
         println!("* CI? {}", self.is_ci);
-        if self.external_test_executor.as_ref().is_some() {
-            println!("* External test executor: {}", self.external_test_executor().unwrap_or_default());
+        if let Some(external_test_executor) = self.external_test_executor.as_ref() {
+            println!("* External test executor: {}", external_test_executor);
             println!("* Driver adapter: {}", self.driver_adapter().unwrap_or_default());
             println!("* Driver adapter url override: {}", self.json_stringify_driver_adapter_config());
         }
@@ -118,7 +133,10 @@ impl TestConfig {
     fn from_env() -> Option<Self> {
         let connector = std::env::var("TEST_CONNECTOR").ok();
         let connector_version = std::env::var("TEST_CONNECTOR_VERSION").ok();
-        let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR").ok();
+        let external_test_executor = std::env::var("EXTERNAL_TEST_EXECUTOR")
+            .map(|value| serde_json::from_str::<TestExecutor>(&value).ok())
+            .unwrap_or_default();
+
         let driver_adapter = std::env::var("DRIVER_ADAPTER").ok();
         let driver_adapter_config = std::env::var("DRIVER_ADAPTER_CONFIG")
             .map(|config| serde_json::from_str::<serde_json::Value>(config.as_str()).ok())
@@ -155,31 +173,24 @@ impl TestConfig {
         })
     }
 
-    /// if the loaded value for external_test_executor is "default" (case insensitive),
-    /// and the workspace_root is set, then use the default external test executor.
-    fn fill_defaults(&mut self) {
+    fn workspace_root() -> Option<PathBuf> {
+        env::var("WORKSPACE_ROOT").ok().map(PathBuf::from)
+    }
+
+    pub fn external_test_executor_path(&self) -> Option<String> {
         const DEFAULT_TEST_EXECUTOR: &str =
             "query-engine/driver-adapters/connector-test-kit-executor/script/start_node.sh";
-
-        if self
-            .external_test_executor
+        self.external_test_executor
             .as_ref()
-            .filter(|s| s.eq_ignore_ascii_case("default"))
-            .is_some()
-        {
-            self.external_test_executor = Self::workspace_root()
-                .map(|path| path.join(DEFAULT_TEST_EXECUTOR))
-                .or_else(|| {
+            .and_then(|_| {
+                Self::workspace_root().or_else(|| {
                     exit_with_message(
                         "WORKSPACE_ROOT needs to be correctly set to the root of the prisma-engines repository",
                     )
                 })
-                .and_then(|path| path.to_str().map(|s| s.to_owned()));
-        }
-    }
-
-    fn workspace_root() -> Option<PathBuf> {
-        env::var("WORKSPACE_ROOT").ok().map(PathBuf::from)
+            })
+            .map(|path| path.join(DEFAULT_TEST_EXECUTOR))
+            .and_then(|path| path.to_str().map(|s| s.to_owned()))
     }
 
     fn validate(&self) {
@@ -206,7 +217,7 @@ impl TestConfig {
             Err(err) => exit_with_message(&err.to_string()),
         }
 
-        if let Some(file) = self.external_test_executor.as_ref() {
+        if let Some(file) = self.external_test_executor_path().as_ref() {
             let path = PathBuf::from(file);
             let md = path.metadata();
             if !path.exists() || md.is_err() || !md.unwrap().is_file() {
@@ -259,8 +270,8 @@ impl TestConfig {
         self.is_ci
     }
 
-    pub fn external_test_executor(&self) -> Option<&str> {
-        self.external_test_executor.as_deref()
+    pub fn external_test_executor(&self) -> Option<TestExecutor> {
+        self.external_test_executor.clone()
     }
 
     pub fn driver_adapter(&self) -> Option<&str> {
@@ -294,11 +305,16 @@ impl TestConfig {
         vec!(
             (
                 "DRIVER_ADAPTER".to_string(), 
-                self.driver_adapter.clone().unwrap_or_default()),
+                self.driver_adapter.clone().unwrap_or_default()
+            ),
             (
                 "DRIVER_ADAPTER_CONFIG".to_string(),
                 self.json_stringify_driver_adapter_config()
             ),
+            (
+                "EXTERNAL_TEST_EXECUTOR".to_string(),
+                self.external_test_executor.clone().unwrap_or(TestExecutor::Napi).to_string(),
+            ),
             (
                 "PRISMA_DISABLE_QUAINT_EXECUTORS".to_string(),
                 "1".to_string(),

diff --git a/...y-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs b/...y-engine/connector-test-kit-rs/query-tests-setup/src/connector_tag/js/external_process.rs
@@ -74,7 +74,7 @@ impl ExecutorProcess {
         };
 
         self.task_handle.send((method_call, sender)).await?;
-        let raw_response = receiver.await?;
+        let raw_response = receiver.await??;
         tracing::debug!(%raw_response);
         let response = serde_json::from_value(raw_response)?;
         Ok(response)
@@ -91,22 +91,25 @@ pub(super) static EXTERNAL_PROCESS: Lazy<ExecutorProcess> =
         }
     });
 
-type ReqImpl = (jsonrpc_core::MethodCall, oneshot::Sender<serde_json::value::Value>);
+type ReqImpl = (
+    jsonrpc_core::MethodCall,
+    oneshot::Sender<Result<serde_json::value::Value>>,
+);
 
 fn start_rpc_thread(mut receiver: mpsc::Receiver<ReqImpl>) -> Result<()> {
     use std::process::Stdio;
     use tokio::process::Command;
 
     let path = crate::CONFIG
-        .external_test_executor()
+        .external_test_executor_path()
         .unwrap_or_else(|| exit_with_message(1, "start_rpc_thread() error: external test executor is not set"));
 
     tokio::runtime::Builder::new_current_thread()
         .enable_io()
         .build()
         .unwrap()
         .block_on(async move {
-            let process = match Command::new(path)
+            let process = match Command::new(&path)
                 .envs(CONFIG.for_external_executor())
                 .stdin(Stdio::piped())
                 .stdout(Stdio::piped())
@@ -119,7 +122,7 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver<ReqImpl>) -> Result<()> {
 
             let mut stdout = BufReader::new(process.stdout.unwrap()).lines();
             let mut stdin = process.stdin.unwrap();
-            let mut pending_requests: HashMap<jsonrpc_core::Id, oneshot::Sender<serde_json::value::Value>> =
+            let mut pending_requests: HashMap<jsonrpc_core::Id, oneshot::Sender<Result<serde_json::value::Value>>> =
                 HashMap::new();
 
             loop {
@@ -140,10 +143,11 @@ fn start_rpc_thread(mut receiver: mpsc::Receiver<ReqImpl>) -> Result<()> {
                                                 // The other end may be dropped if the whole
                                                 // request future was dropped and not polled to
                                                 // completion, so we ignore send errors here.
-                                                _ = sender.send(success.result);
+                                                _ = sender.send(Ok(success.result));
                                             }
                                             jsonrpc_core::Output::Failure(err) => {
-                                                panic!("error response from jsonrpc: {err:?}")
+                                                tracing::error!("error response from jsonrpc: {err:?}");
+                                                _ = sender.send(Err(Box::new(err.error)));
                                             }
                                         }
                                     }

diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite
@@ -1,5 +1,5 @@
 {
     "connector": "sqlite",
     "driver_adapter": "libsql",
-    "external_test_executor": "default"
+    "external_test_executor": "Napi"
 }
diff --git a/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm b/query-engine/connector-test-kit-rs/test-configs/libsql-sqlite-wasm
@@ -0,0 +1,5 @@
+{
+    "connector": "sqlite",
+    "driver_adapter": "libsql",
+    "external_test_executor": "Wasm"
+}