qe-wasm: split by connector (#4681)

* qe-wasm: Split query engine by connector

Intoroduces separate features for mysql, postgresql and sqlite
connectors and builds 3 individual engines for each. All engines are put
and published as a single package (with connector-specific) engine in
subdirectory. It is up to a client to pick the correct one.

Contributes to prisma/team-orm#891

* feat: some of the review comments

* Fix size job

* Fix size again

.github/workflows/test-unit-tests.yml

@@ -28,7 +28,7 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
 
       - run: |
-          cargo test --workspace \
+          cargo test --workspace --all-features \
                 --exclude=quaint \
                 --exclude=query-engine \
                 --exclude=query-engine-node-api \

.github/workflows/wasm-size.yml

@@ -1,14 +1,15 @@
 name: "QE: WASM size"
 on:
-  pull_request:
-    paths-ignore:
-      - ".github/**"
-      - "!.github/workflows/wasm-size.yml"
-      - ".buildkite/**"
-      - "*.md"
-      - "LICENSE"
-      - "CODEOWNERS"
-      - "renovate.json"
+  workflow_dispatch:
+  # pull_request:
+  #   paths-ignore:
+  #     - ".github/**"
+  #     - "!.github/workflows/wasm-size.yml"
+  #     - ".buildkite/**"
+  #     - "*.md"
+  #     - "LICENSE"
+  #     - "CODEOWNERS"
+  #     - "renovate.json"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

libs/user-facing-errors/Cargo.toml

@@ -11,7 +11,7 @@ backtrace = "0.3.40"
 tracing = "0.1"
 indoc.workspace = true
 itertools.workspace = true
-quaint = { path = "../../quaint", optional = true }
+quaint = { path = "../../quaint", default-features = false, optional = true }
 
 [features]
 default = []

nix/all-engines.nix

@@ -117,7 +117,7 @@ in
 
   packages.build-engine-wasm = pkgs.writeShellApplication { 
     name = "build-engine-wasm";
-      runtimeInputs = with pkgs; [ git rustup wasm-pack wasm-bindgen-cli binaryen jq iconv];
+      runtimeInputs = with pkgs; [ git rustup wasm-bindgen-cli binaryen jq iconv ];
       text = ''            
       cd query-engine/query-engine-wasm        
       WASM_BUILD_PROFILE=release ./build.sh "$1" "$2"
@@ -128,19 +128,26 @@ in
     ({ profile }: stdenv.mkDerivation {
       name = "query-engine-wasm-gz";
       inherit src;
+      buildInputs = with pkgs; [ iconv ];
 
       buildPhase = ''
       export HOME=$(mktemp -dt wasm-engine-home-XXXX)
       
       OUT_FOLDER=$(mktemp -dt wasm-engine-out-XXXX)
       ${self'.packages.build-engine-wasm}/bin/build-engine-wasm "0.0.0" "$OUT_FOLDER" 
-      gzip -ckn "$OUT_FOLDER/query_engine_bg.wasm" > query_engine_bg.wasm.gz
+
+      for provider in "postgresql" "mysql" "sqlite"; do
+        gzip -ckn "$OUT_FOLDER/$provider/query_engine_bg.wasm" > "query-engine-$provider.wasm.gz"
+      done
       '';
 
       installPhase = ''
+      set +x
       mkdir -p $out
-      cp "$OUT_FOLDER/query_engine_bg.wasm" $out/
-      cp query_engine_bg.wasm.gz $out/
+      for provider in "postgresql" "mysql" "sqlite"; do
+        cp "$OUT_FOLDER/$provider/query_engine_bg.wasm" "$out/query-engine-$provider.wasm"
+        cp "query-engine-$provider.wasm.gz" "$out/"
+      done
       '';
     })
     { profile = "release"; };

nix/publish-engine-size.nix

@@ -46,8 +46,12 @@
       ${self'.packages.update-engine-size}/bin/update-engine-size             \
           ${self'.packages.query-engine-bin-and-lib}/bin/query-engine         \
           ${self'.packages.query-engine-bin-and-lib}/lib/libquery_engine.node \
-          ${self'.packages.query-engine-wasm-gz}/query_engine_bg.wasm.gz      \
-          ${self'.packages.query-engine-wasm-gz}/query_engine_bg.wasm
+          ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm.gz           \
+          ${self'.packages.query-engine-wasm-gz}/query-engine-postgresql.wasm              \
+          ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm.gz                \
+          ${self'.packages.query-engine-wasm-gz}/query-engine-mysql.wasm                   \
+          ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm.gz               \
+          ${self'.packages.query-engine-wasm-gz}/query-engine-sqlite.wasm
 
       git add "$CSV_PATH"
       git commit --quiet -m "update engines size for $CURRENT_COMMIT_SHORT"

quaint/src/ast/function.rs

@@ -44,7 +44,7 @@ pub use minimum::*;
 pub use row_number::*;
 #[cfg(feature = "postgresql")]
 pub use row_to_json::*;
-#[cfg(feature = "mysql")]
+#[cfg(any(feature = "mysql", feature = "postgresql"))]
 pub use search::*;
 pub use sum::*;
 pub use upper::*;

quaint/src/error/mod.rs

@@ -15,8 +15,11 @@ use std::time::Duration;
 #[cfg(not(target_arch = "wasm32"))]
 pub use native::NativeErrorKind;
 
+#[cfg(feature = "mysql")]
 pub use crate::connector::mysql::MysqlError;
+#[cfg(feature = "postgresql")]
 pub use crate::connector::postgres::PostgresError;
+#[cfg(feature = "sqlite")]
 pub use crate::connector::sqlite::SqliteError;
 pub(crate) use name::Name;
 

query-engine/connectors/sql-query-connector/Cargo.toml

@@ -4,7 +4,11 @@ name = "sql-query-connector"
 version = "0.1.0"
 
 [features]
+default = ["postgresql", "sqlite", "mysql"]
 vendored-openssl = ["quaint/vendored-openssl"]
+postgresql = ["quaint/postgresql"]
+sqlite = ["quaint/sqlite"]
+mysql = ["quaint/mysql"]
 
 # Enable Driver Adapters
 driver-adapters = []
@@ -18,7 +22,7 @@ futures = "0.3"
 itertools.workspace = true
 once_cell = "1.3"
 rand = "0.7"
-serde_json = {version = "1.0", features = ["float_roundtrip"]}
+serde_json = { version = "1.0", features = ["float_roundtrip"] }
 thiserror = "1.0"
 tokio = { version = "1.0", features = ["macros", "time"] }
 tracing = "0.1"
@@ -32,7 +36,7 @@ cuid = { git = "https://github.com/prisma/cuid-rust", branch = "wasm32-support"
 quaint.workspace = true
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-quaint = { path = "../../../quaint" }
+quaint = { path = "../../../quaint", default-features = false }
 
 [dependencies.connector-interface]
 package = "query-connector"

query-engine/driver-adapters/Cargo.toml

@@ -3,6 +3,11 @@ name = "driver-adapters"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+mysql = ["quaint/mysql"]
+sqlite = ["quaint/sqlite"]
+postgresql = ["quaint/postgresql"]
+
 [dependencies]
 async-trait = "0.1"
 once_cell = "1.15"
@@ -28,7 +33,7 @@ napi-derive.workspace = true
 quaint.workspace = true
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-quaint = { path = "../../quaint" }
+quaint = { path = "../../quaint", default-features = false }
 js-sys.workspace = true
 serde-wasm-bindgen.workspace = true
 wasm-bindgen.workspace = true

query-engine/driver-adapters/executor/package.json

@@ -12,7 +12,9 @@
   },
   "tsup": {
     "external": [
-      "../../../query-engine-wasm/pkg/query_engine_bg.js"
+      "../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js",
+      "../../../query-engine-wasm/pkg/mysql/query_engine_bg.js",
+      "../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js"
     ]
   },
   "keywords": [],

query-engine/driver-adapters/executor/src/qe.ts

@@ -35,7 +35,8 @@ export async function initQueryEngine(
   const options = queryEngineOptions(datamodel);
 
   if (engineType === "Wasm") {
-    const { WasmQueryEngine } = await import("./wasm");
+    const { getEngineForProvider } = await import("./wasm");
+    const WasmQueryEngine = await getEngineForProvider(adapter.provider)
     return new WasmQueryEngine(options, logCallback, adapter);
   } else {
     const { QueryEngine } = loadNapiEngine();

query-engine/driver-adapters/executor/src/wasm.ts

@@ -1,13 +1,35 @@
-import * as wasm from '../../../query-engine-wasm/pkg/query_engine_bg.js'
+import * as wasmPostgres from '../../../query-engine-wasm/pkg/postgresql/query_engine_bg.js'
+import * as wasmMysql from '../../../query-engine-wasm/pkg/mysql/query_engine_bg.js'
+import * as wasmSqlite from '../../../query-engine-wasm/pkg/sqlite/query_engine_bg.js'
 import fs from 'node:fs/promises'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
 const dirname = path.dirname(fileURLToPath(import.meta.url))
 
-const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', 'query_engine_bg.wasm'))
-const module = new WebAssembly.Module(bytes) 
-const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': wasm })
-wasm.__wbg_set_wasm(instance.exports);
+const wasm = {
+    postgres: wasmPostgres,
+    mysql: wasmMysql,
+    sqlite: wasmSqlite
+}
 
-export const WasmQueryEngine = wasm.QueryEngine
+type EngineName = keyof typeof wasm;
+
+const initializedModules = new Set<EngineName>()
+
+
+
+export async function getEngineForProvider(provider: EngineName) {
+    const engine = wasm[provider]
+    if (!initializedModules.has(provider)) {
+        const subDir = provider === 'postgres' ? 'postgresql' : provider
+        const bytes = await fs.readFile(path.resolve(dirname, '..', '..', '..', 'query-engine-wasm', 'pkg', subDir, 'query_engine_bg.wasm'))
+        console.error(bytes)
+        const module = new WebAssembly.Module(bytes) 
+        const instance = new WebAssembly.Instance(module, { './query_engine_bg.js': engine })
+        engine.__wbg_set_wasm(instance.exports);
+        initializedModules.add(provider)
+    }
+
+    return engine.QueryEngine
+}

query-engine/driver-adapters/src/conversion/mod.rs

@@ -1,8 +1,11 @@
 pub(crate) mod js_arg;
 pub(crate) mod js_to_quaint;
 
+#[cfg(feature = "mysql")]
 pub(crate) mod mysql;
+#[cfg(feature = "postgresql")]
 pub(crate) mod postgres;
+#[cfg(feature = "sqlite")]
 pub(crate) mod sqlite;
 
 pub use js_arg::JSArg;

query-engine/driver-adapters/src/error.rs

@@ -1,6 +1,14 @@
-use quaint::error::{MysqlError, PostgresError, SqliteError};
+#[cfg(feature = "mysql")]
+use quaint::error::MysqlError;
+
+#[cfg(feature = "postgresql")]
+use quaint::error::PostgresError;
+
+#[cfg(feature = "sqlite")]
+use quaint::error::SqliteError;
 use serde::Deserialize;
 
+#[cfg(feature = "postgresql")]
 #[derive(Deserialize)]
 #[serde(remote = "PostgresError")]
 pub struct PostgresErrorDef {
@@ -12,6 +20,7 @@ pub struct PostgresErrorDef {
     hint: Option<String>,
 }
 
+#[cfg(feature = "mysql")]
 #[derive(Deserialize)]
 #[serde(remote = "MysqlError")]
 pub struct MysqlErrorDef {
@@ -20,6 +29,7 @@ pub struct MysqlErrorDef {
     pub state: String,
 }
 
+#[cfg(feature = "sqlite")]
 #[derive(Deserialize)]
 #[serde(remote = "SqliteError", rename_all = "camelCase")]
 pub struct SqliteErrorDef {
@@ -32,14 +42,15 @@ pub struct SqliteErrorDef {
 /// Wrapper for JS-side errors
 pub(crate) enum DriverAdapterError {
     /// Unexpected JS exception
-    GenericJs {
-        id: i32,
-    },
+    GenericJs { id: i32 },
     UnsupportedNativeDataType {
         #[serde(rename = "type")]
         native_type: String,
     },
+    #[cfg(feature = "postgresql")]
     Postgres(#[serde(with = "PostgresErrorDef")] PostgresError),
+    #[cfg(feature = "mysql")]
     Mysql(#[serde(with = "MysqlErrorDef")] MysqlError),
+    #[cfg(feature = "sqlite")]
     Sqlite(#[serde(with = "SqliteErrorDef")] SqliteError),
 }

query-engine/driver-adapters/src/lib.rs

@@ -34,8 +34,11 @@ impl From<DriverAdapterError> for QuaintError {
                 .build()
             }
             DriverAdapterError::GenericJs { id } => QuaintError::external_error(id),
+            #[cfg(feature = "postgresql")]
             DriverAdapterError::Postgres(e) => e.into(),
+            #[cfg(feature = "mysql")]
             DriverAdapterError::Mysql(e) => e.into(),
+            #[cfg(feature = "sqlite")]
             DriverAdapterError::Sqlite(e) => e.into(),
             // in future, more error types would be added and we'll need to convert them to proper QuaintErrors here
         }

query-engine/driver-adapters/src/queryable.rs

@@ -41,8 +41,11 @@ impl JsBaseQueryable {
     /// visit a quaint query AST according to the provider of the JS connector
     fn visit_quaint_query<'a>(&self, q: QuaintQuery<'a>) -> quaint::Result<(String, Vec<quaint::Value<'a>>)> {
         match self.provider {
+            #[cfg(feature = "mysql")]
             AdapterFlavour::Mysql => visitor::Mysql::build(q),
+            #[cfg(feature = "postgresql")]
             AdapterFlavour::Postgres => visitor::Postgres::build(q),
+            #[cfg(feature = "sqlite")]
             AdapterFlavour::Sqlite => visitor::Sqlite::build(q),
         }
     }
@@ -51,8 +54,11 @@ impl JsBaseQueryable {
         let sql: String = sql.to_string();
 
         let converter = match self.provider {
+            #[cfg(feature = "postgresql")]
             AdapterFlavour::Postgres => conversion::postgres::value_to_js_arg,
+            #[cfg(feature = "sqlite")]
             AdapterFlavour::Sqlite => conversion::sqlite::value_to_js_arg,
+            #[cfg(feature = "mysql")]
             AdapterFlavour::Mysql => conversion::mysql::value_to_js_arg,
         };
 
@@ -125,6 +131,7 @@ impl QuaintQueryable for JsBaseQueryable {
             return Err(Error::builder(ErrorKind::invalid_isolation_level(&isolation_level)).build());
         }
 
+        #[cfg(feature = "sqlite")]
         if self.provider == AdapterFlavour::Sqlite {
             return match isolation_level {
                 IsolationLevel::Serializable => Ok(()),
@@ -138,8 +145,12 @@ impl QuaintQueryable for JsBaseQueryable {
 
     fn requires_isolation_first(&self) -> bool {
         match self.provider {
+            #[cfg(feature = "mysql")]
             AdapterFlavour::Mysql => true,
-            AdapterFlavour::Postgres | AdapterFlavour::Sqlite => false,
+            #[cfg(feature = "postgresql")]
+            AdapterFlavour::Postgres => false,
+            #[cfg(feature = "sqlite")]
+            AdapterFlavour::Sqlite => false,
         }
     }
 }