feat: accelerate bun through caches (#3909)

* all * all * all * all * all * simplify extract
windmill-labs · Jun 13, 2024 · 28afe59 · 28afe59
1 parent 4f0e111
commit 28afe59
Show file tree

Hide file tree

Showing 7 changed files with 140 additions and 140 deletions.
diff --git a/backend/src/main.rs b/backend/src/main.rs
@@ -53,9 +53,9 @@ use windmill_common::METRICS_ADDR;
 use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
 
 use windmill_worker::{
-    BUN_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
-    GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
-    TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
+    BUN_CACHE_DIR, BUN_TAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
+    GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
+    POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
 };
 
 use crate::monitor::{
@@ -709,6 +709,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
         BUN_CACHE_DIR,
+        BUN_TAR_CACHE_DIR,
         GO_CACHE_DIR,
         GO_BIN_CACHE_DIR,
         HUB_CACHE_DIR,

diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs
@@ -28,8 +28,8 @@ use windmill_common::worker::TMP_DIR;
 use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH;
 use windmill_common::variables::get_workspace_key;
 
-use crate::concurrency_groups::join_concurrency_key;
 use crate::add_webhook_allowed_origin;
+use crate::concurrency_groups::join_concurrency_key;
 use crate::db::ApiAuthed;
 
 use crate::{
@@ -1327,9 +1327,7 @@ async fn list_filtered_uuids(
 ) -> error::JsonResult<Vec<Uuid>> {
     require_admin(authed.is_admin, &authed.username)?;
 
-    let mut sqlb = SqlBuilder::select_from("queue")
-        .fields(&["id"])
-        .clone();
+    let mut sqlb = SqlBuilder::select_from("queue").fields(&["id"]).clone();
 
     sqlb = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb);
 
@@ -3935,13 +3933,16 @@ async fn add_batch_jobs(
         )
         .fetch_all(&db)
         .await?;
-    sqlx::query!(
-        "INSERT INTO concurrency_key (job_id, key) SELECT id, $1 FROM unnest($2::uuid[]) as id",
-        custom_concurrency_key,
-        &uuids
-    )
-    .execute(&db)
-    .await?;
+
+    if let Some(custom_concurrency_key) = custom_concurrency_key {
+        sqlx::query!(
+            "INSERT INTO concurrency_key (job_id, key) SELECT id, $1 FROM unnest($2::uuid[]) as id",
+            custom_concurrency_key,
+            &uuids
+        )
+        .execute(&db)
+        .await?;
+    }
 
     Ok(Json(uuids))
 }

diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml
@@ -11,10 +11,10 @@ path = "src/lib.rs"
 [features]
 default = []
 prometheus = ["dep:prometheus", "windmill-common/prometheus"]
-enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
+enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
 benchmark = ["windmill-queue/benchmark"]
 flamegraph = []
-parquet = ["windmill-common/parquet", "dep:object_store", "dep:tar"]
+parquet = ["windmill-common/parquet", "dep:object_store"]
 flow_testing = []
 cloud = []
 
@@ -71,7 +71,7 @@ base64.workspace = true
 gcp_auth = { workspace = true, optional = true }
 rust_decimal.workspace = true
 jsonwebtoken = { workspace = true, optional = true }
-sha2 = { workspace = true, optional = true }
+sha2.workspace = true
 pem = { workspace = true, optional = true }
 urlencoding.workspace = true
 nix.workspace = true
@@ -81,7 +81,7 @@ hex.workspace = true
 tiberius = { workspace = true, optional = true }
 tokio-util = { workspace = true, optional = true }
 openidconnect = { workspace = true, optional = true}
-tar = { workspace = true, optional = true}
+tar.workspace = true
 object_store = { workspace = true, optional = true}
 convert_case.workspace = true